Implement max-connect facility which makes the server sleep if a
[yazproxy-moved-to-github.git] / src / yaz-proxy.cpp
1 /* $Id: yaz-proxy.cpp,v 1.45 2006-03-30 10:35:15 adam Exp $
2    Copyright (c) 1998-2006, Index Data.
3
4 This file is part of the yazproxy.
5
6 YAZ proxy is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 2, or (at your option) any later
9 version.
10
11 YAZ proxy is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
14 for more details.
15
16 You should have received a copy of the GNU General Public License
17 along with YAZ proxy; see the file LICENSE.  If not, write to the
18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
19 02111-1307, USA.
20  */
21
22 #ifdef WIN32
23 #define HAVE_SYS_STAT_H 1
24 #define HAVE_SYS_TYPES_H 1
25 #endif
26
27 #if HAVE_UNISTD_H
28 #include <unistd.h>
29 #endif
30 #if HAVE_SYS_TIME_H
31 #include <sys/time.h>
32 #endif
33 #if HAVE_SYS_TYPES_H
34 #include <sys/types.h>
35 #endif
36 #if HAVE_SYS_STAT_H
37 #include <sys/stat.h>
38 #endif
39
40 #include <assert.h>
41 #include <stdlib.h>
42 #include <time.h>
43 #include <fcntl.h>
44
45 #include <yaz/srw.h>
46 #include <yaz/marcdisp.h>
47 #include <yaz/yaz-iconv.h>
48 #include <yaz/log.h>
49 #include <yaz/diagbib1.h>
50 #include "proxyp.h"
51 #include <yaz/pquery.h>
52 #include <yaz/otherinfo.h>
53 #include <yaz/charneg.h>
54 #include "msg-thread.h"
55
56 using namespace yazpp_1;
57
58 #ifdef WIN32
59 #define strncasecmp _strnicmp
60 #endif
61
62 #define USE_AUTH_MSG 1
63
64 #if USE_AUTH_MSG
65 class Auth_Msg : public IMsg_Thread {
66 public:
67     int m_ret;
68     IMsg_Thread *handle();
69     void result();
70     Yaz_Proxy *m_proxy;
71     NMEM m_nmem;
72     char *m_apdu_buf;
73     int m_apdu_len;
74     Auth_Msg();
75     virtual ~Auth_Msg();
76 };
77
78 Auth_Msg::Auth_Msg()
79 {
80     m_nmem = nmem_create();
81 }
82
83 Auth_Msg::~Auth_Msg()
84 {
85     nmem_destroy(m_nmem);
86 }
87
88 IMsg_Thread *Auth_Msg::handle()
89 {
90     ODR decode = odr_createmem(ODR_DECODE);
91     Z_APDU *apdu;
92
93     odr_setbuf(decode, m_apdu_buf, m_apdu_len, 0);
94     int r = z_APDU(decode, &apdu, 0, 0);
95     if (!r)
96     {
97         yaz_log(YLOG_WARN, "decode failed in Auth_Msg::handle");
98     }
99     else
100     {
101         m_ret = m_proxy->handle_authentication(apdu);
102     }
103     odr_destroy(decode);
104     return this;
105 }
106
107 void Auth_Msg::result()
108 {
109     if (m_proxy->dec_ref(false))
110     {
111         yaz_log(YLOG_LOG, "Auth_Msg::proxy deleted meanwhile");
112     }
113     else
114     {
115         odr_setbuf(m_proxy->odr_decode(), m_apdu_buf, m_apdu_len, 0);
116         Z_APDU *apdu = 0;
117         int r = z_APDU(m_proxy->odr_decode(), &apdu, 0, 0);
118         if (!r)
119             yaz_log(YLOG_LOG, "Auth_Msg::result z_APDU failed");
120         m_proxy->result_authentication(apdu, m_ret);
121     }
122     delete this;
123 }
124
125 #endif
126
127 void Yaz_Proxy::result_authentication(Z_APDU *apdu, int ret)
128 {
129     if (apdu == 0 || ret == 0)
130     {
131         Z_APDU *apdu_reject = zget_APDU(odr_encode(), Z_APDU_initResponse);
132         *apdu_reject->u.initResponse->result = 0;
133         send_to_client(apdu_reject);
134         dec_ref(false);
135     }
136     else
137     {
138         if (apdu->which == Z_APDU_initRequest)
139         {
140             Yaz_ProxyConfig *cfg = check_reconfigure();
141             if (cfg)
142                 cfg->target_authentication(m_default_target, odr_encode(), 
143                                            apdu->u.initRequest);
144         }
145         handle_incoming_Z_PDU_2(apdu);
146     }
147 }
148
149 static const char *apdu_name(Z_APDU *apdu)
150 {
151     switch (apdu->which)
152     {
153     case Z_APDU_initRequest:
154         return "initRequest";
155     case Z_APDU_initResponse:
156         return "initResponse";
157     case Z_APDU_searchRequest:
158         return "searchRequest";
159     case Z_APDU_searchResponse:
160         return "searchResponse";
161     case Z_APDU_presentRequest:
162         return "presentRequest";
163     case Z_APDU_presentResponse:
164         return "presentResponse";
165     case Z_APDU_deleteResultSetRequest:
166         return "deleteResultSetRequest";
167     case Z_APDU_deleteResultSetResponse:
168         return "deleteResultSetResponse";
169     case Z_APDU_scanRequest:
170         return "scanRequest";
171     case Z_APDU_scanResponse:
172         return "scanResponse";
173     case Z_APDU_sortRequest:
174         return "sortRequest";
175     case Z_APDU_sortResponse:
176         return "sortResponse";
177     case Z_APDU_extendedServicesRequest:
178         return "extendedServicesRequest";
179     case Z_APDU_extendedServicesResponse:
180         return "extendedServicesResponse";
181     case Z_APDU_close:
182         return "close";
183     }
184     return "other";
185 }
186
187 static const char *gdu_name(Z_GDU *gdu)
188 {
189     switch(gdu->which)
190     {
191     case Z_GDU_Z3950:
192         return apdu_name(gdu->u.z3950);
193     case Z_GDU_HTTP_Request:
194         return "HTTP Request";
195     case Z_GDU_HTTP_Response:
196         return "HTTP Response";
197     }
198     return "Unknown request/response";
199 }
200
201 Yaz_Proxy::Yaz_Proxy(IPDU_Observable *the_PDU_Observable,
202                      ISocketObservable *the_socket_observable,
203                      Yaz_Proxy *parent)
204     :
205     Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60)
206 {
207     m_PDU_Observable = the_PDU_Observable;
208     m_socket_observable = the_socket_observable;
209     m_client = 0;
210     m_parent = parent;
211     m_clientPool = 0;
212     m_seqno = 1;
213     m_keepalive_limit_bw = 500000;
214     m_keepalive_limit_pdu = 1000;
215     m_proxyTarget = 0;
216     m_default_target = 0;
217     m_proxy_negotiation_charset = 0;
218     m_proxy_negotiation_lang = 0;
219     m_proxy_negotiation_default_charset = 0;
220     m_charset_converter = new Yaz_CharsetConverter;
221     m_max_clients = 150;
222     m_log_mask = 0;
223     m_seed = time(0);
224     m_client_idletime = 600;
225     m_target_idletime = 600;
226     m_optimize = xstrdup ("1");
227     strcpy(m_session_str, "0 ");
228     m_session_no = 0;
229     m_bytes_sent = 0;
230     m_bytes_recv = 0;
231     m_bw_max = 0;
232     m_pdu_max = 0;
233     m_search_max = 0;
234     m_connect_max = 0;
235     m_timeout_mode = timeout_normal;
236     m_timeout_gdu = 0;
237     m_max_record_retrieve = 0;
238     m_reconfig_flag = 0;
239     m_config_fname = 0;
240     m_request_no = 0;
241     m_flag_invalid_session = 0;
242     m_referenceId = 0;
243     m_referenceId_mem = nmem_create();
244     m_config = 0;
245     m_marcxml_mode = none;
246     m_stylesheet_xsp = 0;
247     m_stylesheet_nprl = 0;
248     m_stylesheet_apdu = 0;
249     m_s2z_stylesheet = 0;
250     m_s2z_database = 0;
251     m_schema = 0;
252     m_backend_type = 0;
253     m_backend_charset = 0;
254     m_frontend_type = 0;
255     m_initRequest_apdu = 0;
256     m_initRequest_mem = 0;
257     m_initRequest_preferredMessageSize = 0;
258     m_initRequest_maximumRecordSize = 0;
259     m_initRequest_options = 0;
260     m_initRequest_version = 0;
261     m_initRequest_oi_negotiation_charsets = 0;
262     m_initRequest_oi_negotiation_num_charsets = 0;
263     m_initRequest_oi_negotiation_langs = 0;
264     m_initRequest_oi_negotiation_num_langs = 0;
265     m_initRequest_oi_negotiation_selected = 0;
266     m_apdu_invalid_session = 0;
267     m_mem_invalid_session = 0;
268     m_s2z_odr_init = 0;
269     m_s2z_odr_search = 0;
270     m_s2z_init_apdu = 0;
271     m_s2z_search_apdu = 0;
272     m_s2z_present_apdu = 0;
273     m_http_keepalive = 0;
274     m_http_version = 0;
275     m_soap_ns = 0;
276     m_s2z_packing = Z_SRW_recordPacking_string;
277 #if HAVE_GETTIMEOFDAY
278     m_time_tv = xmalloc(sizeof(struct timeval));
279     struct timeval *tv = (struct timeval *) m_time_tv;
280     tv->tv_sec = 0;
281     tv->tv_usec = 0;
282 #else
283     m_time_tv = 0;
284 #endif
285     m_usemarcon_ini_stage1 = 0;
286     m_usemarcon_ini_stage2 = 0;
287     m_usemarcon = new Yaz_usemarcon();
288     if (!m_parent)
289         low_socket_open();
290     m_my_thread = 0;
291     m_ref_count = 1;
292     m_main_ptr_dec = false;
293     m_peername = 0;
294 }
295
296 void Yaz_Proxy::inc_ref()
297 {
298     m_ref_count++;
299 }
300
301 Yaz_Proxy::~Yaz_Proxy()
302 {
303     yaz_log(YLOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
304             m_bytes_sent, m_bytes_recv);
305     nmem_destroy(m_initRequest_mem);
306     nmem_destroy(m_mem_invalid_session);
307     nmem_destroy(m_referenceId_mem);
308
309     xfree(m_proxyTarget);
310     xfree(m_default_target);
311     xfree(m_proxy_negotiation_charset);
312     xfree(m_proxy_negotiation_lang);
313     xfree(m_proxy_negotiation_default_charset);
314     delete m_charset_converter;
315     xfree(m_optimize);
316
317 #if HAVE_XSLT
318     if (m_stylesheet_xsp)
319         xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
320 #endif
321     xfree (m_time_tv);
322
323     xfree (m_peername);
324     xfree (m_schema);
325     xfree (m_backend_type);
326     xfree (m_backend_charset);
327     xfree (m_usemarcon_ini_stage1);
328     xfree (m_usemarcon_ini_stage2);
329     delete m_usemarcon;
330     if (m_s2z_odr_init)
331         odr_destroy(m_s2z_odr_init);
332     if (m_s2z_odr_search)
333         odr_destroy(m_s2z_odr_search);
334     if (!m_parent)
335         low_socket_close();
336     if (!m_parent)
337         delete m_my_thread;
338     delete m_config;
339 }
340
341 void Yaz_Proxy::set_debug_mode(int mode)
342 {
343     m_debug_mode = mode;
344 }
345
346 int Yaz_Proxy::set_config(const char *config)
347 {
348     delete m_config;
349     m_config = new Yaz_ProxyConfig();
350     xfree(m_config_fname);
351     m_config_fname = xstrdup(config);
352     int r = m_config->read_xml(config);
353     if (!r)
354         m_config->get_generic_info(&m_log_mask, &m_max_clients,
355                                    &m_connect_max);
356     return r;
357 }
358
359 void Yaz_Proxy::set_default_target(const char *target)
360 {
361     xfree (m_default_target);
362     m_default_target = 0;
363     if (target)
364         m_default_target = (char *) xstrdup (target);
365 }
366
367 void Yaz_Proxy::set_proxy_negotiation (const char *charset, const char *lang,
368                                        const char *default_charset)
369 {
370     yaz_log(YLOG_LOG, "%sSet the proxy negotiation: charset to '%s', "
371         "default charset to '%s', language to '%s'", m_session_str, 
372         charset?charset:"none",
373         default_charset?default_charset:"none",
374         lang?lang:"none");
375     xfree (m_proxy_negotiation_charset);
376     xfree (m_proxy_negotiation_lang);
377     m_proxy_negotiation_charset = m_proxy_negotiation_lang = 0;
378     if (charset)
379         m_proxy_negotiation_charset = (char *) xstrdup (charset);
380     if (lang)
381         m_proxy_negotiation_lang = (char *) xstrdup (lang);
382     if (default_charset)
383         m_proxy_negotiation_default_charset =
384             (char *) xstrdup (default_charset);
385 }
386
387 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
388 {
389     if (m_parent)
390         return m_parent->check_reconfigure();
391
392     Yaz_ProxyConfig *cfg = m_config;
393     if (m_reconfig_flag)
394     {
395         yaz_log(YLOG_LOG, "reconfigure");
396         yaz_log_reopen();
397         if (m_config_fname && cfg)
398         {
399             yaz_log(YLOG_LOG, "reconfigure config %s", m_config_fname);
400             int r = cfg->read_xml(m_config_fname);
401             if (r)
402                 yaz_log(YLOG_WARN, "reconfigure failed");
403             else
404             {
405                 m_log_mask = 0;
406                 cfg->get_generic_info(&m_log_mask, &m_max_clients,
407                                       &m_connect_max);
408             }
409         }
410         else
411             yaz_log(YLOG_LOG, "reconfigure");
412         m_reconfig_flag = 0;
413     }
414     return cfg;
415 }
416
417 IPDU_Observer *Yaz_Proxy::sessionNotify(IPDU_Observable
418                                         *the_PDU_Observable, int fd)
419 {
420     check_reconfigure();
421     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable,
422                                          m_socket_observable, this);
423     new_proxy->m_config = 0;
424     new_proxy->m_config_fname = 0;
425     new_proxy->timeout(m_client_idletime);
426     new_proxy->m_target_idletime = m_target_idletime;
427     new_proxy->set_default_target(m_default_target);
428     new_proxy->m_max_clients = m_max_clients;
429     new_proxy->m_log_mask = m_log_mask;
430     new_proxy->set_APDU_log(get_APDU_log());
431     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
432         new_proxy->set_APDU_yazlog(1);
433     else
434         new_proxy->set_APDU_yazlog(0);
435     sprintf(new_proxy->m_session_str, "%ld:%d ", (long) time(0), m_session_no);
436     m_session_no++;
437     new_proxy->m_peername = xstrdup(the_PDU_Observable->getpeername());
438     yaz_log (YLOG_LOG, "%sNew session %s", new_proxy->m_session_str,
439              new_proxy->m_peername);
440     new_proxy->set_proxy_negotiation(m_proxy_negotiation_charset,
441         m_proxy_negotiation_lang, m_proxy_negotiation_default_charset);
442     // create thread object the first time we get an incoming connection
443     if (!m_my_thread)
444         m_my_thread = new Msg_Thread(m_socket_observable, 1);
445     new_proxy->m_my_thread = m_my_thread;
446     return new_proxy;
447 }
448
449 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
450 {
451     int oid[OID_SIZE];
452     Z_OtherInformationUnit *oi;
453     struct oident ent;
454     ent.proto = PROTO_Z3950;
455     ent.oclass = CLASS_USERINFO;
456     ent.value = (oid_value) VAL_COOKIE;
457     assert (oid_ent_to_oid (&ent, oid));
458
459     if (oid_ent_to_oid (&ent, oid) &&
460         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
461         oi->which == Z_OtherInfo_characterInfo)
462         return oi->information.characterInfo;
463     return 0;
464 }
465 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
466 {
467     int oid[OID_SIZE];
468     Z_OtherInformationUnit *oi;
469     struct oident ent;
470     ent.proto = PROTO_Z3950;
471     ent.oclass = CLASS_USERINFO;
472     ent.value = (oid_value) VAL_PROXY;
473     if (oid_ent_to_oid (&ent, oid) &&
474         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
475         oi->which == Z_OtherInfo_characterInfo)
476         return oi->information.characterInfo;
477     return 0;
478 }
479 const char *Yaz_Proxy::load_balance(const char **url)
480 {
481     int zurl_in_use[MAX_ZURL_PLEX];
482     int zurl_in_spare[MAX_ZURL_PLEX];
483     Yaz_ProxyClient *c;
484     int i;
485
486     for (i = 0; i<MAX_ZURL_PLEX; i++)
487     {
488         zurl_in_use[i] = 0;
489         zurl_in_spare[i] = 0;
490     }
491     for (c = m_parent->m_clientPool; c; c = c->m_next)
492     {
493         for (i = 0; url[i]; i++)
494             if (!strcmp(url[i], c->get_hostname()))
495             {
496                 zurl_in_use[i]++;
497                 if (c->m_cookie == 0 && c->m_server == 0 && c->m_waiting == 0)
498                     zurl_in_spare[i]++;
499             }
500     }
501     int min_use = 100000;
502     int spare_for_min = 0;
503     int max_spare = 0;
504     const char *ret_min = 0;
505     const char *ret_spare = 0;
506     for (i = 0; url[i]; i++)
507     {
508         yaz_log(YLOG_DEBUG, "%szurl=%s use=%d spare=%d",
509                 m_session_str, url[i], zurl_in_use[i], zurl_in_spare[i]);
510         if (min_use > zurl_in_use[i])
511         {
512             ret_min = url[i];
513             min_use = zurl_in_use[i];
514             spare_for_min = zurl_in_spare[i];
515         }
516         if (max_spare < zurl_in_spare[i])
517         {
518             ret_spare = url[i];
519             max_spare = zurl_in_spare[i];
520         }
521     }
522     return ret_min;
523 }
524
525 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
526                                        const char *proxy_host)
527 {
528     assert (m_parent);
529     Yaz_Proxy *parent = m_parent;
530     Yaz_ProxyClient *c = m_client;
531
532     if (!m_proxyTarget)
533     {
534         const char *url[MAX_ZURL_PLEX];
535         Yaz_ProxyConfig *cfg = check_reconfigure();
536         if (proxy_host)
537         {
538             if (parent && parent->m_debug_mode)
539             {
540                 // only to be enabled for debugging...
541                 if (!strcmp(proxy_host, "stop"))
542                     exit(0);
543             }
544             xfree(m_default_target);
545             m_default_target = xstrdup(proxy_host);
546         }
547         proxy_host = m_default_target;
548         int client_idletime = -1;
549         const char *cql2rpn_fname = 0;
550         const char *negotiation_charset = 0;
551         const char *negotiation_lang = 0;
552         const char *query_charset = 0;
553         const char *default_client_query_charset = 0;
554         url[0] = m_default_target;
555         url[1] = 0;
556         if (cfg)
557         {
558             int pre_init = 0;
559             cfg->get_target_info(proxy_host, url, &m_bw_max,
560                                  &m_pdu_max, &m_max_record_retrieve,
561                                  &m_search_max,
562                                  &m_target_idletime, &client_idletime,
563                                  &parent->m_max_clients,
564                                  &m_keepalive_limit_bw,
565                                  &m_keepalive_limit_pdu,
566                                  &pre_init,
567                                  &cql2rpn_fname,
568                                  &negotiation_charset,
569                                  &negotiation_lang,
570                                  &query_charset,
571                                  &default_client_query_charset);
572         }
573         if (client_idletime != -1)
574         {
575             m_client_idletime = client_idletime;
576             timeout(m_client_idletime);
577         }
578         if (cql2rpn_fname)
579             m_cql2rpn.set_pqf_file(cql2rpn_fname);
580         if (negotiation_charset || negotiation_lang || default_client_query_charset)
581         {
582             set_proxy_negotiation(negotiation_charset,
583                 negotiation_lang, default_client_query_charset);
584         }
585         m_charset_converter->set_target_query_charset(query_charset);
586         if (!url[0])
587         {
588             yaz_log(YLOG_LOG, "%sNo default target", m_session_str);
589             return 0;
590         }
591         // we don't handle multiplexing for cookie session, so we just
592         // pick the first one in this case (anonymous users will be able
593         // to use any backend)
594         if (cookie && *cookie)
595             m_proxyTarget = (char*) xstrdup(url[0]);
596         else
597             m_proxyTarget = (char*) xstrdup(load_balance(url));
598     }
599     if (cookie && *cookie)
600     {   // search in sessions with a cookie
601         for (c = parent->m_clientPool; c; c = c->m_next)
602         {
603             assert (c->m_prev);
604             assert (*c->m_prev == c);
605             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
606                 !strcmp(m_proxyTarget, c->get_hostname()))
607             {
608                 // Found it in cache
609                 // The following handles "cancel"
610                 // If connection is busy (waiting for PDU) and
611                 // we have an initRequest we can safely do re-open
612                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
613                 {
614                     yaz_log (YLOG_LOG, "%s REOPEN target=%s", m_session_str,
615                              c->get_hostname());
616                     c->close();
617                     c->m_init_flag = 0;
618
619                     c->m_last_ok = 0;
620                     c->m_cache.clear();
621                     c->m_last_resultCount = 0;
622                     c->m_sr_transform = 0;
623                     c->m_waiting = 0;
624                     c->m_resultSetStartPoint = 0;
625                     c->m_target_idletime = m_target_idletime;
626                     if (c->client(m_proxyTarget))
627                     {
628                         delete c;
629                         return 0;
630                     }
631                     c->timeout(30);
632                 }
633                 c->m_seqno = parent->m_seqno;
634                 if (c->m_server && c->m_server != this)
635                     c->m_server->m_client = 0;
636                 c->m_server = this;
637                 (parent->m_seqno)++;
638                 yaz_log (YLOG_DEBUG, "get_client 1 %p %p", this, c);
639                 return c;
640             }
641         }
642     }
643     else if (!c && apdu->which == Z_APDU_initRequest )
644     {
645         // anonymous sessions without cookie.
646         // if authentication is set it is NOT anonymous se we can't share them.
647         // If charset and lang negotiation is use it is NOT anonymous session too.
648         for (c = parent->m_clientPool; c; c = c->m_next)
649         {
650             assert(c->m_prev);
651             assert(*c->m_prev == c);
652             if (c->m_server == 0 && c->m_cookie == 0 &&  c->m_waiting == 0 
653                 && c->compare_idAuthentication(apdu)
654                 && c->compare_charset(apdu)
655                 && !strcmp(m_proxyTarget, c->get_hostname()))
656             {
657                 // found it in cache
658                 yaz_log (YLOG_LOG, "%sREUSE %d %s",
659                          m_session_str, parent->m_seqno, c->get_hostname());
660                 
661                 c->m_seqno = parent->m_seqno;
662                 assert(c->m_server == 0);
663                 c->m_server = this;
664
665                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
666                     c->set_APDU_yazlog(1);
667                 else
668                     c->set_APDU_yazlog(0);
669
670                 (parent->m_seqno)++;
671
672                 parent->pre_init();
673
674                 return c;
675             }
676         }
677     }
678     if (!m_client)
679     {
680         if (apdu->which != Z_APDU_initRequest)
681         {
682             yaz_log (YLOG_LOG, "%sno init request as first PDU", m_session_str);
683             return 0;
684         }
685         // go through list of clients - and find the lowest/oldest one.
686         Yaz_ProxyClient *c_min = 0;
687         int min_seq = -1;
688         int no_of_clients = 0;
689         if (parent->m_clientPool)
690             yaz_log (YLOG_DEBUG, "Existing sessions");
691         for (c = parent->m_clientPool; c; c = c->m_next)
692         {
693             yaz_log (YLOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
694                                c->m_waiting, c->get_hostname(),
695                                c->m_cookie ? c->m_cookie : "");
696             no_of_clients++;
697             if (min_seq < 0 || c->m_seqno < min_seq)
698             {
699                 min_seq = c->m_seqno;
700                 c_min = c;
701             }
702         }
703         if (no_of_clients >= parent->m_max_clients)
704         {
705             c = c_min;
706             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
707             {
708                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Destroy %d",
709                          m_session_str, parent->m_max_clients, c->m_seqno);
710                 if (c->m_server && c->m_server != this)
711                     delete c->m_server;   // PROBLEM: m_ref_count!
712                 c->m_server = 0;
713             }
714             else
715             {
716                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Reuse %d %d %s",
717                          m_session_str, parent->m_max_clients,
718                          c->m_seqno, parent->m_seqno, c->get_hostname());
719                 xfree (c->m_cookie);
720                 c->m_cookie = 0;
721                 if (cookie)
722                     c->m_cookie = xstrdup(cookie);
723                 c->m_seqno = parent->m_seqno;
724                 if (c->m_server && c->m_server != this)
725                 {
726                     c->m_server->m_client = 0;
727                     delete c->m_server;   // PROBLEM: m_ref_count!
728                 }
729                 (parent->m_seqno)++;
730                 c->m_target_idletime = m_target_idletime;
731                 c->timeout(m_target_idletime);
732
733                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
734                     c->set_APDU_yazlog(1);
735                 else
736                     c->set_APDU_yazlog(0);
737
738                 return c;
739             }
740         }
741         else
742         {
743             yaz_log (YLOG_LOG, "%sNEW %d %s",
744                      m_session_str, parent->m_seqno, m_proxyTarget);
745             c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
746             c->m_next = parent->m_clientPool;
747             if (c->m_next)
748                 c->m_next->m_prev = &c->m_next;
749             parent->m_clientPool = c;
750             c->m_prev = &parent->m_clientPool;
751         }
752
753         xfree (c->m_cookie);
754         c->m_cookie = 0;
755         if (cookie)
756             c->m_cookie = xstrdup(cookie);
757
758         c->m_seqno = parent->m_seqno;
759         c->m_init_flag = 0;
760         c->m_last_resultCount = 0;
761         c->m_last_ok = 0;
762         c->m_cache.clear();
763         c->m_sr_transform = 0;
764         c->m_waiting = 0;
765         c->m_resultSetStartPoint = 0;
766         (parent->m_seqno)++;
767         if (c->client(m_proxyTarget))
768         {
769             delete c;
770             return 0;
771         }
772         c->m_target_idletime = m_target_idletime;
773         c->timeout(30);
774
775         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
776             c->set_APDU_yazlog(1);
777         else
778             c->set_APDU_yazlog(0);
779
780         c->set_idAuthentication(apdu);
781     }
782     yaz_log (YLOG_DEBUG, "get_client 3 %p %p", this, c);
783     return c;
784 }
785
786 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
787 {
788     int i;
789     for (i = 0; i<num; i++)
790     {
791         oident *ent;
792         Z_DefaultDiagFormat *r;
793         Z_DiagRec *p = pp[i];
794         if (p->which != Z_DiagRec_defaultFormat)
795         {
796             yaz_log(YLOG_LOG, "%sError no diagnostics", m_session_str);
797             return;
798         }
799         else
800             r = p->u.defaultFormat;
801         if (!(ent = oid_getentbyoid(r->diagnosticSetId)) ||
802             ent->oclass != CLASS_DIAGSET || ent->value != VAL_BIB1)
803             yaz_log(YLOG_LOG, "%sError unknown diagnostic set", m_session_str);
804         switch (r->which)
805         {
806         case Z_DefaultDiagFormat_v2Addinfo:
807             yaz_log(YLOG_LOG, "%sError %d %s:%s",
808                     m_session_str,
809                     *r->condition, diagbib1_str(*r->condition),
810                     r->u.v2Addinfo);
811             break;
812         case Z_DefaultDiagFormat_v3Addinfo:
813             yaz_log(YLOG_LOG, "%sError %d %s:%s",
814                     m_session_str,
815                     *r->condition, diagbib1_str(*r->condition),
816                     r->u.v3Addinfo);
817             break;
818         }
819     }
820 }
821
822 int Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p, Z_APDU *apdu)
823 {
824     if (!m_stylesheet_xsp || p->num_records <= 0)
825     {
826         return 0;  /* no XSLT to be done ... */
827     }
828
829     m_stylesheet_offset = 0;
830     m_stylesheet_nprl = p;
831     m_stylesheet_apdu = apdu;
832     m_timeout_mode = timeout_xsl;
833
834     timeout(0);
835     return 1;
836 }
837
838 void Yaz_Proxy::convert_xsl_delay()
839 {
840 #if HAVE_XSLT
841     Z_NamePlusRecord *npr = m_stylesheet_nprl->records[m_stylesheet_offset];
842     if (npr->which == Z_NamePlusRecord_databaseRecord)
843     {
844         Z_External *r = npr->u.databaseRecord;
845         if (r->which == Z_External_octet)
846         {
847 #if 0
848             fwrite((char*) r->u.octet_aligned->buf, 1, r->u.octet_aligned->len, stdout);
849 #endif
850             xmlDocPtr res, doc = xmlParseMemory(
851                 (char*) r->u.octet_aligned->buf,
852                 r->u.octet_aligned->len);
853
854
855             yaz_log(YLOG_LOG, "%sXSLT convert %d",
856                     m_session_str, m_stylesheet_offset);
857             res = xsltApplyStylesheet((xsltStylesheetPtr) m_stylesheet_xsp,
858                                       doc, 0);
859
860             if (res)
861             {
862                 xmlChar *out_buf;
863                 int out_len;
864                 xmlDocDumpFormatMemory (res, &out_buf, &out_len, 1);
865
866                 m_stylesheet_nprl->records[m_stylesheet_offset]->
867                     u.databaseRecord =
868                     z_ext_record(odr_encode(), VAL_TEXT_XML,
869                                  (char*) out_buf, out_len);
870                 xmlFree(out_buf);
871                 xmlFreeDoc(res);
872             }
873
874             xmlFreeDoc(doc);
875         }
876     }
877 #endif
878     m_stylesheet_offset++;
879     if (m_stylesheet_offset == m_stylesheet_nprl->num_records)
880     {
881         m_timeout_mode = timeout_normal;
882         m_stylesheet_nprl = 0;
883 #if HAVE_XSLT
884         if (m_stylesheet_xsp)
885             xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
886 #endif
887         m_stylesheet_xsp = 0;
888         timeout(m_client_idletime);
889         send_PDU_convert(m_stylesheet_apdu);
890     }
891     else
892         timeout(0);
893 }
894
895 void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p)
896 {
897     if (m_frontend_type != VAL_NONE)
898     {
899         int i;
900         for (i = 0; i < p->num_records; i++)
901         {
902             Z_NamePlusRecord *npr = p->records[i];
903             if (npr->which == Z_NamePlusRecord_databaseRecord)
904             {
905                 Z_External *r = npr->u.databaseRecord;
906                 if (r->which == Z_External_octet)
907                 {
908 #if HAVE_USEMARCON
909                     if (m_usemarcon_ini_stage1 && *m_usemarcon_ini_stage1)
910                     {
911                         if (!m_usemarcon->m_stage1)
912                         {
913                             m_usemarcon->m_stage1 = new CDetails();
914                         }
915                         m_usemarcon->m_stage1->SetIniFileName(m_usemarcon_ini_stage1);
916                         m_usemarcon->m_stage1->SetMarcRecord((char*) r->u.octet_aligned->buf, r->u.octet_aligned->len);
917                         int res = m_usemarcon->m_stage1->Start();
918                         if (res == 0)
919                         {
920                             char *converted;
921                             int convlen;
922                             m_usemarcon->m_stage1->GetMarcRecord(converted, convlen);
923                             if (m_usemarcon_ini_stage2 && *m_usemarcon_ini_stage2)
924                             {
925                                 if (!m_usemarcon->m_stage2)
926                                 {
927                                     m_usemarcon->m_stage2 = new CDetails();
928                                 }
929                                 m_usemarcon->m_stage2->SetIniFileName(m_usemarcon_ini_stage2);
930                                 m_usemarcon->m_stage2->SetMarcRecord(converted, convlen);
931                                 res = m_usemarcon->m_stage2->Start();
932                                 if (res == 0)
933                                 {
934                                     free(converted);
935                                     m_usemarcon->m_stage2->GetMarcRecord(converted, convlen);
936                                 }
937                                 else
938                                 {
939                                     yaz_log(YLOG_LOG, "%sUSEMARCON stage 2 error %d", m_session_str, res);
940                                 }
941                             }
942                             npr->u.databaseRecord =
943                                 z_ext_record(odr_encode(),
944                                              m_frontend_type,
945                                              converted,
946                                              strlen(converted));
947                             free(converted);
948                         }
949                         else
950                         {
951                             yaz_log(YLOG_LOG, "%sUSEMARCON stage 1 error %d", m_session_str, res);
952                         }
953                         continue;
954                     }
955 #endif
956 /* HAVE_USEMARCON */
957                     npr->u.databaseRecord =
958                         z_ext_record(odr_encode(),
959                                      m_frontend_type,
960                                      (char*) r->u.octet_aligned->buf,
961                                      r->u.octet_aligned->len);
962                 }
963             }
964         }
965     }
966 }
967
968 void Yaz_Proxy::convert_records_charset(Z_NamePlusRecordList *p,
969                                         const char *backend_charset)
970 {
971     yaz_log(YLOG_LOG, "%sconvert_to_marc", m_session_str);
972     int sel =   m_charset_converter->get_client_charset_selected();
973     const char *client_record_charset =
974         m_charset_converter->get_client_query_charset();
975     if (sel && backend_charset && client_record_charset &&
976         strcmp(backend_charset, client_record_charset))
977     {
978         int i;
979         yaz_iconv_t cd = yaz_iconv_open(client_record_charset,
980                                         backend_charset);
981         yaz_marc_t mt = yaz_marc_create();
982         yaz_marc_xml(mt, YAZ_MARC_ISO2709);
983         yaz_marc_iconv(mt, cd);
984         for (i = 0; i < p->num_records; i++)
985         {
986             Z_NamePlusRecord *npr = p->records[i];
987             if (npr->which == Z_NamePlusRecord_databaseRecord)
988             {
989                 Z_External *r = npr->u.databaseRecord;
990                 oident *ent = oid_getentbyoid(r->direct_reference);
991                 if (!ent || ent->value == VAL_NONE)
992                     continue;
993
994                 if (ent->value == VAL_SUTRS)
995                 {
996                     WRBUF w = wrbuf_alloc();
997
998                     wrbuf_iconv_write(w, cd,  (char*) r->u.octet_aligned->buf,
999                                       r->u.octet_aligned->len);
1000                     npr->u.databaseRecord =
1001                         z_ext_record(odr_encode(), ent->value, wrbuf_buf(w),
1002                                      wrbuf_len(w));
1003                     wrbuf_free(w, 1);
1004                 }
1005                 else if (ent->value == VAL_TEXT_XML)
1006                 {
1007                     ;
1008                 }
1009                 else if (r->which == Z_External_octet)
1010                 {
1011                     int rlen;
1012                     char *result;
1013                     if (yaz_marc_decode_buf(mt,
1014                                             (char*) r->u.octet_aligned->buf,
1015                                             r->u.octet_aligned->len,
1016                                             &result, &rlen))
1017                     {
1018                         npr->u.databaseRecord =
1019                             z_ext_record(odr_encode(), ent->value, result, rlen);
1020                         yaz_log(YLOG_LOG, "%sRecoding MARC record",
1021                                 m_session_str);
1022                     }
1023                 }
1024             }
1025         }
1026         if (cd)
1027             yaz_iconv_close(cd);
1028         yaz_marc_destroy(mt);
1029     }
1030     else
1031     {
1032         yaz_log(YLOG_LOG, "%sSkipping marc convert", m_session_str);
1033     }
1034 }
1035
1036 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p,
1037                                    const char *backend_charset)
1038 {
1039     int i;
1040     if (!backend_charset)
1041         backend_charset = "MARC-8";
1042     yaz_iconv_t cd = yaz_iconv_open("UTF-8", backend_charset);
1043     yaz_marc_t mt = yaz_marc_create();
1044     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
1045     yaz_marc_iconv(mt, cd);
1046     for (i = 0; i < p->num_records; i++)
1047     {
1048         Z_NamePlusRecord *npr = p->records[i];
1049         if (npr->which == Z_NamePlusRecord_databaseRecord)
1050         {
1051             Z_External *r = npr->u.databaseRecord;
1052             if (r->which == Z_External_OPAC)
1053             {
1054                 WRBUF w = wrbuf_alloc();
1055
1056                 yaz_opac_decode_wrbuf(mt, r->u.opac, w);
1057                 npr->u.databaseRecord = z_ext_record(
1058                     odr_encode(), VAL_TEXT_XML,
1059                     wrbuf_buf(w), wrbuf_len(w)
1060                     );
1061                 wrbuf_free(w, 1);
1062             }
1063             else if (r->which == Z_External_octet)
1064             {
1065                 int rlen;
1066                 char *result;
1067                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
1068                                         r->u.octet_aligned->len,
1069                                         &result, &rlen))
1070                 {
1071                     npr->u.databaseRecord =
1072                         z_ext_record(odr_encode(), VAL_TEXT_XML, result, rlen);
1073                 }
1074             }
1075         }
1076     }
1077     if (cd)
1078         yaz_iconv_close(cd);
1079     yaz_marc_destroy(mt);
1080 }
1081
1082 void Yaz_Proxy::logtime()
1083 {
1084 #if HAVE_GETTIMEOFDAY
1085     struct timeval *tv = (struct timeval*) m_time_tv;
1086     if (tv->tv_sec)
1087     {
1088         struct timeval tv1;
1089         gettimeofday(&tv1, 0);
1090         long diff = (tv1.tv_sec - tv->tv_sec)*1000000 +
1091             (tv1.tv_usec - tv->tv_usec);
1092         if (diff >= 0)
1093             yaz_log(YLOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
1094                     diff/1000000, (diff/1000)%1000);
1095     }
1096     tv->tv_sec = 0;
1097     tv->tv_usec = 0;
1098 #endif
1099 }
1100
1101 int Yaz_Proxy::send_http_response(int code)
1102 {
1103     ODR o = odr_encode();
1104     Z_GDU *gdu = z_get_HTTP_Response(o, code);
1105     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1106     if (m_http_version)
1107         hres->version = odr_strdup(o, m_http_version);
1108     if (m_http_keepalive)
1109         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1110     else
1111         timeout(0);
1112
1113     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1114     {
1115         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1116                  gdu_name(gdu));
1117     }
1118     int len;
1119     int r = send_GDU(gdu, &len);
1120     m_bytes_sent += len;
1121     m_bw_stat.add_bytes(len);
1122     logtime();
1123
1124     recv_GDU_more(true);
1125
1126     return r;
1127 }
1128
1129 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu, int http_code /* = 200 */)
1130 {
1131     ODR o = odr_encode();
1132     const char *ctype = "text/xml";
1133     Z_GDU *gdu = z_get_HTTP_Response(o, http_code);
1134     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1135     if (m_http_version)
1136         hres->version = odr_strdup(o, m_http_version);
1137     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
1138     if (m_http_keepalive)
1139         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1140     else
1141         timeout(0);
1142     if (http_code == 401)
1143         z_HTTP_header_add(o, &hres->headers, "WWW-Authenticate", "Basic realm=\"YAZ Proxy\"");
1144
1145     static Z_SOAP_Handler soap_handlers[2] = {
1146 #if HAVE_XSLT
1147         {"http://www.loc.gov/zing/srw/", 0,
1148          (Z_SOAP_fun) yaz_srw_codec},
1149 #endif
1150         {0, 0, 0}
1151     };
1152
1153     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
1154     soap_package->which = Z_SOAP_generic;
1155     soap_package->u.generic =
1156         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
1157     soap_package->u.generic->no = 0;
1158     soap_package->u.generic->ns = soap_handlers[0].ns;
1159     soap_package->u.generic->p = (void *) srw_pdu;
1160     soap_package->ns = m_soap_ns;
1161     z_soap_codec_enc_xsl(o, &soap_package,
1162                          &hres->content_buf, &hres->content_len,
1163                          soap_handlers, 0, m_s2z_stylesheet);
1164     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1165     {
1166         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1167                  gdu_name(gdu));
1168     }
1169     int len;
1170     int r = send_GDU(gdu, &len);
1171     m_bytes_sent += len;
1172     m_bw_stat.add_bytes(len);
1173     logtime();
1174
1175     recv_GDU_more(true);
1176
1177     return r;
1178 }
1179
1180 int Yaz_Proxy::send_to_srw_client_error(int srw_error, const char *add)
1181 {
1182     ODR o = odr_encode();
1183     Z_SRW_diagnostic *diagnostic = (Z_SRW_diagnostic *)
1184         odr_malloc(o, sizeof(*diagnostic));
1185     int num_diagnostic = 1;
1186     yaz_mk_std_diagnostic(o, diagnostic, srw_error, add);
1187     return send_srw_search_response(diagnostic, num_diagnostic, srw_error == 3 ? 401 : 200);
1188 }
1189
1190 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
1191                              Z_DefaultDiagFormat *ddf)
1192 {
1193     int bib1_code = *ddf->condition;
1194     if (bib1_code == 109)
1195         return 404;
1196     srw_res->num_diagnostics = 1;
1197     srw_res->diagnostics = (Z_SRW_diagnostic *)
1198         odr_malloc(o, sizeof(*srw_res->diagnostics));
1199     yaz_mk_std_diagnostic(o, srw_res->diagnostics,
1200                           yaz_diag_bib1_to_srw(*ddf->condition),
1201                           ddf->u.v2Addinfo);
1202     return 0;
1203 }
1204
1205 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
1206 {
1207     ODR o = odr_encode();
1208     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1209     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1210
1211     srw_res->numberOfRecords = odr_intdup (o, hits);
1212     if (records && records->which == Z_Records_DBOSD)
1213     {
1214         srw_res->num_records =
1215             records->u.databaseOrSurDiagnostics->num_records;
1216         int i;
1217         srw_res->records = (Z_SRW_record *)
1218             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
1219         for (i = 0; i < srw_res->num_records; i++)
1220         {
1221             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
1222             if (npr->which != Z_NamePlusRecord_databaseRecord)
1223             {
1224                 srw_res->records[i].recordSchema = "diagnostic";
1225                 srw_res->records[i].recordPacking = m_s2z_packing;
1226                 srw_res->records[i].recordData_buf = "67";
1227                 srw_res->records[i].recordData_len = 2;
1228                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1229                 continue;
1230             }
1231             Z_External *r = npr->u.databaseRecord;
1232             oident *ent = oid_getentbyoid(r->direct_reference);
1233             if (r->which == Z_External_octet && ent->value == VAL_TEXT_XML)
1234             {
1235                 srw_res->records[i].recordSchema = m_schema;
1236                 srw_res->records[i].recordPacking = m_s2z_packing;
1237                 srw_res->records[i].recordData_buf = (char*)
1238                     r->u.octet_aligned->buf;
1239                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
1240                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1241             }
1242             else
1243             {
1244                 srw_res->records[i].recordSchema = "diagnostic";
1245                 srw_res->records[i].recordPacking = m_s2z_packing;
1246                 srw_res->records[i].recordData_buf = "67";
1247                 srw_res->records[i].recordData_len = 2;
1248                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1249             }
1250         }
1251     }
1252     if (records && records->which == Z_Records_NSD)
1253     {
1254         int http_code;
1255         http_code = z_to_srw_diag(odr_encode(), srw_res,
1256                                    records->u.nonSurrogateDiagnostic);
1257         if (http_code)
1258             return send_http_response(http_code);
1259     }
1260     return send_srw_response(srw_pdu);
1261
1262 }
1263
1264 int Yaz_Proxy::send_srw_search_response(Z_SRW_diagnostic *diagnostics,
1265                                         int num_diagnostics, int http_code /* = 200 */)
1266 {
1267     ODR o = odr_encode();
1268     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1269     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1270
1271     srw_res->num_diagnostics = num_diagnostics;
1272     srw_res->diagnostics = diagnostics;
1273     return send_srw_response(srw_pdu, http_code);
1274 }
1275
1276 int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics,
1277                                         int num_diagnostics)
1278 {
1279     Yaz_ProxyConfig *cfg = check_reconfigure();
1280     if (cfg)
1281     {
1282         int len;
1283         char *b = cfg->get_explain_doc(odr_encode(), 0 /* target */,
1284                                        m_s2z_database, &len);
1285         if (b)
1286         {
1287             Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
1288             Z_SRW_explainResponse *er = res->u.explain_response;
1289
1290             er->record.recordData_buf = b;
1291             er->record.recordData_len = len;
1292             er->record.recordPacking = m_s2z_packing;
1293             er->record.recordSchema = "http://explain.z3950.org/dtd/2.0/";
1294
1295             er->diagnostics = diagnostics;
1296             er->num_diagnostics = num_diagnostics;
1297             return send_srw_response(res);
1298         }
1299     }
1300     return send_http_response(404);
1301 }
1302
1303 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu)
1304 {
1305     if (m_http_version)
1306     {
1307         if (apdu->which == Z_APDU_initResponse)
1308         {
1309             Z_InitResponse *res = apdu->u.initResponse;
1310             if (*res->result == 0)
1311             {
1312                 send_to_srw_client_error(3, 0);
1313             }
1314             else if (!m_s2z_search_apdu)
1315             {
1316                 send_srw_explain_response(0, 0);
1317             }
1318             else
1319             {
1320                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1321             }
1322         }
1323         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
1324         {
1325             m_s2z_search_apdu = 0;
1326             Z_SearchResponse *res = apdu->u.searchResponse;
1327             m_s2z_hit_count = *res->resultCount;
1328             if (res->records && res->records->which == Z_Records_NSD)
1329             {
1330                 send_to_srw_client_ok(0, res->records, 1);
1331             }
1332             else if (m_s2z_present_apdu && m_s2z_hit_count > 0)
1333             {
1334                 // adjust
1335                 Z_PresentRequest *pr = m_s2z_present_apdu->u.presentRequest;
1336
1337                 if (*pr->resultSetStartPoint <= m_s2z_hit_count)
1338                 {
1339                     if (*pr->numberOfRecordsRequested+ *pr->resultSetStartPoint
1340                         > m_s2z_hit_count)
1341                         *pr->numberOfRecordsRequested =
1342                             1 + m_s2z_hit_count - *pr->resultSetStartPoint;
1343                 }
1344                 handle_incoming_Z_PDU(m_s2z_present_apdu);
1345             }
1346             else
1347             {
1348                 m_s2z_present_apdu = 0;
1349                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
1350             }
1351         }
1352         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
1353         {
1354             int start =
1355                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
1356
1357             m_s2z_present_apdu = 0;
1358             Z_PresentResponse *res = apdu->u.presentResponse;
1359             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
1360         }
1361     }
1362     else
1363     {
1364         int len = 0;
1365         if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1366             yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1367                      apdu_name(apdu));
1368         int r = send_Z_PDU(apdu, &len);
1369         m_bytes_sent += len;
1370         m_bw_stat.add_bytes(len);
1371         logtime();
1372         return r;
1373     }
1374     return 0;
1375 }
1376
1377 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
1378 {
1379     int kill_session = 0;
1380     Z_ReferenceId **new_id = get_referenceIdP(apdu);
1381
1382     if (new_id)
1383         *new_id = m_referenceId;
1384
1385     if (apdu->which == Z_APDU_searchResponse)
1386     {
1387         Z_SearchResponse *sr = apdu->u.searchResponse;
1388         Z_Records *p = sr->records;
1389         if (p && p->which == Z_Records_NSD)
1390         {
1391             Z_DiagRec dr, *dr_p = &dr;
1392             dr.which = Z_DiagRec_defaultFormat;
1393             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1394
1395             *sr->searchStatus = 0;
1396             display_diagrecs(&dr_p, 1);
1397         }
1398         else
1399         {
1400             if (p && p->which == Z_Records_DBOSD)
1401             {
1402                 if (m_backend_type
1403 #if HAVE_USEMARCON
1404                     || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1405 #endif
1406                     )
1407                     convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1408                 if (m_marcxml_mode == marcxml)
1409                     convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1410                                        m_backend_charset);
1411                 else
1412                     convert_records_charset(p->u.databaseOrSurDiagnostics,
1413                                             m_backend_charset);
1414                 if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1415                     return 0;
1416
1417             }
1418             if (sr->resultCount)
1419             {
1420                 yaz_log(YLOG_LOG, "%s%d hits", m_session_str,
1421                         *sr->resultCount);
1422                 if (*sr->resultCount < 0)
1423                 {
1424                     m_flag_invalid_session = 1;
1425                     kill_session = 1;
1426
1427                     *sr->searchStatus = 0;
1428                     sr->records =
1429                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
1430                     *sr->resultCount = 0;
1431                 }
1432             }
1433         }
1434     }
1435     else if (apdu->which == Z_APDU_presentResponse)
1436     {
1437         Z_PresentResponse *sr = apdu->u.presentResponse;
1438         Z_Records *p = sr->records;
1439         if (p && p->which == Z_Records_NSD)
1440         {
1441             Z_DiagRec dr, *dr_p = &dr;
1442             dr.which = Z_DiagRec_defaultFormat;
1443             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1444             if (*sr->presentStatus == Z_PresentStatus_success)
1445                 *sr->presentStatus = Z_PresentStatus_failure;
1446             display_diagrecs(&dr_p, 1);
1447         }
1448         if (p && p->which == Z_Records_DBOSD)
1449         {
1450             if (m_backend_type
1451 #if HAVE_USEMARCON
1452                 || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1453 #endif
1454                 )
1455                 convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1456             if (m_marcxml_mode == marcxml)
1457                 convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1458                                    m_backend_charset);
1459             else
1460                 convert_records_charset(p->u.databaseOrSurDiagnostics,
1461                                         m_backend_charset);
1462             if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1463                 return 0;
1464         }
1465     }
1466     else if (apdu->which == Z_APDU_initResponse)
1467     {
1468         //Get and check negotiation record
1469         //from init response.
1470         handle_charset_lang_negotiation(apdu);
1471
1472         if (m_initRequest_options)
1473         {
1474             Z_Options *nopt =
1475                 (Odr_bitmask *)odr_malloc(odr_encode(),
1476                                           sizeof(Odr_bitmask));
1477             ODR_MASK_ZERO(nopt);
1478
1479             int i;
1480             for (i = 0; i<24; i++)
1481                 if (ODR_MASK_GET(m_initRequest_options, i) &&
1482                     ODR_MASK_GET(apdu->u.initResponse->options, i))
1483                     ODR_MASK_SET(nopt, i);
1484             apdu->u.initResponse->options = nopt;
1485         }
1486         if (m_initRequest_version)
1487         {
1488             Z_ProtocolVersion *nopt =
1489                 (Odr_bitmask *)odr_malloc(odr_encode(),
1490                                           sizeof(Odr_bitmask));
1491             ODR_MASK_ZERO(nopt);
1492
1493             int i;
1494             for (i = 0; i<8; i++)
1495                 if (ODR_MASK_GET(m_initRequest_version, i) &&
1496                     ODR_MASK_GET(apdu->u.initResponse->protocolVersion, i))
1497                     ODR_MASK_SET(nopt, i);
1498             apdu->u.initResponse->protocolVersion = nopt;
1499         }
1500         apdu->u.initResponse->preferredMessageSize =
1501             odr_intdup(odr_encode(),
1502                        m_client->m_initResponse_preferredMessageSize >
1503                        m_initRequest_preferredMessageSize ?
1504                        m_initRequest_preferredMessageSize :
1505                        m_client->m_initResponse_preferredMessageSize);
1506         apdu->u.initResponse->maximumRecordSize =
1507             odr_intdup(odr_encode(),
1508                        m_client->m_initResponse_maximumRecordSize >
1509                        m_initRequest_maximumRecordSize ?
1510                        m_initRequest_maximumRecordSize :
1511                        m_client->m_initResponse_maximumRecordSize);
1512     }
1513
1514     int r = send_PDU_convert(apdu);
1515     if (r)
1516         return r;
1517     if (kill_session)
1518     {
1519         delete m_client;
1520         m_client = 0;
1521         m_parent->pre_init();
1522     }
1523     return r;
1524 }
1525
1526 void Yaz_ProxyClient::set_idAuthentication(Z_APDU *apdu)
1527 {
1528     Z_IdAuthentication *t = apdu->u.initRequest->idAuthentication;
1529     
1530     odr_reset(m_idAuthentication_odr);
1531     z_IdAuthentication(m_idAuthentication_odr, &t, 1, 0);
1532     m_idAuthentication_ber_buf =
1533         odr_getbuf(m_idAuthentication_odr, 
1534                    &m_idAuthentication_ber_size, 0);
1535 }
1536
1537 bool Yaz_ProxyClient::compare_charset(Z_APDU *apdu)
1538 {
1539     return true;
1540 }
1541
1542 bool Yaz_ProxyClient::compare_idAuthentication(Z_APDU *apdu)
1543 {
1544     Z_IdAuthentication *t = apdu->u.initRequest->idAuthentication;
1545     ODR odr = odr_createmem(ODR_ENCODE);
1546
1547     z_IdAuthentication(odr, &t, 1, 0);
1548     int sz;
1549     char *buf = odr_getbuf(odr, &sz, 0);
1550     if (buf && m_idAuthentication_ber_buf
1551         && sz == m_idAuthentication_ber_size
1552         && !memcmp(m_idAuthentication_ber_buf, buf, sz))
1553     {
1554         odr_destroy(odr);
1555         return true;
1556     }
1557     odr_destroy(odr);
1558     if (!buf && !m_idAuthentication_ber_buf)
1559         return true;
1560     return false;
1561 }
1562
1563 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
1564 {
1565     int len = 0;
1566     const char *apdu_name_tmp = apdu_name(apdu);
1567     int r = send_Z_PDU(apdu, &len);
1568     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
1569         yaz_log (YLOG_LOG, "%sSending %s to %s %d bytes",
1570                  get_session_str(),
1571                  apdu_name_tmp, get_hostname(), len);
1572     m_bytes_sent += len;
1573     return r;
1574 }
1575
1576 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
1577 {
1578     if (apdu->which == Z_APDU_presentRequest)
1579     {
1580         Z_PresentRequest *pr = apdu->u.presentRequest;
1581         int toget = *pr->numberOfRecordsRequested;
1582         int start = *pr->resultSetStartPoint;
1583
1584         yaz_log(YLOG_LOG, "%sPresent %s %d+%d", m_session_str,
1585                 pr->resultSetId, start, toget);
1586
1587         if (*m_parent->m_optimize == '0')
1588             return apdu;
1589
1590         if (!m_client->m_last_resultSetId)
1591         {
1592             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1593             new_apdu->u.presentResponse->records =
1594                 create_nonSurrogateDiagnostics(odr_encode(), 30,
1595                                                pr->resultSetId);
1596             send_to_client(new_apdu);
1597             return 0;
1598         }
1599         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1600         {
1601             if (start+toget-1 > m_client->m_last_resultCount)
1602             {
1603                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1604                 new_apdu->u.presentResponse->records =
1605                     create_nonSurrogateDiagnostics(odr_encode(), 13, 0);
1606                 send_to_client(new_apdu);
1607                 return 0;
1608             }
1609             Z_NamePlusRecordList *npr;
1610 #if 0
1611             yaz_log(YLOG_LOG, "%sCache lookup %d+%d syntax=%s",
1612                     m_session_str, start, toget, yaz_z3950oid_to_str(
1613                         pr->preferredRecordSyntax, &oclass));
1614 #endif
1615             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1616                                           pr->preferredRecordSyntax,
1617                                           pr->recordComposition))
1618             {
1619                 yaz_log (YLOG_LOG, "%sReturned cached records for present request",
1620                          m_session_str);
1621                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1622                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1623
1624                 new_apdu->u.presentResponse->numberOfRecordsReturned
1625                     = odr_intdup(odr_encode(), toget);
1626
1627                 new_apdu->u.presentResponse->records = (Z_Records*)
1628                     odr_malloc(odr_encode(), sizeof(Z_Records));
1629                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1630                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1631                 new_apdu->u.presentResponse->nextResultSetPosition =
1632                     odr_intdup(odr_encode(), start+toget);
1633
1634                 send_to_client(new_apdu);
1635                 return 0;
1636             }
1637         }
1638     }
1639
1640     if (apdu->which != Z_APDU_searchRequest)
1641         return apdu;
1642     Z_SearchRequest *sr = apdu->u.searchRequest;
1643     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1644     Yaz_Z_Databases this_databases;
1645
1646     this_databases.set(sr->num_databaseNames, (const char **)
1647                        sr->databaseNames);
1648
1649     this_query->set_Z_Query(sr->query);
1650
1651     char query_str[120];
1652     this_query->print(query_str, sizeof(query_str)-1);
1653     yaz_log(YLOG_LOG, "%sSearch %s", m_session_str, query_str);
1654
1655     if (*m_parent->m_optimize != '0' &&
1656         m_client->m_last_ok && m_client->m_last_query &&
1657         m_client->m_last_query->match(this_query) &&
1658         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1659         m_client->m_last_databases.match(this_databases))
1660     {
1661         delete this_query;
1662         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1663             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1664         {
1665             Z_NamePlusRecordList *npr;
1666             int toget = *sr->mediumSetPresentNumber;
1667             Z_RecordComposition *comp = 0;
1668
1669             if (toget > m_client->m_last_resultCount)
1670                 toget = m_client->m_last_resultCount;
1671
1672             if (sr->mediumSetElementSetNames)
1673             {
1674                 comp = (Z_RecordComposition *)
1675                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1676                 comp->which = Z_RecordComp_simple;
1677                 comp->u.simple = sr->mediumSetElementSetNames;
1678             }
1679
1680             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1681                                           sr->preferredRecordSyntax, comp))
1682             {
1683                 yaz_log (YLOG_LOG, "%sReturned cached records for medium set",
1684                          m_session_str);
1685                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1686                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1687                 new_apdu->u.searchResponse->resultCount =
1688                     &m_client->m_last_resultCount;
1689
1690                 new_apdu->u.searchResponse->numberOfRecordsReturned
1691                     = odr_intdup(odr_encode(), toget);
1692
1693                 new_apdu->u.searchResponse->presentStatus =
1694                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1695                 new_apdu->u.searchResponse->records = (Z_Records*)
1696                     odr_malloc(odr_encode(), sizeof(Z_Records));
1697                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1698                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1699                 new_apdu->u.searchResponse->nextResultSetPosition =
1700                     odr_intdup(odr_encode(), toget+1);
1701                 send_to_client(new_apdu);
1702                 return 0;
1703             }
1704             else
1705             {
1706                 // medium Set
1707                 // send present request (medium size)
1708                 yaz_log (YLOG_LOG, "%sOptimizing search for medium set",
1709                          m_session_str);
1710
1711                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1712                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1713                 pr->referenceId = sr->referenceId;
1714                 pr->resultSetId = sr->resultSetName;
1715                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1716                 *pr->numberOfRecordsRequested = toget;
1717                 pr->recordComposition = comp;
1718                 m_client->m_sr_transform = 1;
1719                 return new_apdu;
1720             }
1721         }
1722         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1723             m_client->m_last_resultCount <= 0)
1724         {
1725             // large set. Return pseudo-search response immediately
1726             yaz_log (YLOG_LOG, "%sOptimizing search for large set",
1727                      m_session_str);
1728             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1729             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1730             new_apdu->u.searchResponse->resultCount =
1731                 &m_client->m_last_resultCount;
1732             send_to_client(new_apdu);
1733             return 0;
1734         }
1735         else
1736         {
1737             Z_NamePlusRecordList *npr;
1738             int toget = m_client->m_last_resultCount;
1739             Z_RecordComposition *comp = 0;
1740             // small set
1741             // send a present request (small set)
1742
1743             if (sr->smallSetElementSetNames)
1744             {
1745                 comp = (Z_RecordComposition *)
1746                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1747                 comp->which = Z_RecordComp_simple;
1748                 comp->u.simple = sr->smallSetElementSetNames;
1749             }
1750
1751             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1752                                           sr->preferredRecordSyntax, comp))
1753             {
1754                 yaz_log (YLOG_LOG, "%sReturned cached records for small set",
1755                          m_session_str);
1756                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1757                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1758                 new_apdu->u.searchResponse->resultCount =
1759                     &m_client->m_last_resultCount;
1760
1761                 new_apdu->u.searchResponse->numberOfRecordsReturned
1762                     = odr_intdup(odr_encode(), toget);
1763
1764                 new_apdu->u.searchResponse->presentStatus =
1765                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1766                 new_apdu->u.searchResponse->records = (Z_Records*)
1767                     odr_malloc(odr_encode(), sizeof(Z_Records));
1768                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1769                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1770                 new_apdu->u.searchResponse->nextResultSetPosition =
1771                     odr_intdup(odr_encode(), toget+1);
1772                 send_to_client(new_apdu);
1773                 return 0;
1774             }
1775             else
1776             {
1777                 yaz_log (YLOG_LOG, "%sOptimizing search for small set",
1778                          m_session_str);
1779                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1780                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1781                 pr->referenceId = sr->referenceId;
1782                 pr->resultSetId = sr->resultSetName;
1783                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1784                 *pr->numberOfRecordsRequested = toget;
1785                 pr->recordComposition = comp;
1786                 m_client->m_sr_transform = 1;
1787                 return new_apdu;
1788             }
1789         }
1790     }
1791     else  // query doesn't match
1792     {
1793         delete m_client->m_last_query;
1794         m_client->m_last_query = this_query;
1795         m_client->m_last_ok = 0;
1796         m_client->m_cache.clear();
1797         m_client->m_resultSetStartPoint = 0;
1798
1799         xfree (m_client->m_last_resultSetId);
1800         m_client->m_last_resultSetId = xstrdup (sr->resultSetName);
1801
1802         m_client->m_last_databases.set(sr->num_databaseNames,
1803                                        (const char **) sr->databaseNames);
1804     }
1805     return apdu;
1806 }
1807
1808
1809 void Yaz_Proxy::inc_request_no()
1810 {
1811     char *cp = strchr(m_session_str, ' ');
1812     m_request_no++;
1813     if (cp)
1814         sprintf(cp+1, "%d ", m_request_no);
1815 }
1816
1817 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
1818 {
1819     inc_request_no();
1820
1821     m_bytes_recv += len;
1822
1823     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1824         yaz_log (YLOG_LOG, "%sReceiving %s from client %d bytes",
1825                  m_session_str, gdu_name(apdu), len);
1826
1827 #if HAVE_GETTIMEOFDAY
1828     gettimeofday((struct timeval *) m_time_tv, 0);
1829 #endif
1830     m_bw_stat.add_bytes(len);
1831     m_pdu_stat.add_bytes(1);
1832
1833     GDU *gdu = new GDU(apdu);
1834     m_in_queue.enqueue(gdu);
1835
1836     recv_GDU_more(false);
1837 }
1838
1839 void Yaz_Proxy::recv_GDU_reduce(GDU *gdu)
1840 {
1841     int reduce = 0;
1842
1843     if (1)
1844     {
1845         m_parent->m_connect.add_connect(m_peername);
1846         int connect_total = m_parent->m_connect.get_total(m_peername);
1847         int connect_max = m_parent->m_connect_max;
1848
1849         if (connect_max && connect_total > connect_max)
1850         {
1851             yaz_log(YLOG_LOG, "%sconnect delay total=%d max=%d",
1852                     m_session_str, connect_total, connect_max);
1853             reduce = connect_total / connect_max;
1854         }
1855         else
1856             yaz_log(YLOG_LOG, "%sconnect OK total=%d", m_session_str,
1857                     connect_total);
1858         m_parent->m_connect.cleanup(false);
1859     }
1860
1861
1862     int bw_total = m_bw_stat.get_total();
1863     int pdu_total = m_pdu_stat.get_total();
1864
1865     assert(m_timeout_mode == timeout_busy);
1866     assert(m_timeout_gdu == 0);
1867
1868     if (m_bw_max)
1869     {
1870         if (bw_total > m_bw_max)
1871         {
1872             reduce = (bw_total/m_bw_max);
1873         }
1874     }
1875     if (m_pdu_max)
1876     {
1877         if (pdu_total > m_pdu_max)
1878         {
1879             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
1880             reduce = (reduce > nreduce) ? reduce : nreduce;
1881         }
1882     }
1883     m_http_version = 0;
1884
1885 #if 0
1886     /* uncomment to force a big reduce */
1887     m_timeout_mode = timeout_reduce;
1888     m_timeout_gdu = gdu;
1889     timeout(3);       // call us reduce seconds later
1890     return;
1891 #endif
1892     if (reduce)
1893     {
1894         yaz_log(YLOG_LOG, "%sdelay=%d bw=%d pdu=%d limit-bw=%d limit-pdu=%d",
1895                 m_session_str, reduce, bw_total, pdu_total,
1896                 m_bw_max, m_pdu_max);
1897
1898         m_timeout_mode = timeout_reduce;
1899         m_timeout_gdu = gdu;
1900         timeout(reduce);       // call us reduce seconds later
1901     }
1902     else
1903         recv_GDU_normal(gdu);
1904 }
1905
1906 void Yaz_Proxy::recv_GDU_more(bool normal)
1907 {
1908     GDU *g;
1909     if (normal && m_timeout_mode == timeout_busy)
1910         m_timeout_mode = timeout_normal;
1911     while (m_timeout_mode == timeout_normal && (g = m_in_queue.dequeue()))
1912     {
1913         m_timeout_mode = timeout_busy;
1914         recv_GDU_reduce(g);
1915     }
1916 }
1917
1918 void Yaz_Proxy::recv_GDU_normal(GDU *gdu)
1919 {
1920     Z_GDU *apdu = 0;
1921     gdu->move_away_gdu(odr_decode(), &apdu);
1922     delete gdu;
1923
1924     if (apdu->which == Z_GDU_Z3950)
1925         handle_incoming_Z_PDU(apdu->u.z3950);
1926     else if (apdu->which == Z_GDU_HTTP_Request)
1927         handle_incoming_HTTP(apdu->u.HTTP_Request);
1928 }
1929
1930 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
1931 {
1932     if (m_max_record_retrieve)
1933     {
1934         if (apdu->which == Z_APDU_presentRequest)
1935         {
1936             Z_PresentRequest *pr = apdu->u.presentRequest;
1937             if (pr->numberOfRecordsRequested &&
1938                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
1939                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
1940         }
1941     }
1942 }
1943
1944 void Yaz_Proxy::handle_charset_lang_negotiation(Z_APDU *apdu)
1945 {
1946     if (apdu->which == Z_APDU_initRequest)
1947     {
1948         if (m_initRequest_options &&
1949             !ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel) &&
1950             (m_proxy_negotiation_charset || m_proxy_negotiation_lang))
1951         {
1952             // There is no negotiation proposal from
1953             // client's side. OK. The proxy negotiation
1954             // in use, only.
1955             Z_InitRequest *initRequest = apdu->u.initRequest;
1956             Z_OtherInformation **otherInfo;
1957             Z_OtherInformationUnit *oi;
1958             get_otherInfoAPDU(apdu, &otherInfo);
1959             oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
1960             if (oi)
1961             {
1962                 ODR_MASK_SET(initRequest->options,
1963                     Z_Options_negotiationModel);
1964                 oi->which = Z_OtherInfo_externallyDefinedInfo;
1965                 oi->information.externallyDefinedInfo =
1966                 yaz_set_proposal_charneg(odr_encode(),
1967                     (const char**)&m_proxy_negotiation_charset,
1968                     m_proxy_negotiation_charset ? 1:0,
1969                     (const char**)&m_proxy_negotiation_lang,
1970                     m_proxy_negotiation_lang ? 1:0,
1971                     1);
1972             }
1973         }
1974         else if (m_initRequest_options &&
1975                  ODR_MASK_GET(m_initRequest_options,
1976                               Z_Options_negotiationModel) &&
1977                  m_charset_converter->get_target_query_charset())
1978         {
1979             yaz_log(YLOG_LOG, "%sManaged charset negotiation: charset=%s",
1980                     m_session_str,
1981                     m_charset_converter->get_target_query_charset());
1982             Z_InitRequest *initRequest = apdu->u.initRequest;
1983             Z_CharSetandLanguageNegotiation *negotiation =
1984                 yaz_get_charneg_record (initRequest->otherInfo);
1985             if (negotiation &&
1986                 negotiation->which == Z_CharSetandLanguageNegotiation_proposal)
1987             {
1988                 NMEM nmem = nmem_create();
1989                 char **charsets = 0;
1990                 int num_charsets = 0;
1991                 char **langs = 0;
1992                 int num_langs = 0;
1993                 int selected = 0;
1994                 yaz_get_proposal_charneg (nmem, negotiation,
1995                                           &charsets, &num_charsets,
1996                                           &langs, &num_langs, &selected);
1997                 int i;
1998                 for (i = 0; i<num_charsets; i++)
1999                     yaz_log(YLOG_LOG, "%scharset %s", m_session_str,
2000                             charsets[i]);
2001                 for (i = 0; i<num_langs; i++)
2002                     yaz_log(YLOG_LOG, "%slang %s", m_session_str,
2003                             langs[i]);
2004
2005                 const char *t_charset =
2006                     m_charset_converter->get_target_query_charset();
2007                 // sweep through charsets and pick the first supported
2008                 // conversion
2009                 for (i = 0; i<num_charsets; i++)
2010                 {
2011                     const char *c_charset = charsets[i];
2012                     if (!odr_set_charset(odr_decode(), t_charset, c_charset))
2013                         break;
2014                 }
2015                 if (i != num_charsets)
2016                 {
2017                     // got one .. set up ODR for reverse direction
2018                     const char *c_charset = charsets[i];
2019                     odr_set_charset(odr_encode(), c_charset, t_charset);
2020                     m_charset_converter->set_client_query_charset(c_charset);
2021                     m_charset_converter->set_client_charset_selected(selected);
2022                 }
2023                 nmem_destroy(nmem);
2024                 ODR_MASK_CLEAR(m_initRequest_options,
2025                                Z_Options_negotiationModel);
2026                 yaz_del_charneg_record(&initRequest->otherInfo);
2027             }
2028             else
2029             {
2030                 yaz_log(YLOG_WARN, "%sUnable to decode charset package",
2031                         m_session_str);
2032             }
2033         }
2034         else if (m_charset_converter->get_target_query_charset() &&
2035             m_proxy_negotiation_default_charset)
2036         {
2037             m_charset_converter->
2038                 set_client_query_charset(m_proxy_negotiation_default_charset);
2039         }
2040     }
2041     else if (apdu->which == Z_APDU_initResponse)
2042     {
2043         Z_InitResponse *initResponse = apdu->u.initResponse;
2044         Z_OtherInformation **otherInfo;
2045         get_otherInfoAPDU(apdu, &otherInfo);
2046         
2047         Z_CharSetandLanguageNegotiation *charneg = 0;
2048
2049         if (otherInfo && *otherInfo && 
2050             ODR_MASK_GET(initResponse->options, Z_Options_negotiationModel)
2051             && (charneg = yaz_get_charneg_record(*otherInfo)))
2052         {
2053             char *charset = 0;
2054             char *lang = 0;
2055             int selected = 0;
2056
2057             yaz_get_response_charneg(m_referenceId_mem, charneg,
2058                 &charset, &lang, &selected);
2059
2060             yaz_log(YLOG_LOG, "%sAccepted charset - '%s' and lang - '%s'",
2061                 m_session_str, (charset)?charset:"none", (lang)?lang:"none");
2062
2063             if (m_initRequest_options &&
2064                 ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel))
2065             {
2066                 yaz_log(YLOG_LOG, "%sClient's negotiation record in use",
2067                     m_session_str);
2068             }
2069             else if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
2070             {
2071                 // negotiation-charset, negotiation-lang
2072                 // elements of config file in use.
2073
2074                 yaz_log(YLOG_LOG, "%sProxy's negotiation record in use",
2075                     m_session_str);
2076
2077                 // clear negotiation option.
2078                 ODR_MASK_CLEAR(initResponse->options, Z_Options_negotiationModel);
2079
2080                 // Delete negotiation (charneg-3) entry.
2081                 yaz_del_charneg_record(otherInfo);
2082             }
2083         }
2084         else
2085         {
2086             if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
2087             {
2088                 yaz_log(YLOG_LOG, "%sTarget did not honor negotiation",
2089                         m_session_str);
2090             }
2091             else if (m_charset_converter->get_client_query_charset())
2092             {
2093                 Z_OtherInformation **otherInfo;
2094                 Z_OtherInformationUnit *oi;
2095                 get_otherInfoAPDU(apdu, &otherInfo);
2096                 oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
2097                 if (oi)
2098                 {
2099                     ODR_MASK_SET(initResponse->options,
2100                                  Z_Options_negotiationModel);
2101                     if (m_initRequest_options)
2102                         ODR_MASK_SET(m_initRequest_options,
2103                                      Z_Options_negotiationModel);
2104                     
2105                     oi->which = Z_OtherInfo_externallyDefinedInfo;    
2106                     oi->information.externallyDefinedInfo =
2107                         yaz_set_response_charneg(
2108                             odr_encode(),
2109                             m_charset_converter->get_client_query_charset(),
2110                             0 /* no lang */,
2111                             m_charset_converter->get_client_charset_selected());
2112                 }
2113             }
2114         }
2115     }
2116 }
2117
2118 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
2119                                                      int error,
2120                                                      const char *addinfo)
2121 {
2122     Z_Records *rec = (Z_Records *)
2123         odr_malloc (odr, sizeof(*rec));
2124     int *err = (int *)
2125         odr_malloc (odr, sizeof(*err));
2126     Z_DiagRec *drec = (Z_DiagRec *)
2127         odr_malloc (odr, sizeof(*drec));
2128     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
2129         odr_malloc (odr, sizeof(*dr));
2130     *err = error;
2131     rec->which = Z_Records_NSD;
2132     rec->u.nonSurrogateDiagnostic = dr;
2133     dr->diagnosticSetId =
2134         yaz_oidval_to_z3950oid (odr, CLASS_DIAGSET, VAL_BIB1);
2135     dr->condition = err;
2136     dr->which = Z_DefaultDiagFormat_v2Addinfo;
2137     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
2138     return rec;
2139 }
2140
2141 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
2142 {
2143     if (apdu->which == Z_APDU_searchRequest &&
2144         apdu->u.searchRequest->query &&
2145         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
2146         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
2147     {
2148         Z_RPNQuery *rpnquery = 0;
2149         Z_SearchRequest *sr = apdu->u.searchRequest;
2150         char *addinfo = 0;
2151
2152         yaz_log(YLOG_LOG, "%sCQL: %s", m_session_str,
2153                 sr->query->u.type_104->u.cql);
2154
2155         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
2156                                           &rpnquery, odr_encode(),
2157                                           &addinfo);
2158         if (r == -3)
2159             yaz_log(YLOG_LOG, "%sNo CQL to RPN table", m_session_str);
2160         else if (r)
2161         {
2162             yaz_log(YLOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
2163             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2164
2165             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2166             new_apdu->u.searchResponse->records =
2167                 create_nonSurrogateDiagnostics(odr_encode(),
2168                                                yaz_diag_srw_to_bib1(r),
2169                                                addinfo);
2170             *new_apdu->u.searchResponse->searchStatus = 0;
2171
2172             send_to_client(new_apdu);
2173
2174             return 0;
2175         }
2176         else
2177         {
2178             sr->query->which = Z_Query_type_1;
2179             sr->query->u.type_1 = rpnquery;
2180         }
2181         return apdu;
2182     }
2183     return apdu;
2184 }
2185
2186 Z_APDU *Yaz_Proxy::handle_target_charset_conversion(Z_APDU *apdu)
2187 {
2188     if (apdu->which == Z_APDU_searchRequest &&
2189         apdu->u.searchRequest->query)
2190     {
2191         if (apdu->u.searchRequest->query->which == Z_Query_type_1
2192             || apdu->u.searchRequest->query->which == Z_Query_type_101)
2193         {
2194             if (m_http_version)
2195                 m_charset_converter->set_client_query_charset("UTF-8");
2196             Z_RPNQuery *rpnquery = apdu->u.searchRequest->query->u.type_1;
2197             m_charset_converter->convert_type_1(rpnquery, odr_encode());
2198         }
2199     }
2200     return apdu;
2201 }
2202
2203
2204 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
2205 {
2206     if (apdu->which == Z_APDU_searchRequest)
2207     {
2208         Z_SearchRequest *sr = apdu->u.searchRequest;
2209         int err = 0;
2210         char *addinfo = 0;
2211
2212         Yaz_ProxyConfig *cfg = check_reconfigure();
2213         if (cfg)
2214             err = cfg->check_query(odr_encode(), m_default_target,
2215                                    sr->query, &addinfo);
2216         if (err)
2217         {
2218             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2219
2220             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2221             new_apdu->u.searchResponse->records =
2222                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2223             *new_apdu->u.searchResponse->searchStatus = 0;
2224
2225             send_to_client(new_apdu);
2226
2227             return 0;
2228         }
2229     }
2230     return apdu;
2231 }
2232
2233 int Yaz_Proxy::handle_authentication(Z_APDU *apdu)
2234 {
2235     if (apdu->which != Z_APDU_initRequest)
2236         return 1;  // pass if no init request
2237     Z_InitRequest *req = apdu->u.initRequest;
2238
2239     Yaz_ProxyConfig *cfg = check_reconfigure();
2240     if (!cfg)
2241         return 1;  // pass if no config
2242
2243     int ret;
2244     if (req->idAuthentication == 0)
2245     {
2246         ret = cfg->client_authentication(m_default_target, 0, 0, 0,
2247                                          m_peername);
2248     }
2249     else if (req->idAuthentication->which == Z_IdAuthentication_idPass)
2250     {
2251         ret = cfg->client_authentication(
2252             m_default_target,
2253             req->idAuthentication->u.idPass->userId,
2254             req->idAuthentication->u.idPass->groupId,
2255             req->idAuthentication->u.idPass->password,
2256             m_peername);
2257     }
2258     else if (req->idAuthentication->which == Z_IdAuthentication_open)
2259     {
2260         char user[64], pass[64];
2261         *user = '\0';
2262         *pass = '\0';
2263         sscanf(req->idAuthentication->u.open, "%63[^/]/%63s", user, pass);
2264         ret = cfg->client_authentication(m_default_target, user, 0, pass,
2265                                          m_peername);
2266     }
2267     else
2268         ret = cfg->client_authentication(m_default_target, 0, 0, 0,
2269                                          m_peername);
2270     return ret;
2271 }
2272
2273 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
2274 {
2275     m_marcxml_mode = none;
2276     if (apdu->which == Z_APDU_searchRequest)
2277     {
2278         Z_SearchRequest *sr = apdu->u.searchRequest;
2279         int err = 0;
2280         char *addinfo = 0;
2281         Yaz_ProxyConfig *cfg = check_reconfigure();
2282
2283         Z_RecordComposition rc_temp, *rc = 0;
2284         if (sr->smallSetElementSetNames)
2285         {
2286             rc_temp.which = Z_RecordComp_simple;
2287             rc_temp.u.simple = sr->smallSetElementSetNames;
2288             rc = &rc_temp;
2289         }
2290
2291         if (sr->preferredRecordSyntax)
2292         {
2293             struct oident *ent;
2294             ent = oid_getentbyoid(sr->preferredRecordSyntax);
2295             m_frontend_type = ent->value;
2296         }
2297         else
2298             m_frontend_type = VAL_NONE;
2299
2300         char *stylesheet_name = 0;
2301         if (cfg)
2302             err = cfg->check_syntax(odr_encode(),
2303                                     m_default_target,
2304                                     sr->preferredRecordSyntax, rc,
2305                                     &addinfo, &stylesheet_name, &m_schema,
2306                                     &m_backend_type, &m_backend_charset,
2307                                     &m_usemarcon_ini_stage1,
2308                                     &m_usemarcon_ini_stage2);
2309         if (stylesheet_name)
2310         {
2311             m_parent->low_socket_close();
2312
2313 #if HAVE_XSLT
2314             if (m_stylesheet_xsp)
2315                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2316             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2317                                                        stylesheet_name);
2318 #endif
2319             m_stylesheet_offset = 0;
2320             xfree(stylesheet_name);
2321
2322             m_parent->low_socket_open();
2323         }
2324         if (err == -1)
2325         {
2326             sr->smallSetElementSetNames = 0;
2327             sr->mediumSetElementSetNames = 0;
2328             m_marcxml_mode = marcxml;
2329             if (m_backend_type)
2330             {
2331
2332                 sr->preferredRecordSyntax =
2333                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
2334                                         m_backend_type);
2335             }
2336             else
2337                 sr->preferredRecordSyntax =
2338                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
2339                                            VAL_USMARC);
2340         }
2341         else if (err)
2342         {
2343             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2344
2345             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2346             new_apdu->u.searchResponse->records =
2347                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2348             *new_apdu->u.searchResponse->searchStatus = 0;
2349
2350             send_to_client(new_apdu);
2351
2352             return 0;
2353         }
2354         else if (m_backend_type)
2355         {
2356             sr->preferredRecordSyntax =
2357                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
2358         }
2359     }
2360     else if (apdu->which == Z_APDU_presentRequest)
2361     {
2362         Z_PresentRequest *pr = apdu->u.presentRequest;
2363         int err = 0;
2364         char *addinfo = 0;
2365         Yaz_ProxyConfig *cfg = check_reconfigure();
2366
2367         if (pr->preferredRecordSyntax)
2368         {
2369             struct oident *ent;
2370             ent = oid_getentbyoid(pr->preferredRecordSyntax);
2371             m_frontend_type = ent->value;
2372         }
2373         else
2374             m_frontend_type = VAL_NONE;
2375
2376         char *stylesheet_name = 0;
2377         if (cfg)
2378             err = cfg->check_syntax(odr_encode(), m_default_target,
2379                                     pr->preferredRecordSyntax,
2380                                     pr->recordComposition,
2381                                     &addinfo, &stylesheet_name, &m_schema,
2382                                     &m_backend_type, &m_backend_charset,
2383                                     &m_usemarcon_ini_stage1,
2384                                     &m_usemarcon_ini_stage2
2385                                     );
2386         if (stylesheet_name)
2387         {
2388             m_parent->low_socket_close();
2389
2390 #if HAVE_XSLT
2391             if (m_stylesheet_xsp)
2392                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2393             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2394                                                        stylesheet_name);
2395 #endif
2396             m_stylesheet_offset = 0;
2397             xfree(stylesheet_name);
2398
2399             m_parent->low_socket_open();
2400         }
2401         if (err == -1)
2402         {
2403             pr->recordComposition = 0;
2404             m_marcxml_mode = marcxml;
2405             if (m_backend_type)
2406             {
2407
2408                 pr->preferredRecordSyntax =
2409                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
2410                                         m_backend_type);
2411             }
2412             else
2413                 pr->preferredRecordSyntax =
2414                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
2415                                            VAL_USMARC);
2416         }
2417         else if (err)
2418         {
2419             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
2420
2421             new_apdu->u.presentResponse->referenceId = pr->referenceId;
2422             new_apdu->u.presentResponse->records =
2423                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2424             *new_apdu->u.presentResponse->presentStatus =
2425                 Z_PresentStatus_failure;
2426
2427             send_to_client(new_apdu);
2428
2429             return 0;
2430         }
2431         else if (m_backend_type)
2432         {
2433             pr->preferredRecordSyntax =
2434                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
2435         }
2436     }
2437     return apdu;
2438 }
2439
2440 Z_ElementSetNames *Yaz_Proxy::mk_esn_from_schema(ODR o, const char *schema)
2441 {
2442     if (!schema)
2443         return 0;
2444     Z_ElementSetNames *esn = (Z_ElementSetNames *)
2445         odr_malloc(o, sizeof(Z_ElementSetNames));
2446     esn->which = Z_ElementSetNames_generic;
2447     esn->u.generic = odr_strdup(o, schema);
2448     return esn;
2449 }
2450
2451 void Yaz_Proxy::srw_get_client(const char *db, const char **backend_db)
2452 {
2453     const char *t = 0;
2454     Yaz_ProxyConfig *cfg = check_reconfigure();
2455     if (cfg)
2456         t = cfg->get_explain_name(db, backend_db);
2457
2458     if (m_client && m_default_target && t && strcmp(m_default_target, t))
2459     {
2460         releaseClient();
2461     }
2462
2463     if (t)
2464     {
2465         xfree(m_default_target);
2466         m_default_target = xstrdup(t);
2467     }
2468 }
2469
2470 int Yaz_Proxy::file_access(Z_HTTP_Request *hreq)
2471 {
2472     struct stat sbuf;
2473     if (strcmp(hreq->method, "GET"))
2474         return 0;
2475     if (hreq->path[0] != '/')
2476     {
2477         yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
2478         return 0;
2479     }
2480     const char *cp = hreq->path;
2481     while (*cp)
2482     {
2483         if (*cp == '/' && strchr("/.", cp[1]))
2484         {
2485             yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
2486             return 0;
2487         }
2488         cp++;
2489     }
2490     const char *fname = hreq->path+1;
2491     if (stat(fname, &sbuf))
2492     {
2493         yaz_log(YLOG_WARN|YLOG_ERRNO, "%s: stat failed", fname);
2494         return 0;
2495     }
2496     if ((sbuf.st_mode & S_IFMT) != S_IFREG)
2497     {
2498         yaz_log(YLOG_WARN, "%s: not a regular file", fname);
2499         return 0;
2500     }
2501     if (sbuf.st_size > (off_t) 1000000)
2502     {
2503         yaz_log(YLOG_WARN, "%s: too large for transfer", fname);
2504         return 0;
2505     }
2506
2507     ODR o = odr_encode();
2508     Yaz_ProxyConfig *cfg = check_reconfigure();
2509     const char *ctype = cfg->check_mime_type(fname);
2510     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
2511     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
2512     if (m_http_version)
2513         hres->version = odr_strdup(o, m_http_version);
2514     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
2515     if (m_http_keepalive)
2516         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
2517     else
2518         timeout(0);
2519
2520     hres->content_len = sbuf.st_size;
2521     hres->content_buf = (char*) odr_malloc(o, hres->content_len);
2522     FILE *f = fopen(fname, "rb");
2523     if (f)
2524     {
2525         fread(hres->content_buf, 1, hres->content_len, f);
2526         fclose(f);
2527     }
2528     else
2529     {
2530         return 0;
2531     }
2532     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
2533     {
2534         yaz_log (YLOG_LOG, "%sSending file %s to client", m_session_str,
2535                  fname);
2536     }
2537     int len;
2538     send_GDU(gdu, &len);
2539     recv_GDU_more(true);
2540     return 1;
2541 }
2542
2543 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
2544 {
2545     if (m_s2z_odr_init)
2546     {
2547         odr_destroy(m_s2z_odr_init);
2548         m_s2z_odr_init = 0;
2549     }
2550     if (m_s2z_odr_search)
2551     {
2552         odr_destroy(m_s2z_odr_search);
2553         m_s2z_odr_search = 0;
2554     }
2555
2556     m_http_keepalive = 0;
2557     m_http_version = 0;
2558     if (!strcmp(hreq->version, "1.0"))
2559     {
2560         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2561         if (v && !strcmp(v, "Keep-Alive"))
2562             m_http_keepalive = 1;
2563         else
2564             m_http_keepalive = 0;
2565         m_http_version = "1.0";
2566     }
2567     else
2568     {
2569         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2570         if (v && !strcmp(v, "close"))
2571             m_http_keepalive = 0;
2572         else
2573             m_http_keepalive = 1;
2574         m_http_version = "1.1";
2575     }
2576
2577     const char *a = z_HTTP_header_lookup(hreq->headers, "Authorization");
2578     char authorization_str[255];
2579     *authorization_str = '\0';
2580     if (a && strncasecmp(a, "Basic ", 6) == 0)
2581         base64_decode(a + 6, authorization_str, 254);
2582
2583     Z_SRW_PDU *srw_pdu = 0;
2584     Z_SOAP *soap_package = 0;
2585     char *charset = 0;
2586     Z_SRW_diagnostic *diagnostic = 0;
2587     int num_diagnostic = 0;
2588
2589     if (file_access(hreq))
2590     {
2591         return;
2592     }
2593     else if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2594                             &charset) == 0
2595              || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2596                                &charset, &diagnostic, &num_diagnostic) == 0)
2597     {
2598         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
2599         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
2600         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns);
2601         m_s2z_init_apdu = 0;
2602         m_s2z_search_apdu = 0;
2603         m_s2z_present_apdu = 0;
2604
2605         m_s2z_stylesheet = 0;
2606
2607         Z_IdAuthentication *auth = NULL;
2608         if (*authorization_str)
2609         {
2610             auth = (Z_IdAuthentication *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdAuthentication));
2611             auth->which = Z_IdAuthentication_idPass;
2612             auth->u.idPass = (Z_IdPass *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdPass));
2613             auth->u.idPass->groupId = NULL;
2614             char *p = strchr(authorization_str, ':');
2615             if (p)
2616             {
2617                 *p = '\0';
2618                 p++;
2619                 auth->u.idPass->password = odr_strdup(m_s2z_odr_init, p);
2620             }
2621             auth->u.idPass->userId = odr_strdup(m_s2z_odr_init, authorization_str);
2622         }
2623
2624         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
2625         {
2626             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
2627
2628             const char *backend_db = srw_req->database;
2629             srw_get_client(srw_req->database, &backend_db);
2630
2631             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2632             // recordXPath unsupported.
2633             if (srw_req->recordXPath)
2634             {
2635                 yaz_add_srw_diagnostic(odr_decode(),
2636                                        &diagnostic, &num_diagnostic,
2637                                        72, 0);
2638             }
2639             // sort unsupported
2640             if (srw_req->sort_type != Z_SRW_sort_type_none)
2641             {
2642                 yaz_add_srw_diagnostic(odr_decode(),
2643                                        &diagnostic, &num_diagnostic,
2644                                        80, 0);
2645             }
2646             // save stylesheet
2647             if (srw_req->stylesheet)
2648                 m_s2z_stylesheet =
2649                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2650
2651             // set packing for response records ..
2652             if (srw_req->recordPacking &&
2653                 !strcmp(srw_req->recordPacking, "xml"))
2654                 m_s2z_packing = Z_SRW_recordPacking_XML;
2655             else
2656                 m_s2z_packing = Z_SRW_recordPacking_string;
2657
2658             if (num_diagnostic)
2659             {
2660                 Z_SRW_PDU *srw_pdu =
2661                     yaz_srw_get(odr_encode(),
2662                                 Z_SRW_searchRetrieve_response);
2663                 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
2664
2665                 srw_res->diagnostics = diagnostic;
2666                 srw_res->num_diagnostics = num_diagnostic;
2667                 send_srw_response(srw_pdu);
2668                 return;
2669             }
2670
2671             // prepare search PDU
2672             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
2673                                           Z_APDU_searchRequest);
2674             Z_SearchRequest *z_searchRequest =
2675                 m_s2z_search_apdu->u.searchRequest;
2676
2677             z_searchRequest->num_databaseNames = 1;
2678             z_searchRequest->databaseNames = (char**)
2679                 odr_malloc(m_s2z_odr_search, sizeof(char *));
2680             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
2681                                                            backend_db);
2682
2683             // query transformation
2684             Z_Query *query = (Z_Query *)
2685                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
2686             z_searchRequest->query = query;
2687
2688             if (srw_req->query_type == Z_SRW_query_type_cql)
2689             {
2690                 Z_External *ext = (Z_External *)
2691                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
2692                 ext->direct_reference =
2693                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
2694                 ext->indirect_reference = 0;
2695                 ext->descriptor = 0;
2696                 ext->which = Z_External_CQL;
2697                 ext->u.cql = srw_req->query.cql;
2698
2699                 query->which = Z_Query_type_104;
2700                 query->u.type_104 =  ext;
2701             }
2702             else if (srw_req->query_type == Z_SRW_query_type_pqf)
2703             {
2704                 Z_RPNQuery *RPNquery;
2705                 YAZ_PQF_Parser pqf_parser;
2706
2707                 pqf_parser = yaz_pqf_create ();
2708
2709                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
2710                                           srw_req->query.pqf);
2711                 if (!RPNquery)
2712                 {
2713                     const char *pqf_msg;
2714                     size_t off;
2715                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
2716                     int ioff = off;
2717                     yaz_log(YLOG_LOG, "%*s^\n", ioff+4, "");
2718                     yaz_log(YLOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
2719
2720                     send_to_srw_client_error(10, 0);
2721                     return;
2722                 }
2723                 query->which = Z_Query_type_1;
2724                 query->u.type_1 =  RPNquery;
2725
2726                 yaz_pqf_destroy (pqf_parser);
2727             }
2728             else
2729             {
2730                 send_to_srw_client_error(7, "query");
2731                 return;
2732             }
2733
2734             // present
2735             m_s2z_present_apdu = 0;
2736             int max = 0;
2737             if (srw_req->maximumRecords)
2738                 max = *srw_req->maximumRecords;
2739             int start = 1;
2740             if (srw_req->startRecord)
2741                 start = *srw_req->startRecord;
2742             if (max > 0)
2743             {
2744                 // Some backend, such as Voyager doesn't honor piggyback
2745                 // So we use present always (0 &&).
2746                 if (0 && start <= 1)  // Z39.50 piggyback
2747                 {
2748                     *z_searchRequest->smallSetUpperBound = max;
2749                     *z_searchRequest->mediumSetPresentNumber = max;
2750                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
2751
2752                     z_searchRequest->preferredRecordSyntax =
2753                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2754                                                VAL_TEXT_XML);
2755                     if (srw_req->recordSchema)
2756                     {
2757                         z_searchRequest->smallSetElementSetNames =
2758                             z_searchRequest->mediumSetElementSetNames =
2759                             mk_esn_from_schema(m_s2z_odr_search,
2760                                                srw_req->recordSchema);
2761                     }
2762                 }
2763                 else   // Z39.50 present
2764                 {
2765                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
2766                                                    Z_APDU_presentRequest);
2767                     Z_PresentRequest *z_presentRequest =
2768                         m_s2z_present_apdu->u.presentRequest;
2769                     *z_presentRequest->resultSetStartPoint = start;
2770                     *z_presentRequest->numberOfRecordsRequested = max;
2771                     z_presentRequest->preferredRecordSyntax =
2772                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2773                                                VAL_TEXT_XML);
2774                     if (srw_req->recordSchema)
2775                     {
2776                         z_presentRequest->recordComposition =
2777                             (Z_RecordComposition *)
2778                             odr_malloc(m_s2z_odr_search,
2779                                        sizeof(Z_RecordComposition));
2780                         z_presentRequest->recordComposition->which =
2781                             Z_RecordComp_simple;
2782                         z_presentRequest->recordComposition->u.simple =
2783                             mk_esn_from_schema(m_s2z_odr_search,
2784                                                srw_req->recordSchema);
2785                     }
2786                 }
2787             }
2788             if (!m_client)
2789             {
2790                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2791                                             Z_APDU_initRequest);
2792
2793                 m_s2z_init_apdu->u.initRequest->idAuthentication = auth;
2794
2795                 // prevent m_initRequest_apdu memory from being grabbed
2796                 // in Yaz_Proxy::handle_incoming_Z_PDU
2797                 m_initRequest_apdu = m_s2z_init_apdu;
2798                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2799                 return;
2800             }
2801             else
2802             {
2803                 handle_incoming_Z_PDU(m_s2z_search_apdu);
2804                 return;
2805             }
2806         }
2807         else if (srw_pdu->which == Z_SRW_explain_request)
2808         {
2809             Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
2810
2811             const char *backend_db = srw_req->database;
2812             srw_get_client(srw_req->database, &backend_db);
2813
2814             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2815
2816             // save stylesheet
2817             if (srw_req->stylesheet)
2818                 m_s2z_stylesheet =
2819                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2820
2821             if (srw_req->recordPacking &&
2822                 !strcmp(srw_req->recordPacking, "xml"))
2823                 m_s2z_packing = Z_SRW_recordPacking_XML;
2824             else
2825                 m_s2z_packing = Z_SRW_recordPacking_string;
2826
2827             if (num_diagnostic)
2828             {
2829                 send_srw_explain_response(diagnostic, num_diagnostic);
2830                 return;
2831             }
2832
2833             if (!m_client)
2834             {
2835                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2836                                             Z_APDU_initRequest);
2837
2838                 m_s2z_init_apdu->u.initRequest->idAuthentication = auth;
2839                 
2840                 // prevent m_initRequest_apdu memory from being grabbed
2841                 // in Yaz_Proxy::handle_incoming_Z_PDU
2842                 m_initRequest_apdu = m_s2z_init_apdu;
2843                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2844             }
2845             else
2846                 send_srw_explain_response(0, 0);
2847             return;
2848         }
2849         else if (srw_pdu->which == Z_SRW_scan_request)
2850         {
2851             m_s2z_database = odr_strdup(m_s2z_odr_init,
2852                                         srw_pdu->u.scan_request->database);
2853
2854             yaz_add_srw_diagnostic(odr_decode(),
2855                                    &diagnostic, &num_diagnostic,
2856                                    4, "scan");
2857             Z_SRW_PDU *srw_pdu =
2858                 yaz_srw_get(odr_encode(),
2859                             Z_SRW_scan_response);
2860             Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
2861
2862             srw_res->diagnostics = diagnostic;
2863             srw_res->num_diagnostics = num_diagnostic;
2864             send_srw_response(srw_pdu);
2865             return;
2866         }
2867         else
2868         {
2869             m_s2z_database = 0;
2870
2871             send_to_srw_client_error(4, 0);
2872         }
2873     }
2874     send_http_response(400);
2875 }
2876
2877 void Yaz_Proxy::handle_init(Z_APDU *apdu)
2878 {
2879
2880     Z_OtherInformation **oi;
2881     get_otherInfoAPDU(apdu, &oi);
2882
2883     if (apdu->u.initRequest->implementationId)
2884         yaz_log(YLOG_LOG, "%simplementationId: %s",
2885                 m_session_str, apdu->u.initRequest->implementationId);
2886     if (apdu->u.initRequest->implementationName)
2887         yaz_log(YLOG_LOG, "%simplementationName: %s",
2888                 m_session_str, apdu->u.initRequest->implementationName);
2889     if (apdu->u.initRequest->implementationVersion)
2890         yaz_log(YLOG_LOG, "%simplementationVersion: %s",
2891                 m_session_str, apdu->u.initRequest->implementationVersion);
2892     if (m_initRequest_apdu == 0)
2893     {
2894         if (m_initRequest_mem)
2895             nmem_destroy(m_initRequest_mem);
2896
2897         m_initRequest_apdu = apdu;
2898         m_initRequest_mem = odr_extract_mem(odr_decode());
2899
2900         m_initRequest_preferredMessageSize = *apdu->u.initRequest->
2901             preferredMessageSize;
2902         *apdu->u.initRequest->preferredMessageSize = 1024*1024;
2903         m_initRequest_maximumRecordSize = *apdu->u.initRequest->
2904             maximumRecordSize;
2905         *apdu->u.initRequest->maximumRecordSize = 1024*1024;
2906
2907         Z_CharSetandLanguageNegotiation *charSetandLangRecord =
2908             yaz_get_charneg_record(*oi);
2909
2910         // Save proposal charsets and langs.
2911         if (ODR_MASK_GET(apdu->u.initRequest->options,
2912                          Z_Options_negotiationModel)
2913             && charSetandLangRecord)
2914         {
2915
2916             yaz_get_proposal_charneg(m_referenceId_mem,
2917                                      charSetandLangRecord,
2918                                      &m_initRequest_oi_negotiation_charsets,
2919                                      &m_initRequest_oi_negotiation_num_charsets,
2920                                      &m_initRequest_oi_negotiation_langs,
2921                                      &m_initRequest_oi_negotiation_num_langs,
2922                                      &m_initRequest_oi_negotiation_selected);
2923
2924             for (int i = 0; i<m_initRequest_oi_negotiation_num_charsets; i++)
2925             {
2926                 yaz_log(YLOG_LOG, "%scharacters set proposal: %s",
2927                         m_session_str,(m_initRequest_oi_negotiation_charsets[i])?
2928                         m_initRequest_oi_negotiation_charsets[i]:"none");
2929             }
2930             for (int i=0; i<m_initRequest_oi_negotiation_num_langs; i++)
2931             {
2932                 yaz_log(YLOG_LOG, "%slanguages proposal: %s",
2933                         m_session_str, (m_initRequest_oi_negotiation_langs[i])?
2934                         m_initRequest_oi_negotiation_langs[i]:"none");
2935             }
2936             yaz_log(YLOG_LOG, "%sselected proposal: %d (boolean)",
2937                     m_session_str, m_initRequest_oi_negotiation_selected);
2938         }
2939         // save init options for the response..
2940         m_initRequest_options = apdu->u.initRequest->options;
2941
2942         apdu->u.initRequest->options =
2943             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2944                                        sizeof(Odr_bitmask));
2945         ODR_MASK_ZERO(apdu->u.initRequest->options);
2946         int i;
2947         for (i = 0; i<= 24; i++)
2948             ODR_MASK_SET(apdu->u.initRequest->options, i);
2949         // check negotiation option
2950         if (!ODR_MASK_GET(m_initRequest_options,
2951                           Z_Options_negotiationModel))
2952         {
2953             ODR_MASK_CLEAR(apdu->u.initRequest->options,
2954                            Z_Options_negotiationModel);
2955         }
2956         ODR_MASK_CLEAR(apdu->u.initRequest->options,
2957                        Z_Options_concurrentOperations);
2958         // make new version
2959         m_initRequest_version = apdu->u.initRequest->protocolVersion;
2960         apdu->u.initRequest->protocolVersion =
2961             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2962                                        sizeof(Odr_bitmask));
2963         ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
2964
2965         for (i = 0; i<= 8; i++)
2966             ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
2967     }
2968     handle_charset_lang_negotiation(apdu);
2969     if (m_client->m_init_flag)
2970     {
2971         if (handle_init_response_for_invalid_session(apdu))
2972             return;
2973         if (m_client->m_initResponse)
2974         {
2975             Z_APDU *apdu2 = m_client->m_initResponse;
2976             apdu2->u.initResponse->otherInfo = 0;
2977             if (m_client->m_cookie && *m_client->m_cookie)
2978                 set_otherInformationString(apdu2, VAL_COOKIE, 1,
2979                                            m_client->m_cookie);
2980             apdu2->u.initResponse->referenceId =
2981                 apdu->u.initRequest->referenceId;
2982             apdu2->u.initResponse->options = m_client->m_initResponse_options;
2983             apdu2->u.initResponse->protocolVersion =
2984                 m_client->m_initResponse_version;
2985
2986             handle_charset_lang_negotiation(apdu2);
2987
2988             send_to_client(apdu2);
2989             m_timeout_mode = timeout_normal;
2990             return;
2991         }
2992     }
2993     m_client->m_init_flag = 1;
2994
2995 #if USE_AUTH_MSG
2996     Auth_Msg *m = new Auth_Msg;
2997     m->m_proxy = this;
2998     z_APDU(odr_encode(), &apdu, 0, "encode");
2999     char *apdu_buf = odr_getbuf(odr_encode(), &m->m_apdu_len, 0);
3000     m->m_apdu_buf = (char*) nmem_malloc(m->m_nmem, m->m_apdu_len);
3001     memcpy(m->m_apdu_buf, apdu_buf, m->m_apdu_len);
3002     odr_reset(odr_encode());
3003
3004     inc_ref();
3005     m_my_thread->put(m);
3006 #else
3007     int ret = handle_authentication(apdu);
3008     result_authentication(apdu, ret);
3009 #endif
3010 }
3011
3012 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
3013 {
3014     Z_ReferenceId **refid = get_referenceIdP(apdu);
3015     nmem_reset(m_referenceId_mem);
3016     if (refid && *refid)
3017     {
3018         m_referenceId = (Z_ReferenceId *)
3019             nmem_malloc(m_referenceId_mem, sizeof(*m_referenceId));
3020         m_referenceId->len = m_referenceId->size = (*refid)->len;
3021         m_referenceId->buf = (unsigned char *)
3022             nmem_malloc(m_referenceId_mem, (*refid)->len);
3023         memcpy(m_referenceId->buf, (*refid)->buf, (*refid)->len);
3024     }
3025     else
3026         m_referenceId = 0;
3027
3028     if (!m_client && m_flag_invalid_session)
3029     {
3030         // Got request for a session that is invalid..
3031         m_apdu_invalid_session = apdu; // save package
3032         m_mem_invalid_session = odr_extract_mem(odr_decode());
3033         apdu = m_initRequest_apdu;     // but throw an init to the target
3034     }
3035
3036     // Determine our client.
3037     Z_OtherInformation **oi;
3038     get_otherInfoAPDU(apdu, &oi);
3039     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi));
3040     if (!m_client)
3041     {
3042         if (m_http_version)
3043         {   // HTTP. Send not found
3044             send_http_response(404);
3045             return;
3046         }
3047         else
3048         {
3049             // Z39.50 just shutdown
3050             delete this;
3051             return;
3052         }
3053     }
3054
3055     m_client->m_server = this;
3056
3057     if (apdu->which == Z_APDU_initRequest)
3058         handle_init(apdu);
3059     else
3060         handle_incoming_Z_PDU_2(apdu);
3061 }
3062
3063 void Yaz_Proxy::handle_incoming_Z_PDU_2(Z_APDU *apdu)
3064 {
3065     handle_max_record_retrieve(apdu);
3066
3067     if (apdu)
3068         apdu = handle_syntax_validation(apdu);
3069
3070     if (apdu)
3071         apdu = handle_query_transformation(apdu);
3072
3073     if (apdu)
3074         apdu = handle_target_charset_conversion(apdu);
3075
3076     if (apdu)
3077         apdu = handle_query_validation(apdu);
3078
3079     if (apdu)
3080         apdu = result_set_optimize(apdu);
3081
3082     if (!apdu)
3083     {
3084         m_client->timeout(m_target_idletime);  // mark it active even
3085         recv_GDU_more(true);
3086         // though we didn't use it
3087         return;
3088     }
3089
3090     // delete other info construct completely if 0 elements
3091     Z_OtherInformation **oi;
3092     get_otherInfoAPDU(apdu, &oi);
3093     if (oi && *oi && (*oi)->num_elements == 0)
3094         *oi = 0;
3095
3096     if (apdu->which == Z_APDU_presentRequest &&
3097         m_client->m_resultSetStartPoint == 0)
3098     {
3099         Z_PresentRequest *pr = apdu->u.presentRequest;
3100         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
3101         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
3102     } else {
3103         m_client->m_resultSetStartPoint = 0;
3104     }
3105     if (m_client->send_to_target(apdu) < 0)
3106     {
3107         m_client->shutdown();
3108     }
3109     else
3110         m_client->m_waiting = 1;
3111 }
3112
3113 void Yaz_Proxy::connectNotify()
3114 {
3115 }
3116
3117 void Yaz_Proxy::releaseClient()
3118 {
3119     xfree(m_proxyTarget);
3120     m_proxyTarget = 0;
3121     m_flag_invalid_session = 0;
3122     // only keep if keep_alive flag is set...
3123     if (m_client &&
3124         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
3125         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
3126         m_client->m_waiting == 0)
3127     {
3128         yaz_log(YLOG_LOG, "%sShutdown (client to proxy) keepalive %s",
3129                  m_session_str,
3130                  m_client->get_hostname());
3131         yaz_log(YLOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
3132                 m_session_str, m_client->m_pdu_recv,
3133                 m_client->m_bytes_sent + m_client->m_bytes_recv,
3134                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
3135         assert (m_client->m_waiting != 2);
3136         // Tell client (if any) that no server connection is there..
3137         m_client->m_server = 0;
3138         m_client = 0;
3139     }
3140     else if (m_client)
3141     {
3142         yaz_log (YLOG_LOG, "%sShutdown (client to proxy) close %s",
3143                  m_session_str,
3144                  m_client->get_hostname());
3145         assert (m_client->m_waiting != 2);
3146         delete m_client;
3147         m_client = 0;
3148     }
3149     else if (!m_parent)
3150     {
3151         yaz_log (YLOG_LOG, "%sshutdown (client to proxy) bad state",
3152                  m_session_str);
3153         assert (m_parent);
3154     }
3155     else
3156     {
3157         yaz_log (YLOG_LOG, "%sShutdown (client to proxy)",
3158                  m_session_str);
3159     }
3160     if (m_parent)
3161         m_parent->pre_init();
3162 }
3163
3164 bool Yaz_Proxy::dec_ref(bool main_ptr)
3165 {
3166     assert(m_ref_count > 0);
3167     if (main_ptr)
3168     {
3169         if (m_main_ptr_dec)
3170             return false;
3171         m_main_ptr_dec = true;
3172     }
3173
3174     m_http_keepalive = 0;
3175
3176     --m_ref_count;
3177     if (m_ref_count > 0)
3178         return false;
3179
3180     releaseClient();
3181
3182     delete this;
3183     return true;
3184 }
3185
3186 const char *Yaz_ProxyClient::get_session_str()
3187 {
3188     if (!m_server)
3189         return "0 ";
3190     return m_server->get_session_str();
3191 }
3192
3193 void Yaz_ProxyClient::shutdown()
3194 {
3195     yaz_log (YLOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
3196              get_hostname());
3197
3198     if (m_server)
3199     {
3200         m_waiting = 1;      // ensure it's released from Proxy in releaseClient
3201         m_server->dec_ref(true);
3202     }
3203     else
3204         delete this;
3205 }
3206
3207 void Yaz_Proxy::failNotify()
3208 {
3209     inc_request_no();
3210     yaz_log (YLOG_LOG, "%sConnection closed by client",
3211              get_session_str());
3212     dec_ref(true);
3213 }
3214
3215 void Yaz_Proxy::send_response_fail_client(const char *addr)
3216 {
3217     if (m_http_version)
3218     {
3219         Z_SRW_diagnostic *diagnostic = 0;
3220         int num_diagnostic = 0;
3221         
3222         yaz_add_srw_diagnostic(odr_encode(),
3223                                &diagnostic, &num_diagnostic,
3224                                YAZ_SRW_SYSTEM_TEMPORARILY_UNAVAILABLE, addr);
3225         if (m_s2z_search_apdu)
3226             send_srw_search_response(diagnostic, num_diagnostic);
3227         else
3228             send_srw_explain_response(diagnostic, num_diagnostic);
3229     }            
3230 }
3231 void Yaz_ProxyClient::failNotify()
3232 {
3233     if (m_server)
3234         m_server->inc_request_no();
3235     yaz_log (YLOG_LOG, "%sConnection closed by target %s",
3236              get_session_str(), get_hostname());
3237
3238     if (m_server)
3239         m_server->send_response_fail_client(get_hostname());
3240     shutdown();
3241 }
3242
3243 void Yaz_ProxyClient::connectNotify()
3244 {
3245     const char *s = get_session_str();
3246     const char *h = get_hostname();
3247     yaz_log (YLOG_LOG, "%sConnection accepted by %s timeout=%d", s, h,
3248              m_target_idletime);
3249     timeout(m_target_idletime);
3250     if (!m_server)
3251         pre_init_client();
3252 }
3253
3254 IPDU_Observer *Yaz_ProxyClient::sessionNotify(IPDU_Observable
3255                                               *the_PDU_Observable, int fd)
3256 {
3257     return new Yaz_ProxyClient(the_PDU_Observable, 0);
3258 }
3259
3260 Yaz_ProxyClient::~Yaz_ProxyClient()
3261 {
3262     if (m_prev)
3263         *m_prev = m_next;
3264     if (m_next)
3265         m_next->m_prev = m_prev;
3266     m_waiting = 2;     // for debugging purposes only.
3267     odr_destroy(m_init_odr);
3268     odr_destroy(m_idAuthentication_odr);
3269     delete m_last_query;
3270     xfree (m_last_resultSetId);
3271     xfree (m_cookie);
3272 }
3273
3274 void Yaz_ProxyClient::pre_init_client()
3275 {
3276     Z_APDU *apdu = create_Z_PDU(Z_APDU_initRequest);
3277     Z_InitRequest *req = apdu->u.initRequest;
3278
3279     int i;
3280     for (i = 0; i<= 24; i++)
3281         ODR_MASK_SET(req->options, i);
3282     ODR_MASK_CLEAR(apdu->u.initRequest->options,
3283                    Z_Options_negotiationModel);
3284     ODR_MASK_CLEAR(apdu->u.initRequest->options,
3285                    Z_Options_concurrentOperations);
3286     for (i = 0; i<= 10; i++)
3287         ODR_MASK_SET(req->protocolVersion, i);
3288
3289     if (send_to_target(apdu) < 0)
3290     {
3291         delete this;
3292     }
3293     else
3294     {
3295         m_waiting = 1;
3296         m_init_flag = 1;
3297     }
3298 }
3299
3300 void Yaz_Proxy::pre_init()
3301 {
3302     int i;
3303     const char *name = 0;
3304     const char *zurl_in_use[MAX_ZURL_PLEX];
3305     int limit_bw, limit_pdu, limit_req, limit_search;
3306     int target_idletime, client_idletime;
3307     int max_clients;
3308     int keepalive_limit_bw, keepalive_limit_pdu;
3309     int pre_init;
3310     const char *cql2rpn = 0;
3311     const char *authentication = 0;
3312     const char *negotiation_charset = 0;
3313     const char *negotiation_lang = 0;
3314
3315     Yaz_ProxyConfig *cfg = check_reconfigure();
3316
3317     zurl_in_use[0] = 0;
3318
3319     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
3320         set_APDU_yazlog(1);
3321     else
3322         set_APDU_yazlog(0);
3323
3324     for (i = 0; cfg && cfg->get_target_no(i, &name, zurl_in_use,
3325                                           &limit_bw, &limit_pdu, &limit_req,
3326                                           &limit_search,
3327                                           &target_idletime, &client_idletime,
3328                                           &max_clients,
3329                                           &keepalive_limit_bw,
3330                                           &keepalive_limit_pdu,
3331                                           &pre_init,
3332                                           &cql2rpn,
3333                                           &authentication,
3334                                           &negotiation_charset,
3335                                           &negotiation_lang,
3336                                           0,
3337                                           0) ; i++)
3338     {
3339         if (pre_init)
3340         {
3341             int j;
3342             for (j = 0; zurl_in_use[j]; j++)
3343             {
3344                 Yaz_ProxyClient *c;
3345                 int spare = 0;
3346                 int spare_waiting = 0;
3347                 int in_use = 0;
3348                 int other = 0;
3349                 for (c = m_clientPool; c; c = c->m_next)
3350                 {
3351                     if (!strcmp(zurl_in_use[j], c->get_hostname()))
3352                     {
3353                         if (c->m_cookie == 0)
3354                         {
3355                             if (c->m_server == 0)
3356                  &nbs