Fixed timeout bug for SRW/SRU operation : Could occur if XSLT
[yazproxy-moved-to-github.git] / src / yaz-proxy.cpp
1 /* $Id: yaz-proxy.cpp,v 1.63 2006-04-27 00:04:42 adam Exp $
2    Copyright (c) 1998-2006, Index Data.
3
4 This file is part of the yazproxy.
5
6 YAZ proxy is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 2, or (at your option) any later
9 version.
10
11 YAZ proxy is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
14 for more details.
15
16 You should have received a copy of the GNU General Public License
17 along with YAZ proxy; see the file LICENSE.  If not, write to the
18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
19 02111-1307, USA.
20  */
21
22 #ifdef WIN32
23 #define HAVE_SYS_STAT_H 1
24 #define HAVE_SYS_TYPES_H 1
25 #endif
26
27 #if HAVE_UNISTD_H
28 #include <unistd.h>
29 #endif
30 #if HAVE_SYS_TIME_H
31 #include <sys/time.h>
32 #endif
33 #if HAVE_SYS_TYPES_H
34 #include <sys/types.h>
35 #endif
36 #if HAVE_SYS_STAT_H
37 #include <sys/stat.h>
38 #endif
39
40 #include <assert.h>
41 #include <stdlib.h>
42 #include <time.h>
43 #include <fcntl.h>
44
45 #include <yaz/srw.h>
46 #include <yaz/marcdisp.h>
47 #include <yaz/yaz-iconv.h>
48 #include <yaz/log.h>
49 #include <yaz/diagbib1.h>
50 #include "proxyp.h"
51 #include <yaz/pquery.h>
52 #include <yaz/otherinfo.h>
53 #include <yaz/charneg.h>
54 #include "msg-thread.h"
55
56 using namespace yazpp_1;
57
58 #ifdef WIN32
59 #define strncasecmp _strnicmp
60 #endif
61
62 #define USE_AUTH_MSG 1
63
64 #if USE_AUTH_MSG
65 class YAZ_EXPORT Auth_Msg : public IMsg_Thread {
66 public:
67     int m_ret;
68     IMsg_Thread *handle();
69     void result();
70     Yaz_Proxy *m_proxy;
71     NMEM m_nmem;
72     char *m_apdu_buf;
73     int m_apdu_len;
74     Auth_Msg();
75     virtual ~Auth_Msg();
76 };
77
78 Auth_Msg::Auth_Msg()
79 {
80     m_nmem = nmem_create();
81 }
82
83 Auth_Msg::~Auth_Msg()
84 {
85     nmem_destroy(m_nmem);
86 }
87
88 IMsg_Thread *Auth_Msg::handle()
89 {
90     ODR decode = odr_createmem(ODR_DECODE);
91     Z_APDU *apdu;
92
93     odr_setbuf(decode, m_apdu_buf, m_apdu_len, 0);
94     int r = z_APDU(decode, &apdu, 0, 0);
95     if (!r)
96     {
97         yaz_log(YLOG_WARN, "decode failed in Auth_Msg::handle");
98     }
99     else
100     {
101         m_ret = m_proxy->handle_authentication(apdu);
102     }
103     odr_destroy(decode);
104     return this;
105 }
106
107 void Auth_Msg::result()
108 {
109     if (m_proxy->dec_ref())
110     {
111         yaz_log(YLOG_LOG, "Auth_Msg::proxy deleted meanwhile");
112     }
113     else
114     {
115         odr_setbuf(m_proxy->odr_decode(), m_apdu_buf, m_apdu_len, 0);
116         Z_APDU *apdu = 0;
117         int r = z_APDU(m_proxy->odr_decode(), &apdu, 0, 0);
118         if (!r)
119             yaz_log(YLOG_LOG, "Auth_Msg::result z_APDU failed");
120         m_proxy->result_authentication(apdu, m_ret);
121     }
122     delete this;
123 }
124
125 #endif
126
127 void Yaz_Proxy::result_authentication(Z_APDU *apdu, int ret)
128 {
129     if (apdu == 0 || ret == 0)
130     {
131         Z_APDU *apdu_reject = zget_APDU(odr_encode(), Z_APDU_initResponse);
132         *apdu_reject->u.initResponse->result = 0;
133         send_to_client(apdu_reject);
134         dec_ref();
135     }
136     else
137     {
138         if (apdu->which == Z_APDU_initRequest)
139         {
140             Yaz_ProxyConfig *cfg = check_reconfigure();
141             if (cfg)
142                 cfg->target_authentication(m_default_target, odr_encode(), 
143                                            apdu->u.initRequest);
144         }
145         handle_incoming_Z_PDU_2(apdu);
146     }
147 }
148
149 static const char *apdu_name(Z_APDU *apdu)
150 {
151     switch (apdu->which)
152     {
153     case Z_APDU_initRequest:
154         return "initRequest";
155     case Z_APDU_initResponse:
156         return "initResponse";
157     case Z_APDU_searchRequest:
158         return "searchRequest";
159     case Z_APDU_searchResponse:
160         return "searchResponse";
161     case Z_APDU_presentRequest:
162         return "presentRequest";
163     case Z_APDU_presentResponse:
164         return "presentResponse";
165     case Z_APDU_deleteResultSetRequest:
166         return "deleteResultSetRequest";
167     case Z_APDU_deleteResultSetResponse:
168         return "deleteResultSetResponse";
169     case Z_APDU_scanRequest:
170         return "scanRequest";
171     case Z_APDU_scanResponse:
172         return "scanResponse";
173     case Z_APDU_sortRequest:
174         return "sortRequest";
175     case Z_APDU_sortResponse:
176         return "sortResponse";
177     case Z_APDU_extendedServicesRequest:
178         return "extendedServicesRequest";
179     case Z_APDU_extendedServicesResponse:
180         return "extendedServicesResponse";
181     case Z_APDU_close:
182         return "close";
183     }
184     return "other";
185 }
186
187 static const char *gdu_name(Z_GDU *gdu)
188 {
189     switch(gdu->which)
190     {
191     case Z_GDU_Z3950:
192         return apdu_name(gdu->u.z3950);
193     case Z_GDU_HTTP_Request:
194         return "HTTP Request";
195     case Z_GDU_HTTP_Response:
196         return "HTTP Response";
197     }
198     return "Unknown request/response";
199 }
200
201 Yaz_Proxy::Yaz_Proxy(IPDU_Observable *the_PDU_Observable,
202                      ISocketObservable *the_socket_observable,
203                      Yaz_Proxy *parent)
204     :
205     Z_Assoc(the_PDU_Observable),
206     m_bw_stat(60), m_pdu_stat(60), m_search_stat(60)
207 {
208     m_PDU_Observable = the_PDU_Observable;
209     m_socket_observable = the_socket_observable;
210     m_client = 0;
211     m_parent = parent;
212     m_clientPool = 0;
213     m_seqno = 1;
214     m_keepalive_limit_bw = 500000;
215     m_keepalive_limit_pdu = 1000;
216     m_proxyTarget = 0;
217     m_default_target = 0;
218     m_proxy_negotiation_charset = 0;
219     m_proxy_negotiation_lang = 0;
220     m_proxy_negotiation_default_charset = 0;
221     m_charset_converter = new Yaz_CharsetConverter;
222     m_max_clients = 150;
223     m_log_mask = 0;
224     m_seed = time(0);
225     m_client_idletime = 600;
226     m_target_idletime = 600;
227     m_optimize = xstrdup ("1");
228     strcpy(m_session_str, "0 ");
229     m_session_no = 0;
230     m_bytes_sent = 0;
231     m_bytes_recv = 0;
232     m_bw_max = 0;
233     m_pdu_max = 0;
234     m_search_max = 0;
235     m_max_connect = 0;
236     m_max_connect_period = 0;
237     m_limit_connect = 0;
238     m_limit_connect_period = 0;
239     m_timeout_mode = timeout_normal;
240     m_timeout_gdu = 0;
241     m_max_record_retrieve = 0;
242     m_reconfig_flag = 0;
243     m_config_fname = 0;
244     m_request_no = 0;
245     m_flag_invalid_session = 0;
246     m_referenceId = 0;
247     m_referenceId_mem = nmem_create();
248     m_config = 0;
249     m_marcxml_mode = none;
250     m_stylesheet_xsp = 0;
251     m_stylesheet_nprl = 0;
252     m_stylesheet_apdu = 0;
253     m_s2z_stylesheet = 0;
254     m_s2z_database = 0;
255     m_schema = 0;
256     m_backend_type = 0;
257     m_backend_charset = 0;
258     m_frontend_type = 0;
259     m_initRequest_apdu = 0;
260     m_initRequest_mem = 0;
261     m_initRequest_preferredMessageSize = 0;
262     m_initRequest_maximumRecordSize = 0;
263     m_initRequest_options = 0;
264     m_initRequest_version = 0;
265     m_initRequest_oi_negotiation_charsets = 0;
266     m_initRequest_oi_negotiation_num_charsets = 0;
267     m_initRequest_oi_negotiation_langs = 0;
268     m_initRequest_oi_negotiation_num_langs = 0;
269     m_initRequest_oi_negotiation_selected = 0;
270     m_apdu_invalid_session = 0;
271     m_mem_invalid_session = 0;
272     m_s2z_odr_init = 0;
273     m_s2z_odr_search = 0;
274     m_s2z_init_apdu = 0;
275     m_s2z_search_apdu = 0;
276     m_s2z_present_apdu = 0;
277     m_http_keepalive = 0;
278     m_http_version = 0;
279     m_soap_ns = 0;
280     m_s2z_packing = Z_SRW_recordPacking_string;
281 #if HAVE_GETTIMEOFDAY
282     m_time_tv = xmalloc(sizeof(struct timeval));
283     struct timeval *tv = (struct timeval *) m_time_tv;
284     tv->tv_sec = 0;
285     tv->tv_usec = 0;
286 #else
287     m_time_tv = 0;
288 #endif
289     m_usemarcon_ini_stage1 = 0;
290     m_usemarcon_ini_stage2 = 0;
291     m_usemarcon = new Yaz_usemarcon();
292     if (!m_parent)
293         low_socket_open();
294     m_my_thread = 0;
295     m_ref_count = 1;
296     m_main_ptr_dec = false;
297     m_peername = 0;
298 }
299
300 void Yaz_Proxy::inc_ref()
301 {
302     m_ref_count++;
303 }
304
305 Yaz_Proxy::~Yaz_Proxy()
306 {
307     yaz_log(YLOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
308             m_bytes_sent, m_bytes_recv);
309     nmem_destroy(m_initRequest_mem);
310     nmem_destroy(m_mem_invalid_session);
311     nmem_destroy(m_referenceId_mem);
312
313     xfree(m_proxyTarget);
314     xfree(m_default_target);
315     xfree(m_proxy_negotiation_charset);
316     xfree(m_proxy_negotiation_lang);
317     xfree(m_proxy_negotiation_default_charset);
318     delete m_charset_converter;
319     xfree(m_optimize);
320
321 #if HAVE_XSLT
322     if (m_stylesheet_xsp)
323         xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
324 #endif
325     xfree (m_time_tv);
326
327     xfree (m_peername);
328     xfree (m_schema);
329     xfree (m_backend_type);
330     xfree (m_backend_charset);
331     xfree (m_usemarcon_ini_stage1);
332     xfree (m_usemarcon_ini_stage2);
333     delete m_usemarcon;
334     if (m_s2z_odr_init)
335         odr_destroy(m_s2z_odr_init);
336     if (m_s2z_odr_search)
337         odr_destroy(m_s2z_odr_search);
338     if (!m_parent)
339         low_socket_close();
340     if (!m_parent)
341         delete m_my_thread;
342     delete m_config;
343 }
344
345 void Yaz_Proxy::set_debug_mode(int mode)
346 {
347     m_debug_mode = mode;
348 }
349
350 int Yaz_Proxy::set_config(const char *config)
351 {
352     delete m_config;
353     m_config = new Yaz_ProxyConfig();
354     xfree(m_config_fname);
355     m_config_fname = xstrdup(config);
356     int r = m_config->read_xml(config);
357     if (!r)
358     {
359         int period = 60;
360         m_config->get_generic_info(&m_log_mask, &m_max_clients,
361                                    &m_max_connect, &m_limit_connect, &period);
362         m_connect.set_period(period);
363     }
364     return r;
365 }
366
367 void Yaz_Proxy::set_default_target(const char *target)
368 {
369     xfree (m_default_target);
370     m_default_target = 0;
371     if (target)
372         m_default_target = (char *) xstrdup (target);
373 }
374
375 void Yaz_Proxy::set_proxy_negotiation (const char *charset, const char *lang,
376                                        const char *default_charset)
377 {
378     yaz_log(YLOG_DEBUG, "%sSet the proxy negotiation: charset to '%s', "
379         "default charset to '%s', language to '%s'", m_session_str, 
380         charset?charset:"none",
381         default_charset?default_charset:"none",
382         lang?lang:"none");
383     xfree (m_proxy_negotiation_charset);
384     xfree (m_proxy_negotiation_lang);
385     m_proxy_negotiation_charset = m_proxy_negotiation_lang = 0;
386     if (charset)
387         m_proxy_negotiation_charset = (char *) xstrdup (charset);
388     if (lang)
389         m_proxy_negotiation_lang = (char *) xstrdup (lang);
390     if (default_charset)
391         m_proxy_negotiation_default_charset =
392             (char *) xstrdup (default_charset);
393 }
394
395 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
396 {
397     if (m_parent)
398         return m_parent->check_reconfigure();
399
400     Yaz_ProxyConfig *cfg = m_config;
401     if (m_reconfig_flag)
402     {
403         yaz_log(YLOG_LOG, "reconfigure");
404         yaz_log_reopen();
405         if (m_config_fname && cfg)
406         {
407             yaz_log(YLOG_LOG, "reconfigure config %s", m_config_fname);
408             int r = cfg->read_xml(m_config_fname);
409             if (r)
410                 yaz_log(YLOG_WARN, "reconfigure failed");
411             else
412             {
413                 m_log_mask = 0;
414                 int period = 60;
415                 cfg->get_generic_info(&m_log_mask, &m_max_clients,
416                                       &m_max_connect, &m_limit_connect,
417                                       &period);
418                 m_connect.set_period(period);
419             }
420         }
421         else
422             yaz_log(YLOG_LOG, "reconfigure");
423         m_reconfig_flag = 0;
424     }
425     return cfg;
426 }
427
428 IPDU_Observer *Yaz_Proxy::sessionNotify(IPDU_Observable
429                                         *the_PDU_Observable, int fd)
430 {
431     check_reconfigure();
432
433     char session_str[200];
434     const char *peername = the_PDU_Observable->getpeername();
435     if (m_log_mask & PROXY_LOG_IP_CLIENT)
436         sprintf(session_str, "%ld:%d %.80s %d ",
437                 (long) time(0), m_session_no, peername, 0);
438     else
439         sprintf(session_str, "%ld:%d %d ",
440                 (long) time(0), m_session_no, 0);
441     m_session_no++;
442
443     yaz_log (YLOG_LOG, "%sNew session %s", session_str, peername);
444
445     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable,
446                                          m_socket_observable, this);
447
448     new_proxy->m_config = 0;
449     new_proxy->m_config_fname = 0;
450     new_proxy->timeout(m_client_idletime);
451     new_proxy->m_target_idletime = m_target_idletime;
452     new_proxy->set_default_target(m_default_target);
453     new_proxy->m_max_clients = m_max_clients;
454     new_proxy->m_log_mask = m_log_mask;
455
456     if (!strcmp(peername, "tcp:163.121.19.82")) // NIS GROUP
457         new_proxy->m_log_mask = 255;
458
459     new_proxy->set_APDU_log(get_APDU_log());
460     if (new_proxy->m_log_mask & PROXY_LOG_APDU_CLIENT)
461         new_proxy->set_APDU_yazlog(1);
462     else
463         new_proxy->set_APDU_yazlog(0);
464     strcpy(new_proxy->m_session_str, session_str);
465     new_proxy->m_peername = xstrdup(peername);
466     new_proxy->set_proxy_negotiation(m_proxy_negotiation_charset,
467         m_proxy_negotiation_lang, m_proxy_negotiation_default_charset);
468     // create thread object the first time we get an incoming connection
469     if (!m_my_thread)
470         m_my_thread = new Msg_Thread(m_socket_observable, 1);
471     new_proxy->m_my_thread = m_my_thread;
472     return new_proxy;
473 }
474
475 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
476 {
477     int oid[OID_SIZE];
478     Z_OtherInformationUnit *oi;
479     struct oident ent;
480     ent.proto = PROTO_Z3950;
481     ent.oclass = CLASS_USERINFO;
482     ent.value = (oid_value) VAL_COOKIE;
483     assert (oid_ent_to_oid (&ent, oid));
484
485     if (oid_ent_to_oid (&ent, oid) &&
486         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
487         oi->which == Z_OtherInfo_characterInfo)
488         return oi->information.characterInfo;
489     return 0;
490 }
491 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
492 {
493     int oid[OID_SIZE];
494     Z_OtherInformationUnit *oi;
495     struct oident ent;
496     ent.proto = PROTO_Z3950;
497     ent.oclass = CLASS_USERINFO;
498     ent.value = (oid_value) VAL_PROXY;
499     if (oid_ent_to_oid (&ent, oid) &&
500         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
501         oi->which == Z_OtherInfo_characterInfo)
502         return oi->information.characterInfo;
503     return 0;
504 }
505 const char *Yaz_Proxy::load_balance(const char **url)
506 {
507     int zurl_in_use[MAX_ZURL_PLEX];
508     int zurl_in_spare[MAX_ZURL_PLEX];
509     Yaz_ProxyClient *c;
510     int i;
511
512     for (i = 0; i<MAX_ZURL_PLEX; i++)
513     {
514         zurl_in_use[i] = 0;
515         zurl_in_spare[i] = 0;
516     }
517     for (c = m_parent->m_clientPool; c; c = c->m_next)
518     {
519         for (i = 0; url[i]; i++)
520             if (!strcmp(url[i], c->get_hostname()))
521             {
522                 zurl_in_use[i]++;
523                 if (c->m_cookie == 0 && c->m_server == 0 && c->m_waiting == 0)
524                     zurl_in_spare[i]++;
525             }
526     }
527     int min_use = 100000;
528     int spare_for_min = 0;
529     int max_spare = 0;
530     const char *ret_min = 0;
531     const char *ret_spare = 0;
532     for (i = 0; url[i]; i++)
533     {
534         yaz_log(YLOG_DEBUG, "%szurl=%s use=%d spare=%d",
535                 m_session_str, url[i], zurl_in_use[i], zurl_in_spare[i]);
536         if (min_use > zurl_in_use[i])
537         {
538             ret_min = url[i];
539             min_use = zurl_in_use[i];
540             spare_for_min = zurl_in_spare[i];
541         }
542         if (max_spare < zurl_in_spare[i])
543         {
544             ret_spare = url[i];
545             max_spare = zurl_in_spare[i];
546         }
547     }
548     return ret_min;
549 }
550
551 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
552                                        const char *proxy_host)
553 {
554     assert (m_parent);
555     Yaz_Proxy *parent = m_parent;
556     Yaz_ProxyClient *c = m_client;
557
558     if (!m_proxyTarget)
559     {
560         const char *url[MAX_ZURL_PLEX];
561         Yaz_ProxyConfig *cfg = check_reconfigure();
562         if (proxy_host)
563         {
564             if (parent && parent->m_debug_mode)
565             {
566                 // only to be enabled for debugging...
567                 if (!strcmp(proxy_host, "stop"))
568                     exit(0);
569             }
570             xfree(m_default_target);
571             m_default_target = xstrdup(proxy_host);
572         }
573         proxy_host = m_default_target;
574         int client_idletime = -1;
575         const char *cql2rpn_fname = 0;
576         const char *negotiation_charset = 0;
577         const char *negotiation_lang = 0;
578         const char *query_charset = 0;
579         const char *default_client_query_charset = 0;
580         url[0] = m_default_target;
581         url[1] = 0;
582         if (cfg)
583         {
584             int pre_init = 0;
585             cfg->get_target_info(proxy_host, url, &m_bw_max,
586                                  &m_pdu_max, &m_max_record_retrieve,
587                                  &m_search_max,
588                                  &m_target_idletime, &client_idletime,
589                                  &parent->m_max_clients,
590                                  &m_keepalive_limit_bw,
591                                  &m_keepalive_limit_pdu,
592                                  &pre_init,
593                                  &cql2rpn_fname,
594                                  &negotiation_charset,
595                                  &negotiation_lang,
596                                  &query_charset,
597                                  &default_client_query_charset);
598         }
599         if (client_idletime != -1)
600         {
601             m_client_idletime = client_idletime;
602             timeout(m_client_idletime);
603         }
604
605         // get those FILE descriptors available 
606         m_parent->low_socket_close();
607         if (cql2rpn_fname)
608             m_cql2rpn.set_pqf_file(cql2rpn_fname);
609         // reserve them again
610         m_parent->low_socket_open();
611         
612         if (negotiation_charset || negotiation_lang || default_client_query_charset)
613         {
614             set_proxy_negotiation(negotiation_charset,
615                 negotiation_lang, default_client_query_charset);
616         }
617         m_charset_converter->set_target_query_charset(query_charset);
618         if (!url[0])
619         {
620             yaz_log(YLOG_LOG, "%sNo default target", m_session_str);
621             return 0;
622         }
623         // we don't handle multiplexing for cookie session, so we just
624         // pick the first one in this case (anonymous users will be able
625         // to use any backend)
626         if (cookie && *cookie)
627             m_proxyTarget = (char*) xstrdup(url[0]);
628         else
629             m_proxyTarget = (char*) xstrdup(load_balance(url));
630     }
631     if (cookie && *cookie)
632     {   // search in sessions with a cookie
633         for (c = parent->m_clientPool; c; c = c->m_next)
634         {
635             assert (c->m_prev);
636             assert (*c->m_prev == c);
637             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
638                 !strcmp(m_proxyTarget, c->get_hostname()))
639             {
640                 // Found it in cache
641                 // The following handles "cancel"
642                 // If connection is busy (waiting for PDU) and
643                 // we have an initRequest we can safely do re-open
644                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
645                 {
646                     yaz_log (YLOG_LOG, "%s REOPEN target=%s", m_session_str,
647                              c->get_hostname());
648                     c->close();
649                     c->m_init_flag = 0;
650
651                     c->m_last_ok = 0;
652                     c->m_cache.clear();
653                     c->m_last_resultCount = 0;
654                     c->m_sr_transform = 0;
655                     c->m_waiting = 0;
656                     c->m_resultSetStartPoint = 0;
657                     c->m_target_idletime = m_target_idletime;
658                     if (c->client(m_proxyTarget))
659                     {
660                         delete c;
661                         return 0;
662                     }
663                     c->timeout(30);
664                 }
665                 c->m_seqno = parent->m_seqno;
666                 if (c->m_server && c->m_server != this)
667                     c->m_server->m_client = 0;
668                 c->m_server = this;
669                 (parent->m_seqno)++;
670                 yaz_log (YLOG_DEBUG, "get_client 1 %p %p", this, c);
671                 return c;
672             }
673         }
674     }
675     else if (!c && apdu->which == Z_APDU_initRequest )
676     {
677         // anonymous sessions without cookie.
678         // if authentication is set it is NOT anonymous se we can't share them.
679         // If charset and lang negotiation is use it is NOT anonymous session too.
680         for (c = parent->m_clientPool; c; c = c->m_next)
681         {
682             assert(c->m_prev);
683             assert(*c->m_prev == c);
684             if (c->m_server == 0 && c->m_cookie == 0 &&  c->m_waiting == 0 
685                 && c->compare_idAuthentication(apdu)
686                 && c->compare_charset(apdu)
687                 && !strcmp(m_proxyTarget, c->get_hostname()))
688             {
689                 // found it in cache
690                 yaz_log (YLOG_LOG, "%sREUSE %d %s",
691                          m_session_str, parent->m_seqno, c->get_hostname());
692                 
693                 c->m_seqno = parent->m_seqno;
694                 assert(c->m_server == 0);
695                 c->m_server = this;
696
697                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
698                     c->set_APDU_yazlog(1);
699                 else
700                     c->set_APDU_yazlog(0);
701
702                 (parent->m_seqno)++;
703
704                 parent->pre_init();
705
706                 return c;
707             }
708         }
709     }
710     if (!m_client)
711     {
712         if (apdu->which != Z_APDU_initRequest)
713         {
714             yaz_log (YLOG_LOG, "%sno init request as first PDU", m_session_str);
715             return 0;
716         }
717         // go through list of clients - and find the lowest/oldest one.
718         Yaz_ProxyClient *c_min = 0;
719         int min_seq = -1;
720         int no_of_clients = 0;
721         if (parent->m_clientPool)
722             yaz_log (YLOG_DEBUG, "Existing sessions");
723         for (c = parent->m_clientPool; c; c = c->m_next)
724         {
725             yaz_log (YLOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
726                                c->m_waiting, c->get_hostname(),
727                                c->m_cookie ? c->m_cookie : "");
728             no_of_clients++;
729             if (min_seq < 0 || c->m_seqno < min_seq)
730             {
731                 min_seq = c->m_seqno;
732                 c_min = c;
733             }
734         }
735         if (no_of_clients >= parent->m_max_clients)
736         {
737             c = c_min;
738             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
739             {
740                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Destroy %d",
741                          m_session_str, parent->m_max_clients, c->m_seqno);
742                 if (c->m_server && c->m_server != this)
743                     c->m_server->dec_ref();
744             }
745             else
746             {
747                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Reuse %d %d %s",
748                          m_session_str, parent->m_max_clients,
749                          c->m_seqno, parent->m_seqno, c->get_hostname());
750                 xfree (c->m_cookie);
751                 c->m_cookie = 0;
752                 if (cookie)
753                     c->m_cookie = xstrdup(cookie);
754                 c->m_seqno = parent->m_seqno;
755                 if (c->m_server && c->m_server != this)
756                 {
757                     c->m_server->m_client = 0;
758                     c->m_server->dec_ref();
759                 }
760                 (parent->m_seqno)++;
761                 c->m_target_idletime = m_target_idletime;
762                 c->timeout(m_target_idletime);
763
764                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
765                     c->set_APDU_yazlog(1);
766                 else
767                     c->set_APDU_yazlog(0);
768
769                 return c;
770             }
771         }
772         else
773         {
774             yaz_log (YLOG_LOG, "%sNEW %d %s",
775                      m_session_str, parent->m_seqno, m_proxyTarget);
776             c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
777             c->m_next = parent->m_clientPool;
778             if (c->m_next)
779                 c->m_next->m_prev = &c->m_next;
780             parent->m_clientPool = c;
781             c->m_prev = &parent->m_clientPool;
782         }
783
784         xfree (c->m_cookie);
785         c->m_cookie = 0;
786         if (cookie)
787             c->m_cookie = xstrdup(cookie);
788
789         c->m_seqno = parent->m_seqno;
790         c->m_init_flag = 0;
791         c->m_last_resultCount = 0;
792         c->m_last_ok = 0;
793         c->m_cache.clear();
794         c->m_sr_transform = 0;
795         c->m_waiting = 0;
796         c->m_resultSetStartPoint = 0;
797         (parent->m_seqno)++;
798         if (c->client(m_proxyTarget))
799         {
800             delete c;
801             return 0;
802         }
803         c->m_target_idletime = m_target_idletime;
804         c->timeout(30);
805
806         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
807             c->set_APDU_yazlog(1);
808         else
809             c->set_APDU_yazlog(0);
810
811         c->set_idAuthentication(apdu);
812     }
813     yaz_log (YLOG_DEBUG, "get_client 3 %p %p", this, c);
814     return c;
815 }
816
817 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
818 {
819     int i;
820     for (i = 0; i<num; i++)
821     {
822         oident *ent;
823         Z_DefaultDiagFormat *r;
824         Z_DiagRec *p = pp[i];
825         if (p->which != Z_DiagRec_defaultFormat)
826         {
827             yaz_log(YLOG_LOG, "%sError no diagnostics", m_session_str);
828             return;
829         }
830         else
831             r = p->u.defaultFormat;
832         if (!(ent = oid_getentbyoid(r->diagnosticSetId)) ||
833             ent->oclass != CLASS_DIAGSET || ent->value != VAL_BIB1)
834             yaz_log(YLOG_LOG, "%sError unknown diagnostic set", m_session_str);
835         switch (r->which)
836         {
837         case Z_DefaultDiagFormat_v2Addinfo:
838             yaz_log(YLOG_LOG, "%sError %d %s:%s",
839                     m_session_str,
840                     *r->condition, diagbib1_str(*r->condition),
841                     r->u.v2Addinfo);
842             break;
843         case Z_DefaultDiagFormat_v3Addinfo:
844             yaz_log(YLOG_LOG, "%sError %d %s:%s",
845                     m_session_str,
846                     *r->condition, diagbib1_str(*r->condition),
847                     r->u.v3Addinfo);
848             break;
849         }
850     }
851 }
852
853 int Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p, Z_APDU *apdu)
854 {
855     if (!m_stylesheet_xsp || p->num_records <= 0)
856     {
857         return 0;  /* no XSLT to be done ... */
858     }
859
860     m_stylesheet_offset = 0;
861     m_stylesheet_nprl = p;
862     m_stylesheet_apdu = apdu;
863     m_timeout_mode = timeout_xsl;
864
865     timeout(0);
866     return 1;
867 }
868
869 void Yaz_Proxy::convert_xsl_delay()
870 {
871 #if HAVE_XSLT
872     Z_NamePlusRecord *npr = m_stylesheet_nprl->records[m_stylesheet_offset];
873     if (npr->which == Z_NamePlusRecord_databaseRecord)
874     {
875         Z_External *r = npr->u.databaseRecord;
876         if (r->which == Z_External_octet)
877         {
878 #if 0
879             fwrite((char*) r->u.octet_aligned->buf, 1, r->u.octet_aligned->len, stdout);
880 #endif
881             xmlDocPtr res, doc = xmlParseMemory(
882                 (char*) r->u.octet_aligned->buf,
883                 r->u.octet_aligned->len);
884
885
886             yaz_log(YLOG_LOG, "%sXSLT convert %d",
887                     m_session_str, m_stylesheet_offset);
888             res = xsltApplyStylesheet((xsltStylesheetPtr) m_stylesheet_xsp,
889                                       doc, 0);
890
891             if (res)
892             {
893                 xmlChar *out_buf;
894                 int out_len;
895                 xmlDocDumpFormatMemory (res, &out_buf, &out_len, 1);
896
897                 m_stylesheet_nprl->records[m_stylesheet_offset]->
898                     u.databaseRecord =
899                     z_ext_record(odr_encode(), VAL_TEXT_XML,
900                                  (char*) out_buf, out_len);
901                 xmlFree(out_buf);
902                 xmlFreeDoc(res);
903             }
904
905             xmlFreeDoc(doc);
906         }
907     }
908 #endif
909     m_stylesheet_offset++;
910     if (m_stylesheet_offset == m_stylesheet_nprl->num_records)
911     {
912         m_timeout_mode = timeout_normal;
913         m_stylesheet_nprl = 0;
914 #if HAVE_XSLT
915         if (m_stylesheet_xsp)
916             xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
917 #endif
918         m_stylesheet_xsp = 0;
919         timeout(m_client_idletime);
920         send_PDU_convert(m_stylesheet_apdu);
921     }
922     else
923         timeout(0);
924 }
925
926 void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p)
927 {
928     if (m_frontend_type != VAL_NONE)
929     {
930         int i;
931         for (i = 0; i < p->num_records; i++)
932         {
933             Z_NamePlusRecord *npr = p->records[i];
934             if (npr->which == Z_NamePlusRecord_databaseRecord)
935             {
936                 Z_External *r = npr->u.databaseRecord;
937                 if (r->which == Z_External_octet)
938                 {
939 #if HAVE_USEMARCON
940                     if (m_usemarcon_ini_stage1 && *m_usemarcon_ini_stage1)
941                     {
942                         if (!m_usemarcon->m_stage1)
943                         {
944                             m_usemarcon->m_stage1 = new CDetails();
945                         }
946                         m_usemarcon->m_stage1->SetIniFileName(m_usemarcon_ini_stage1);
947                         m_usemarcon->m_stage1->SetMarcRecord((char*) r->u.octet_aligned->buf, r->u.octet_aligned->len);
948                         int res = m_usemarcon->m_stage1->Start();
949                         if (res == 0)
950                         {
951                             char *converted;
952                             int convlen;
953                             m_usemarcon->m_stage1->GetMarcRecord(converted, convlen);
954                             if (m_usemarcon_ini_stage2 && *m_usemarcon_ini_stage2)
955                             {
956                                 if (!m_usemarcon->m_stage2)
957                                 {
958                                     m_usemarcon->m_stage2 = new CDetails();
959                                 }
960                                 m_usemarcon->m_stage2->SetIniFileName(m_usemarcon_ini_stage2);
961                                 m_usemarcon->m_stage2->SetMarcRecord(converted, convlen);
962                                 res = m_usemarcon->m_stage2->Start();
963                                 if (res == 0)
964                                 {
965                                     free(converted);
966                                     m_usemarcon->m_stage2->GetMarcRecord(converted, convlen);
967                                 }
968                                 else
969                                 {
970                                     yaz_log(YLOG_LOG, "%sUSEMARCON stage 2 error %d", m_session_str, res);
971                                 }
972                             }
973                             npr->u.databaseRecord =
974                                 z_ext_record(odr_encode(),
975                                              m_frontend_type,
976                                              converted,
977                                              strlen(converted));
978                             free(converted);
979                         }
980                         else
981                         {
982                             yaz_log(YLOG_LOG, "%sUSEMARCON stage 1 error %d", m_session_str, res);
983                         }
984                         continue;
985                     }
986 #endif
987 /* HAVE_USEMARCON */
988                     npr->u.databaseRecord =
989                         z_ext_record(odr_encode(),
990                                      m_frontend_type,
991                                      (char*) r->u.octet_aligned->buf,
992                                      r->u.octet_aligned->len);
993                 }
994             }
995         }
996     }
997 }
998
999 void Yaz_Proxy::convert_records_charset(Z_NamePlusRecordList *p,
1000                                         const char *backend_charset)
1001 {
1002     int sel =   m_charset_converter->get_client_charset_selected();
1003     const char *client_record_charset =
1004         m_charset_converter->get_client_query_charset();
1005     if (sel && backend_charset && client_record_charset &&
1006         strcmp(backend_charset, client_record_charset))
1007     {
1008         int i;
1009         yaz_iconv_t cd = yaz_iconv_open(client_record_charset,
1010                                         backend_charset);
1011         yaz_marc_t mt = yaz_marc_create();
1012         yaz_marc_xml(mt, YAZ_MARC_ISO2709);
1013         yaz_marc_iconv(mt, cd);
1014         for (i = 0; i < p->num_records; i++)
1015         {
1016             Z_NamePlusRecord *npr = p->records[i];
1017             if (npr->which == Z_NamePlusRecord_databaseRecord)
1018             {
1019                 Z_External *r = npr->u.databaseRecord;
1020                 oident *ent = oid_getentbyoid(r->direct_reference);
1021                 if (!ent || ent->value == VAL_NONE)
1022                     continue;
1023
1024                 if (ent->value == VAL_SUTRS)
1025                 {
1026                     WRBUF w = wrbuf_alloc();
1027
1028                     wrbuf_iconv_write(w, cd,  (char*) r->u.octet_aligned->buf,
1029                                       r->u.octet_aligned->len);
1030                     npr->u.databaseRecord =
1031                         z_ext_record(odr_encode(), ent->value, wrbuf_buf(w),
1032                                      wrbuf_len(w));
1033                     wrbuf_free(w, 1);
1034                 }
1035                 else if (ent->value == VAL_TEXT_XML)
1036                 {
1037                     ;
1038                 }
1039                 else if (r->which == Z_External_octet)
1040                 {
1041                     int rlen;
1042                     char *result;
1043                     if (yaz_marc_decode_buf(mt,
1044                                             (char*) r->u.octet_aligned->buf,
1045                                             r->u.octet_aligned->len,
1046                                             &result, &rlen))
1047                     {
1048                         npr->u.databaseRecord =
1049                             z_ext_record(odr_encode(), ent->value, result, rlen);
1050                         yaz_log(YLOG_LOG, "%sRecoding MARC record",
1051                                 m_session_str);
1052                     }
1053                 }
1054             }
1055         }
1056         if (cd)
1057             yaz_iconv_close(cd);
1058         yaz_marc_destroy(mt);
1059     }
1060 }
1061
1062 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p,
1063                                    const char *backend_charset)
1064 {
1065     int i;
1066     if (!backend_charset)
1067         backend_charset = "MARC-8";
1068     yaz_iconv_t cd = yaz_iconv_open("UTF-8", backend_charset);
1069     yaz_marc_t mt = yaz_marc_create();
1070     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
1071     yaz_marc_iconv(mt, cd);
1072     for (i = 0; i < p->num_records; i++)
1073     {
1074         Z_NamePlusRecord *npr = p->records[i];
1075         if (npr->which == Z_NamePlusRecord_databaseRecord)
1076         {
1077             Z_External *r = npr->u.databaseRecord;
1078             if (r->which == Z_External_OPAC)
1079             {
1080                 WRBUF w = wrbuf_alloc();
1081
1082                 yaz_opac_decode_wrbuf(mt, r->u.opac, w);
1083                 npr->u.databaseRecord = z_ext_record(
1084                     odr_encode(), VAL_TEXT_XML,
1085                     wrbuf_buf(w), wrbuf_len(w)
1086                     );
1087                 wrbuf_free(w, 1);
1088             }
1089             else if (r->which == Z_External_octet)
1090             {
1091                 int rlen;
1092                 char *result;
1093                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
1094                                         r->u.octet_aligned->len,
1095                                         &result, &rlen))
1096                 {
1097                     npr->u.databaseRecord =
1098                         z_ext_record(odr_encode(), VAL_TEXT_XML, result, rlen);
1099                 }
1100             }
1101         }
1102     }
1103     if (cd)
1104         yaz_iconv_close(cd);
1105     yaz_marc_destroy(mt);
1106 }
1107
1108 void Yaz_Proxy::logtime()
1109 {
1110 #if HAVE_GETTIMEOFDAY
1111     struct timeval *tv = (struct timeval*) m_time_tv;
1112     if (tv->tv_sec)
1113     {
1114         struct timeval tv1;
1115         gettimeofday(&tv1, 0);
1116         long diff = (tv1.tv_sec - tv->tv_sec)*1000000 +
1117             (tv1.tv_usec - tv->tv_usec);
1118         if (diff >= 0)
1119             yaz_log(YLOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
1120                     diff/1000000, (diff/1000)%1000);
1121     }
1122     tv->tv_sec = 0;
1123     tv->tv_usec = 0;
1124 #endif
1125 }
1126
1127 int Yaz_Proxy::send_http_response(int code)
1128 {
1129     ODR o = odr_encode();
1130     Z_GDU *gdu = z_get_HTTP_Response(o, code);
1131     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1132     if (m_http_version)
1133         hres->version = odr_strdup(o, m_http_version);
1134     if (m_http_keepalive)
1135         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1136     else
1137         timeout(0);
1138
1139     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1140     {
1141         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1142                  gdu_name(gdu));
1143     }
1144     int len;
1145     int r = send_GDU(gdu, &len);
1146     m_bytes_sent += len;
1147     m_bw_stat.add_bytes(len);
1148     logtime();
1149
1150     recv_GDU_more(true);
1151
1152     return r;
1153 }
1154
1155 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu, int http_code /* = 200 */)
1156 {
1157     ODR o = odr_encode();
1158     const char *ctype = "text/xml";
1159     Z_GDU *gdu = z_get_HTTP_Response(o, http_code);
1160     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1161     if (m_http_version)
1162         hres->version = odr_strdup(o, m_http_version);
1163     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
1164     if (m_http_keepalive)
1165         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1166     else
1167         timeout(0);
1168     if (http_code == 401)
1169         z_HTTP_header_add(o, &hres->headers, "WWW-Authenticate", "Basic realm=\"YAZ Proxy\"");
1170
1171     static Z_SOAP_Handler soap_handlers[2] = {
1172 #if HAVE_XSLT
1173         {"http://www.loc.gov/zing/srw/", 0,
1174          (Z_SOAP_fun) yaz_srw_codec},
1175 #endif
1176         {0, 0, 0}
1177     };
1178
1179     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
1180     soap_package->which = Z_SOAP_generic;
1181     soap_package->u.generic =
1182         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
1183     soap_package->u.generic->no = 0;
1184     soap_package->u.generic->ns = soap_handlers[0].ns;
1185     soap_package->u.generic->p = (void *) srw_pdu;
1186     soap_package->ns = m_soap_ns;
1187     z_soap_codec_enc_xsl(o, &soap_package,
1188                          &hres->content_buf, &hres->content_len,
1189                          soap_handlers, 0, m_s2z_stylesheet);
1190     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1191     {
1192         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1193                  gdu_name(gdu));
1194     }
1195     int len;
1196     int r = send_GDU(gdu, &len);
1197     m_bytes_sent += len;
1198     m_bw_stat.add_bytes(len);
1199     logtime();
1200
1201     recv_GDU_more(true);
1202
1203     return r;
1204 }
1205
1206 int Yaz_Proxy::send_to_srw_client_error(int srw_error, const char *add)
1207 {
1208     ODR o = odr_encode();
1209     Z_SRW_diagnostic *diagnostic = (Z_SRW_diagnostic *)
1210         odr_malloc(o, sizeof(*diagnostic));
1211     int num_diagnostic = 1;
1212     yaz_mk_std_diagnostic(o, diagnostic, srw_error, add);
1213     return send_srw_search_response(diagnostic, num_diagnostic, srw_error == 3 ? 401 : 200);
1214 }
1215
1216 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
1217                              Z_DefaultDiagFormat *ddf)
1218 {
1219     int bib1_code = *ddf->condition;
1220     if (bib1_code == 109)
1221         return 404;
1222     srw_res->num_diagnostics = 1;
1223     srw_res->diagnostics = (Z_SRW_diagnostic *)
1224         odr_malloc(o, sizeof(*srw_res->diagnostics));
1225     yaz_mk_std_diagnostic(o, srw_res->diagnostics,
1226                           yaz_diag_bib1_to_srw(*ddf->condition),
1227                           ddf->u.v2Addinfo);
1228     return 0;
1229 }
1230
1231 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
1232 {
1233     ODR o = odr_encode();
1234     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1235     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1236
1237     srw_res->numberOfRecords = odr_intdup (o, hits);
1238     if (records && records->which == Z_Records_DBOSD)
1239     {
1240         srw_res->num_records =
1241             records->u.databaseOrSurDiagnostics->num_records;
1242         int i;
1243         srw_res->records = (Z_SRW_record *)
1244             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
1245         for (i = 0; i < srw_res->num_records; i++)
1246         {
1247             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
1248             if (npr->which != Z_NamePlusRecord_databaseRecord)
1249             {
1250                 srw_res->records[i].recordSchema = "diagnostic";
1251                 srw_res->records[i].recordPacking = m_s2z_packing;
1252                 srw_res->records[i].recordData_buf = "67";
1253                 srw_res->records[i].recordData_len = 2;
1254                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1255                 continue;
1256             }
1257             Z_External *r = npr->u.databaseRecord;
1258             oident *ent = oid_getentbyoid(r->direct_reference);
1259             if (r->which == Z_External_octet && ent->value == VAL_TEXT_XML)
1260             {
1261                 srw_res->records[i].recordSchema = m_schema;
1262                 srw_res->records[i].recordPacking = m_s2z_packing;
1263                 srw_res->records[i].recordData_buf = (char*)
1264                     r->u.octet_aligned->buf;
1265                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
1266                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1267             }
1268             else
1269             {
1270                 srw_res->records[i].recordSchema = "diagnostic";
1271                 srw_res->records[i].recordPacking = m_s2z_packing;
1272                 srw_res->records[i].recordData_buf = "67";
1273                 srw_res->records[i].recordData_len = 2;
1274                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1275             }
1276         }
1277     }
1278     if (records && records->which == Z_Records_NSD)
1279     {
1280         int http_code;
1281         http_code = z_to_srw_diag(odr_encode(), srw_res,
1282                                    records->u.nonSurrogateDiagnostic);
1283         if (http_code)
1284             return send_http_response(http_code);
1285     }
1286     return send_srw_response(srw_pdu);
1287
1288 }
1289
1290 int Yaz_Proxy::send_srw_search_response(Z_SRW_diagnostic *diagnostics,
1291                                         int num_diagnostics, int http_code /* = 200 */)
1292 {
1293     ODR o = odr_encode();
1294     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1295     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1296
1297     srw_res->num_diagnostics = num_diagnostics;
1298     srw_res->diagnostics = diagnostics;
1299     return send_srw_response(srw_pdu, http_code);
1300 }
1301
1302 int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics,
1303                                         int num_diagnostics)
1304 {
1305     Yaz_ProxyConfig *cfg = check_reconfigure();
1306     if (cfg)
1307     {
1308         int len;
1309         char *b = cfg->get_explain_doc(odr_encode(), 0 /* target */,
1310                                        m_s2z_database, &len);
1311         if (b)
1312         {
1313             Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
1314             Z_SRW_explainResponse *er = res->u.explain_response;
1315
1316             er->record.recordData_buf = b;
1317             er->record.recordData_len = len;
1318             er->record.recordPacking = m_s2z_packing;
1319             er->record.recordSchema = "http://explain.z3950.org/dtd/2.0/";
1320
1321             er->diagnostics = diagnostics;
1322             er->num_diagnostics = num_diagnostics;
1323             return send_srw_response(res);
1324         }
1325     }
1326     return send_http_response(404);
1327 }
1328
1329 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu)
1330 {
1331     if (m_http_version)
1332     {
1333         if (apdu->which == Z_APDU_initResponse)
1334         {
1335             Z_InitResponse *res = apdu->u.initResponse;
1336             if (*res->result == 0)
1337             {
1338                 send_to_srw_client_error(3, 0);
1339             }
1340             else if (!m_s2z_search_apdu)
1341             {
1342                 send_srw_explain_response(0, 0);
1343             }
1344             else
1345             {
1346                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1347             }
1348         }
1349         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
1350         {
1351             m_s2z_search_apdu = 0;
1352             Z_SearchResponse *res = apdu->u.searchResponse;
1353             m_s2z_hit_count = *res->resultCount;
1354             if (res->records && res->records->which == Z_Records_NSD)
1355             {
1356                 send_to_srw_client_ok(0, res->records, 1);
1357             }
1358             else if (m_s2z_present_apdu && m_s2z_hit_count > 0)
1359             {
1360                 // adjust
1361                 Z_PresentRequest *pr = m_s2z_present_apdu->u.presentRequest;
1362
1363                 if (*pr->resultSetStartPoint <= m_s2z_hit_count)
1364                 {
1365                     if (*pr->numberOfRecordsRequested+ *pr->resultSetStartPoint
1366                         > m_s2z_hit_count)
1367                         *pr->numberOfRecordsRequested =
1368                             1 + m_s2z_hit_count - *pr->resultSetStartPoint;
1369                 }
1370                 handle_incoming_Z_PDU(m_s2z_present_apdu);
1371             }
1372             else
1373             {
1374                 m_s2z_present_apdu = 0;
1375                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
1376             }
1377         }
1378         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
1379         {
1380             int start =
1381                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
1382
1383             m_s2z_present_apdu = 0;
1384             Z_PresentResponse *res = apdu->u.presentResponse;
1385             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
1386         }
1387     }
1388     else
1389     {
1390         int len = 0;
1391         if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1392             yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1393                      apdu_name(apdu));
1394         int r = send_Z_PDU(apdu, &len);
1395         m_bytes_sent += len;
1396         m_bw_stat.add_bytes(len);
1397         logtime();
1398         return r;
1399     }
1400     return 0;
1401 }
1402
1403 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
1404 {
1405     int kill_session = 0;
1406     Z_ReferenceId **new_id = get_referenceIdP(apdu);
1407
1408     if (new_id)
1409         *new_id = m_referenceId;
1410
1411     if (apdu->which == Z_APDU_searchResponse)
1412     {
1413         Z_SearchResponse *sr = apdu->u.searchResponse;
1414         Z_Records *p = sr->records;
1415         if (p && p->which == Z_Records_NSD)
1416         {
1417             Z_DiagRec dr, *dr_p = &dr;
1418             dr.which = Z_DiagRec_defaultFormat;
1419             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1420
1421             *sr->searchStatus = 0;
1422             display_diagrecs(&dr_p, 1);
1423         }
1424         else
1425         {
1426             if (p && p->which == Z_Records_DBOSD)
1427             {
1428                 if (m_backend_type
1429 #if HAVE_USEMARCON
1430                     || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1431 #endif
1432                     )
1433                     convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1434                 if (m_marcxml_mode == marcxml)
1435                     convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1436                                        m_backend_charset);
1437                 else
1438                     convert_records_charset(p->u.databaseOrSurDiagnostics,
1439                                             m_backend_charset);
1440                 if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1441                     return 0;
1442
1443             }
1444             if (sr->resultCount)
1445             {
1446                 yaz_log(YLOG_LOG, "%s%d hits", m_session_str,
1447                         *sr->resultCount);
1448                 if (*sr->resultCount < 0)
1449                 {
1450                     m_flag_invalid_session = 1;
1451                     kill_session = 1;
1452
1453                     *sr->searchStatus = 0;
1454                     sr->records =
1455                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
1456                     *sr->resultCount = 0;
1457                 }
1458             }
1459         }
1460     }
1461     else if (apdu->which == Z_APDU_presentResponse)
1462     {
1463         Z_PresentResponse *sr = apdu->u.presentResponse;
1464         Z_Records *p = sr->records;
1465         if (p && p->which == Z_Records_NSD)
1466         {
1467             Z_DiagRec dr, *dr_p = &dr;
1468             dr.which = Z_DiagRec_defaultFormat;
1469             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1470             if (*sr->presentStatus == Z_PresentStatus_success)
1471                 *sr->presentStatus = Z_PresentStatus_failure;
1472             display_diagrecs(&dr_p, 1);
1473         }
1474         if (p && p->which == Z_Records_DBOSD)
1475         {
1476             if (m_backend_type
1477 #if HAVE_USEMARCON
1478                 || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1479 #endif
1480                 )
1481                 convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1482             if (m_marcxml_mode == marcxml)
1483                 convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1484                                    m_backend_charset);
1485             else
1486                 convert_records_charset(p->u.databaseOrSurDiagnostics,
1487                                         m_backend_charset);
1488             if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1489                 return 0;
1490         }
1491     }
1492     else if (apdu->which == Z_APDU_initResponse)
1493     {
1494         //Get and check negotiation record
1495         //from init response.
1496         handle_charset_lang_negotiation(apdu);
1497
1498         if (m_initRequest_options)
1499         {
1500             Z_Options *nopt =
1501                 (Odr_bitmask *)odr_malloc(odr_encode(),
1502                                           sizeof(Odr_bitmask));
1503             ODR_MASK_ZERO(nopt);
1504
1505             int i;
1506             for (i = 0; i<24; i++)
1507                 if (ODR_MASK_GET(m_initRequest_options, i) &&
1508                     ODR_MASK_GET(apdu->u.initResponse->options, i))
1509                     ODR_MASK_SET(nopt, i);
1510             apdu->u.initResponse->options = nopt;
1511         }
1512         if (m_initRequest_version)
1513         {
1514             Z_ProtocolVersion *nopt =
1515                 (Odr_bitmask *)odr_malloc(odr_encode(),
1516                                           sizeof(Odr_bitmask));
1517             ODR_MASK_ZERO(nopt);
1518
1519             int i;
1520             for (i = 0; i<8; i++)
1521                 if (ODR_MASK_GET(m_initRequest_version, i) &&
1522                     ODR_MASK_GET(apdu->u.initResponse->protocolVersion, i))
1523                     ODR_MASK_SET(nopt, i);
1524             apdu->u.initResponse->protocolVersion = nopt;
1525         }
1526         apdu->u.initResponse->preferredMessageSize =
1527             odr_intdup(odr_encode(),
1528                        m_client->m_initResponse_preferredMessageSize >
1529                        m_initRequest_preferredMessageSize ?
1530                        m_initRequest_preferredMessageSize :
1531                        m_client->m_initResponse_preferredMessageSize);
1532         apdu->u.initResponse->maximumRecordSize =
1533             odr_intdup(odr_encode(),
1534                        m_client->m_initResponse_maximumRecordSize >
1535                        m_initRequest_maximumRecordSize ?
1536                        m_initRequest_maximumRecordSize :
1537                        m_client->m_initResponse_maximumRecordSize);
1538     }
1539
1540     int r = send_PDU_convert(apdu);
1541     if (r)
1542         return r;
1543     if (kill_session)
1544     {
1545         delete m_client;
1546         m_client = 0;
1547         m_parent->pre_init();
1548     }
1549     return r;
1550 }
1551
1552 void Yaz_ProxyClient::set_idAuthentication(Z_APDU *apdu)
1553 {
1554     Z_IdAuthentication *t = apdu->u.initRequest->idAuthentication;
1555     
1556     odr_reset(m_idAuthentication_odr);
1557     z_IdAuthentication(m_idAuthentication_odr, &t, 1, 0);
1558     m_idAuthentication_ber_buf =
1559         odr_getbuf(m_idAuthentication_odr, 
1560                    &m_idAuthentication_ber_size, 0);
1561 }
1562
1563 bool Yaz_ProxyClient::compare_charset(Z_APDU *apdu)
1564 {
1565     return true;
1566 }
1567
1568 bool Yaz_ProxyClient::compare_idAuthentication(Z_APDU *apdu)
1569 {
1570     Z_IdAuthentication *t = apdu->u.initRequest->idAuthentication;
1571     ODR odr = odr_createmem(ODR_ENCODE);
1572
1573     z_IdAuthentication(odr, &t, 1, 0);
1574     int sz;
1575     char *buf = odr_getbuf(odr, &sz, 0);
1576     if (buf && m_idAuthentication_ber_buf
1577         && sz == m_idAuthentication_ber_size
1578         && !memcmp(m_idAuthentication_ber_buf, buf, sz))
1579     {
1580         odr_destroy(odr);
1581         return true;
1582     }
1583     odr_destroy(odr);
1584     if (!buf && !m_idAuthentication_ber_buf)
1585         return true;
1586     return false;
1587 }
1588
1589 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
1590 {
1591     int len = 0;
1592     const char *apdu_name_tmp = apdu_name(apdu);
1593     int r = send_Z_PDU(apdu, &len);
1594     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
1595         yaz_log (YLOG_LOG, "%sSending %s to %s %d bytes",
1596                  get_session_str(),
1597                  apdu_name_tmp, get_hostname(), len);
1598     m_bytes_sent += len;
1599     return r;
1600 }
1601
1602 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
1603 {
1604     if (apdu->which == Z_APDU_presentRequest)
1605     {
1606         Z_PresentRequest *pr = apdu->u.presentRequest;
1607         int toget = *pr->numberOfRecordsRequested;
1608         int start = *pr->resultSetStartPoint;
1609
1610         yaz_log(YLOG_LOG, "%sPresent %s %d+%d", m_session_str,
1611                 pr->resultSetId, start, toget);
1612
1613         if (*m_parent->m_optimize == '0')
1614             return apdu;
1615
1616         if (!m_client->m_last_resultSetId)
1617         {
1618             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1619             new_apdu->u.presentResponse->records =
1620                 create_nonSurrogateDiagnostics(
1621                     odr_encode(), 
1622                     YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
1623                     pr->resultSetId);
1624             send_to_client(new_apdu);
1625             return 0;
1626         }
1627         if (start < 1 || toget < 0)
1628         {
1629             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1630             new_apdu->u.presentResponse->records =
1631                 create_nonSurrogateDiagnostics(
1632                     odr_encode(), 
1633                     YAZ_BIB1_PRESENT_REQUEST_OUT_OF_RANGE, 
1634                     0);
1635             send_to_client(new_apdu);
1636             return 0;
1637         }
1638         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1639         {
1640             if (start+toget-1 > m_client->m_last_resultCount)
1641             {
1642                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1643                 new_apdu->u.presentResponse->records =
1644                     create_nonSurrogateDiagnostics(
1645                         odr_encode(), 
1646                         YAZ_BIB1_PRESENT_REQUEST_OUT_OF_RANGE,
1647                         0);
1648                 send_to_client(new_apdu);
1649                 return 0;
1650             }
1651             Z_NamePlusRecordList *npr;
1652 #if 0
1653             yaz_log(YLOG_LOG, "%sCache lookup %d+%d syntax=%s",
1654                     m_session_str, start, toget, yaz_z3950oid_to_str(
1655                         pr->preferredRecordSyntax, &oclass));
1656 #endif
1657             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1658                                           pr->preferredRecordSyntax,
1659                                           pr->recordComposition))
1660             {
1661                 yaz_log (YLOG_LOG, "%sReturned cached records for present request",
1662                          m_session_str);
1663                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1664                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1665
1666                 new_apdu->u.presentResponse->numberOfRecordsReturned
1667                     = odr_intdup(odr_encode(), toget);
1668
1669                 new_apdu->u.presentResponse->records = (Z_Records*)
1670                     odr_malloc(odr_encode(), sizeof(Z_Records));
1671                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1672                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1673                 new_apdu->u.presentResponse->nextResultSetPosition =
1674                     odr_intdup(odr_encode(), start+toget);
1675
1676                 send_to_client(new_apdu);
1677                 return 0;
1678             }
1679         }
1680     }
1681
1682     if (apdu->which != Z_APDU_searchRequest)
1683         return apdu;
1684     Z_SearchRequest *sr = apdu->u.searchRequest;
1685     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1686     Yaz_Z_Databases this_databases;
1687
1688     this_databases.set(sr->num_databaseNames, (const char **)
1689                        sr->databaseNames);
1690
1691     this_query->set_Z_Query(sr->query);
1692
1693     // Check for non-negative piggyback params.
1694     if (*sr->smallSetUpperBound < 0
1695         || *sr->largeSetLowerBound < 0
1696         || *sr->mediumSetPresentNumber < 0)
1697     {
1698         Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1699         // Not a present request.. But can't find better diagnostic
1700         new_apdu->u.searchResponse->records =
1701             create_nonSurrogateDiagnostics(
1702                 odr_encode(), 
1703                 YAZ_BIB1_PRESENT_REQUEST_OUT_OF_RANGE, 0);
1704         send_to_client(new_apdu);
1705         return 0;
1706     }
1707
1708     char query_str[120];
1709     this_query->print(query_str, sizeof(query_str)-1);
1710     yaz_log(YLOG_LOG, "%sSearch %s", m_session_str, query_str);
1711
1712     if (*m_parent->m_optimize != '0' &&
1713         m_client->m_last_ok && m_client->m_last_query &&
1714         m_client->m_last_query->match(this_query) &&
1715         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1716         m_client->m_last_databases.match(this_databases))
1717     {
1718         delete this_query;
1719         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1720             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1721         {
1722             Z_NamePlusRecordList *npr;
1723             int toget = *sr->mediumSetPresentNumber;
1724             Z_RecordComposition *comp = 0;
1725
1726             if (toget > m_client->m_last_resultCount)
1727                 toget = m_client->m_last_resultCount;
1728             
1729             if (sr->mediumSetElementSetNames)
1730             {
1731                 comp = (Z_RecordComposition *)
1732                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1733                 comp->which = Z_RecordComp_simple;
1734                 comp->u.simple = sr->mediumSetElementSetNames;
1735             }
1736
1737             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1738                                           sr->preferredRecordSyntax, comp))
1739             {
1740                 yaz_log (YLOG_LOG, "%sReturned cached records for medium set",
1741                          m_session_str);
1742                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1743                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1744                 new_apdu->u.searchResponse->resultCount =
1745                     &m_client->m_last_resultCount;
1746
1747                 new_apdu->u.searchResponse->numberOfRecordsReturned
1748                     = odr_intdup(odr_encode(), toget);
1749
1750                 new_apdu->u.searchResponse->presentStatus =
1751                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1752                 new_apdu->u.searchResponse->records = (Z_Records*)
1753                     odr_malloc(odr_encode(), sizeof(Z_Records));
1754                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1755                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1756                 new_apdu->u.searchResponse->nextResultSetPosition =
1757                     odr_intdup(odr_encode(), toget+1);
1758                 send_to_client(new_apdu);
1759                 return 0;
1760             }
1761             else
1762             {
1763                 // medium Set
1764                 // send present request (medium size)
1765                 yaz_log (YLOG_LOG, "%sOptimizing search for medium set",
1766                          m_session_str);
1767
1768                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1769                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1770                 pr->referenceId = sr->referenceId;
1771                 pr->resultSetId = sr->resultSetName;
1772                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1773                 *pr->numberOfRecordsRequested = toget;
1774                 pr->recordComposition = comp;
1775                 m_client->m_sr_transform = 1;
1776                 return new_apdu;
1777             }
1778         }
1779         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1780             m_client->m_last_resultCount <= 0)
1781         {
1782             // large set. Return pseudo-search response immediately
1783             yaz_log (YLOG_LOG, "%sOptimizing search for large set",
1784                      m_session_str);
1785             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1786             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1787             new_apdu->u.searchResponse->resultCount =
1788                 &m_client->m_last_resultCount;
1789             send_to_client(new_apdu);
1790             return 0;
1791         }
1792         else
1793         {
1794             Z_NamePlusRecordList *npr;
1795             int toget = m_client->m_last_resultCount;
1796             Z_RecordComposition *comp = 0;
1797             // small set
1798             // send a present request (small set)
1799
1800             if (sr->smallSetElementSetNames)
1801             {
1802                 comp = (Z_RecordComposition *)
1803                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1804                 comp->which = Z_RecordComp_simple;
1805                 comp->u.simple = sr->smallSetElementSetNames;
1806             }
1807
1808             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1809                                           sr->preferredRecordSyntax, comp))
1810             {
1811                 yaz_log (YLOG_LOG, "%sReturned cached records for small set",
1812                          m_session_str);
1813                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1814                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1815                 new_apdu->u.searchResponse->resultCount =
1816                     &m_client->m_last_resultCount;
1817
1818                 new_apdu->u.searchResponse->numberOfRecordsReturned
1819                     = odr_intdup(odr_encode(), toget);
1820
1821                 new_apdu->u.searchResponse->presentStatus =
1822                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1823                 new_apdu->u.searchResponse->records = (Z_Records*)
1824                     odr_malloc(odr_encode(), sizeof(Z_Records));
1825                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1826                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1827                 new_apdu->u.searchResponse->nextResultSetPosition =
1828                     odr_intdup(odr_encode(), toget+1);
1829                 send_to_client(new_apdu);
1830                 return 0;
1831             }
1832             else
1833             {
1834                 yaz_log (YLOG_LOG, "%sOptimizing search for small set",
1835                          m_session_str);
1836                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1837                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1838                 pr->referenceId = sr->referenceId;
1839                 pr->resultSetId = sr->resultSetName;
1840                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1841                 *pr->numberOfRecordsRequested = toget;
1842                 pr->recordComposition = comp;
1843                 m_client->m_sr_transform = 1;
1844                 return new_apdu;
1845             }
1846         }
1847     }
1848     else  // query doesn't match
1849     {
1850         delete m_client->m_last_query;
1851         m_client->m_last_query = this_query;
1852         m_client->m_last_ok = 0;
1853         m_client->m_cache.clear();
1854         m_client->m_resultSetStartPoint = 0;
1855
1856         xfree (m_client->m_last_resultSetId);
1857         m_client->m_last_resultSetId = xstrdup (sr->resultSetName);
1858
1859         m_client->m_last_databases.set(sr->num_databaseNames,
1860                                        (const char **) sr->databaseNames);
1861     }
1862     return apdu;
1863 }
1864
1865
1866 void Yaz_Proxy::inc_request_no()
1867 {
1868     m_request_no++;
1869     char *cp = m_session_str + strlen(m_session_str)-1;
1870     if (*cp == ' ')
1871         cp--;
1872     while (*cp && *cp != ' ')
1873         cp--;
1874     if (*cp)
1875         sprintf(cp+1, "%d ", m_request_no);
1876 }
1877
1878 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
1879 {
1880     inc_request_no();
1881
1882     m_bytes_recv += len;
1883
1884     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1885         yaz_log (YLOG_LOG, "%sReceiving %s from client %d bytes",
1886                  m_session_str, gdu_name(apdu), len);
1887
1888 #if 0
1889     // try to make a _bad_ attribute set ID .. Don't enable this in prod.
1890     if (apdu->which == Z_GDU_Z3950 
1891         && apdu->u.z3950->which == Z_APDU_searchRequest)
1892     {
1893         Z_SearchRequest *req = apdu->u.z3950->u.searchRequest;
1894         if (req->query && req->query->which == Z_Query_type_1)
1895         {
1896             Z_RPNQuery *rpnquery = req->query->u.type_1;
1897             if (rpnquery->attributeSetId)
1898             {
1899                 rpnquery->attributeSetId[0] = -2;
1900                 rpnquery->attributeSetId[1] = -1;
1901                 yaz_log(YLOG_WARN, "%sBAD FIXUP TEST", m_session_str);
1902             }
1903         }
1904     }
1905 #endif
1906
1907 #if HAVE_GETTIMEOFDAY
1908     gettimeofday((struct timeval *) m_time_tv, 0);
1909 #endif
1910     m_bw_stat.add_bytes(len);
1911     m_pdu_stat.add_bytes(1);
1912
1913     GDU *gdu = new GDU(apdu);
1914
1915     if (gdu->get() == 0)
1916     {
1917         delete gdu;
1918         yaz_log(YLOG_LOG, "%sUnable to encode package", m_session_str);
1919         m_in_queue.clear();
1920         dec_ref();
1921         return;
1922     }
1923     m_in_queue.enqueue(gdu);
1924     recv_GDU_more(false);
1925 }
1926
1927 void Yaz_Proxy::HTTP_Forwarded(Z_GDU *z_gdu)
1928 {
1929     if (z_gdu->which == Z_GDU_HTTP_Request)
1930     {
1931         Z_HTTP_Request *hreq = z_gdu->u.HTTP_Request;
1932         const char *x_forwarded_for =
1933             z_HTTP_header_lookup(hreq->headers, "X-Forwarded-For");
1934         if (x_forwarded_for)
1935         {
1936             xfree(m_peername);
1937             m_peername = (char*) xmalloc(strlen(x_forwarded_for)+5);
1938             sprintf(m_peername, "tcp:%s", x_forwarded_for);
1939             
1940             yaz_log(YLOG_LOG, "%sHTTP Forwarded from %s", m_session_str,
1941                     m_peername);
1942             if (m_log_mask & PROXY_LOG_IP_CLIENT)
1943                 sprintf(m_session_str, "%ld:%d %.80s %d ",
1944                         (long) time(0), m_session_no, m_peername, m_request_no);
1945             else
1946                 sprintf(m_session_str, "%ld:%d %d ",
1947                         (long) time(0), m_session_no, m_request_no);
1948         }
1949     }
1950 }
1951
1952 void Yaz_Proxy::connect_stat(bool &block, int &reduce)
1953 {
1954
1955     m_parent->m_connect.cleanup(false);
1956     m_parent->m_connect.add_connect(m_peername);
1957
1958     int connect_total = m_parent->m_connect.get_total(m_peername);
1959     int max_connect = m_parent->m_max_connect;
1960
1961     if (max_connect && connect_total > max_connect)
1962     {
1963         yaz_log(YLOG_LOG, "%sconnect not accepted total=%d max=%d",
1964                 m_session_str, connect_total, max_connect);
1965         block = true;
1966     }
1967     else 
1968         block = false;
1969     yaz_log(YLOG_LOG, "%sconnect accepted total=%d", m_session_str,
1970             connect_total);
1971     
1972     int limit_connect = m_parent->m_limit_connect;
1973     if (limit_connect)
1974         reduce = connect_total / limit_connect;
1975     else
1976         reduce = 0;
1977 }
1978
1979 void Yaz_Proxy::recv_GDU_reduce(GDU *gdu)
1980 {
1981     HTTP_Forwarded(gdu->get());
1982
1983     int reduce = 0;
1984     
1985     if (m_request_no == 1)
1986     {
1987         bool block = false;
1988         
1989         connect_stat(block, reduce);
1990
1991         if (block)
1992         {
1993             m_timeout_mode = timeout_busy;
1994             timeout(0);
1995             return;
1996         }
1997     }
1998
1999     int bw_total = m_bw_stat.get_total();
2000     int pdu_total = m_pdu_stat.get_total();
2001     int search_total = m_search_stat.get_total();
2002
2003     assert(m_timeout_mode == timeout_busy);
2004     assert(m_timeout_gdu == 0);
2005
2006     if (m_search_max)
2007         reduce += search_total / m_search_max;
2008     if (m_bw_max)
2009         reduce += (bw_total/m_bw_max);
2010     if (m_pdu_max)
2011     {
2012         if (pdu_total > m_pdu_max)
2013         {
2014             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
2015             reduce = (reduce > nreduce) ? reduce : nreduce;
2016         }
2017     }
2018     m_http_version = 0;
2019
2020 #if 0
2021     /* uncomment to force a big reduce */
2022     m_timeout_mode = timeout_reduce;
2023     m_timeout_gdu = gdu;
2024     timeout(3);       // call us reduce seconds later
2025     return;
2026 #endif
2027     if (reduce)
2028     {
2029         yaz_log(YLOG_LOG, "%sdelay=%d bw=%d pdu=%d search=%d limit-bw=%d limit-pdu=%d limit-search=%d",
2030                 m_session_str, reduce, bw_total, pdu_total, search_total,
2031                 m_bw_max, m_pdu_max, m_search_max);
2032
2033         m_timeout_mode = timeout_reduce;
2034         m_timeout_gdu = gdu;
2035         timeout(reduce);       // call us reduce seconds later
2036     }
2037     else
2038         recv_GDU_normal(gdu);
2039 }
2040
2041 void Yaz_Proxy::recv_GDU_more(bool normal)
2042 {
2043     GDU *g;
2044     if (normal && m_timeout_mode == timeout_busy)
2045         m_timeout_mode = timeout_normal;
2046     while (m_timeout_mode == timeout_normal && (g = m_in_queue.dequeue()))
2047     {
2048         m_timeout_mode = timeout_busy;
2049         inc_ref();
2050         recv_GDU_reduce(g);
2051         if (dec_ref())
2052             break;
2053     }
2054 }
2055
2056 void Yaz_Proxy::recv_GDU_normal(GDU *gdu)
2057 {
2058     Z_GDU *apdu = 0;
2059     gdu->move_away_gdu(odr_decode(), &apdu);
2060     delete gdu;
2061
2062     if (apdu->which == Z_GDU_Z3950)
2063         handle_incoming_Z_PDU(apdu->u.z3950);
2064     else if (apdu->which == Z_GDU_HTTP_Request)
2065         handle_incoming_HTTP(apdu->u.HTTP_Request);
2066 }
2067
2068 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
2069 {
2070     if (m_max_record_retrieve)
2071     {
2072         if (apdu->which == Z_APDU_presentRequest)
2073         {
2074             Z_PresentRequest *pr = apdu->u.presentRequest;
2075             if (pr->numberOfRecordsRequested &&
2076                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
2077                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
2078         }
2079     }
2080 }
2081
2082 void Yaz_Proxy::handle_charset_lang_negotiation(Z_APDU *apdu)
2083 {
2084     if (apdu->which == Z_APDU_initRequest)
2085     {
2086         if (m_initRequest_options &&
2087             !ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel) &&
2088             (m_proxy_negotiation_charset || m_proxy_negotiation_lang))
2089         {
2090             // There is no negotiation proposal from
2091             // client's side. OK. The proxy negotiation
2092             // in use, only.
2093             Z_InitRequest *initRequest = apdu->u.initRequest;
2094             Z_OtherInformation **otherInfo;
2095             Z_OtherInformationUnit *oi;
2096             get_otherInfoAPDU(apdu, &otherInfo);
2097             oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
2098             if (oi)
2099             {
2100                 ODR_MASK_SET(initRequest->options,
2101                     Z_Options_negotiationModel);
2102                 oi->which = Z_OtherInfo_externallyDefinedInfo;
2103                 oi->information.externallyDefinedInfo =
2104                 yaz_set_proposal_charneg(odr_encode(),
2105                     (const char**)&m_proxy_negotiation_charset,
2106                     m_proxy_negotiation_charset ? 1:0,
2107                     (const char**)&m_proxy_negotiation_lang,
2108                     m_proxy_negotiation_lang ? 1:0,
2109                     1);
2110             }
2111         }
2112         else if (m_initRequest_options &&
2113                  ODR_MASK_GET(m_initRequest_options,
2114                               Z_Options_negotiationModel) &&
2115                  m_charset_converter->get_target_query_charset())
2116         {
2117             yaz_log(YLOG_LOG, "%sManaged charset negotiation: charset=%s",
2118                     m_session_str,
2119                     m_charset_converter->get_target_query_charset());
2120             Z_InitRequest *initRequest = apdu->u.initRequest;
2121             Z_CharSetandLanguageNegotiation *negotiation =
2122                 yaz_get_charneg_record (initRequest->otherInfo);
2123             if (negotiation &&
2124                 negotiation->which == Z_CharSetandLanguageNegotiation_proposal)
2125             {
2126                 NMEM nmem = nmem_create();
2127                 char **charsets = 0;
2128                 int num_charsets = 0;
2129                 char **langs = 0;
2130                 int num_langs = 0;
2131                 int selected = 0;
2132                 yaz_get_proposal_charneg (nmem, negotiation,
2133                                           &charsets, &num_charsets,
2134                                           &langs, &num_langs, &selected);
2135                 int i;
2136                 for (i = 0; i<num_charsets; i++)
2137                     yaz_log(YLOG_LOG, "%scharset %s", m_session_str,
2138                             charsets[i]);
2139                 for (i = 0; i<num_langs; i++)
2140                     yaz_log(YLOG_LOG, "%slang %s", m_session_str,
2141                             langs[i]);
2142
2143                 const char *t_charset =
2144                     m_charset_converter->get_target_query_charset();
2145                 // sweep through charsets and pick the first supported
2146                 // conversion
2147                 for (i = 0; i<num_charsets; i++)
2148                 {
2149                     const char *c_charset = charsets[i];
2150                     if (!odr_set_charset(odr_decode(), t_charset, c_charset))
2151                         break;
2152                 }
2153                 if (i != num_charsets)
2154                 {
2155                     // got one .. set up ODR for reverse direction
2156                     const char *c_charset = charsets[i];
2157                     odr_set_charset(odr_encode(), c_charset, t_charset);
2158                     m_charset_converter->set_client_query_charset(c_charset);
2159                     m_charset_converter->set_client_charset_selected(selected);
2160                 }
2161                 nmem_destroy(nmem);
2162                 ODR_MASK_CLEAR(m_initRequest_options,
2163                                Z_Options_negotiationModel);
2164                 yaz_del_charneg_record(&initRequest->otherInfo);
2165             }
2166             else
2167             {
2168                 yaz_log(YLOG_WARN, "%sUnable to decode charset package",
2169                         m_session_str);
2170             }
2171         }
2172         else if (m_charset_converter->get_target_query_charset() &&
2173             m_proxy_negotiation_default_charset)
2174         {
2175             m_charset_converter->
2176                 set_client_query_charset(m_proxy_negotiation_default_charset);
2177         }
2178     }
2179     else if (apdu->which == Z_APDU_initResponse)
2180     {
2181         Z_InitResponse *initResponse = apdu->u.initResponse;
2182         Z_OtherInformation **otherInfo;
2183         get_otherInfoAPDU(apdu, &otherInfo);
2184         
2185         Z_CharSetandLanguageNegotiation *charneg = 0;
2186
2187         if (otherInfo && *otherInfo && 
2188             ODR_MASK_GET(initResponse->options, Z_Options_negotiationModel)
2189             && (charneg = yaz_get_charneg_record(*otherInfo)))
2190         {
2191             char *charset = 0;
2192             char *lang = 0;
2193             int selected = 0;
2194
2195             yaz_get_response_charneg(m_referenceId_mem, charneg,
2196                 &charset, &lang, &selected);
2197
2198             yaz_log(YLOG_LOG, "%sAccepted charset - '%s' and lang - '%s'",
2199                 m_session_str, (charset)?charset:"none", (lang)?lang:"none");
2200
2201             if (m_initRequest_options &&
2202                 ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel))
2203             {
2204                 yaz_log(YLOG_LOG, "%sClient's negotiation record in use",
2205                     m_session_str);
2206             }
2207             else if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
2208             {
2209                 // negotiation-charset, negotiation-lang
2210                 // elements of config file in use.
2211
2212                 yaz_log(YLOG_LOG, "%sProxy's negotiation record in use",
2213                     m_session_str);
2214
2215                 // clear negotiation option.
2216                 ODR_MASK_CLEAR(initResponse->options, Z_Options_negotiationModel);
2217
2218                 // Delete negotiation (charneg-3) entry.
2219                 yaz_del_charneg_record(otherInfo);
2220             }
2221         }
2222         else
2223         {
2224             if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
2225             {
2226                 yaz_log(YLOG_LOG, "%sTarget did not honor negotiation",
2227                         m_session_str);
2228             }
2229             else if (m_charset_converter->get_client_query_charset())
2230             {
2231                 Z_OtherInformation **otherInfo;
2232                 Z_OtherInformationUnit *oi;
2233                 get_otherInfoAPDU(apdu, &otherInfo);
2234                 oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
2235                 if (oi)
2236                 {
2237                     ODR_MASK_SET(initResponse->options,
2238                                  Z_Options_negotiationModel);
2239                     if (m_initRequest_options)
2240                         ODR_MASK_SET(m_initRequest_options,
2241                                      Z_Options_negotiationModel);
2242                     
2243                     oi->which = Z_OtherInfo_externallyDefinedInfo;    
2244                     oi->information.externallyDefinedInfo =
2245                         yaz_set_response_charneg(
2246                             odr_encode(),
2247                             m_charset_converter->get_client_query_charset(),
2248                             0 /* no lang */,
2249                             m_charset_converter->get_client_charset_selected());
2250                 }
2251             }
2252         }
2253     }
2254 }
2255
2256 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
2257                                                      int error,
2258                                                      const char *addinfo)
2259 {
2260     Z_Records *rec = (Z_Records *)
2261         odr_malloc (odr, sizeof(*rec));
2262     int *err = (int *)
2263         odr_malloc (odr, sizeof(*err));
2264     Z_DiagRec *drec = (Z_DiagRec *)
2265         odr_malloc (odr, sizeof(*drec));
2266     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
2267         odr_malloc (odr, sizeof(*dr));
2268     *err = error;
2269     rec->which = Z_Records_NSD;
2270     rec->u.nonSurrogateDiagnostic = dr;
2271     dr->diagnosticSetId =
2272         yaz_oidval_to_z3950oid (odr, CLASS_DIAGSET, VAL_BIB1);
2273     dr->condition = err;
2274     dr->which = Z_DefaultDiagFormat_v2Addinfo;
2275     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
2276     return rec;
2277 }
2278
2279 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
2280 {
2281     if (apdu->which == Z_APDU_searchRequest &&
2282         apdu->u.searchRequest->query &&
2283         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
2284         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
2285     {
2286         Z_RPNQuery *rpnquery = 0;
2287         Z_SearchRequest *sr = apdu->u.searchRequest;
2288         char *addinfo = 0;
2289
2290         yaz_log(YLOG_LOG, "%sCQL: %s", m_session_str,
2291                 sr->query->u.type_104->u.cql);
2292
2293         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
2294                                           &rpnquery, odr_encode(),
2295                                           &addinfo);
2296         if (r == -3)
2297             yaz_log(YLOG_LOG, "%sNo CQL to RPN table", m_session_str);
2298         else if (r)
2299         {
2300             yaz_log(YLOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
2301             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2302
2303             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2304             new_apdu->u.searchResponse->records =
2305                 create_nonSurrogateDiagnostics(odr_encode(),
2306                                                yaz_diag_srw_to_bib1(r),
2307                                                addinfo);
2308             *new_apdu->u.searchResponse->searchStatus = 0;
2309
2310             send_to_client(new_apdu);
2311
2312             return 0;
2313         }
2314         else
2315         {
2316             sr->query->which = Z_Query_type_1;
2317             sr->query->u.type_1 = rpnquery;
2318         }
2319         return apdu;
2320     }
2321     return apdu;
2322 }
2323
2324 Z_APDU *Yaz_Proxy::handle_target_charset_conversion(Z_APDU *apdu)
2325 {
2326     if (apdu->which == Z_APDU_searchRequest &&
2327         apdu->u.searchRequest->query)
2328     {
2329         if (apdu->u.searchRequest->query->which == Z_Query_type_1
2330             || apdu->u.searchRequest->query->which == Z_Query_type_101)
2331         {
2332             if (m_http_version)
2333                 m_charset_converter->set_client_query_charset("UTF-8");
2334             Z_RPNQuery *rpnquery = apdu->u.searchRequest->query->u.type_1;
2335             m_charset_converter->convert_type_1(rpnquery, odr_encode());
2336         }
2337     }
2338     return apdu;
2339 }
2340
2341
2342 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
2343 {
2344     if (apdu->which == Z_APDU_searchRequest)
2345     {
2346         Z_SearchRequest *sr = apdu->u.searchRequest;
2347         int err = 0;
2348         char *addinfo = 0;
2349
2350         Yaz_ProxyConfig *cfg = check_reconfigure();
2351         if (cfg)
2352             err = cfg->check_query(odr_encode(), m_default_target,
2353                                    sr->query, &addinfo);
2354         if (err)
2355         {
2356             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2357
2358             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2359             new_apdu->u.searchResponse->records =
2360                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2361             *new_apdu->u.searchResponse->searchStatus = 0;
2362
2363             send_to_client(new_apdu);
2364
2365             return 0;
2366         }
2367     }
2368     return apdu;
2369 }
2370
2371 int Yaz_Proxy::handle_authentication(Z_APDU *apdu)
2372 {
2373     if (apdu->which != Z_APDU_initRequest)
2374         return 1;  // pass if no init request
2375     Z_InitRequest *req = apdu->u.initRequest;
2376
2377     Yaz_ProxyConfig *cfg = check_reconfigure();
2378     if (!cfg)
2379         return 1;  // pass if no config
2380
2381     int ret;
2382     if (req->idAuthentication == 0)
2383     {
2384         ret = cfg->client_authentication(m_default_target, 0, 0, 0,
2385                                          m_peername);
2386     }
2387     else if (req->idAuthentication->which == Z_IdAuthentication_idPass)
2388     {
2389         ret = cfg->client_authentication(
2390             m_default_target,
2391             req->idAuthentication->u.idPass->userId,
2392             req->idAuthentication->u.idPass->groupId,
2393             req->idAuthentication->u.idPass->password,
2394             m_peername);
2395     }
2396     else if (req->idAuthentication->which == Z_IdAuthentication_open)
2397     {
2398         char user[64], pass[64];
2399         *user = '\0';
2400         *pass = '\0';
2401         sscanf(req->idAuthentication->u.open, "%63[^/]/%63s", user, pass);
2402         ret = cfg->client_authentication(m_default_target, user, 0, pass,
2403                                          m_peername);
2404     }
2405     else
2406         ret = cfg->client_authentication(m_default_target, 0, 0, 0,
2407                                          m_peername);
2408     return ret;
2409 }
2410
2411 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
2412 {
2413     m_marcxml_mode = none;
2414     if (apdu->which == Z_APDU_searchRequest)
2415     {
2416         Z_SearchRequest *sr = apdu->u.searchRequest;
2417         int err = 0;
2418         char *addinfo = 0;
2419         Yaz_ProxyConfig *cfg = check_reconfigure();
2420
2421         Z_RecordComposition rc_temp, *rc = 0;
2422         if (sr->smallSetElementSetNames)
2423         {
2424             rc_temp.which = Z_RecordComp_simple;
2425             rc_temp.u.simple = sr->smallSetElementSetNames;
2426             rc = &rc_temp;
2427         }
2428
2429         if (sr->preferredRecordSyntax)
2430         {
2431             struct oident *ent;
2432             ent = oid_getentbyoid(sr->preferredRecordSyntax);
2433             m_frontend_type = ent->value;
2434         }
2435         else
2436             m_frontend_type = VAL_NONE;
2437
2438         char *stylesheet_name = 0;
2439         if (cfg)
2440             err = cfg->check_syntax(odr_encode(),
2441                                     m_default_target,
2442                                     sr->preferredRecordSyntax, rc,
2443                                     &addinfo, &stylesheet_name, &m_schema,
2444                                     &m_backend_type, &m_backend_charset,
2445                                     &m_usemarcon_ini_stage1,
2446                                     &m_usemarcon_ini_stage2);
2447         if (stylesheet_name)
2448         {
2449             m_parent->low_socket_close();
2450
2451 #if HAVE_XSLT
2452             if (m_stylesheet_xsp)
2453                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2454             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2455                                                        stylesheet_name);
2456 #endif
2457             m_stylesheet_offset = 0;
2458             xfree(stylesheet_name);
2459
2460             m_parent->low_socket_open();
2461         }
2462         if (err == -1)
2463         {
2464             sr->smallSetElementSetNames = 0;
2465             sr->mediumSetElementSetNames = 0;
2466             m_marcxml_mode = marcxml;
2467             if (m_backend_type)
2468             {
2469
2470                 sr->preferredRecordSyntax =
2471                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
2472                                         m_backend_type);
2473             }
2474             else
2475                 sr->preferredRecordSyntax =
2476                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
2477                                            VAL_USMARC);
2478         }
2479         else if (err)
2480         {
2481             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2482
2483             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2484             new_apdu->u.searchResponse->records =
2485                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2486             *new_apdu->u.searchResponse->searchStatus = 0;
2487
2488             send_to_client(new_apdu);
2489
2490             return 0;
2491         }
2492         else if (m_backend_type)
2493         {
2494             sr->preferredRecordSyntax =
2495                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
2496         }
2497     }
2498     else if (apdu->which == Z_APDU_presentRequest)
2499     {
2500         Z_PresentRequest *pr = apdu->u.presentRequest;
2501         int err = 0;
2502         char *addinfo = 0;
2503         Yaz_ProxyConfig *cfg = check_reconfigure();
2504
2505         if (pr->preferredRecordSyntax)
2506         {
2507             struct oident *ent;
2508             ent = oid_getentbyoid(pr->preferredRecordSyntax);
2509             m_frontend_type = ent->value;
2510         }
2511         else
2512             m_frontend_type = VAL_NONE;
2513
2514         char *stylesheet_name = 0;
2515         if (cfg)
2516             err = cfg->check_syntax(odr_encode(), m_default_target,
2517                                     pr->preferredRecordSyntax,
2518                                     pr->recordComposition,
2519                                     &addinfo, &stylesheet_name, &m_schema,
2520                                     &m_backend_type, &m_backend_charset,
2521                                     &m_usemarcon_ini_stage1,
2522                                     &m_usemarcon_ini_stage2
2523                                     );
2524         if (stylesheet_name)
2525         {
2526             m_parent->low_socket_close();
2527
2528 #if HAVE_XSLT
2529             if (m_stylesheet_xsp)
2530                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2531             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2532                                                        stylesheet_name);
2533 #endif
2534             m_stylesheet_offset = 0;
2535             xfree(stylesheet_name);
2536
2537             m_parent->low_socket_open();
2538         }
2539         if (err == -1)
2540         {
2541             pr->recordComposition = 0;
2542             m_marcxml_mode = marcxml;
2543             if (m_backend_type)
2544             {
2545
2546                 pr->preferredRecordSyntax =
2547                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
2548                                         m_backend_type);
2549             }
2550             else
2551                 pr->preferredRecordSyntax =
2552                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
2553                                            VAL_USMARC);
2554         }
2555         else if (err)
2556         {
2557             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
2558
2559             new_apdu->u.presentResponse->referenceId = pr->referenceId;
2560             new_apdu->u.presentResponse->records =
2561                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2562             *new_apdu->u.presentResponse->presentStatus =
2563                 Z_PresentStatus_failure;
2564
2565             send_to_client(new_apdu);
2566
2567             return 0;
2568         }
2569         else if (m_backend_type)
2570         {
2571             pr->preferredRecordSyntax =
2572                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
2573         }
2574     }
2575     return apdu;
2576 }
2577
2578 Z_ElementSetNames *Yaz_Proxy::mk_esn_from_schema(ODR o, const char *schema)
2579 {
2580     if (!schema)
2581         return 0;
2582     Z_ElementSetNames *esn = (Z_ElementSetNames *)
2583         odr_malloc(o, sizeof(Z_ElementSetNames));
2584     esn->which = Z_ElementSetNames_generic;
2585     esn->u.generic = odr_strdup(o, schema);
2586     return esn;
2587 }
2588
2589 void Yaz_Proxy::srw_get_client(const char *db, const char **backend_db)
2590 {
2591     const char *t = 0;
2592     Yaz_ProxyConfig *cfg = check_reconfigure();
2593     if (cfg)
2594         t = cfg->get_explain_name(db, backend_db);
2595
2596     if (m_client && m_default_target && t && strcmp(m_default_target, t))
2597     {
2598         releaseClient();
2599     }
2600
2601     if (t)
2602     {
2603         xfree(m_default_target);
2604         m_default_target = xstrdup(t);
2605     }
2606 }
2607
2608 int Yaz_Proxy::file_access(Z_HTTP_Request *hreq)
2609 {
2610     struct stat sbuf;
2611     if (strcmp(hreq->method, "GET"))
2612         return 0;
2613     if (hreq->path[0] != '/')
2614         return 0;
2615     const char *cp = hreq->path;
2616     while (*cp)
2617     {
2618         if (*cp == '/' && strchr("/.", cp[1]))
2619             return 0;
2620         cp++;
2621     }
2622
2623     Yaz_ProxyConfig *cfg = check_reconfigure();
2624
2625     if (!cfg->get_file_access_info(hreq->path+1))
2626         return 0;
2627
2628     const char *fname = hreq->path+1;
2629     if (stat(fname, &sbuf))
2630     {
2631         yaz_log(YLOG_LOG|YLOG_ERRNO, "%sstat failed for %s", m_session_str,
2632                 fname);
2633         return 0;
2634     }
2635     if ((sbuf.st_mode & S_IFMT) != S_IFREG)
2636     {
2637         yaz_log(YLOG_LOG, "%sNot a regular file %s", m_session_str, fname);
2638         return 0;
2639     }
2640     if (sbuf.st_size > (off_t) 1000000)
2641     {
2642         yaz_log(YLOG_WARN, "%sFile %s too large for transfer", m_session_str,
2643                 fname);
2644         return 0;
2645     }
2646
2647     ODR o = odr_encode();
2648
2649     const char *ctype = cfg->check_mime_type(fname);
2650     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
2651     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
2652     if (m_http_version)
2653         hres->version = odr_strdup(o, m_http_version);
2654     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
2655     if (m_http_keepalive)
2656         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
2657     else
2658         timeout(0);
2659
2660     hres->content_len = sbuf.st_size;
2661     hres->content_buf = (char*) odr_malloc(o, hres->content_len);
2662     FILE *f = fopen(fname, "rb");
2663     if (f)
2664     {
2665         fread(hres->content_buf, 1, hres->content_len, f);
2666         fclose(f);
2667     }
2668     else
2669     {
2670         return 0;
2671     }
2672     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
2673     {
2674         yaz_log (YLOG_LOG, "%sSending file %s to client", m_session_str,
2675                  fname);
2676     }
2677     int len;
2678     send_GDU(gdu, &len);
2679     recv_GDU_more(true);
2680     return 1;
2681 }
2682
2683 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
2684 {
2685     if (m_s2z_odr_init)
2686     {
2687         odr_destroy(m_s2z_odr_init);
2688         m_s2z_odr_init = 0;
2689     }
2690     if (m_s2z_odr_search)
2691     {
2692         odr_destroy(m_s2z_odr_search);
2693         m_s2z_odr_search = 0;
2694     }
2695
2696     m_http_keepalive = 0;
2697     m_http_version = 0;
2698     if (!strcmp(hreq->version, "1.0"))
2699     {
2700         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2701         if (v && !strcmp(v, "Keep-Alive"))
2702             m_http_keepalive = 1;
2703         else
2704             m_http_keepalive = 0;
2705         m_http_version = "1.0";
2706     }
2707     else
2708     {
2709         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2710         if (v && !strcmp(v, "close"))
2711             m_http_keepalive = 0;
2712         else
2713             m_http_keepalive = 1;
2714         m_http_version = "1.1";
2715     }
2716
2717     const char *a = z_HTTP_header_lookup(hreq->headers, "Authorization");
2718     char authorization_str[255];
2719     *authorization_str = '\0';
2720     if (a && strncasecmp(a, "Basic ", 6) == 0)
2721         base64_decode(a + 6, authorization_str, 254);
2722
2723     Z_SRW_PDU *srw_pdu = 0;
2724     Z_SOAP *soap_package = 0;
2725     char *charset = 0;
2726     Z_SRW_diagnostic *diagnostic = 0;
2727     int num_diagnostic = 0;
2728
2729     yaz_log(YLOG_LOG, "%s%s %s", m_session_str, hreq->method, hreq->path);
2730
2731     if (file_access(hreq))
2732     {
2733         return;
2734     }
2735     else if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2736                             &charset) == 0
2737              || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2738                                &charset, &diagnostic, &num_diagnostic) == 0)
2739     {
2740         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
2741         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
2742         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns);
2743         m_s2z_init_apdu = 0;
2744         m_s2z_search_apdu = 0;
2745         m_s2z_present_apdu = 0;
2746
2747         m_s2z_stylesheet = 0;
2748
2749         Z_IdAuthentication *auth = NULL;
2750         if (*authorization_str)
2751         {
2752             auth = (Z_IdAuthentication *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdAuthentication));
2753             auth->which = Z_IdAuthentication_idPass;
2754             auth->u.idPass = (Z_IdPass *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdPass));
2755             auth->u.idPass->groupId = NULL;
2756             char *p = strchr(authorization_str, ':');
2757             if (p)
2758             {
2759                 *p = '\0';
2760                 p++;
2761                 auth->u.idPass->password = odr_strdup(m_s2z_odr_init, p);
2762             }
2763             auth->u.idPass->userId = odr_strdup(m_s2z_odr_init, authorization_str);
2764         }
2765
2766         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
2767         {
2768
2769             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
2770
2771             const char *backend_db = srw_req->database;
2772             srw_get_client(srw_req->database, &backend_db);
2773
2774             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2775             // recordXPath unsupported.
2776             if (srw_req->recordXPath)
2777             {
2778                 yaz_add_srw_diagnostic(odr_decode(),
2779                                        &diagnostic, &num_diagnostic,
2780                                        72, 0);
2781             }
2782             // sort unsupported
2783             if (srw_req->sort_type != Z_SRW_sort_type_none)
2784             {
2785                 yaz_add_srw_diagnostic(odr_decode(),
2786                                        &diagnostic, &num_diagnostic,
2787                                        80, 0);
2788             }
2789             // save stylesheet
2790             if (srw_req->stylesheet)
2791                 m_s2z_stylesheet =
2792                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2793
2794             // set packing for response records ..
2795             if (srw_req->recordPacking &&
2796                 !strcmp(srw_req->recordPacking, "xml"))
2797                 m_s2z_packing = Z_SRW_recordPacking_XML;
2798             else
2799                 m_s2z_packing = Z_SRW_recordPacking_string;
2800
2801             if (num_diagnostic)
2802             {
2803                 Z_SRW_PDU *srw_pdu =
2804                     yaz_srw_get(odr_encode(),
2805                                 Z_SRW_searchRetrieve_response);
2806                 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
2807
2808                 srw_res->diagnostics = diagnostic;
2809                 srw_res->num_diagnostics = num_diagnostic;
2810                 send_srw_response(srw_pdu);
2811                 return;
2812             }
2813
2814             // prepare search PDU
2815             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
2816                                           Z_APDU_searchRequest);
2817             Z_SearchRequest *z_searchRequest =
2818                 m_s2z_search_apdu->u.searchRequest;
2819
2820             z_searchRequest->num_databaseNames = 1;
2821             z_searchRequest->databaseNames = (char**)
2822                 odr_malloc(m_s2z_odr_search, sizeof(char *));
2823             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
2824                                                            backend_db);
2825
2826             // query transformation
2827             Z_Query *query = (Z_Query *)
2828                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
2829             z_searchRequest->query = query;
2830
2831             if (srw_req->query_type == Z_SRW_query_type_cql)
2832             {
2833                 Z_External *ext = (Z_External *)
2834                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
2835                 ext->direct_reference =
2836                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
2837                 ext->indirect_reference = 0;
2838                 ext->descriptor = 0;
2839                 ext->which = Z_External_CQL;
2840                 ext->u.cql = srw_req->query.cql;
2841
2842                 query->which = Z_Query_type_104;
2843                 query->u.type_104 =  ext;
2844             }
2845             else if (srw_req->query_type == Z_SRW_query_type_pqf)
2846             {
2847                 Z_RPNQuery *RPNquery;
2848                 YAZ_PQF_Parser pqf_parser;
2849
2850                 pqf_parser = yaz_pqf_create ();
2851
2852                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
2853                                           srw_req->query.pqf);
2854                 if (!RPNquery)
2855                 {
2856                     const char *pqf_msg;
2857                     size_t off;
2858                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
2859                     int ioff = off;
2860                     yaz_log(YLOG_LOG, "%*s^\n", ioff+4, "");
2861                     yaz_log(YLOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
2862
2863                     send_to_srw_client_error(10, 0);
2864                     return;
2865                 }
2866                 query->which = Z_Query_type_1;
2867                 query->u.type_1 =  RPNquery;
2868
2869                 yaz_pqf_destroy (pqf_parser);
2870             }
2871             else
2872             {
2873                 send_to_srw_client_error(7, "query");
2874                 return;
2875             }
2876
2877             // present
2878             m_s2z_present_apdu = 0;
2879             int max = 0;
2880             if (srw_req->maximumRecords)
2881                 max = *srw_req->maximumRecords;
2882             int start = 1;
2883             if (srw_req->startRecord)
2884                 start = *srw_req->startRecord;
2885             if (max > 0)
2886             {
2887                 // Some backend, such as Voyager doesn't honor piggyback
2888                 // So we use present always (0 &&).
2889                 if (0 && start <= 1)  // Z39.50 piggyback
2890                 {
2891                     *z_searchRequest->smallSetUpperBound = max;
2892                     *z_searchRequest->mediumSetPresentNumber = max;
2893                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
2894
2895                     z_searchRequest->preferredRecordSyntax =
2896                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2897                                                VAL_TEXT_XML);
2898                     if (srw_req->recordSchema)
2899                     {
2900                         z_searchRequest->smallSetElementSetNames =
2901                             z_searchRequest->mediumSetElementSetNames =
2902                             mk_esn_from_schema(m_s2z_odr_search,
2903                                                srw_req->recordSchema);
2904                     }
2905                 }
2906                 else   // Z39.50 present
2907                 {
2908                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
2909                                                    Z_APDU_presentRequest);
2910                     Z_PresentRequest *z_presentRequest =
2911                         m_s2z_present_apdu->u.presentRequest;
2912                     *z_presentRequest->resultSetStartPoint = start;
2913                     *z_presentRequest->numberOfRecordsRequested = max;
2914                     z_presentRequest->preferredRecordSyntax =
2915                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2916                                                VAL_TEXT_XML);
2917                     if (srw_req->recordSchema)
2918                     {
2919                         z_presentRequest->recordComposition =
2920                             (Z_RecordComposition *)
2921                             odr_malloc(m_s2z_odr_search,
2922                                        sizeof(Z_RecordComposition));
2923                         z_presentRequest->recordComposition->which =
2924                             Z_RecordComp_simple;
2925                         z_presentRequest->recordComposition->u.simple =
2926                             mk_esn_from_schema(m_s2z_odr_search,
2927                                                srw_req->recordSchema);
2928                     }
2929                 }
2930             }
2931             if (!m_client)
2932             {
2933                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2934                                             Z_APDU_initRequest);
2935
2936                 m_s2z_init_apdu->u.initRequest->idAuthentication = auth;
2937
2938                 // prevent m_initRequest_apdu memory from being grabbed
2939                 // in Yaz_Proxy::handle_incoming_Z_PDU
2940                 m_initRequest_apdu = m_s2z_init_apdu;
2941                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2942                 return;
2943             }
2944             else
2945             {
2946                 handle_incoming_Z_PDU(m_s2z_search_apdu);
2947                 return;
2948             }
2949         }
2950         else if (srw_pdu->which == Z_SRW_explain_request)
2951         {
2952             Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
2953
2954             const char *backend_db = srw_req->database;
2955             srw_get_client(srw_req->database, &backend_db);
2956
2957             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2958
2959             // save stylesheet
2960             if (srw_req->stylesheet)
2961                 m_s2z_stylesheet =
2962                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2963
2964             if (srw_req->recordPacking &&
2965                 !strcmp(srw_req->recordPacking, "xml"))
2966                 m_s2z_packing = Z_SRW_recordPacking_XML;
2967             else
2968                 m_s2z_packing = Z_SRW_recordPacking_string;
2969
2970             if (num_diagnostic)
2971             {
2972                 send_srw_explain_response(diagnostic, num_diagnostic);
2973                 return;
2974             }
2975
2976             if (!m_client)
2977             {
2978                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2979                                             Z_APDU_initRequest);
2980
2981                 m_s2z_init_apdu->u.initRequest->idAuthentication = auth;
2982                 
2983                 // prevent m_initRequest_apdu memory from being grabbed
2984                 // in Yaz_Proxy::handle_incoming_Z_PDU
2985                 m_initRequest_apdu = m_s2z_init_apdu;
2986                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2987             }
2988             else
2989                 send_srw_explain_response(0, 0);
2990             return;
2991         }
2992         else if (srw_pdu->which == Z_SRW_scan_request)
2993         {
2994             m_s2z_database = odr_strdup(m_s2z_odr_init,
2995                                         srw_pdu->u.scan_request->database);
2996
2997             yaz_add_srw_diagnostic(odr_decode(),
2998                                    &diagnostic, &num_diagnostic,
2999                                    4, "scan");
3000             Z_SRW_PDU *srw_pdu =
3001                 yaz_srw_get(odr_encode(),
3002                             Z_SRW_scan_response);
3003             Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
3004
3005             srw_res->diagnostics = diagnostic;
3006             srw_res->num_diagnostics = num_diagnostic;
3007             send_srw_response(srw_pdu);
3008             return;
3009         }
3010         else
3011         {
3012             m_s2z_database = 0;
3013
3014             send_to_srw_client_error(4, 0);
3015         }
3016     }
3017     send_http_response(400);
3018 }
3019
3020 void Yaz_Proxy::handle_init(Z_APDU *apdu)
3021 {
3022
3023     Z_OtherInformation **oi;
3024     get_otherInfoAPDU(apdu, &oi);
3025
3026     if (apdu->u.initRequest->implementationId)
3027         yaz_log(YLOG_LOG, "%simplementationId: %s",
3028                 m_session_str, apdu->u.initRequest->implementationId);
3029     if (apdu->u.initRequest->implementationName)
3030         yaz_log(YLOG_LOG, "%simplementationName: %s",
3031                 m_session_str, apdu->u.initRequest->implementationName);
3032     if (apdu->u.initRequest->implementationVersion)
3033         yaz_log(YLOG_LOG, "%simplementationVersion: %s",
3034                 m_session_str, apdu->u.initRequest->implementationVersion);
3035     if (m_initRequest_apdu == 0)
3036     {
3037         if (m_initRequest_mem)
3038             nmem_destroy(m_initRequest_mem);
3039
3040         m_initRequest_apdu = apdu;
3041         m_initRequest_mem = odr_extract_mem(odr_decode());
3042
3043         m_initRequest_preferredMessageSize = *apdu->u.initRequest->
3044             preferredMessageSize;
3045         *apdu->u.initRequest->preferredMessageSize = 1024*1024;
3046         m_initRequest_maximumRecordSize = *apdu->u.initRequest->
3047             maximumRecordSize;
3048         *apdu->u.initRequest->maximumRecordSize = 1024*1024;
3049
3050         Z_CharSetandLanguageNegotiation *charSetandLangRecord =
3051             yaz_get_charneg_record(*oi);
3052
3053         // Save proposal charsets and langs.
3054         if (ODR_MASK_GET(apdu->u.initRequest->options,
3055                          Z_Options_negotiationModel)
3056             && charSetandLangRecord)
3057         {
3058
3059             yaz_get_proposal_charneg(m_referenceId_mem,
3060                                      charSetandLangRecord,
3061                                      &m_initRequest_oi_negotiation_charsets,
3062                                      &m_initRequest_oi_negotiation_num_charsets,
3063                                      &m_initRequest_oi_negotiation_langs,
3064                                      &m_initRequest_oi_negotiation_num_langs,
3065                                      &m_initRequest_oi_negotiation_selected);
3066
3067             for (int i = 0; i<m_initRequest_oi_negotiation_num_charsets; i++)
3068             {
3069                 yaz_log(YLOG_LOG, "%scharacters set proposal: %s",
3070                         m_session_str,(m_initRequest_oi_negotiation_charsets[i])?
3071                         m_initRequest_oi_negotiation_charsets[i]:"none");
3072             }
3073             for (int i=0; i<m_initRequest_oi_negotiation_num_langs; i++)
3074             {
3075                 yaz_log(YLOG_LOG, "%slanguages proposal: %s",
3076                         m_session_str, (m_initRequest_oi_negotiation_langs[i])?
3077                         m_initRequest_oi_negotiation_langs[i]:"none");
3078             }
3079             yaz_log(YLOG_LOG, "%sselected proposal: %d (boolean)",
3080                     m_session_str, m_initRequest_oi_negotiation_selected);
3081         }
3082         // save init options for the response..
3083         m_initRequest_options = apdu->u.initRequest->options;
3084
3085         apdu->u.initRequest->options =
3086             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
3087                                        sizeof(Odr_bitmask));
3088         ODR_MASK_ZERO(apdu->u.initRequest->options);
3089         int i;
3090         for (i = 0; i<= 24; i++)
3091             ODR_MASK_SET(apdu->u.initRequest->options, i);
3092         // check negotiation option
3093         if (!ODR_MASK_GET(m_initRequest_options,
3094                           Z_Options_negotiationModel))
3095         {
3096             ODR_MASK_CLEAR(apdu->u.initRequest->options,
3097                            Z_Options_negotiationModel);
3098         }
3099         ODR_MASK_CLEAR(apdu->u.initRequest->options,
3100                        Z_Options_concurrentOperations);
3101         // make new version
3102         m_initRequest_version = apdu->u.initRequest->protocolVersion;
3103         apdu->u.initRequest->protocolVersion =
3104             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
3105                                        sizeof(Odr_bitmask));
3106         ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
3107
3108         for (i = 0; i<= 8; i++)
3109             ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
3110     }
3111     handle_charset_lang_negotiation(apdu);
3112     if (m_client->m_init_flag)
3113     {
3114         if (handle_init_response_for_invalid_session(apdu))
3115             return;
3116         if (m_client->m_initResponse)
3117         {
3118             Z_APDU *apdu2 = m_client->m_initResponse;
3119             apdu2->u.initResponse->otherInfo = 0;
3120             if (m_client->m_cookie && *m_client->m_cookie)
3121                 set_otherInformationString(apdu2, VAL_COOKIE, 1,
3122                                            m_client->m_cookie);
3123             apdu2->u.initResponse->referenceId =
3124                 apdu->u.initRequest->referenceId;
3125             apdu2->u.initResponse->options = m_client->m_initResponse_options;
3126             apdu2->u.initResponse->protocolVersion =
3127                 m_client->m_initResponse_version;
3128
3129             handle_charset_lang_negotiation(apdu2);
3130
3131             if (m_timeout_mode == timeout_busy)
3132                 m_timeout_mode = timeout_normal;
3133             send_to_client(apdu2);
3134             return;
3135         }
3136     }
3137     m_client->m_init_flag = 1;
3138
3139 #if USE_AUTH_MSG
3140     Auth_Msg *m = new Auth_Msg;
3141     m->m_proxy = this;
3142     z_APDU(odr_encode(), &apdu, 0, "encode");
3143     char *apdu_buf = odr_getbuf(odr_encode(), &m->m_apdu_len, 0);
3144     m->m_apdu_buf = (char*) nmem_malloc(m->m_nmem, m->m_apdu_len);
3145     memcpy(m->m_apdu_buf, apdu_buf, m->m_apdu_len);
3146     odr_reset(odr_encode());
3147
3148     inc_ref();
3149     m_my_thread->put(m);
3150 #else
3151     int ret = handle_authentication(apdu);
3152     result_authentication(apdu, ret);
3153 #endif
3154 }
3155
3156 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
3157 {
3158     Z_ReferenceId **refid = get_referenceIdP(apdu);
3159     nmem_reset(m_referenceId_mem);
3160     if (refid && *refid)
3161     {
3162         m_referenceId = (Z_ReferenceId *)
3163             nmem_malloc(m_referenceId_mem, sizeof(*m_referenceId));
3164         m_referenceId->len = m_referenceId->size = (*refid)->len;
3165         m_referenceId->buf = (unsigned char *)
3166             nmem_malloc(m_referenceId_mem, (*refid)->len);
3167         memcpy(m_referenceId->buf, (*refid)->buf, (*refid)->len);
3168     }
3169     else
3170         m_referenceId = 0;
3171
3172     if (!m_client && m_flag_invalid_session)
3173     {
3174         // Got request for a session that is invalid..
3175         m_apdu_invalid_session = apdu; // save package
3176         m_mem_invalid_session = odr_extract_mem(odr_decode());
3177         apdu = m_initRequest_apdu;     // but throw an init to the target
3178     }
3179
3180     if (apdu->which == Z_APDU_searchRequest)
3181         m_search_stat.add_bytes(1);
3182
3183     // Determine our client.
3184     Z_OtherInformation **oi;
3185     get_otherInfoAPDU(apdu, &oi);
3186     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi));
3187     if (!m_client)
3188     {
3189         if (m_http_version)
3190         {   // HTTP. Send not found
3191             send_http_response(404);
3192             return;
3193         }
3194         else
3195         {
3196             // Z39.50 just shutdown
3197             timeout(0);
3198             return;
3199         }
3200     }
3201
3202     m_client->m_server = this;
3203
3204     if (apdu->which == Z_APDU_initRequest)
3205         handle_init(apdu);
3206     else
3207         handle_incoming_Z_PDU_2(apdu);
3208 }
3209
3210 void Yaz_Proxy::handle_incoming_Z_PDU_2(Z_APDU *apdu)
3211 {
3212     handle_max_record_retrieve(apdu);
3213
3214     if (apdu)
3215         apdu = handle_syntax_validation(apdu);
3216
3217     if (apdu)
3218         apdu = handle_query_transformation(apdu);
3219
3220     if (apdu)
3221         apdu = handle_target_charset_conversion(apdu);
3222
3223     if (apdu)
3224         apdu = handle_query_validation(apdu);
3225
3226     if (apdu)
3227         apdu = result_set_optimize(apdu);
3228
3229     if (!apdu)
3230     {
3231         m_client->timeout(m_target_idletime);  // mark it active even
3232         recv_GDU_more(true);
3233         // though we didn't use it
3234         return;
3235     }
3236
3237     // delete other info construct completely if 0 elements
3238     Z_OtherInformation **oi;
3239     get_otherInfoAPDU(apdu, &oi);
3240     if (oi && *oi && (*oi)->num_elements == 0)
3241         *oi = 0;
3242
3243     if (apdu->which == Z_APDU_presentRequest &&
3244         m_client->m_resultSetStartPoint == 0)
3245     {
3246         Z_PresentRequest *pr = apdu->u.presentRequest;
3247         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
3248         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
3249     } else {
3250         m_client->m_resultSetStartPoint = 0;
3251     }
3252     if (m_client->send_to_target(apdu) < 0)
3253     {
3254         m_client->shutdown();
3255     }
3256     else
3257         m_client->m_waiting = 1;
3258 }
3259
3260 void Yaz_Proxy::connectNotify()
3261 {
3262 }
3263
3264 void Yaz_Proxy::releaseClient()
3265 {
3266     xfree(m_proxyTarget);
3267     m_proxyTarget = 0;
3268     m_flag_invalid_session = 0;
3269     // only keep if keep_alive flag is set...
3270     if (m_client &&
3271         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
3272         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
3273         m_client->m_waiting == 0)
3274     {
3275         yaz_log(YLOG_LOG, "%sShutdown (client to proxy) keepalive %s",
3276                  m_session_str,
3277                  m_client->get_hostname());
3278         yaz_log(YLOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
3279                 m_session_str, m_client->m_pdu_recv,
3280                 m_client->m_bytes_sent + m_client->m_bytes_recv,
3281                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
3282         assert (m_client->m_waiting != 2);
3283         // Tell client (if any) that no server connection is there..
3284         m_client->m_server = 0;
3285         m_client = 0;
3286     }
3287     else if (m_client)
3288     {
3289         yaz_log (YLOG_LOG, "%sShutdown (client to proxy) close %s",
3290                  m_session_str,
3291                  m_client->get_hostname());
3292         assert (m_client->m_waiting != 2);
3293         delete m_client;
3294         m_client = 0;
3295     }
3296     else if (!m_parent)
3297     {
3298         yaz_log (YLOG_LOG, "%sshutdown (client to proxy) bad state",
3299                  m_session_str);
3300         assert (m_parent);
3301     }
3302     else
3303     {
3304         yaz_log (YLOG_LOG, "%sShutdown (client to proxy)",
3305                  m_session_str);
3306     }
3307     if (m_parent)
3308         m_parent->pre_init();
3309 }
3310
3311 bool Yaz_Proxy::dec_ref()
3312 {
3313     m_http_keepalive = 0;
3314
3315     --m_ref_count;
3316     if (m_ref_count > 0)
3317         return false;
3318
3319     releaseClient();
3320
3321     delete this;
3322     return true;
3323 }
3324
3325 const char *Yaz_ProxyClient::get_session_str()
3326 {
3327     if (!m_server)
3328         return "0 ";
3329     return m_server->get_session_str();
3330 }
3331
3332 void Yaz_ProxyClient::shutdown()
3333 {
3334     yaz_log (YLOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
3335              get_hostname());
3336
3337     if (m_server)
3338     {
3339         m_waiting = 1;   // ensure it's released from Yaz_Proxy::releaseClient
3340         m_server->dec_ref();
3341     }
3342     else
3343         delete this;
3344 }
3345
3346 void Yaz_Proxy::failNotify()
3347 {
3348     inc_request_no();
3349     yaz_log (YLOG_LOG, "%sConnection closed by client", get_session_str());
3350     dec_ref();
3351 }
3352
3353 void Yaz_Proxy::send_response_fail_client(const char *addr)
3354 {
3355     if (m_http_version)
3356     {
3357         Z_SRW_diagnostic *diagnostic = 0;
3358         int num_diagnostic = 0;
3359         
3360         yaz_add_srw_diagnostic(odr_encode(),
3361