Enable USE_AUTH_MSG again
[yazproxy-moved-to-github.git] / src / yaz-proxy.cpp
1 /* $Id: yaz-proxy.cpp,v 1.48 2006-03-30 14:22:06 adam Exp $
2    Copyright (c) 1998-2006, Index Data.
3
4 This file is part of the yazproxy.
5
6 YAZ proxy is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 2, or (at your option) any later
9 version.
10
11 YAZ proxy is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
14 for more details.
15
16 You should have received a copy of the GNU General Public License
17 along with YAZ proxy; see the file LICENSE.  If not, write to the
18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
19 02111-1307, USA.
20  */
21
22 #ifdef WIN32
23 #define HAVE_SYS_STAT_H 1
24 #define HAVE_SYS_TYPES_H 1
25 #endif
26
27 #if HAVE_UNISTD_H
28 #include <unistd.h>
29 #endif
30 #if HAVE_SYS_TIME_H
31 #include <sys/time.h>
32 #endif
33 #if HAVE_SYS_TYPES_H
34 #include <sys/types.h>
35 #endif
36 #if HAVE_SYS_STAT_H
37 #include <sys/stat.h>
38 #endif
39
40 #include <assert.h>
41 #include <stdlib.h>
42 #include <time.h>
43 #include <fcntl.h>
44
45 #include <yaz/srw.h>
46 #include <yaz/marcdisp.h>
47 #include <yaz/yaz-iconv.h>
48 #include <yaz/log.h>
49 #include <yaz/diagbib1.h>
50 #include "proxyp.h"
51 #include <yaz/pquery.h>
52 #include <yaz/otherinfo.h>
53 #include <yaz/charneg.h>
54 #include "msg-thread.h"
55
56 using namespace yazpp_1;
57
58 #ifdef WIN32
59 #define strncasecmp _strnicmp
60 #endif
61
62 #define USE_AUTH_MSG 1
63
64 #if USE_AUTH_MSG
65 class YAZ_EXPORT Auth_Msg : public IMsg_Thread {
66 public:
67     int m_ret;
68     IMsg_Thread *handle();
69     void result();
70     Yaz_Proxy *m_proxy;
71     NMEM m_nmem;
72     char *m_apdu_buf;
73     int m_apdu_len;
74     Auth_Msg();
75     virtual ~Auth_Msg();
76 };
77
78 Auth_Msg::Auth_Msg()
79 {
80     m_nmem = nmem_create();
81 }
82
83 Auth_Msg::~Auth_Msg()
84 {
85     nmem_destroy(m_nmem);
86 }
87
88 IMsg_Thread *Auth_Msg::handle()
89 {
90     ODR decode = odr_createmem(ODR_DECODE);
91     Z_APDU *apdu;
92
93     odr_setbuf(decode, m_apdu_buf, m_apdu_len, 0);
94     int r = z_APDU(decode, &apdu, 0, 0);
95     if (!r)
96     {
97         yaz_log(YLOG_WARN, "decode failed in Auth_Msg::handle");
98     }
99     else
100     {
101         m_ret = m_proxy->handle_authentication(apdu);
102     }
103     odr_destroy(decode);
104     return this;
105 }
106
107 void Auth_Msg::result()
108 {
109     if (m_proxy->dec_ref(false))
110     {
111         yaz_log(YLOG_LOG, "Auth_Msg::proxy deleted meanwhile");
112     }
113     else
114     {
115         odr_setbuf(m_proxy->odr_decode(), m_apdu_buf, m_apdu_len, 0);
116         Z_APDU *apdu = 0;
117         int r = z_APDU(m_proxy->odr_decode(), &apdu, 0, 0);
118         if (!r)
119             yaz_log(YLOG_LOG, "Auth_Msg::result z_APDU failed");
120         m_proxy->result_authentication(apdu, m_ret);
121     }
122     delete this;
123 }
124
125 #endif
126
127 void Yaz_Proxy::result_authentication(Z_APDU *apdu, int ret)
128 {
129     if (apdu == 0 || ret == 0)
130     {
131         Z_APDU *apdu_reject = zget_APDU(odr_encode(), Z_APDU_initResponse);
132         *apdu_reject->u.initResponse->result = 0;
133         send_to_client(apdu_reject);
134         dec_ref(false);
135     }
136     else
137     {
138         if (apdu->which == Z_APDU_initRequest)
139         {
140             Yaz_ProxyConfig *cfg = check_reconfigure();
141             if (cfg)
142                 cfg->target_authentication(m_default_target, odr_encode(), 
143                                            apdu->u.initRequest);
144         }
145         handle_incoming_Z_PDU_2(apdu);
146     }
147 }
148
149 static const char *apdu_name(Z_APDU *apdu)
150 {
151     switch (apdu->which)
152     {
153     case Z_APDU_initRequest:
154         return "initRequest";
155     case Z_APDU_initResponse:
156         return "initResponse";
157     case Z_APDU_searchRequest:
158         return "searchRequest";
159     case Z_APDU_searchResponse:
160         return "searchResponse";
161     case Z_APDU_presentRequest:
162         return "presentRequest";
163     case Z_APDU_presentResponse:
164         return "presentResponse";
165     case Z_APDU_deleteResultSetRequest:
166         return "deleteResultSetRequest";
167     case Z_APDU_deleteResultSetResponse:
168         return "deleteResultSetResponse";
169     case Z_APDU_scanRequest:
170         return "scanRequest";
171     case Z_APDU_scanResponse:
172         return "scanResponse";
173     case Z_APDU_sortRequest:
174         return "sortRequest";
175     case Z_APDU_sortResponse:
176         return "sortResponse";
177     case Z_APDU_extendedServicesRequest:
178         return "extendedServicesRequest";
179     case Z_APDU_extendedServicesResponse:
180         return "extendedServicesResponse";
181     case Z_APDU_close:
182         return "close";
183     }
184     return "other";
185 }
186
187 static const char *gdu_name(Z_GDU *gdu)
188 {
189     switch(gdu->which)
190     {
191     case Z_GDU_Z3950:
192         return apdu_name(gdu->u.z3950);
193     case Z_GDU_HTTP_Request:
194         return "HTTP Request";
195     case Z_GDU_HTTP_Response:
196         return "HTTP Response";
197     }
198     return "Unknown request/response";
199 }
200
201 Yaz_Proxy::Yaz_Proxy(IPDU_Observable *the_PDU_Observable,
202                      ISocketObservable *the_socket_observable,
203                      Yaz_Proxy *parent)
204     :
205     Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60)
206 {
207     m_PDU_Observable = the_PDU_Observable;
208     m_socket_observable = the_socket_observable;
209     m_client = 0;
210     m_parent = parent;
211     m_clientPool = 0;
212     m_seqno = 1;
213     m_keepalive_limit_bw = 500000;
214     m_keepalive_limit_pdu = 1000;
215     m_proxyTarget = 0;
216     m_default_target = 0;
217     m_proxy_negotiation_charset = 0;
218     m_proxy_negotiation_lang = 0;
219     m_proxy_negotiation_default_charset = 0;
220     m_charset_converter = new Yaz_CharsetConverter;
221     m_max_clients = 150;
222     m_log_mask = 0;
223     m_seed = time(0);
224     m_client_idletime = 600;
225     m_target_idletime = 600;
226     m_optimize = xstrdup ("1");
227     strcpy(m_session_str, "0 ");
228     m_session_no = 0;
229     m_bytes_sent = 0;
230     m_bytes_recv = 0;
231     m_bw_max = 0;
232     m_pdu_max = 0;
233     m_search_max = 0;
234     m_connect_max = 0;
235     m_timeout_mode = timeout_normal;
236     m_timeout_gdu = 0;
237     m_max_record_retrieve = 0;
238     m_reconfig_flag = 0;
239     m_config_fname = 0;
240     m_request_no = 0;
241     m_flag_invalid_session = 0;
242     m_referenceId = 0;
243     m_referenceId_mem = nmem_create();
244     m_config = 0;
245     m_marcxml_mode = none;
246     m_stylesheet_xsp = 0;
247     m_stylesheet_nprl = 0;
248     m_stylesheet_apdu = 0;
249     m_s2z_stylesheet = 0;
250     m_s2z_database = 0;
251     m_schema = 0;
252     m_backend_type = 0;
253     m_backend_charset = 0;
254     m_frontend_type = 0;
255     m_initRequest_apdu = 0;
256     m_initRequest_mem = 0;
257     m_initRequest_preferredMessageSize = 0;
258     m_initRequest_maximumRecordSize = 0;
259     m_initRequest_options = 0;
260     m_initRequest_version = 0;
261     m_initRequest_oi_negotiation_charsets = 0;
262     m_initRequest_oi_negotiation_num_charsets = 0;
263     m_initRequest_oi_negotiation_langs = 0;
264     m_initRequest_oi_negotiation_num_langs = 0;
265     m_initRequest_oi_negotiation_selected = 0;
266     m_apdu_invalid_session = 0;
267     m_mem_invalid_session = 0;
268     m_s2z_odr_init = 0;
269     m_s2z_odr_search = 0;
270     m_s2z_init_apdu = 0;
271     m_s2z_search_apdu = 0;
272     m_s2z_present_apdu = 0;
273     m_http_keepalive = 0;
274     m_http_version = 0;
275     m_soap_ns = 0;
276     m_s2z_packing = Z_SRW_recordPacking_string;
277 #if HAVE_GETTIMEOFDAY
278     m_time_tv = xmalloc(sizeof(struct timeval));
279     struct timeval *tv = (struct timeval *) m_time_tv;
280     tv->tv_sec = 0;
281     tv->tv_usec = 0;
282 #else
283     m_time_tv = 0;
284 #endif
285     m_usemarcon_ini_stage1 = 0;
286     m_usemarcon_ini_stage2 = 0;
287     m_usemarcon = new Yaz_usemarcon();
288     if (!m_parent)
289         low_socket_open();
290     m_my_thread = 0;
291     m_ref_count = 1;
292     m_main_ptr_dec = false;
293     m_peername = 0;
294 }
295
296 void Yaz_Proxy::inc_ref()
297 {
298     m_ref_count++;
299 }
300
301 Yaz_Proxy::~Yaz_Proxy()
302 {
303     yaz_log(YLOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
304             m_bytes_sent, m_bytes_recv);
305     nmem_destroy(m_initRequest_mem);
306     nmem_destroy(m_mem_invalid_session);
307     nmem_destroy(m_referenceId_mem);
308
309     xfree(m_proxyTarget);
310     xfree(m_default_target);
311     xfree(m_proxy_negotiation_charset);
312     xfree(m_proxy_negotiation_lang);
313     xfree(m_proxy_negotiation_default_charset);
314     delete m_charset_converter;
315     xfree(m_optimize);
316
317 #if HAVE_XSLT
318     if (m_stylesheet_xsp)
319         xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
320 #endif
321     xfree (m_time_tv);
322
323     xfree (m_peername);
324     xfree (m_schema);
325     xfree (m_backend_type);
326     xfree (m_backend_charset);
327     xfree (m_usemarcon_ini_stage1);
328     xfree (m_usemarcon_ini_stage2);
329     delete m_usemarcon;
330     if (m_s2z_odr_init)
331         odr_destroy(m_s2z_odr_init);
332     if (m_s2z_odr_search)
333         odr_destroy(m_s2z_odr_search);
334     if (!m_parent)
335         low_socket_close();
336     if (!m_parent)
337         delete m_my_thread;
338     delete m_config;
339 }
340
341 void Yaz_Proxy::set_debug_mode(int mode)
342 {
343     m_debug_mode = mode;
344 }
345
346 int Yaz_Proxy::set_config(const char *config)
347 {
348     delete m_config;
349     m_config = new Yaz_ProxyConfig();
350     xfree(m_config_fname);
351     m_config_fname = xstrdup(config);
352     int r = m_config->read_xml(config);
353     if (!r)
354         m_config->get_generic_info(&m_log_mask, &m_max_clients,
355                                    &m_connect_max);
356     return r;
357 }
358
359 void Yaz_Proxy::set_default_target(const char *target)
360 {
361     xfree (m_default_target);
362     m_default_target = 0;
363     if (target)
364         m_default_target = (char *) xstrdup (target);
365 }
366
367 void Yaz_Proxy::set_proxy_negotiation (const char *charset, const char *lang,
368                                        const char *default_charset)
369 {
370     yaz_log(YLOG_LOG, "%sSet the proxy negotiation: charset to '%s', "
371         "default charset to '%s', language to '%s'", m_session_str, 
372         charset?charset:"none",
373         default_charset?default_charset:"none",
374         lang?lang:"none");
375     xfree (m_proxy_negotiation_charset);
376     xfree (m_proxy_negotiation_lang);
377     m_proxy_negotiation_charset = m_proxy_negotiation_lang = 0;
378     if (charset)
379         m_proxy_negotiation_charset = (char *) xstrdup (charset);
380     if (lang)
381         m_proxy_negotiation_lang = (char *) xstrdup (lang);
382     if (default_charset)
383         m_proxy_negotiation_default_charset =
384             (char *) xstrdup (default_charset);
385 }
386
387 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
388 {
389     if (m_parent)
390         return m_parent->check_reconfigure();
391
392     Yaz_ProxyConfig *cfg = m_config;
393     if (m_reconfig_flag)
394     {
395         yaz_log(YLOG_LOG, "reconfigure");
396         yaz_log_reopen();
397         if (m_config_fname && cfg)
398         {
399             yaz_log(YLOG_LOG, "reconfigure config %s", m_config_fname);
400             int r = cfg->read_xml(m_config_fname);
401             if (r)
402                 yaz_log(YLOG_WARN, "reconfigure failed");
403             else
404             {
405                 m_log_mask = 0;
406                 cfg->get_generic_info(&m_log_mask, &m_max_clients,
407                                       &m_connect_max);
408             }
409         }
410         else
411             yaz_log(YLOG_LOG, "reconfigure");
412         m_reconfig_flag = 0;
413     }
414     return cfg;
415 }
416
417 IPDU_Observer *Yaz_Proxy::sessionNotify(IPDU_Observable
418                                         *the_PDU_Observable, int fd)
419 {
420     check_reconfigure();
421
422     char session_str[200];
423     sprintf(session_str, "%ld:%d ", (long) time(0), m_session_no);
424     m_session_no++;
425
426     const char *peername = the_PDU_Observable->getpeername();
427     yaz_log (YLOG_LOG, "%sNew session %s", session_str, peername);
428
429     m_connect.cleanup(false);
430     m_connect.add_connect(peername);
431
432     int connect_total = m_connect.get_total(peername);
433     int connect_max = m_connect_max;
434     if (connect_max && connect_total > connect_max)
435     {
436         yaz_log(YLOG_LOG, "%sconnect not accepted total=%d max=%d",
437                 session_str, connect_total, connect_max);
438         return 0;
439     }
440     yaz_log(YLOG_LOG, "%sconnect accepted total=%d", session_str,
441             connect_total);
442     
443     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable,
444                                          m_socket_observable, this);
445     new_proxy->m_config = 0;
446     new_proxy->m_config_fname = 0;
447     new_proxy->timeout(m_client_idletime);
448     new_proxy->m_target_idletime = m_target_idletime;
449     new_proxy->set_default_target(m_default_target);
450     new_proxy->m_max_clients = m_max_clients;
451     new_proxy->m_log_mask = m_log_mask;
452     new_proxy->set_APDU_log(get_APDU_log());
453     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
454         new_proxy->set_APDU_yazlog(1);
455     else
456         new_proxy->set_APDU_yazlog(0);
457     strcpy(new_proxy->m_session_str, session_str);
458     new_proxy->m_peername = xstrdup(peername);
459     new_proxy->set_proxy_negotiation(m_proxy_negotiation_charset,
460         m_proxy_negotiation_lang, m_proxy_negotiation_default_charset);
461     // create thread object the first time we get an incoming connection
462     if (!m_my_thread)
463         m_my_thread = new Msg_Thread(m_socket_observable, 1);
464     new_proxy->m_my_thread = m_my_thread;
465     return new_proxy;
466 }
467
468 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
469 {
470     int oid[OID_SIZE];
471     Z_OtherInformationUnit *oi;
472     struct oident ent;
473     ent.proto = PROTO_Z3950;
474     ent.oclass = CLASS_USERINFO;
475     ent.value = (oid_value) VAL_COOKIE;
476     assert (oid_ent_to_oid (&ent, oid));
477
478     if (oid_ent_to_oid (&ent, oid) &&
479         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
480         oi->which == Z_OtherInfo_characterInfo)
481         return oi->information.characterInfo;
482     return 0;
483 }
484 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
485 {
486     int oid[OID_SIZE];
487     Z_OtherInformationUnit *oi;
488     struct oident ent;
489     ent.proto = PROTO_Z3950;
490     ent.oclass = CLASS_USERINFO;
491     ent.value = (oid_value) VAL_PROXY;
492     if (oid_ent_to_oid (&ent, oid) &&
493         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
494         oi->which == Z_OtherInfo_characterInfo)
495         return oi->information.characterInfo;
496     return 0;
497 }
498 const char *Yaz_Proxy::load_balance(const char **url)
499 {
500     int zurl_in_use[MAX_ZURL_PLEX];
501     int zurl_in_spare[MAX_ZURL_PLEX];
502     Yaz_ProxyClient *c;
503     int i;
504
505     for (i = 0; i<MAX_ZURL_PLEX; i++)
506     {
507         zurl_in_use[i] = 0;
508         zurl_in_spare[i] = 0;
509     }
510     for (c = m_parent->m_clientPool; c; c = c->m_next)
511     {
512         for (i = 0; url[i]; i++)
513             if (!strcmp(url[i], c->get_hostname()))
514             {
515                 zurl_in_use[i]++;
516                 if (c->m_cookie == 0 && c->m_server == 0 && c->m_waiting == 0)
517                     zurl_in_spare[i]++;
518             }
519     }
520     int min_use = 100000;
521     int spare_for_min = 0;
522     int max_spare = 0;
523     const char *ret_min = 0;
524     const char *ret_spare = 0;
525     for (i = 0; url[i]; i++)
526     {
527         yaz_log(YLOG_DEBUG, "%szurl=%s use=%d spare=%d",
528                 m_session_str, url[i], zurl_in_use[i], zurl_in_spare[i]);
529         if (min_use > zurl_in_use[i])
530         {
531             ret_min = url[i];
532             min_use = zurl_in_use[i];
533             spare_for_min = zurl_in_spare[i];
534         }
535         if (max_spare < zurl_in_spare[i])
536         {
537             ret_spare = url[i];
538             max_spare = zurl_in_spare[i];
539         }
540     }
541     return ret_min;
542 }
543
544 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
545                                        const char *proxy_host)
546 {
547     assert (m_parent);
548     Yaz_Proxy *parent = m_parent;
549     Yaz_ProxyClient *c = m_client;
550
551     if (!m_proxyTarget)
552     {
553         const char *url[MAX_ZURL_PLEX];
554         Yaz_ProxyConfig *cfg = check_reconfigure();
555         if (proxy_host)
556         {
557             if (parent && parent->m_debug_mode)
558             {
559                 // only to be enabled for debugging...
560                 if (!strcmp(proxy_host, "stop"))
561                     exit(0);
562             }
563             xfree(m_default_target);
564             m_default_target = xstrdup(proxy_host);
565         }
566         proxy_host = m_default_target;
567         int client_idletime = -1;
568         const char *cql2rpn_fname = 0;
569         const char *negotiation_charset = 0;
570         const char *negotiation_lang = 0;
571         const char *query_charset = 0;
572         const char *default_client_query_charset = 0;
573         url[0] = m_default_target;
574         url[1] = 0;
575         if (cfg)
576         {
577             int pre_init = 0;
578             cfg->get_target_info(proxy_host, url, &m_bw_max,
579                                  &m_pdu_max, &m_max_record_retrieve,
580                                  &m_search_max,
581                                  &m_target_idletime, &client_idletime,
582                                  &parent->m_max_clients,
583                                  &m_keepalive_limit_bw,
584                                  &m_keepalive_limit_pdu,
585                                  &pre_init,
586                                  &cql2rpn_fname,
587                                  &negotiation_charset,
588                                  &negotiation_lang,
589                                  &query_charset,
590                                  &default_client_query_charset);
591         }
592         if (client_idletime != -1)
593         {
594             m_client_idletime = client_idletime;
595             timeout(m_client_idletime);
596         }
597         if (cql2rpn_fname)
598             m_cql2rpn.set_pqf_file(cql2rpn_fname);
599         if (negotiation_charset || negotiation_lang || default_client_query_charset)
600         {
601             set_proxy_negotiation(negotiation_charset,
602                 negotiation_lang, default_client_query_charset);
603         }
604         m_charset_converter->set_target_query_charset(query_charset);
605         if (!url[0])
606         {
607             yaz_log(YLOG_LOG, "%sNo default target", m_session_str);
608             return 0;
609         }
610         // we don't handle multiplexing for cookie session, so we just
611         // pick the first one in this case (anonymous users will be able
612         // to use any backend)
613         if (cookie && *cookie)
614             m_proxyTarget = (char*) xstrdup(url[0]);
615         else
616             m_proxyTarget = (char*) xstrdup(load_balance(url));
617     }
618     if (cookie && *cookie)
619     {   // search in sessions with a cookie
620         for (c = parent->m_clientPool; c; c = c->m_next)
621         {
622             assert (c->m_prev);
623             assert (*c->m_prev == c);
624             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
625                 !strcmp(m_proxyTarget, c->get_hostname()))
626             {
627                 // Found it in cache
628                 // The following handles "cancel"
629                 // If connection is busy (waiting for PDU) and
630                 // we have an initRequest we can safely do re-open
631                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
632                 {
633                     yaz_log (YLOG_LOG, "%s REOPEN target=%s", m_session_str,
634                              c->get_hostname());
635                     c->close();
636                     c->m_init_flag = 0;
637
638                     c->m_last_ok = 0;
639                     c->m_cache.clear();
640                     c->m_last_resultCount = 0;
641                     c->m_sr_transform = 0;
642                     c->m_waiting = 0;
643                     c->m_resultSetStartPoint = 0;
644                     c->m_target_idletime = m_target_idletime;
645                     if (c->client(m_proxyTarget))
646                     {
647                         delete c;
648                         return 0;
649                     }
650                     c->timeout(30);
651                 }
652                 c->m_seqno = parent->m_seqno;
653                 if (c->m_server && c->m_server != this)
654                     c->m_server->m_client = 0;
655                 c->m_server = this;
656                 (parent->m_seqno)++;
657                 yaz_log (YLOG_DEBUG, "get_client 1 %p %p", this, c);
658                 return c;
659             }
660         }
661     }
662     else if (!c && apdu->which == Z_APDU_initRequest )
663     {
664         // anonymous sessions without cookie.
665         // if authentication is set it is NOT anonymous se we can't share them.
666         // If charset and lang negotiation is use it is NOT anonymous session too.
667         for (c = parent->m_clientPool; c; c = c->m_next)
668         {
669             assert(c->m_prev);
670             assert(*c->m_prev == c);
671             if (c->m_server == 0 && c->m_cookie == 0 &&  c->m_waiting == 0 
672                 && c->compare_idAuthentication(apdu)
673                 && c->compare_charset(apdu)
674                 && !strcmp(m_proxyTarget, c->get_hostname()))
675             {
676                 // found it in cache
677                 yaz_log (YLOG_LOG, "%sREUSE %d %s",
678                          m_session_str, parent->m_seqno, c->get_hostname());
679                 
680                 c->m_seqno = parent->m_seqno;
681                 assert(c->m_server == 0);
682                 c->m_server = this;
683
684                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
685                     c->set_APDU_yazlog(1);
686                 else
687                     c->set_APDU_yazlog(0);
688
689                 (parent->m_seqno)++;
690
691                 parent->pre_init();
692
693                 return c;
694             }
695         }
696     }
697     if (!m_client)
698     {
699         if (apdu->which != Z_APDU_initRequest)
700         {
701             yaz_log (YLOG_LOG, "%sno init request as first PDU", m_session_str);
702             return 0;
703         }
704         // go through list of clients - and find the lowest/oldest one.
705         Yaz_ProxyClient *c_min = 0;
706         int min_seq = -1;
707         int no_of_clients = 0;
708         if (parent->m_clientPool)
709             yaz_log (YLOG_DEBUG, "Existing sessions");
710         for (c = parent->m_clientPool; c; c = c->m_next)
711         {
712             yaz_log (YLOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
713                                c->m_waiting, c->get_hostname(),
714                                c->m_cookie ? c->m_cookie : "");
715             no_of_clients++;
716             if (min_seq < 0 || c->m_seqno < min_seq)
717             {
718                 min_seq = c->m_seqno;
719                 c_min = c;
720             }
721         }
722         if (no_of_clients >= parent->m_max_clients)
723         {
724             c = c_min;
725             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
726             {
727                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Destroy %d",
728                          m_session_str, parent->m_max_clients, c->m_seqno);
729                 if (c->m_server && c->m_server != this)
730                     delete c->m_server;   // PROBLEM: m_ref_count!
731                 c->m_server = 0;
732             }
733             else
734             {
735                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Reuse %d %d %s",
736                          m_session_str, parent->m_max_clients,
737                          c->m_seqno, parent->m_seqno, c->get_hostname());
738                 xfree (c->m_cookie);
739                 c->m_cookie = 0;
740                 if (cookie)
741                     c->m_cookie = xstrdup(cookie);
742                 c->m_seqno = parent->m_seqno;
743                 if (c->m_server && c->m_server != this)
744                 {
745                     c->m_server->m_client = 0;
746                     delete c->m_server;   // PROBLEM: m_ref_count!
747                 }
748                 (parent->m_seqno)++;
749                 c->m_target_idletime = m_target_idletime;
750                 c->timeout(m_target_idletime);
751
752                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
753                     c->set_APDU_yazlog(1);
754                 else
755                     c->set_APDU_yazlog(0);
756
757                 return c;
758             }
759         }
760         else
761         {
762             yaz_log (YLOG_LOG, "%sNEW %d %s",
763                      m_session_str, parent->m_seqno, m_proxyTarget);
764             c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
765             c->m_next = parent->m_clientPool;
766             if (c->m_next)
767                 c->m_next->m_prev = &c->m_next;
768             parent->m_clientPool = c;
769             c->m_prev = &parent->m_clientPool;
770         }
771
772         xfree (c->m_cookie);
773         c->m_cookie = 0;
774         if (cookie)
775             c->m_cookie = xstrdup(cookie);
776
777         c->m_seqno = parent->m_seqno;
778         c->m_init_flag = 0;
779         c->m_last_resultCount = 0;
780         c->m_last_ok = 0;
781         c->m_cache.clear();
782         c->m_sr_transform = 0;
783         c->m_waiting = 0;
784         c->m_resultSetStartPoint = 0;
785         (parent->m_seqno)++;
786         if (c->client(m_proxyTarget))
787         {
788             delete c;
789             return 0;
790         }
791         c->m_target_idletime = m_target_idletime;
792         c->timeout(30);
793
794         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
795             c->set_APDU_yazlog(1);
796         else
797             c->set_APDU_yazlog(0);
798
799         c->set_idAuthentication(apdu);
800     }
801     yaz_log (YLOG_DEBUG, "get_client 3 %p %p", this, c);
802     return c;
803 }
804
805 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
806 {
807     int i;
808     for (i = 0; i<num; i++)
809     {
810         oident *ent;
811         Z_DefaultDiagFormat *r;
812         Z_DiagRec *p = pp[i];
813         if (p->which != Z_DiagRec_defaultFormat)
814         {
815             yaz_log(YLOG_LOG, "%sError no diagnostics", m_session_str);
816             return;
817         }
818         else
819             r = p->u.defaultFormat;
820         if (!(ent = oid_getentbyoid(r->diagnosticSetId)) ||
821             ent->oclass != CLASS_DIAGSET || ent->value != VAL_BIB1)
822             yaz_log(YLOG_LOG, "%sError unknown diagnostic set", m_session_str);
823         switch (r->which)
824         {
825         case Z_DefaultDiagFormat_v2Addinfo:
826             yaz_log(YLOG_LOG, "%sError %d %s:%s",
827                     m_session_str,
828                     *r->condition, diagbib1_str(*r->condition),
829                     r->u.v2Addinfo);
830             break;
831         case Z_DefaultDiagFormat_v3Addinfo:
832             yaz_log(YLOG_LOG, "%sError %d %s:%s",
833                     m_session_str,
834                     *r->condition, diagbib1_str(*r->condition),
835                     r->u.v3Addinfo);
836             break;
837         }
838     }
839 }
840
841 int Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p, Z_APDU *apdu)
842 {
843     if (!m_stylesheet_xsp || p->num_records <= 0)
844     {
845         return 0;  /* no XSLT to be done ... */
846     }
847
848     m_stylesheet_offset = 0;
849     m_stylesheet_nprl = p;
850     m_stylesheet_apdu = apdu;
851     m_timeout_mode = timeout_xsl;
852
853     timeout(0);
854     return 1;
855 }
856
857 void Yaz_Proxy::convert_xsl_delay()
858 {
859 #if HAVE_XSLT
860     Z_NamePlusRecord *npr = m_stylesheet_nprl->records[m_stylesheet_offset];
861     if (npr->which == Z_NamePlusRecord_databaseRecord)
862     {
863         Z_External *r = npr->u.databaseRecord;
864         if (r->which == Z_External_octet)
865         {
866 #if 0
867             fwrite((char*) r->u.octet_aligned->buf, 1, r->u.octet_aligned->len, stdout);
868 #endif
869             xmlDocPtr res, doc = xmlParseMemory(
870                 (char*) r->u.octet_aligned->buf,
871                 r->u.octet_aligned->len);
872
873
874             yaz_log(YLOG_LOG, "%sXSLT convert %d",
875                     m_session_str, m_stylesheet_offset);
876             res = xsltApplyStylesheet((xsltStylesheetPtr) m_stylesheet_xsp,
877                                       doc, 0);
878
879             if (res)
880             {
881                 xmlChar *out_buf;
882                 int out_len;
883                 xmlDocDumpFormatMemory (res, &out_buf, &out_len, 1);
884
885                 m_stylesheet_nprl->records[m_stylesheet_offset]->
886                     u.databaseRecord =
887                     z_ext_record(odr_encode(), VAL_TEXT_XML,
888                                  (char*) out_buf, out_len);
889                 xmlFree(out_buf);
890                 xmlFreeDoc(res);
891             }
892
893             xmlFreeDoc(doc);
894         }
895     }
896 #endif
897     m_stylesheet_offset++;
898     if (m_stylesheet_offset == m_stylesheet_nprl->num_records)
899     {
900         m_timeout_mode = timeout_normal;
901         m_stylesheet_nprl = 0;
902 #if HAVE_XSLT
903         if (m_stylesheet_xsp)
904             xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
905 #endif
906         m_stylesheet_xsp = 0;
907         timeout(m_client_idletime);
908         send_PDU_convert(m_stylesheet_apdu);
909     }
910     else
911         timeout(0);
912 }
913
914 void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p)
915 {
916     if (m_frontend_type != VAL_NONE)
917     {
918         int i;
919         for (i = 0; i < p->num_records; i++)
920         {
921             Z_NamePlusRecord *npr = p->records[i];
922             if (npr->which == Z_NamePlusRecord_databaseRecord)
923             {
924                 Z_External *r = npr->u.databaseRecord;
925                 if (r->which == Z_External_octet)
926                 {
927 #if HAVE_USEMARCON
928                     if (m_usemarcon_ini_stage1 && *m_usemarcon_ini_stage1)
929                     {
930                         if (!m_usemarcon->m_stage1)
931                         {
932                             m_usemarcon->m_stage1 = new CDetails();
933                         }
934                         m_usemarcon->m_stage1->SetIniFileName(m_usemarcon_ini_stage1);
935                         m_usemarcon->m_stage1->SetMarcRecord((char*) r->u.octet_aligned->buf, r->u.octet_aligned->len);
936                         int res = m_usemarcon->m_stage1->Start();
937                         if (res == 0)
938                         {
939                             char *converted;
940                             int convlen;
941                             m_usemarcon->m_stage1->GetMarcRecord(converted, convlen);
942                             if (m_usemarcon_ini_stage2 && *m_usemarcon_ini_stage2)
943                             {
944                                 if (!m_usemarcon->m_stage2)
945                                 {
946                                     m_usemarcon->m_stage2 = new CDetails();
947                                 }
948                                 m_usemarcon->m_stage2->SetIniFileName(m_usemarcon_ini_stage2);
949                                 m_usemarcon->m_stage2->SetMarcRecord(converted, convlen);
950                                 res = m_usemarcon->m_stage2->Start();
951                                 if (res == 0)
952                                 {
953                                     free(converted);
954                                     m_usemarcon->m_stage2->GetMarcRecord(converted, convlen);
955                                 }
956                                 else
957                                 {
958                                     yaz_log(YLOG_LOG, "%sUSEMARCON stage 2 error %d", m_session_str, res);
959                                 }
960                             }
961                             npr->u.databaseRecord =
962                                 z_ext_record(odr_encode(),
963                                              m_frontend_type,
964                                              converted,
965                                              strlen(converted));
966                             free(converted);
967                         }
968                         else
969                         {
970                             yaz_log(YLOG_LOG, "%sUSEMARCON stage 1 error %d", m_session_str, res);
971                         }
972                         continue;
973                     }
974 #endif
975 /* HAVE_USEMARCON */
976                     npr->u.databaseRecord =
977                         z_ext_record(odr_encode(),
978                                      m_frontend_type,
979                                      (char*) r->u.octet_aligned->buf,
980                                      r->u.octet_aligned->len);
981                 }
982             }
983         }
984     }
985 }
986
987 void Yaz_Proxy::convert_records_charset(Z_NamePlusRecordList *p,
988                                         const char *backend_charset)
989 {
990     yaz_log(YLOG_LOG, "%sconvert_to_marc", m_session_str);
991     int sel =   m_charset_converter->get_client_charset_selected();
992     const char *client_record_charset =
993         m_charset_converter->get_client_query_charset();
994     if (sel && backend_charset && client_record_charset &&
995         strcmp(backend_charset, client_record_charset))
996     {
997         int i;
998         yaz_iconv_t cd = yaz_iconv_open(client_record_charset,
999                                         backend_charset);
1000         yaz_marc_t mt = yaz_marc_create();
1001         yaz_marc_xml(mt, YAZ_MARC_ISO2709);
1002         yaz_marc_iconv(mt, cd);
1003         for (i = 0; i < p->num_records; i++)
1004         {
1005             Z_NamePlusRecord *npr = p->records[i];
1006             if (npr->which == Z_NamePlusRecord_databaseRecord)
1007             {
1008                 Z_External *r = npr->u.databaseRecord;
1009                 oident *ent = oid_getentbyoid(r->direct_reference);
1010                 if (!ent || ent->value == VAL_NONE)
1011                     continue;
1012
1013                 if (ent->value == VAL_SUTRS)
1014                 {
1015                     WRBUF w = wrbuf_alloc();
1016
1017                     wrbuf_iconv_write(w, cd,  (char*) r->u.octet_aligned->buf,
1018                                       r->u.octet_aligned->len);
1019                     npr->u.databaseRecord =
1020                         z_ext_record(odr_encode(), ent->value, wrbuf_buf(w),
1021                                      wrbuf_len(w));
1022                     wrbuf_free(w, 1);
1023                 }
1024                 else if (ent->value == VAL_TEXT_XML)
1025                 {
1026                     ;
1027                 }
1028                 else if (r->which == Z_External_octet)
1029                 {
1030                     int rlen;
1031                     char *result;
1032                     if (yaz_marc_decode_buf(mt,
1033                                             (char*) r->u.octet_aligned->buf,
1034                                             r->u.octet_aligned->len,
1035                                             &result, &rlen))
1036                     {
1037                         npr->u.databaseRecord =
1038                             z_ext_record(odr_encode(), ent->value, result, rlen);
1039                         yaz_log(YLOG_LOG, "%sRecoding MARC record",
1040                                 m_session_str);
1041                     }
1042                 }
1043             }
1044         }
1045         if (cd)
1046             yaz_iconv_close(cd);
1047         yaz_marc_destroy(mt);
1048     }
1049     else
1050     {
1051         yaz_log(YLOG_LOG, "%sSkipping marc convert", m_session_str);
1052     }
1053 }
1054
1055 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p,
1056                                    const char *backend_charset)
1057 {
1058     int i;
1059     if (!backend_charset)
1060         backend_charset = "MARC-8";
1061     yaz_iconv_t cd = yaz_iconv_open("UTF-8", backend_charset);
1062     yaz_marc_t mt = yaz_marc_create();
1063     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
1064     yaz_marc_iconv(mt, cd);
1065     for (i = 0; i < p->num_records; i++)
1066     {
1067         Z_NamePlusRecord *npr = p->records[i];
1068         if (npr->which == Z_NamePlusRecord_databaseRecord)
1069         {
1070             Z_External *r = npr->u.databaseRecord;
1071             if (r->which == Z_External_OPAC)
1072             {
1073                 WRBUF w = wrbuf_alloc();
1074
1075                 yaz_opac_decode_wrbuf(mt, r->u.opac, w);
1076                 npr->u.databaseRecord = z_ext_record(
1077                     odr_encode(), VAL_TEXT_XML,
1078                     wrbuf_buf(w), wrbuf_len(w)
1079                     );
1080                 wrbuf_free(w, 1);
1081             }
1082             else if (r->which == Z_External_octet)
1083             {
1084                 int rlen;
1085                 char *result;
1086                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
1087                                         r->u.octet_aligned->len,
1088                                         &result, &rlen))
1089                 {
1090                     npr->u.databaseRecord =
1091                         z_ext_record(odr_encode(), VAL_TEXT_XML, result, rlen);
1092                 }
1093             }
1094         }
1095     }
1096     if (cd)
1097         yaz_iconv_close(cd);
1098     yaz_marc_destroy(mt);
1099 }
1100
1101 void Yaz_Proxy::logtime()
1102 {
1103 #if HAVE_GETTIMEOFDAY
1104     struct timeval *tv = (struct timeval*) m_time_tv;
1105     if (tv->tv_sec)
1106     {
1107         struct timeval tv1;
1108         gettimeofday(&tv1, 0);
1109         long diff = (tv1.tv_sec - tv->tv_sec)*1000000 +
1110             (tv1.tv_usec - tv->tv_usec);
1111         if (diff >= 0)
1112             yaz_log(YLOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
1113                     diff/1000000, (diff/1000)%1000);
1114     }
1115     tv->tv_sec = 0;
1116     tv->tv_usec = 0;
1117 #endif
1118 }
1119
1120 int Yaz_Proxy::send_http_response(int code)
1121 {
1122     ODR o = odr_encode();
1123     Z_GDU *gdu = z_get_HTTP_Response(o, code);
1124     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1125     if (m_http_version)
1126         hres->version = odr_strdup(o, m_http_version);
1127     if (m_http_keepalive)
1128         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1129     else
1130         timeout(0);
1131
1132     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1133     {
1134         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1135                  gdu_name(gdu));
1136     }
1137     int len;
1138     int r = send_GDU(gdu, &len);
1139     m_bytes_sent += len;
1140     m_bw_stat.add_bytes(len);
1141     logtime();
1142
1143     recv_GDU_more(true);
1144
1145     return r;
1146 }
1147
1148 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu, int http_code /* = 200 */)
1149 {
1150     ODR o = odr_encode();
1151     const char *ctype = "text/xml";
1152     Z_GDU *gdu = z_get_HTTP_Response(o, http_code);
1153     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1154     if (m_http_version)
1155         hres->version = odr_strdup(o, m_http_version);
1156     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
1157     if (m_http_keepalive)
1158         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1159     else
1160         timeout(0);
1161     if (http_code == 401)
1162         z_HTTP_header_add(o, &hres->headers, "WWW-Authenticate", "Basic realm=\"YAZ Proxy\"");
1163
1164     static Z_SOAP_Handler soap_handlers[2] = {
1165 #if HAVE_XSLT
1166         {"http://www.loc.gov/zing/srw/", 0,
1167          (Z_SOAP_fun) yaz_srw_codec},
1168 #endif
1169         {0, 0, 0}
1170     };
1171
1172     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
1173     soap_package->which = Z_SOAP_generic;
1174     soap_package->u.generic =
1175         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
1176     soap_package->u.generic->no = 0;
1177     soap_package->u.generic->ns = soap_handlers[0].ns;
1178     soap_package->u.generic->p = (void *) srw_pdu;
1179     soap_package->ns = m_soap_ns;
1180     z_soap_codec_enc_xsl(o, &soap_package,
1181                          &hres->content_buf, &hres->content_len,
1182                          soap_handlers, 0, m_s2z_stylesheet);
1183     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1184     {
1185         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1186                  gdu_name(gdu));
1187     }
1188     int len;
1189     int r = send_GDU(gdu, &len);
1190     m_bytes_sent += len;
1191     m_bw_stat.add_bytes(len);
1192     logtime();
1193
1194     recv_GDU_more(true);
1195
1196     return r;
1197 }
1198
1199 int Yaz_Proxy::send_to_srw_client_error(int srw_error, const char *add)
1200 {
1201     ODR o = odr_encode();
1202     Z_SRW_diagnostic *diagnostic = (Z_SRW_diagnostic *)
1203         odr_malloc(o, sizeof(*diagnostic));
1204     int num_diagnostic = 1;
1205     yaz_mk_std_diagnostic(o, diagnostic, srw_error, add);
1206     return send_srw_search_response(diagnostic, num_diagnostic, srw_error == 3 ? 401 : 200);
1207 }
1208
1209 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
1210                              Z_DefaultDiagFormat *ddf)
1211 {
1212     int bib1_code = *ddf->condition;
1213     if (bib1_code == 109)
1214         return 404;
1215     srw_res->num_diagnostics = 1;
1216     srw_res->diagnostics = (Z_SRW_diagnostic *)
1217         odr_malloc(o, sizeof(*srw_res->diagnostics));
1218     yaz_mk_std_diagnostic(o, srw_res->diagnostics,
1219                           yaz_diag_bib1_to_srw(*ddf->condition),
1220                           ddf->u.v2Addinfo);
1221     return 0;
1222 }
1223
1224 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
1225 {
1226     ODR o = odr_encode();
1227     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1228     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1229
1230     srw_res->numberOfRecords = odr_intdup (o, hits);
1231     if (records && records->which == Z_Records_DBOSD)
1232     {
1233         srw_res->num_records =
1234             records->u.databaseOrSurDiagnostics->num_records;
1235         int i;
1236         srw_res->records = (Z_SRW_record *)
1237             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
1238         for (i = 0; i < srw_res->num_records; i++)
1239         {
1240             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
1241             if (npr->which != Z_NamePlusRecord_databaseRecord)
1242             {
1243                 srw_res->records[i].recordSchema = "diagnostic";
1244                 srw_res->records[i].recordPacking = m_s2z_packing;
1245                 srw_res->records[i].recordData_buf = "67";
1246                 srw_res->records[i].recordData_len = 2;
1247                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1248                 continue;
1249             }
1250             Z_External *r = npr->u.databaseRecord;
1251             oident *ent = oid_getentbyoid(r->direct_reference);
1252             if (r->which == Z_External_octet && ent->value == VAL_TEXT_XML)
1253             {
1254                 srw_res->records[i].recordSchema = m_schema;
1255                 srw_res->records[i].recordPacking = m_s2z_packing;
1256                 srw_res->records[i].recordData_buf = (char*)
1257                     r->u.octet_aligned->buf;
1258                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
1259                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1260             }
1261             else
1262             {
1263                 srw_res->records[i].recordSchema = "diagnostic";
1264                 srw_res->records[i].recordPacking = m_s2z_packing;
1265                 srw_res->records[i].recordData_buf = "67";
1266                 srw_res->records[i].recordData_len = 2;
1267                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1268             }
1269         }
1270     }
1271     if (records && records->which == Z_Records_NSD)
1272     {
1273         int http_code;
1274         http_code = z_to_srw_diag(odr_encode(), srw_res,
1275                                    records->u.nonSurrogateDiagnostic);
1276         if (http_code)
1277             return send_http_response(http_code);
1278     }
1279     return send_srw_response(srw_pdu);
1280
1281 }
1282
1283 int Yaz_Proxy::send_srw_search_response(Z_SRW_diagnostic *diagnostics,
1284                                         int num_diagnostics, int http_code /* = 200 */)
1285 {
1286     ODR o = odr_encode();
1287     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1288     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1289
1290     srw_res->num_diagnostics = num_diagnostics;
1291     srw_res->diagnostics = diagnostics;
1292     return send_srw_response(srw_pdu, http_code);
1293 }
1294
1295 int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics,
1296                                         int num_diagnostics)
1297 {
1298     Yaz_ProxyConfig *cfg = check_reconfigure();
1299     if (cfg)
1300     {
1301         int len;
1302         char *b = cfg->get_explain_doc(odr_encode(), 0 /* target */,
1303                                        m_s2z_database, &len);
1304         if (b)
1305         {
1306             Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
1307             Z_SRW_explainResponse *er = res->u.explain_response;
1308
1309             er->record.recordData_buf = b;
1310             er->record.recordData_len = len;
1311             er->record.recordPacking = m_s2z_packing;
1312             er->record.recordSchema = "http://explain.z3950.org/dtd/2.0/";
1313
1314             er->diagnostics = diagnostics;
1315             er->num_diagnostics = num_diagnostics;
1316             return send_srw_response(res);
1317         }
1318     }
1319     return send_http_response(404);
1320 }
1321
1322 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu)
1323 {
1324     if (m_http_version)
1325     {
1326         if (apdu->which == Z_APDU_initResponse)
1327         {
1328             Z_InitResponse *res = apdu->u.initResponse;
1329             if (*res->result == 0)
1330             {
1331                 send_to_srw_client_error(3, 0);
1332             }
1333             else if (!m_s2z_search_apdu)
1334             {
1335                 send_srw_explain_response(0, 0);
1336             }
1337             else
1338             {
1339                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1340             }
1341         }
1342         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
1343         {
1344             m_s2z_search_apdu = 0;
1345             Z_SearchResponse *res = apdu->u.searchResponse;
1346             m_s2z_hit_count = *res->resultCount;
1347             if (res->records && res->records->which == Z_Records_NSD)
1348             {
1349                 send_to_srw_client_ok(0, res->records, 1);
1350             }
1351             else if (m_s2z_present_apdu && m_s2z_hit_count > 0)
1352             {
1353                 // adjust
1354                 Z_PresentRequest *pr = m_s2z_present_apdu->u.presentRequest;
1355
1356                 if (*pr->resultSetStartPoint <= m_s2z_hit_count)
1357                 {
1358                     if (*pr->numberOfRecordsRequested+ *pr->resultSetStartPoint
1359                         > m_s2z_hit_count)
1360                         *pr->numberOfRecordsRequested =
1361                             1 + m_s2z_hit_count - *pr->resultSetStartPoint;
1362                 }
1363                 handle_incoming_Z_PDU(m_s2z_present_apdu);
1364             }
1365             else
1366             {
1367                 m_s2z_present_apdu = 0;
1368                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
1369             }
1370         }
1371         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
1372         {
1373             int start =
1374                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
1375
1376             m_s2z_present_apdu = 0;
1377             Z_PresentResponse *res = apdu->u.presentResponse;
1378             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
1379         }
1380     }
1381     else
1382     {
1383         int len = 0;
1384         if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1385             yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1386                      apdu_name(apdu));
1387         int r = send_Z_PDU(apdu, &len);
1388         m_bytes_sent += len;
1389         m_bw_stat.add_bytes(len);
1390         logtime();
1391         return r;
1392     }
1393     return 0;
1394 }
1395
1396 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
1397 {
1398     int kill_session = 0;
1399     Z_ReferenceId **new_id = get_referenceIdP(apdu);
1400
1401     if (new_id)
1402         *new_id = m_referenceId;
1403
1404     if (apdu->which == Z_APDU_searchResponse)
1405     {
1406         Z_SearchResponse *sr = apdu->u.searchResponse;
1407         Z_Records *p = sr->records;
1408         if (p && p->which == Z_Records_NSD)
1409         {
1410             Z_DiagRec dr, *dr_p = &dr;
1411             dr.which = Z_DiagRec_defaultFormat;
1412             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1413
1414             *sr->searchStatus = 0;
1415             display_diagrecs(&dr_p, 1);
1416         }
1417         else
1418         {
1419             if (p && p->which == Z_Records_DBOSD)
1420             {
1421                 if (m_backend_type
1422 #if HAVE_USEMARCON
1423                     || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1424 #endif
1425                     )
1426                     convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1427                 if (m_marcxml_mode == marcxml)
1428                     convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1429                                        m_backend_charset);
1430                 else
1431                     convert_records_charset(p->u.databaseOrSurDiagnostics,
1432                                             m_backend_charset);
1433                 if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1434                     return 0;
1435
1436             }
1437             if (sr->resultCount)
1438             {
1439                 yaz_log(YLOG_LOG, "%s%d hits", m_session_str,
1440                         *sr->resultCount);
1441                 if (*sr->resultCount < 0)
1442                 {
1443                     m_flag_invalid_session = 1;
1444                     kill_session = 1;
1445
1446                     *sr->searchStatus = 0;
1447                     sr->records =
1448                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
1449                     *sr->resultCount = 0;
1450                 }
1451             }
1452         }
1453     }
1454     else if (apdu->which == Z_APDU_presentResponse)
1455     {
1456         Z_PresentResponse *sr = apdu->u.presentResponse;
1457         Z_Records *p = sr->records;
1458         if (p && p->which == Z_Records_NSD)
1459         {
1460             Z_DiagRec dr, *dr_p = &dr;
1461             dr.which = Z_DiagRec_defaultFormat;
1462             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1463             if (*sr->presentStatus == Z_PresentStatus_success)
1464                 *sr->presentStatus = Z_PresentStatus_failure;
1465             display_diagrecs(&dr_p, 1);
1466         }
1467         if (p && p->which == Z_Records_DBOSD)
1468         {
1469             if (m_backend_type
1470 #if HAVE_USEMARCON
1471                 || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1472 #endif
1473                 )
1474                 convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1475             if (m_marcxml_mode == marcxml)
1476                 convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1477                                    m_backend_charset);
1478             else
1479                 convert_records_charset(p->u.databaseOrSurDiagnostics,
1480                                         m_backend_charset);
1481             if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1482                 return 0;
1483         }
1484     }
1485     else if (apdu->which == Z_APDU_initResponse)
1486     {
1487         //Get and check negotiation record
1488         //from init response.
1489         handle_charset_lang_negotiation(apdu);
1490
1491         if (m_initRequest_options)
1492         {
1493             Z_Options *nopt =
1494                 (Odr_bitmask *)odr_malloc(odr_encode(),
1495                                           sizeof(Odr_bitmask));
1496             ODR_MASK_ZERO(nopt);
1497
1498             int i;
1499             for (i = 0; i<24; i++)
1500                 if (ODR_MASK_GET(m_initRequest_options, i) &&
1501                     ODR_MASK_GET(apdu->u.initResponse->options, i))
1502                     ODR_MASK_SET(nopt, i);
1503             apdu->u.initResponse->options = nopt;
1504         }
1505         if (m_initRequest_version)
1506         {
1507             Z_ProtocolVersion *nopt =
1508                 (Odr_bitmask *)odr_malloc(odr_encode(),
1509                                           sizeof(Odr_bitmask));
1510             ODR_MASK_ZERO(nopt);
1511
1512             int i;
1513             for (i = 0; i<8; i++)
1514                 if (ODR_MASK_GET(m_initRequest_version, i) &&
1515                     ODR_MASK_GET(apdu->u.initResponse->protocolVersion, i))
1516                     ODR_MASK_SET(nopt, i);
1517             apdu->u.initResponse->protocolVersion = nopt;
1518         }
1519         apdu->u.initResponse->preferredMessageSize =
1520             odr_intdup(odr_encode(),
1521                        m_client->m_initResponse_preferredMessageSize >
1522                        m_initRequest_preferredMessageSize ?
1523                        m_initRequest_preferredMessageSize :
1524                        m_client->m_initResponse_preferredMessageSize);
1525         apdu->u.initResponse->maximumRecordSize =
1526             odr_intdup(odr_encode(),
1527                        m_client->m_initResponse_maximumRecordSize >
1528                        m_initRequest_maximumRecordSize ?
1529                        m_initRequest_maximumRecordSize :
1530                        m_client->m_initResponse_maximumRecordSize);
1531     }
1532
1533     int r = send_PDU_convert(apdu);
1534     if (r)
1535         return r;
1536     if (kill_session)
1537     {
1538         delete m_client;
1539         m_client = 0;
1540         m_parent->pre_init();
1541     }
1542     return r;
1543 }
1544
1545 void Yaz_ProxyClient::set_idAuthentication(Z_APDU *apdu)
1546 {
1547     Z_IdAuthentication *t = apdu->u.initRequest->idAuthentication;
1548     
1549     odr_reset(m_idAuthentication_odr);
1550     z_IdAuthentication(m_idAuthentication_odr, &t, 1, 0);
1551     m_idAuthentication_ber_buf =
1552         odr_getbuf(m_idAuthentication_odr, 
1553                    &m_idAuthentication_ber_size, 0);
1554 }
1555
1556 bool Yaz_ProxyClient::compare_charset(Z_APDU *apdu)
1557 {
1558     return true;
1559 }
1560
1561 bool Yaz_ProxyClient::compare_idAuthentication(Z_APDU *apdu)
1562 {
1563     Z_IdAuthentication *t = apdu->u.initRequest->idAuthentication;
1564     ODR odr = odr_createmem(ODR_ENCODE);
1565
1566     z_IdAuthentication(odr, &t, 1, 0);
1567     int sz;
1568     char *buf = odr_getbuf(odr, &sz, 0);
1569     if (buf && m_idAuthentication_ber_buf
1570         && sz == m_idAuthentication_ber_size
1571         && !memcmp(m_idAuthentication_ber_buf, buf, sz))
1572     {
1573         odr_destroy(odr);
1574         return true;
1575     }
1576     odr_destroy(odr);
1577     if (!buf && !m_idAuthentication_ber_buf)
1578         return true;
1579     return false;
1580 }
1581
1582 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
1583 {
1584     int len = 0;
1585     const char *apdu_name_tmp = apdu_name(apdu);
1586     int r = send_Z_PDU(apdu, &len);
1587     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
1588         yaz_log (YLOG_LOG, "%sSending %s to %s %d bytes",
1589                  get_session_str(),
1590                  apdu_name_tmp, get_hostname(), len);
1591     m_bytes_sent += len;
1592     return r;
1593 }
1594
1595 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
1596 {
1597     if (apdu->which == Z_APDU_presentRequest)
1598     {
1599         Z_PresentRequest *pr = apdu->u.presentRequest;
1600         int toget = *pr->numberOfRecordsRequested;
1601         int start = *pr->resultSetStartPoint;
1602
1603         yaz_log(YLOG_LOG, "%sPresent %s %d+%d", m_session_str,
1604                 pr->resultSetId, start, toget);
1605
1606         if (*m_parent->m_optimize == '0')
1607             return apdu;
1608
1609         if (!m_client->m_last_resultSetId)
1610         {
1611             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1612             new_apdu->u.presentResponse->records =
1613                 create_nonSurrogateDiagnostics(odr_encode(), 30,
1614                                                pr->resultSetId);
1615             send_to_client(new_apdu);
1616             return 0;
1617         }
1618         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1619         {
1620             if (start+toget-1 > m_client->m_last_resultCount)
1621             {
1622                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1623                 new_apdu->u.presentResponse->records =
1624                     create_nonSurrogateDiagnostics(odr_encode(), 13, 0);
1625                 send_to_client(new_apdu);
1626                 return 0;
1627             }
1628             Z_NamePlusRecordList *npr;
1629 #if 0
1630             yaz_log(YLOG_LOG, "%sCache lookup %d+%d syntax=%s",
1631                     m_session_str, start, toget, yaz_z3950oid_to_str(
1632                         pr->preferredRecordSyntax, &oclass));
1633 #endif
1634             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1635                                           pr->preferredRecordSyntax,
1636                                           pr->recordComposition))
1637             {
1638                 yaz_log (YLOG_LOG, "%sReturned cached records for present request",
1639                          m_session_str);
1640                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1641                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1642
1643                 new_apdu->u.presentResponse->numberOfRecordsReturned
1644                     = odr_intdup(odr_encode(), toget);
1645
1646                 new_apdu->u.presentResponse->records = (Z_Records*)
1647                     odr_malloc(odr_encode(), sizeof(Z_Records));
1648                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1649                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1650                 new_apdu->u.presentResponse->nextResultSetPosition =
1651                     odr_intdup(odr_encode(), start+toget);
1652
1653                 send_to_client(new_apdu);
1654                 return 0;
1655             }
1656         }
1657     }
1658
1659     if (apdu->which != Z_APDU_searchRequest)
1660         return apdu;
1661     Z_SearchRequest *sr = apdu->u.searchRequest;
1662     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1663     Yaz_Z_Databases this_databases;
1664
1665     this_databases.set(sr->num_databaseNames, (const char **)
1666                        sr->databaseNames);
1667
1668     this_query->set_Z_Query(sr->query);
1669
1670     char query_str[120];
1671     this_query->print(query_str, sizeof(query_str)-1);
1672     yaz_log(YLOG_LOG, "%sSearch %s", m_session_str, query_str);
1673
1674     if (*m_parent->m_optimize != '0' &&
1675         m_client->m_last_ok && m_client->m_last_query &&
1676         m_client->m_last_query->match(this_query) &&
1677         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1678         m_client->m_last_databases.match(this_databases))
1679     {
1680         delete this_query;
1681         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1682             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1683         {
1684             Z_NamePlusRecordList *npr;
1685             int toget = *sr->mediumSetPresentNumber;
1686             Z_RecordComposition *comp = 0;
1687
1688             if (toget > m_client->m_last_resultCount)
1689                 toget = m_client->m_last_resultCount;
1690
1691             if (sr->mediumSetElementSetNames)
1692             {
1693                 comp = (Z_RecordComposition *)
1694                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1695                 comp->which = Z_RecordComp_simple;
1696                 comp->u.simple = sr->mediumSetElementSetNames;
1697             }
1698
1699             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1700                                           sr->preferredRecordSyntax, comp))
1701             {
1702                 yaz_log (YLOG_LOG, "%sReturned cached records for medium set",
1703                          m_session_str);
1704                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1705                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1706                 new_apdu->u.searchResponse->resultCount =
1707                     &m_client->m_last_resultCount;
1708
1709                 new_apdu->u.searchResponse->numberOfRecordsReturned
1710                     = odr_intdup(odr_encode(), toget);
1711
1712                 new_apdu->u.searchResponse->presentStatus =
1713                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1714                 new_apdu->u.searchResponse->records = (Z_Records*)
1715                     odr_malloc(odr_encode(), sizeof(Z_Records));
1716                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1717                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1718                 new_apdu->u.searchResponse->nextResultSetPosition =
1719                     odr_intdup(odr_encode(), toget+1);
1720                 send_to_client(new_apdu);
1721                 return 0;
1722             }
1723             else
1724             {
1725                 // medium Set
1726                 // send present request (medium size)
1727                 yaz_log (YLOG_LOG, "%sOptimizing search for medium set",
1728                          m_session_str);
1729
1730                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1731                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1732                 pr->referenceId = sr->referenceId;
1733                 pr->resultSetId = sr->resultSetName;
1734                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1735                 *pr->numberOfRecordsRequested = toget;
1736                 pr->recordComposition = comp;
1737                 m_client->m_sr_transform = 1;
1738                 return new_apdu;
1739             }
1740         }
1741         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1742             m_client->m_last_resultCount <= 0)
1743         {
1744             // large set. Return pseudo-search response immediately
1745             yaz_log (YLOG_LOG, "%sOptimizing search for large set",
1746                      m_session_str);
1747             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1748             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1749             new_apdu->u.searchResponse->resultCount =
1750                 &m_client->m_last_resultCount;
1751             send_to_client(new_apdu);
1752             return 0;
1753         }
1754         else
1755         {
1756             Z_NamePlusRecordList *npr;
1757             int toget = m_client->m_last_resultCount;
1758             Z_RecordComposition *comp = 0;
1759             // small set
1760             // send a present request (small set)
1761
1762             if (sr->smallSetElementSetNames)
1763             {
1764                 comp = (Z_RecordComposition *)
1765                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1766                 comp->which = Z_RecordComp_simple;
1767                 comp->u.simple = sr->smallSetElementSetNames;
1768             }
1769
1770             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1771                                           sr->preferredRecordSyntax, comp))
1772             {
1773                 yaz_log (YLOG_LOG, "%sReturned cached records for small set",
1774                          m_session_str);
1775                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1776                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1777                 new_apdu->u.searchResponse->resultCount =
1778                     &m_client->m_last_resultCount;
1779
1780                 new_apdu->u.searchResponse->numberOfRecordsReturned
1781                     = odr_intdup(odr_encode(), toget);
1782
1783                 new_apdu->u.searchResponse->presentStatus =
1784                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1785                 new_apdu->u.searchResponse->records = (Z_Records*)
1786                     odr_malloc(odr_encode(), sizeof(Z_Records));
1787                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1788                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1789                 new_apdu->u.searchResponse->nextResultSetPosition =
1790                     odr_intdup(odr_encode(), toget+1);
1791                 send_to_client(new_apdu);
1792                 return 0;
1793             }
1794             else
1795             {
1796                 yaz_log (YLOG_LOG, "%sOptimizing search for small set",
1797                          m_session_str);
1798                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1799                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1800                 pr->referenceId = sr->referenceId;
1801                 pr->resultSetId = sr->resultSetName;
1802                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1803                 *pr->numberOfRecordsRequested = toget;
1804                 pr->recordComposition = comp;
1805                 m_client->m_sr_transform = 1;
1806                 return new_apdu;
1807             }
1808         }
1809     }
1810     else  // query doesn't match
1811     {
1812         delete m_client->m_last_query;
1813         m_client->m_last_query = this_query;
1814         m_client->m_last_ok = 0;
1815         m_client->m_cache.clear();
1816         m_client->m_resultSetStartPoint = 0;
1817
1818         xfree (m_client->m_last_resultSetId);
1819         m_client->m_last_resultSetId = xstrdup (sr->resultSetName);
1820
1821         m_client->m_last_databases.set(sr->num_databaseNames,
1822                                        (const char **) sr->databaseNames);
1823     }
1824     return apdu;
1825 }
1826
1827
1828 void Yaz_Proxy::inc_request_no()
1829 {
1830     char *cp = strchr(m_session_str, ' ');
1831     m_request_no++;
1832     if (cp)
1833         sprintf(cp+1, "%d ", m_request_no);
1834 }
1835
1836 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
1837 {
1838     inc_request_no();
1839
1840     m_bytes_recv += len;
1841
1842     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1843         yaz_log (YLOG_LOG, "%sReceiving %s from client %d bytes",
1844                  m_session_str, gdu_name(apdu), len);
1845
1846 #if HAVE_GETTIMEOFDAY
1847     gettimeofday((struct timeval *) m_time_tv, 0);
1848 #endif
1849     m_bw_stat.add_bytes(len);
1850     m_pdu_stat.add_bytes(1);
1851
1852     GDU *gdu = new GDU(apdu);
1853     m_in_queue.enqueue(gdu);
1854
1855     recv_GDU_more(false);
1856 }
1857
1858 void Yaz_Proxy::recv_GDU_reduce(GDU *gdu)
1859 {
1860     int reduce = 0;
1861
1862     int bw_total = m_bw_stat.get_total();
1863     int pdu_total = m_pdu_stat.get_total();
1864
1865     assert(m_timeout_mode == timeout_busy);
1866     assert(m_timeout_gdu == 0);
1867
1868     if (m_bw_max)
1869     {
1870         if (bw_total > m_bw_max)
1871         {
1872             reduce = (bw_total/m_bw_max);
1873         }
1874     }
1875     if (m_pdu_max)
1876     {
1877         if (pdu_total > m_pdu_max)
1878         {
1879             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
1880             reduce = (reduce > nreduce) ? reduce : nreduce;
1881         }
1882     }
1883     m_http_version = 0;
1884
1885 #if 0
1886     /* uncomment to force a big reduce */
1887     m_timeout_mode = timeout_reduce;
1888     m_timeout_gdu = gdu;
1889     timeout(3);       // call us reduce seconds later
1890     return;
1891 #endif
1892     if (reduce)
1893     {
1894         yaz_log(YLOG_LOG, "%sdelay=%d bw=%d pdu=%d limit-bw=%d limit-pdu=%d",
1895                 m_session_str, reduce, bw_total, pdu_total,
1896                 m_bw_max, m_pdu_max);
1897
1898         m_timeout_mode = timeout_reduce;
1899         m_timeout_gdu = gdu;
1900         timeout(reduce);       // call us reduce seconds later
1901     }
1902     else
1903         recv_GDU_normal(gdu);
1904 }
1905
1906 void Yaz_Proxy::recv_GDU_more(bool normal)
1907 {
1908     GDU *g;
1909     if (normal && m_timeout_mode == timeout_busy)
1910         m_timeout_mode = timeout_normal;
1911     while (m_timeout_mode == timeout_normal && (g = m_in_queue.dequeue()))
1912     {
1913         m_timeout_mode = timeout_busy;
1914         recv_GDU_reduce(g);
1915     }
1916 }
1917
1918 void Yaz_Proxy::recv_GDU_normal(GDU *gdu)
1919 {
1920     Z_GDU *apdu = 0;
1921     gdu->move_away_gdu(odr_decode(), &apdu);
1922     delete gdu;
1923
1924     if (apdu->which == Z_GDU_Z3950)
1925         handle_incoming_Z_PDU(apdu->u.z3950);
1926     else if (apdu->which == Z_GDU_HTTP_Request)
1927         handle_incoming_HTTP(apdu->u.HTTP_Request);
1928 }
1929
1930 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
1931 {
1932     if (m_max_record_retrieve)
1933     {
1934         if (apdu->which == Z_APDU_presentRequest)
1935         {
1936             Z_PresentRequest *pr = apdu->u.presentRequest;
1937             if (pr->numberOfRecordsRequested &&
1938                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
1939                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
1940         }
1941     }
1942 }
1943
1944 void Yaz_Proxy::handle_charset_lang_negotiation(Z_APDU *apdu)
1945 {
1946     if (apdu->which == Z_APDU_initRequest)
1947     {
1948         if (m_initRequest_options &&
1949             !ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel) &&
1950             (m_proxy_negotiation_charset || m_proxy_negotiation_lang))
1951         {
1952             // There is no negotiation proposal from
1953             // client's side. OK. The proxy negotiation
1954             // in use, only.
1955             Z_InitRequest *initRequest = apdu->u.initRequest;
1956             Z_OtherInformation **otherInfo;
1957             Z_OtherInformationUnit *oi;
1958             get_otherInfoAPDU(apdu, &otherInfo);
1959             oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
1960             if (oi)
1961             {
1962                 ODR_MASK_SET(initRequest->options,
1963                     Z_Options_negotiationModel);
1964                 oi->which = Z_OtherInfo_externallyDefinedInfo;
1965                 oi->information.externallyDefinedInfo =
1966                 yaz_set_proposal_charneg(odr_encode(),
1967                     (const char**)&m_proxy_negotiation_charset,
1968                     m_proxy_negotiation_charset ? 1:0,
1969                     (const char**)&m_proxy_negotiation_lang,
1970                     m_proxy_negotiation_lang ? 1:0,
1971                     1);
1972             }
1973         }
1974         else if (m_initRequest_options &&
1975                  ODR_MASK_GET(m_initRequest_options,
1976                               Z_Options_negotiationModel) &&
1977                  m_charset_converter->get_target_query_charset())
1978         {
1979             yaz_log(YLOG_LOG, "%sManaged charset negotiation: charset=%s",
1980                     m_session_str,
1981                     m_charset_converter->get_target_query_charset());
1982             Z_InitRequest *initRequest = apdu->u.initRequest;
1983             Z_CharSetandLanguageNegotiation *negotiation =
1984                 yaz_get_charneg_record (initRequest->otherInfo);
1985             if (negotiation &&
1986                 negotiation->which == Z_CharSetandLanguageNegotiation_proposal)
1987             {
1988                 NMEM nmem = nmem_create();
1989                 char **charsets = 0;
1990                 int num_charsets = 0;
1991                 char **langs = 0;
1992                 int num_langs = 0;
1993                 int selected = 0;
1994                 yaz_get_proposal_charneg (nmem, negotiation,
1995                                           &charsets, &num_charsets,
1996                                           &langs, &num_langs, &selected);
1997                 int i;
1998                 for (i = 0; i<num_charsets; i++)
1999                     yaz_log(YLOG_LOG, "%scharset %s", m_session_str,
2000                             charsets[i]);
2001                 for (i = 0; i<num_langs; i++)
2002                     yaz_log(YLOG_LOG, "%slang %s", m_session_str,
2003                             langs[i]);
2004
2005                 const char *t_charset =
2006                     m_charset_converter->get_target_query_charset();
2007                 // sweep through charsets and pick the first supported
2008                 // conversion
2009                 for (i = 0; i<num_charsets; i++)
2010                 {
2011                     const char *c_charset = charsets[i];
2012                     if (!odr_set_charset(odr_decode(), t_charset, c_charset))
2013                         break;
2014                 }
2015                 if (i != num_charsets)
2016                 {
2017                     // got one .. set up ODR for reverse direction
2018                     const char *c_charset = charsets[i];
2019                     odr_set_charset(odr_encode(), c_charset, t_charset);
2020                     m_charset_converter->set_client_query_charset(c_charset);
2021                     m_charset_converter->set_client_charset_selected(selected);
2022                 }
2023                 nmem_destroy(nmem);
2024                 ODR_MASK_CLEAR(m_initRequest_options,
2025                                Z_Options_negotiationModel);
2026                 yaz_del_charneg_record(&initRequest->otherInfo);
2027             }
2028             else
2029             {
2030                 yaz_log(YLOG_WARN, "%sUnable to decode charset package",
2031                         m_session_str);
2032             }
2033         }
2034         else if (m_charset_converter->get_target_query_charset() &&
2035             m_proxy_negotiation_default_charset)
2036         {
2037             m_charset_converter->
2038                 set_client_query_charset(m_proxy_negotiation_default_charset);
2039         }
2040     }
2041     else if (apdu->which == Z_APDU_initResponse)
2042     {
2043         Z_InitResponse *initResponse = apdu->u.initResponse;
2044         Z_OtherInformation **otherInfo;
2045         get_otherInfoAPDU(apdu, &otherInfo);
2046         
2047         Z_CharSetandLanguageNegotiation *charneg = 0;
2048
2049         if (otherInfo && *otherInfo && 
2050             ODR_MASK_GET(initResponse->options, Z_Options_negotiationModel)
2051             && (charneg = yaz_get_charneg_record(*otherInfo)))
2052         {
2053             char *charset = 0;
2054             char *lang = 0;
2055             int selected = 0;
2056
2057             yaz_get_response_charneg(m_referenceId_mem, charneg,
2058                 &charset, &lang, &selected);
2059
2060             yaz_log(YLOG_LOG, "%sAccepted charset - '%s' and lang - '%s'",
2061                 m_session_str, (charset)?charset:"none", (lang)?lang:"none");
2062
2063             if (m_initRequest_options &&
2064                 ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel))
2065             {
2066                 yaz_log(YLOG_LOG, "%sClient's negotiation record in use",
2067                     m_session_str);
2068             }
2069             else if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
2070             {
2071                 // negotiation-charset, negotiation-lang
2072                 // elements of config file in use.
2073
2074                 yaz_log(YLOG_LOG, "%sProxy's negotiation record in use",
2075                     m_session_str);
2076
2077                 // clear negotiation option.
2078                 ODR_MASK_CLEAR(initResponse->options, Z_Options_negotiationModel);
2079
2080                 // Delete negotiation (charneg-3) entry.
2081                 yaz_del_charneg_record(otherInfo);
2082             }
2083         }
2084         else
2085         {
2086             if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
2087             {
2088                 yaz_log(YLOG_LOG, "%sTarget did not honor negotiation",
2089                         m_session_str);
2090             }
2091             else if (m_charset_converter->get_client_query_charset())
2092             {
2093                 Z_OtherInformation **otherInfo;
2094                 Z_OtherInformationUnit *oi;
2095                 get_otherInfoAPDU(apdu, &otherInfo);
2096                 oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
2097                 if (oi)
2098                 {
2099                     ODR_MASK_SET(initResponse->options,
2100                                  Z_Options_negotiationModel);
2101                     if (m_initRequest_options)
2102                         ODR_MASK_SET(m_initRequest_options,
2103                                      Z_Options_negotiationModel);
2104                     
2105                     oi->which = Z_OtherInfo_externallyDefinedInfo;    
2106                     oi->information.externallyDefinedInfo =
2107                         yaz_set_response_charneg(
2108                             odr_encode(),
2109                             m_charset_converter->get_client_query_charset(),
2110                             0 /* no lang */,
2111                             m_charset_converter->get_client_charset_selected());
2112                 }
2113             }
2114         }
2115     }
2116 }
2117
2118 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
2119                                                      int error,
2120                                                      const char *addinfo)
2121 {
2122     Z_Records *rec = (Z_Records *)
2123         odr_malloc (odr, sizeof(*rec));
2124     int *err = (int *)
2125         odr_malloc (odr, sizeof(*err));
2126     Z_DiagRec *drec = (Z_DiagRec *)
2127         odr_malloc (odr, sizeof(*drec));
2128     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
2129         odr_malloc (odr, sizeof(*dr));
2130     *err = error;
2131     rec->which = Z_Records_NSD;
2132     rec->u.nonSurrogateDiagnostic = dr;
2133     dr->diagnosticSetId =
2134         yaz_oidval_to_z3950oid (odr, CLASS_DIAGSET, VAL_BIB1);
2135     dr->condition = err;
2136     dr->which = Z_DefaultDiagFormat_v2Addinfo;
2137     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
2138     return rec;
2139 }
2140
2141 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
2142 {
2143     if (apdu->which == Z_APDU_searchRequest &&
2144         apdu->u.searchRequest->query &&
2145         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
2146         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
2147     {
2148         Z_RPNQuery *rpnquery = 0;
2149         Z_SearchRequest *sr = apdu->u.searchRequest;
2150         char *addinfo = 0;
2151
2152         yaz_log(YLOG_LOG, "%sCQL: %s", m_session_str,
2153                 sr->query->u.type_104->u.cql);
2154
2155         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
2156                                           &rpnquery, odr_encode(),
2157                                           &addinfo);
2158         if (r == -3)
2159             yaz_log(YLOG_LOG, "%sNo CQL to RPN table", m_session_str);
2160         else if (r)
2161         {
2162             yaz_log(YLOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
2163             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2164
2165             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2166             new_apdu->u.searchResponse->records =
2167                 create_nonSurrogateDiagnostics(odr_encode(),
2168                                                yaz_diag_srw_to_bib1(r),
2169                                                addinfo);
2170             *new_apdu->u.searchResponse->searchStatus = 0;
2171
2172             send_to_client(new_apdu);
2173
2174             return 0;
2175         }
2176         else
2177         {
2178             sr->query->which = Z_Query_type_1;
2179             sr->query->u.type_1 = rpnquery;
2180         }
2181         return apdu;
2182     }
2183     return apdu;
2184 }
2185
2186 Z_APDU *Yaz_Proxy::handle_target_charset_conversion(Z_APDU *apdu)
2187 {
2188     if (apdu->which == Z_APDU_searchRequest &&
2189         apdu->u.searchRequest->query)
2190     {
2191         if (apdu->u.searchRequest->query->which == Z_Query_type_1
2192             || apdu->u.searchRequest->query->which == Z_Query_type_101)
2193         {
2194             if (m_http_version)
2195                 m_charset_converter->set_client_query_charset("UTF-8");
2196             Z_RPNQuery *rpnquery = apdu->u.searchRequest->query->u.type_1;
2197             m_charset_converter->convert_type_1(rpnquery, odr_encode());
2198         }
2199     }
2200     return apdu;
2201 }
2202
2203
2204 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
2205 {
2206     if (apdu->which == Z_APDU_searchRequest)
2207     {
2208         Z_SearchRequest *sr = apdu->u.searchRequest;
2209         int err = 0;
2210         char *addinfo = 0;
2211
2212         Yaz_ProxyConfig *cfg = check_reconfigure();
2213         if (cfg)
2214             err = cfg->check_query(odr_encode(), m_default_target,
2215                                    sr->query, &addinfo);
2216         if (err)
2217         {
2218             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2219
2220             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2221             new_apdu->u.searchResponse->records =
2222                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2223             *new_apdu->u.searchResponse->searchStatus = 0;
2224
2225             send_to_client(new_apdu);
2226
2227             return 0;
2228         }
2229     }
2230     return apdu;
2231 }
2232
2233 int Yaz_Proxy::handle_authentication(Z_APDU *apdu)
2234 {
2235     if (apdu->which != Z_APDU_initRequest)
2236         return 1;  // pass if no init request
2237     Z_InitRequest *req = apdu->u.initRequest;
2238
2239     Yaz_ProxyConfig *cfg = check_reconfigure();
2240     if (!cfg)
2241         return 1;  // pass if no config
2242
2243     int ret;
2244     if (req->idAuthentication == 0)
2245     {
2246         ret = cfg->client_authentication(m_default_target, 0, 0, 0,
2247                                          m_peername);
2248     }
2249     else if (req->idAuthentication->which == Z_IdAuthentication_idPass)
2250     {
2251         ret = cfg->client_authentication(
2252             m_default_target,
2253             req->idAuthentication->u.idPass->userId,
2254             req->idAuthentication->u.idPass->groupId,
2255             req->idAuthentication->u.idPass->password,
2256             m_peername);
2257     }
2258     else if (req->idAuthentication->which == Z_IdAuthentication_open)
2259     {
2260         char user[64], pass[64];
2261         *user = '\0';
2262         *pass = '\0';
2263         sscanf(req->idAuthentication->u.open, "%63[^/]/%63s", user, pass);
2264         ret = cfg->client_authentication(m_default_target, user, 0, pass,
2265                                          m_peername);
2266     }
2267     else
2268         ret = cfg->client_authentication(m_default_target, 0, 0, 0,
2269                                          m_peername);
2270     return ret;
2271 }
2272
2273 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
2274 {
2275     m_marcxml_mode = none;
2276     if (apdu->which == Z_APDU_searchRequest)
2277     {
2278         Z_SearchRequest *sr = apdu->u.searchRequest;
2279         int err = 0;
2280         char *addinfo = 0;
2281         Yaz_ProxyConfig *cfg = check_reconfigure();
2282
2283         Z_RecordComposition rc_temp, *rc = 0;
2284         if (sr->smallSetElementSetNames)
2285         {
2286             rc_temp.which = Z_RecordComp_simple;
2287             rc_temp.u.simple = sr->smallSetElementSetNames;
2288             rc = &rc_temp;
2289         }
2290
2291         if (sr->preferredRecordSyntax)
2292         {
2293             struct oident *ent;
2294             ent = oid_getentbyoid(sr->preferredRecordSyntax);
2295             m_frontend_type = ent->value;
2296         }
2297         else
2298             m_frontend_type = VAL_NONE;
2299
2300         char *stylesheet_name = 0;
2301         if (cfg)
2302             err = cfg->check_syntax(odr_encode(),
2303                                     m_default_target,
2304                                     sr->preferredRecordSyntax, rc,
2305                                     &addinfo, &stylesheet_name, &m_schema,
2306                                     &m_backend_type, &m_backend_charset,
2307                                     &m_usemarcon_ini_stage1,
2308                                     &m_usemarcon_ini_stage2);
2309         if (stylesheet_name)
2310         {
2311             m_parent->low_socket_close();
2312
2313 #if HAVE_XSLT
2314             if (m_stylesheet_xsp)
2315                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2316             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2317                                                        stylesheet_name);
2318 #endif
2319             m_stylesheet_offset = 0;
2320             xfree(stylesheet_name);
2321
2322             m_parent->low_socket_open();
2323         }
2324         if (err == -1)
2325         {
2326             sr->smallSetElementSetNames = 0;
2327             sr->mediumSetElementSetNames = 0;
2328             m_marcxml_mode = marcxml;
2329             if (m_backend_type)
2330             {
2331
2332                 sr->preferredRecordSyntax =
2333                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
2334                                         m_backend_type);
2335             }
2336             else
2337                 sr->preferredRecordSyntax =
2338                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
2339                                            VAL_USMARC);
2340         }
2341         else if (err)
2342         {
2343             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2344
2345             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2346             new_apdu->u.searchResponse->records =
2347                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2348             *new_apdu->u.searchResponse->searchStatus = 0;
2349
2350             send_to_client(new_apdu);
2351
2352             return 0;
2353         }
2354         else if (m_backend_type)
2355         {
2356             sr->preferredRecordSyntax =
2357                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
2358         }
2359     }
2360     else if (apdu->which == Z_APDU_presentRequest)
2361     {
2362         Z_PresentRequest *pr = apdu->u.presentRequest;
2363         int err = 0;
2364         char *addinfo = 0;
2365         Yaz_ProxyConfig *cfg = check_reconfigure();
2366
2367         if (pr->preferredRecordSyntax)
2368         {
2369             struct oident *ent;
2370             ent = oid_getentbyoid(pr->preferredRecordSyntax);
2371             m_frontend_type = ent->value;
2372         }
2373         else
2374             m_frontend_type = VAL_NONE;
2375
2376         char *stylesheet_name = 0;
2377         if (cfg)
2378             err = cfg->check_syntax(odr_encode(), m_default_target,
2379                                     pr->preferredRecordSyntax,
2380                                     pr->recordComposition,
2381                                     &addinfo, &stylesheet_name, &m_schema,
2382                                     &m_backend_type, &m_backend_charset,
2383                                     &m_usemarcon_ini_stage1,
2384                                     &m_usemarcon_ini_stage2
2385                                     );
2386         if (stylesheet_name)
2387         {
2388             m_parent->low_socket_close();
2389
2390 #if HAVE_XSLT
2391             if (m_stylesheet_xsp)
2392                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2393             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2394                                                        stylesheet_name);
2395 #endif
2396             m_stylesheet_offset = 0;
2397             xfree(stylesheet_name);
2398
2399             m_parent->low_socket_open();
2400         }
2401         if (err == -1)
2402         {
2403             pr->recordComposition = 0;
2404             m_marcxml_mode = marcxml;
2405             if (m_backend_type)
2406             {
2407
2408                 pr->preferredRecordSyntax =
2409                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
2410                                         m_backend_type);
2411             }
2412             else
2413                 pr->preferredRecordSyntax =
2414                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
2415                                            VAL_USMARC);
2416         }
2417         else if (err)
2418         {
2419             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
2420
2421             new_apdu->u.presentResponse->referenceId = pr->referenceId;
2422             new_apdu->u.presentResponse->records =
2423                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2424             *new_apdu->u.presentResponse->presentStatus =
2425                 Z_PresentStatus_failure;
2426
2427             send_to_client(new_apdu);
2428
2429             return 0;
2430         }
2431         else if (m_backend_type)
2432         {
2433             pr->preferredRecordSyntax =
2434                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
2435         }
2436     }
2437     return apdu;
2438 }
2439
2440 Z_ElementSetNames *Yaz_Proxy::mk_esn_from_schema(ODR o, const char *schema)
2441 {
2442     if (!schema)
2443         return 0;
2444     Z_ElementSetNames *esn = (Z_ElementSetNames *)
2445         odr_malloc(o, sizeof(Z_ElementSetNames));
2446     esn->which = Z_ElementSetNames_generic;
2447     esn->u.generic = odr_strdup(o, schema);
2448     return esn;
2449 }
2450
2451 void Yaz_Proxy::srw_get_client(const char *db, const char **backend_db)
2452 {
2453     const char *t = 0;
2454     Yaz_ProxyConfig *cfg = check_reconfigure();
2455     if (cfg)
2456         t = cfg->get_explain_name(db, backend_db);
2457
2458     if (m_client && m_default_target && t && strcmp(m_default_target, t))
2459     {
2460         releaseClient();
2461     }
2462
2463     if (t)
2464     {
2465         xfree(m_default_target);
2466         m_default_target = xstrdup(t);
2467     }
2468 }
2469
2470 int Yaz_Proxy::file_access(Z_HTTP_Request *hreq)
2471 {
2472     struct stat sbuf;
2473     if (strcmp(hreq->method, "GET"))
2474         return 0;
2475     if (hreq->path[0] != '/')
2476     {
2477         yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
2478         return 0;
2479     }
2480     const char *cp = hreq->path;
2481     while (*cp)
2482     {
2483         if (*cp == '/' && strchr("/.", cp[1]))
2484         {
2485             yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
2486             return 0;
2487         }
2488         cp++;
2489     }
2490     const char *fname = hreq->path+1;
2491     if (stat(fname, &sbuf))
2492     {
2493         yaz_log(YLOG_WARN|YLOG_ERRNO, "%s: stat failed", fname);
2494         return 0;
2495     }
2496     if ((sbuf.st_mode & S_IFMT) != S_IFREG)
2497     {
2498         yaz_log(YLOG_WARN, "%s: not a regular file", fname);
2499         return 0;
2500     }
2501     if (sbuf.st_size > (off_t) 1000000)
2502     {
2503         yaz_log(YLOG_WARN, "%s: too large for transfer", fname);
2504         return 0;
2505     }
2506
2507     ODR o = odr_encode();
2508     Yaz_ProxyConfig *cfg = check_reconfigure();
2509     const char *ctype = cfg->check_mime_type(fname);
2510     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
2511     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
2512     if (m_http_version)
2513         hres->version = odr_strdup(o, m_http_version);
2514     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
2515     if (m_http_keepalive)
2516         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
2517     else
2518         timeout(0);
2519
2520     hres->content_len = sbuf.st_size;
2521     hres->content_buf = (char*) odr_malloc(o, hres->content_len);
2522     FILE *f = fopen(fname, "rb");
2523     if (f)
2524     {
2525         fread(hres->content_buf, 1, hres->content_len, f);
2526         fclose(f);
2527     }
2528     else
2529     {
2530         return 0;
2531     }
2532     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
2533     {
2534         yaz_log (YLOG_LOG, "%sSending file %s to client", m_session_str,
2535                  fname);
2536     }
2537     int len;
2538     send_GDU(gdu, &len);
2539     recv_GDU_more(true);
2540     return 1;
2541 }
2542
2543 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
2544 {
2545     if (m_s2z_odr_init)
2546     {
2547         odr_destroy(m_s2z_odr_init);
2548         m_s2z_odr_init = 0;
2549     }
2550     if (m_s2z_odr_search)
2551     {
2552         odr_destroy(m_s2z_odr_search);
2553         m_s2z_odr_search = 0;
2554     }
2555
2556     m_http_keepalive = 0;
2557     m_http_version = 0;
2558     if (!strcmp(hreq->version, "1.0"))
2559     {
2560         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2561         if (v && !strcmp(v, "Keep-Alive"))
2562             m_http_keepalive = 1;
2563         else
2564             m_http_keepalive = 0;
2565         m_http_version = "1.0";
2566     }
2567     else
2568     {
2569         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2570         if (v && !strcmp(v, "close"))
2571             m_http_keepalive = 0;
2572         else
2573             m_http_keepalive = 1;
2574         m_http_version = "1.1";
2575     }
2576
2577     const char *a = z_HTTP_header_lookup(hreq->headers, "Authorization");
2578     char authorization_str[255];
2579     *authorization_str = '\0';
2580     if (a && strncasecmp(a, "Basic ", 6) == 0)
2581         base64_decode(a + 6, authorization_str, 254);
2582
2583     Z_SRW_PDU *srw_pdu = 0;
2584     Z_SOAP *soap_package = 0;
2585     char *charset = 0;
2586     Z_SRW_diagnostic *diagnostic = 0;
2587     int num_diagnostic = 0;
2588
2589     if (file_access(hreq))
2590     {
2591         return;
2592     }
2593     else if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2594                             &charset) == 0
2595              || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2596                                &charset, &diagnostic, &num_diagnostic) == 0)
2597     {
2598         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
2599         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
2600         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns);
2601         m_s2z_init_apdu = 0;
2602         m_s2z_search_apdu = 0;
2603         m_s2z_present_apdu = 0;
2604
2605         m_s2z_stylesheet = 0;
2606
2607         Z_IdAuthentication *auth = NULL;
2608         if (*authorization_str)
2609         {
2610             auth = (Z_IdAuthentication *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdAuthentication));
2611             auth->which = Z_IdAuthentication_idPass;
2612             auth->u.idPass = (Z_IdPass *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdPass));
2613             auth->u.idPass->groupId = NULL;
2614             char *p = strchr(authorization_str, ':');
2615             if (p)
2616             {
2617                 *p = '\0';
2618                 p++;
2619                 auth->u.idPass->password = odr_strdup(m_s2z_odr_init, p);
2620             }
2621             auth->u.idPass->userId = odr_strdup(m_s2z_odr_init, authorization_str);
2622         }
2623
2624         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
2625         {
2626             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
2627
2628             const char *backend_db = srw_req->database;
2629             srw_get_client(srw_req->database, &backend_db);
2630
2631             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2632             // recordXPath unsupported.
2633             if (srw_req->recordXPath)
2634             {
2635                 yaz_add_srw_diagnostic(odr_decode(),
2636                                        &diagnostic, &num_diagnostic,
2637                                        72, 0);
2638             }
2639             // sort unsupported
2640             if (srw_req->sort_type != Z_SRW_sort_type_none)
2641             {
2642                 yaz_add_srw_diagnostic(odr_decode(),
2643                                        &diagnostic, &num_diagnostic,
2644                                        80, 0);
2645             }
2646             // save stylesheet
2647             if (srw_req->stylesheet)
2648                 m_s2z_stylesheet =
2649                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2650
2651             // set packing for response records ..
2652             if (srw_req->recordPacking &&
2653                 !strcmp(srw_req->recordPacking, "xml"))
2654                 m_s2z_packing = Z_SRW_recordPacking_XML;
2655             else
2656                 m_s2z_packing = Z_SRW_recordPacking_string;
2657
2658             if (num_diagnostic)
2659             {
2660                 Z_SRW_PDU *srw_pdu =
2661                     yaz_srw_get(odr_encode(),
2662                                 Z_SRW_searchRetrieve_response);
2663                 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
2664
2665                 srw_res->diagnostics = diagnostic;
2666                 srw_res->num_diagnostics = num_diagnostic;
2667                 send_srw_response(srw_pdu);
2668                 return;
2669             }
2670
2671             // prepare search PDU
2672             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
2673                                           Z_APDU_searchRequest);
2674             Z_SearchRequest *z_searchRequest =
2675                 m_s2z_search_apdu->u.searchRequest;
2676
2677             z_searchRequest->num_databaseNames = 1;
2678             z_searchRequest->databaseNames = (char**)
2679                 odr_malloc(m_s2z_odr_search, sizeof(char *));
2680             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
2681                                                            backend_db);
2682
2683             // query transformation
2684             Z_Query *query = (Z_Query *)
2685                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
2686             z_searchRequest->query = query;
2687
2688             if (srw_req->query_type == Z_SRW_query_type_cql)
2689             {
2690                 Z_External *ext = (Z_External *)
2691                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
2692                 ext->direct_reference =
2693                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
2694                 ext->indirect_reference = 0;
2695                 ext->descriptor = 0;
2696                 ext->which = Z_External_CQL;
2697                 ext->u.cql = srw_req->query.cql;
2698
2699                 query->which = Z_Query_type_104;
2700                 query->u.type_104 =  ext;
2701             }
2702             else if (srw_req->query_type == Z_SRW_query_type_pqf)
2703             {
2704                 Z_RPNQuery *RPNquery;
2705                 YAZ_PQF_Parser pqf_parser;
2706
2707                 pqf_parser = yaz_pqf_create ();
2708
2709                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
2710                                           srw_req->query.pqf);
2711                 if (!RPNquery)
2712                 {
2713                     const char *pqf_msg;
2714                     size_t off;
2715                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
2716                     int ioff = off;
2717                     yaz_log(YLOG_LOG, "%*s^\n", ioff+4, "");
2718                     yaz_log(YLOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
2719
2720                     send_to_srw_client_error(10, 0);
2721                     return;
2722                 }
2723                 query->which = Z_Query_type_1;
2724                 query->u.type_1 =  RPNquery;
2725
2726                 yaz_pqf_destroy (pqf_parser);
2727             }
2728             else
2729             {
2730                 send_to_srw_client_error(7, "query");
2731                 return;
2732             }
2733
2734             // present
2735             m_s2z_present_apdu = 0;
2736             int max = 0;
2737             if (srw_req->maximumRecords)
2738                 max = *srw_req->maximumRecords;
2739             int start = 1;
2740             if (srw_req->startRecord)
2741                 start = *srw_req->startRecord;
2742             if (max > 0)
2743             {
2744                 // Some backend, such as Voyager doesn't honor piggyback
2745                 // So we use present always (0 &&).
2746                 if (0 && start <= 1)  // Z39.50 piggyback
2747                 {
2748                     *z_searchRequest->smallSetUpperBound = max;
2749                     *z_searchRequest->mediumSetPresentNumber = max;
2750                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
2751
2752                     z_searchRequest->preferredRecordSyntax =
2753                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2754                                                VAL_TEXT_XML);
2755                     if (srw_req->recordSchema)
2756                     {
2757                         z_searchRequest->smallSetElementSetNames =
2758                             z_searchRequest->mediumSetElementSetNames =
2759                             mk_esn_from_schema(m_s2z_odr_search,
2760                                                srw_req->recordSchema);
2761                     }
2762                 }
2763                 else   // Z39.50 present
2764                 {
2765                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
2766                                                    Z_APDU_presentRequest);
2767                     Z_PresentRequest *z_presentRequest =
2768                         m_s2z_present_apdu->u.presentRequest;
2769                     *z_presentRequest->resultSetStartPoint = start;
2770                     *z_presentRequest->numberOfRecordsRequested = max;
2771                     z_presentRequest->preferredRecordSyntax =
2772                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2773                                                VAL_TEXT_XML);
2774                     if (srw_req->recordSchema)
2775                     {
2776                         z_presentRequest->recordComposition =
2777                             (Z_RecordComposition *)
2778                             odr_malloc(m_s2z_odr_search,
2779                                        sizeof(Z_RecordComposition));
2780                         z_presentRequest->recordComposition->which =
2781                             Z_RecordComp_simple;
2782                         z_presentRequest->recordComposition->u.simple =
2783                             mk_esn_from_schema(m_s2z_odr_search,
2784                                                srw_req->recordSchema);
2785                     }
2786                 }
2787             }
2788             if (!m_client)
2789             {
2790                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2791                                             Z_APDU_initRequest);
2792
2793                 m_s2z_init_apdu->u.initRequest->idAuthentication = auth;
2794
2795                 // prevent m_initRequest_apdu memory from being grabbed
2796                 // in Yaz_Proxy::handle_incoming_Z_PDU
2797                 m_initRequest_apdu = m_s2z_init_apdu;
2798                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2799                 return;
2800             }
2801             else
2802             {
2803                 handle_incoming_Z_PDU(m_s2z_search_apdu);
2804                 return;
2805             }
2806         }
2807         else if (srw_pdu->which == Z_SRW_explain_request)
2808         {
2809             Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
2810
2811             const char *backend_db = srw_req->database;
2812             srw_get_client(srw_req->database, &backend_db);
2813
2814             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2815
2816             // save stylesheet
2817             if (srw_req->stylesheet)
2818                 m_s2z_stylesheet =
2819                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2820
2821             if (srw_req->recordPacking &&
2822                 !strcmp(srw_req->recordPacking, "xml"))
2823                 m_s2z_packing = Z_SRW_recordPacking_XML;
2824             else
2825                 m_s2z_packing = Z_SRW_recordPacking_string;
2826
2827             if (num_diagnostic)
2828             {
2829                 send_srw_explain_response(diagnostic, num_diagnostic);
2830                 return;
2831             }
2832
2833             if (!m_client)
2834             {
2835                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2836                                             Z_APDU_initRequest);
2837
2838                 m_s2z_init_apdu->u.initRequest->idAuthentication = auth;
2839                 
2840                 // prevent m_initRequest_apdu memory from being grabbed
2841                 // in Yaz_Proxy::handle_incoming_Z_PDU
2842                 m_initRequest_apdu = m_s2z_init_apdu;
2843                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2844             }
2845             else
2846                 send_srw_explain_response(0, 0);
2847             return;
2848         }
2849         else if (srw_pdu->which == Z_SRW_scan_request)
2850         {
2851             m_s2z_database = odr_strdup(m_s2z_odr_init,
2852                                         srw_pdu->u.scan_request->database);
2853
2854             yaz_add_srw_diagnostic(odr_decode(),
2855                                    &diagnostic, &num_diagnostic,
2856                                    4, "scan");
2857             Z_SRW_PDU *srw_pdu =
2858                 yaz_srw_get(odr_encode(),
2859                             Z_SRW_scan_response);
2860             Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
2861
2862             srw_res->diagnostics = diagnostic;
2863             srw_res->num_diagnostics = num_diagnostic;
2864             send_srw_response(srw_pdu);
2865             return;
2866         }
2867         else
2868         {
2869             m_s2z_database = 0;
2870
2871             send_to_srw_client_error(4, 0);
2872         }
2873     }
2874     send_http_response(400);
2875 }
2876
2877 void Yaz_Proxy::handle_init(Z_APDU *apdu)
2878 {
2879
2880     Z_OtherInformation **oi;
2881     get_otherInfoAPDU(apdu, &oi);
2882
2883     if (apdu->u.initRequest->implementationId)
2884         yaz_log(YLOG_LOG, "%simplementationId: %s",
2885                 m_session_str, apdu->u.initRequest->implementationId);
2886     if (apdu->u.initRequest->implementationName)
2887         yaz_log(YLOG_LOG, "%simplementationName: %s",
2888                 m_session_str, apdu->u.initRequest->implementationName);
2889     if (apdu->u.initRequest->implementationVersion)
2890         yaz_log(YLOG_LOG, "%simplementationVersion: %s",
2891                 m_session_str, apdu->u.initRequest->implementationVersion);
2892     if (m_initRequest_apdu == 0)
2893     {
2894         if (m_initRequest_mem)
2895             nmem_destroy(m_initRequest_mem);
2896
2897         m_initRequest_apdu = apdu;
2898         m_initRequest_mem = odr_extract_mem(odr_decode());
2899
2900         m_initRequest_preferredMessageSize = *apdu->u.initRequest->
2901             preferredMessageSize;
2902         *apdu->u.initRequest->preferredMessageSize = 1024*1024;
2903         m_initRequest_maximumRecordSize = *apdu->u.initRequest->
2904             maximumRecordSize;
2905         *apdu->u.initRequest->maximumRecordSize = 1024*1024;
2906
2907         Z_CharSetandLanguageNegotiation *charSetandLangRecord =
2908             yaz_get_charneg_record(*oi);
2909
2910         // Save proposal charsets and langs.
2911         if (ODR_MASK_GET(apdu->u.initRequest->options,
2912                          Z_Options_negotiationModel)
2913             && charSetandLangRecord)
2914         {
2915
2916             yaz_get_proposal_charneg(m_referenceId_mem,
2917                                      charSetandLangRecord,
2918                                      &m_initRequest_oi_negotiation_charsets,
2919                                      &m_initRequest_oi_negotiation_num_charsets,
2920                                      &m_initRequest_oi_negotiation_langs,
2921                                      &m_initRequest_oi_negotiation_num_langs,
2922                                      &m_initRequest_oi_negotiation_selected);
2923
2924             for (int i = 0; i<m_initRequest_oi_negotiation_num_charsets; i++)
2925             {
2926                 yaz_log(YLOG_LOG, "%scharacters set proposal: %s",
2927                         m_session_str,(m_initRequest_oi_negotiation_charsets[i])?
2928                         m_initRequest_oi_negotiation_charsets[i]:"none");
2929             }
2930             for (int i=0; i<m_initRequest_oi_negotiation_num_langs; i++)
2931             {
2932                 yaz_log(YLOG_LOG, "%slanguages proposal: %s",
2933                         m_session_str, (m_initRequest_oi_negotiation_langs[i])?
2934                         m_initRequest_oi_negotiation_langs[i]:"none");
2935             }
2936             yaz_log(YLOG_LOG, "%sselected proposal: %d (boolean)",
2937                     m_session_str, m_initRequest_oi_negotiation_selected);
2938         }
2939         // save init options for the response..
2940         m_initRequest_options = apdu->u.initRequest->options;
2941
2942         apdu->u.initRequest->options =
2943             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2944                                        sizeof(Odr_bitmask));
2945         ODR_MASK_ZERO(apdu->u.initRequest->options);
2946         int i;
2947         for (i = 0; i<= 24; i++)
2948             ODR_MASK_SET(apdu->u.initRequest->options, i);
2949         // check negotiation option
2950         if (!ODR_MASK_GET(m_initRequest_options,
2951                           Z_Options_negotiationModel))
2952         {
2953             ODR_MASK_CLEAR(apdu->u.initRequest->options,
2954                            Z_Options_negotiationModel);
2955         }
2956         ODR_MASK_CLEAR(apdu->u.initRequest->options,
2957                        Z_Options_concurrentOperations);
2958         // make new version
2959         m_initRequest_version = apdu->u.initRequest->protocolVersion;
2960         apdu->u.initRequest->protocolVersion =
2961             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2962                                        sizeof(Odr_bitmask));
2963         ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
2964
2965         for (i = 0; i<= 8; i++)
2966             ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
2967     }
2968     handle_charset_lang_negotiation(apdu);
2969     if (m_client->m_init_flag)
2970     {
2971         if (handle_init_response_for_invalid_session(apdu))
2972             return;
2973         if (m_client->m_initResponse)
2974         {
2975             Z_APDU *apdu2 = m_client->m_initResponse;
2976             apdu2->u.initResponse->otherInfo = 0;
2977             if (m_client->m_cookie && *m_client->m_cookie)
2978                 set_otherInformationString(apdu2, VAL_COOKIE, 1,
2979                                            m_client->m_cookie);
2980             apdu2->u.initResponse->referenceId =
2981                 apdu->u.initRequest->referenceId;
2982             apdu2->u.initResponse->options = m_client->m_initResponse_options;
2983             apdu2->u.initResponse->protocolVersion =
2984                 m_client->m_initResponse_version;
2985
2986             handle_charset_lang_negotiation(apdu2);
2987
2988             send_to_client(apdu2);
2989             m_timeout_mode = timeout_normal;
2990             return;
2991         }
2992     }
2993     m_client->m_init_flag = 1;
2994
2995 #if USE_AUTH_MSG
2996     Auth_Msg *m = new Auth_Msg;
2997     m->m_proxy = this;
2998     z_APDU(odr_encode(), &apdu, 0, "encode");
2999     char *apdu_buf = odr_getbuf(odr_encode(), &m->m_apdu_len, 0);
3000     m->m_apdu_buf = (char*) nmem_malloc(m->m_nmem, m->m_apdu_len);
3001     memcpy(m->m_apdu_buf, apdu_buf, m->m_apdu_len);
3002     odr_reset(odr_encode());
3003
3004     inc_ref();
3005     m_my_thread->put(m);
3006 #else
3007     int ret = handle_authentication(apdu);
3008     result_authentication(apdu, ret);
3009 #endif
3010 }
3011
3012 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
3013 {
3014     Z_ReferenceId **refid = get_referenceIdP(apdu);
3015     nmem_reset(m_referenceId_mem);
3016     if (refid && *refid)
3017     {
3018         m_referenceId = (Z_ReferenceId *)
3019             nmem_malloc(m_referenceId_mem, sizeof(*m_referenceId));
3020         m_referenceId->len = m_referenceId->size = (*refid)->len;
3021         m_referenceId->buf = (unsigned char *)
3022             nmem_malloc(m_referenceId_mem, (*refid)->len);
3023         memcpy(m_referenceId->buf, (*refid)->buf, (*refid)->len);
3024     }
3025     else
3026         m_referenceId = 0;
3027
3028     if (!m_client && m_flag_invalid_session)
3029     {
3030         // Got request for a session that is invalid..
3031         m_apdu_invalid_session = apdu; // save package
3032         m_mem_invalid_session = odr_extract_mem(odr_decode());
3033         apdu = m_initRequest_apdu;     // but throw an init to the target
3034     }
3035
3036     // Determine our client.
3037     Z_OtherInformation **oi;
3038     get_otherInfoAPDU(apdu, &oi);
3039     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi));
3040     if (!m_client)
3041     {
3042         if (m_http_version)
3043         {   // HTTP. Send not found
3044             send_http_response(404);
3045             return;
3046         }
3047         else
3048         {
3049             // Z39.50 just shutdown
3050             delete this;
3051             return;
3052         }
3053     }
3054
3055     m_client->m_server = this;
3056
3057     if (apdu->which == Z_APDU_initRequest)
3058         handle_init(apdu);
3059     else
3060         handle_incoming_Z_PDU_2(apdu);
3061 }
3062
3063 void Yaz_Proxy::handle_incoming_Z_PDU_2(Z_APDU *apdu)
3064 {
3065     handle_max_record_retrieve(apdu);
3066
3067     if (apdu)
3068         apdu = handle_syntax_validation(apdu);
3069
3070     if (apdu)
3071         apdu = handle_query_transformation(apdu);
3072
3073     if (apdu)
3074         apdu = handle_target_charset_conversion(apdu);
3075
3076     if (apdu)
3077         apdu = handle_query_validation(apdu);
3078
3079     if (apdu)
3080         apdu = result_set_optimize(apdu);
3081
3082     if (!apdu)
3083     {
3084         m_client->timeout(m_target_idletime);  // mark it active even
3085         recv_GDU_more(true);
3086         // though we didn't use it
3087         return;
3088     }
3089
3090     // delete other info construct completely if 0 elements
3091     Z_OtherInformation **oi;
3092     get_otherInfoAPDU(apdu, &oi);
3093     if (oi && *oi && (*oi)->num_elements == 0)
3094         *oi = 0;
3095
3096     if (apdu->which == Z_APDU_presentRequest &&
3097         m_client->m_resultSetStartPoint == 0)
3098     {
3099         Z_PresentRequest *pr = apdu->u.presentRequest;
3100         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
3101         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
3102     } else {
3103         m_client->m_resultSetStartPoint = 0;
3104     }
3105     if (m_client->send_to_target(apdu) < 0)
3106     {
3107         m_client->shutdown();
3108     }
3109     else
3110         m_client->m_waiting = 1;
3111 }
3112
3113 void Yaz_Proxy::connectNotify()
3114 {
3115 }
3116
3117 void Yaz_Proxy::releaseClient()
3118 {
3119     xfree(m_proxyTarget);
3120     m_proxyTarget = 0;
3121     m_flag_invalid_session = 0;
3122     // only keep if keep_alive flag is set...
3123     if (m_client &&
3124         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
3125         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
3126         m_client->m_waiting == 0)
3127     {
3128         yaz_log(YLOG_LOG, "%sShutdown (client to proxy) keepalive %s",
3129                  m_session_str,
3130                  m_client->get_hostname());
3131         yaz_log(YLOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
3132                 m_session_str, m_client->m_pdu_recv,
3133                 m_client->m_bytes_sent + m_client->m_bytes_recv,
3134                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
3135         assert (m_client->m_waiting != 2);
3136         // Tell client (if any) that no server connection is there..
3137         m_client->m_server = 0;
3138         m_client = 0;
3139     }
3140     else if (m_client)
3141     {
3142         yaz_log (YLOG_LOG, "%sShutdown (client to proxy) close %s",
3143                  m_session_str,
3144                  m_client->get_hostname());
3145         assert (m_client->m_waiting != 2);
3146         delete m_client;
3147         m_client = 0;
3148     }
3149     else if (!m_parent)
3150     {
3151         yaz_log (YLOG_LOG, "%sshutdown (client to proxy) bad state",
3152                  m_session_str);
3153         assert (m_parent);
3154     }
3155     else
3156     {
3157         yaz_log (YLOG_LOG, "%sShutdown (client to proxy)",
3158                  m_session_str);
3159     }
3160     if (m_parent)
3161         m_parent->pre_init();
3162 }
3163
3164 bool Yaz_Proxy::dec_ref(bool main_ptr)
3165 {
3166     assert(m_ref_count > 0);
3167     if (main_ptr)
3168     {
3169         if (m_main_ptr_dec)
3170             return false;
3171         m_main_ptr_dec = true;
3172     }
3173
3174     m_http_keepalive = 0;
3175
3176     --m_ref_count;
3177     if (m_ref_count > 0)
3178         return false;
3179
3180     releaseClient();
3181
3182     delete this;
3183     return true;
3184 }
3185
3186 const char *Yaz_ProxyClient::get_session_str()
3187 {
3188     if (!m_server)
3189         return "0 ";
3190     return m_server->get_session_str();
3191 }
3192
3193 void Yaz_ProxyClient::shutdown()
3194 {
3195     yaz_log (YLOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
3196              get_hostname());
3197
3198     if (m_server)
3199     {
3200         m_waiting = 1;      // ensure it's released from Proxy in releaseClient
3201         m_server->dec_ref(true);
3202     }
3203     else
3204         delete this;
3205 }
3206
3207 void Yaz_Proxy::failNotify()
3208 {
3209     inc_request_no();
3210     yaz_log (YLOG_LOG, "%sConnection closed by client",
3211              get_session_str());
3212     dec_ref(true);
3213 }
3214
3215 void Yaz_Proxy::send_response_fail_client(const char *addr)
3216 {
3217     if (m_http_version)
3218     {
3219         Z_SRW_diagnostic *diagnostic = 0;
3220         int num_diagnostic = 0;
3221         
3222         yaz_add_srw_diagnostic(odr_encode(),
3223                                &diagnostic, &num_diagnostic,
3224                                YAZ_SRW_SYSTEM_TEMPORARILY_UNAVAILABLE, addr);
3225         if (m_s2z_search_apdu)
3226             send_srw_search_response(diagnostic, num_diagnostic);
3227         else
3228             send_srw_explain_response(diagnostic, num_diagnostic);
3229     }            
3230 }
3231 void Yaz_ProxyClient::failNotify()
3232 {
3233     if (m_server)
3234         m_server->inc_request_no();
3235     yaz_log (YLOG_LOG, "%sConnection closed by target %s",
3236              get_session_str(), get_hostname());
3237
3238     if (m_server)
3239         m_server->send_response_fail_client(get_hostname());
3240     shutdown();
3241 }
3242
3243 void Yaz_ProxyClient::connectNotify()
3244 {
3245     const char *s = get_session_str();
3246     const char *h = get_hostname();
3247     yaz_log (YLOG_LOG, "%sConnection accepted by %s timeout=%d", s, h,
3248              m_target_idletime);
3249     timeout(m_target_idletime);
3250     if (!m_server)
3251         pre_init_client();
3252 }
3253
3254 IPDU_Observer *Yaz_ProxyClient::sessionNotify(IPDU_Observable
3255                                               *the_PDU_Observable, int fd)
3256 {
3257     return new Yaz_ProxyClient(the_PDU_Observable, 0);
3258 }
3259
3260 Yaz_ProxyClient::~Yaz_ProxyClient()
3261 {
3262     if (m_prev)
3263         *m_prev = m_next;
3264     if (m_next)
3265         m_next->m_prev = m_prev;
3266     m_waiting = 2;     // for debugging purposes only.
3267     odr_destroy(m_init_odr);
3268     odr_destroy(m_idAuthentication_odr);
3269     delete m_last_query;
3270     xfree (m_last_resultSetId);
3271     xfree (m_cookie);
3272 }
3273
3274 void Yaz_ProxyClient::pre_init_client()
3275 {
3276     Z_APDU *apdu = create_Z_PDU(Z_APDU_initRequest);
3277     Z_InitRequest *req = apdu->u.initRequest;
3278
3279     int i;
3280     for (i = 0; i<= 24; i++)
3281         ODR_MASK_SET(req->options, i);
3282     ODR_MASK_CLEAR(apdu->u.initRequest->options,
3283                    Z_Options_negotiationModel);
3284     ODR_MASK_CLEAR(apdu->u.initRequest->options,
3285                    Z_Options_concurrentOperations);
3286     for (i = 0; i<= 10; i++)
3287         ODR_MASK_SET(req->protocolVersion, i);
3288
3289     if (send_to_target(apdu) < 0)
3290     {
3291         delete this;
3292     }
3293     else
3294     {
3295         m_waiting = 1;
3296         m_init_flag = 1;
3297     }
3298 }
3299
3300 void Yaz_Proxy::pre_init()
3301 {
3302     int i;
3303     const char *name = 0;
3304     const char *zurl_in_use[MAX_ZURL_PLEX];
3305     int limit_bw, limit_pdu, limit_req, limit_search;
3306     int target_idletime, client_idletime;
3307     int max_clients;
3308     int keepalive_limit_bw, keepalive_limit_pdu;
3309     int pre_init;
3310     const char *cql2rpn = 0;
3311     const char *authentication = 0;
3312     const char *negotiation_charset = 0;
3313     const char *negotiation_lang = 0;
3314
3315     Yaz_ProxyConfig *cfg = check_reconfigure();
3316
3317     zurl_in_use[0] = 0;
3318
3319     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
3320         set_APDU_yazlog(1);
3321     else
3322         set_APDU_yazlog(0);
3323
3324     for (i = 0; cfg && cfg->get_target_no(i, &name, zurl_in_use,
3325                                           &limit_bw, &limit_pdu, &limit_req,
3326                                           &limit_search,
3327                                           &target_idletime, &client_idletime,
3328                                           &max_clients,
3329                                           &keepalive_limit_bw,
3330                                           &keepalive_limit_pdu,
3331                                           &pre_init,
3332                                           &cql2rpn,
3333                                           &authentication,
3334                                           &negotiation_charset,
3335                                           &negotiation_lang,
3336                                           0,
3337                                           0) ; i++)
3338     {
3339         if (pre_init)
3340         {
3341             int j;
3342             for (j = 0; zurl_in_use[j]; j++)
3343             {
3344                 Yaz_ProxyClient *c;
3345                 int spare = 0;
3346                 int spare_waiting = 0;
3347                 int in_use = 0;
3348                 int other = 0;
3349                 for (c = m_clientPool; c; c = c->m_next)
3350                 {
3351                     if (!strcmp(zurl_in_use[j], c->get_hostname()))
3352                     {
3353                         if (c->m_cookie == 0)
3354                         {
3355                             if (c->m_server == 0)
3356                                 if (c->m_waiting)
3357                        &