New directive default-client-charset which serves as charset
[yazproxy-moved-to-github.git] / src / yaz-proxy.cpp
1 /* $Id: yaz-proxy.cpp,v 1.44 2006-03-25 10:59:14 adam Exp $
2    Copyright (c) 1998-2005, Index Data.
3
4 This file is part of the yaz-proxy.
5
6 YAZ proxy is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 2, or (at your option) any later
9 version.
10
11 YAZ proxy is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
14 for more details.
15
16 You should have received a copy of the GNU General Public License
17 along with YAZ proxy; see the file LICENSE.  If not, write to the
18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
19 02111-1307, USA.
20  */
21
22 #ifdef WIN32
23 #define HAVE_SYS_STAT_H 1
24 #define HAVE_SYS_TYPES_H 1
25 #endif
26
27 #if HAVE_UNISTD_H
28 #include <unistd.h>
29 #endif
30 #if HAVE_SYS_TIME_H
31 #include <sys/time.h>
32 #endif
33 #if HAVE_SYS_TYPES_H
34 #include <sys/types.h>
35 #endif
36 #if HAVE_SYS_STAT_H
37 #include <sys/stat.h>
38 #endif
39
40 #include <assert.h>
41 #include <stdlib.h>
42 #include <time.h>
43 #include <fcntl.h>
44
45 #include <yaz/srw.h>
46 #include <yaz/marcdisp.h>
47 #include <yaz/yaz-iconv.h>
48 #include <yaz/log.h>
49 #include <yaz/diagbib1.h>
50 #include "proxyp.h"
51 #include <yaz/pquery.h>
52 #include <yaz/otherinfo.h>
53 #include <yaz/charneg.h>
54 #include "msg-thread.h"
55
56 using namespace yazpp_1;
57
58 #ifdef WIN32
59 #define strncasecmp _strnicmp
60 #endif
61
62 #define USE_AUTH_MSG 1
63
64 #if USE_AUTH_MSG
65 class Auth_Msg : public IMsg_Thread {
66 public:
67     int m_ret;
68     IMsg_Thread *handle();
69     void result();
70     Yaz_Proxy *m_proxy;
71     NMEM m_nmem;
72     char *m_apdu_buf;
73     int m_apdu_len;
74     Auth_Msg();
75     virtual ~Auth_Msg();
76 };
77
78 Auth_Msg::Auth_Msg()
79 {
80     m_nmem = nmem_create();
81 }
82
83 Auth_Msg::~Auth_Msg()
84 {
85     nmem_destroy(m_nmem);
86 }
87
88 IMsg_Thread *Auth_Msg::handle()
89 {
90     yaz_log(YLOG_LOG, "Auth_Msg:handle begin");
91     ODR decode = odr_createmem(ODR_DECODE);
92     Z_APDU *apdu;
93
94     odr_setbuf(decode, m_apdu_buf, m_apdu_len, 0);
95     int r = z_APDU(decode, &apdu, 0, 0);
96     if (!r)
97     {
98         yaz_log(YLOG_WARN, "decode failed in Auth_Msg::handle");
99     }
100     else
101     {
102         m_ret = m_proxy->handle_authentication(apdu);
103     }
104     yaz_log(YLOG_LOG, "Auth_Msg:handle end");
105     odr_destroy(decode);
106     return this;
107 }
108
109 void Auth_Msg::result()
110 {
111     yaz_log(YLOG_LOG, "Auth_Msg:result proxy ok buf=%p len=%d",
112             m_apdu_buf, m_apdu_len);
113     if (m_proxy->dec_ref(false))
114         yaz_log(YLOG_LOG, "Auth_Msg::proxy deleted meanwhile");
115     else
116     {
117         yaz_log(YLOG_LOG, "Auth_Msg::proxy still alive");
118         odr_setbuf(m_proxy->odr_decode(), m_apdu_buf, m_apdu_len, 0);
119         Z_APDU *apdu = 0;
120         int r = z_APDU(m_proxy->odr_decode(), &apdu, 0, 0);
121         if (r)
122             yaz_log(YLOG_LOG, "Auth_Msg::result z_APDU OK");
123         else
124             yaz_log(YLOG_LOG, "Auth_Msg::result z_APDU failed");
125         m_proxy->result_authentication(apdu, m_ret);
126     }
127     delete this;
128 }
129
130 #endif
131
132 void Yaz_Proxy::result_authentication(Z_APDU *apdu, int ret)
133 {
134     if (apdu == 0 || ret == 0)
135     {
136         Z_APDU *apdu_reject = zget_APDU(odr_encode(), Z_APDU_initResponse);
137         *apdu_reject->u.initResponse->result = 0;
138         send_to_client(apdu_reject);
139         dec_ref(false);
140     }
141     else
142     {
143         if (apdu->which == Z_APDU_initRequest)
144         {
145             Yaz_ProxyConfig *cfg = check_reconfigure();
146             if (cfg)
147                 cfg->target_authentication(m_default_target, odr_encode(), 
148                                            apdu->u.initRequest);
149         }
150         handle_incoming_Z_PDU_2(apdu);
151     }
152 }
153
154 static const char *apdu_name(Z_APDU *apdu)
155 {
156     switch (apdu->which)
157     {
158     case Z_APDU_initRequest:
159         return "initRequest";
160     case Z_APDU_initResponse:
161         return "initResponse";
162     case Z_APDU_searchRequest:
163         return "searchRequest";
164     case Z_APDU_searchResponse:
165         return "searchResponse";
166     case Z_APDU_presentRequest:
167         return "presentRequest";
168     case Z_APDU_presentResponse:
169         return "presentResponse";
170     case Z_APDU_deleteResultSetRequest:
171         return "deleteResultSetRequest";
172     case Z_APDU_deleteResultSetResponse:
173         return "deleteResultSetResponse";
174     case Z_APDU_scanRequest:
175         return "scanRequest";
176     case Z_APDU_scanResponse:
177         return "scanResponse";
178     case Z_APDU_sortRequest:
179         return "sortRequest";
180     case Z_APDU_sortResponse:
181         return "sortResponse";
182     case Z_APDU_extendedServicesRequest:
183         return "extendedServicesRequest";
184     case Z_APDU_extendedServicesResponse:
185         return "extendedServicesResponse";
186     case Z_APDU_close:
187         return "close";
188     }
189     return "other";
190 }
191
192 static const char *gdu_name(Z_GDU *gdu)
193 {
194     switch(gdu->which)
195     {
196     case Z_GDU_Z3950:
197         return apdu_name(gdu->u.z3950);
198     case Z_GDU_HTTP_Request:
199         return "HTTP Request";
200     case Z_GDU_HTTP_Response:
201         return "HTTP Response";
202     }
203     return "Unknown request/response";
204 }
205
206 Yaz_Proxy::Yaz_Proxy(IPDU_Observable *the_PDU_Observable,
207                      ISocketObservable *the_socket_observable,
208                      Yaz_Proxy *parent)
209     :
210     Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60)
211 {
212     m_PDU_Observable = the_PDU_Observable;
213     m_socket_observable = the_socket_observable;
214     m_client = 0;
215     m_parent = parent;
216     m_clientPool = 0;
217     m_seqno = 1;
218     m_keepalive_limit_bw = 500000;
219     m_keepalive_limit_pdu = 1000;
220     m_proxyTarget = 0;
221     m_default_target = 0;
222     m_proxy_negotiation_charset = 0;
223     m_proxy_negotiation_lang = 0;
224     m_proxy_negotiation_default_charset = 0;
225     m_charset_converter = new Yaz_CharsetConverter;
226     m_max_clients = 150;
227     m_log_mask = 0;
228     m_seed = time(0);
229     m_client_idletime = 600;
230     m_target_idletime = 600;
231     m_optimize = xstrdup ("1");
232     strcpy(m_session_str, "0 ");
233     m_session_no = 0;
234     m_bytes_sent = 0;
235     m_bytes_recv = 0;
236     m_bw_max = 0;
237     m_pdu_max = 0;
238     m_search_max = 0;
239     m_connect_max = 0;
240     m_timeout_mode = timeout_normal;
241     m_timeout_gdu = 0;
242     m_max_record_retrieve = 0;
243     m_reconfig_flag = 0;
244     m_config_fname = 0;
245     m_request_no = 0;
246     m_flag_invalid_session = 0;
247     m_referenceId = 0;
248     m_referenceId_mem = nmem_create();
249     m_config = 0;
250     m_marcxml_mode = none;
251     m_stylesheet_xsp = 0;
252     m_stylesheet_nprl = 0;
253     m_stylesheet_apdu = 0;
254     m_s2z_stylesheet = 0;
255     m_s2z_database = 0;
256     m_schema = 0;
257     m_backend_type = 0;
258     m_backend_charset = 0;
259     m_frontend_type = 0;
260     m_initRequest_apdu = 0;
261     m_initRequest_mem = 0;
262     m_initRequest_preferredMessageSize = 0;
263     m_initRequest_maximumRecordSize = 0;
264     m_initRequest_options = 0;
265     m_initRequest_version = 0;
266     m_initRequest_oi_negotiation_charsets = 0;
267     m_initRequest_oi_negotiation_num_charsets = 0;
268     m_initRequest_oi_negotiation_langs = 0;
269     m_initRequest_oi_negotiation_num_langs = 0;
270     m_initRequest_oi_negotiation_selected = 0;
271     m_apdu_invalid_session = 0;
272     m_mem_invalid_session = 0;
273     m_s2z_odr_init = 0;
274     m_s2z_odr_search = 0;
275     m_s2z_init_apdu = 0;
276     m_s2z_search_apdu = 0;
277     m_s2z_present_apdu = 0;
278     m_http_keepalive = 0;
279     m_http_version = 0;
280     m_soap_ns = 0;
281     m_s2z_packing = Z_SRW_recordPacking_string;
282 #if HAVE_GETTIMEOFDAY
283     m_time_tv = xmalloc(sizeof(struct timeval));
284     struct timeval *tv = (struct timeval *) m_time_tv;
285     tv->tv_sec = 0;
286     tv->tv_usec = 0;
287 #else
288     m_time_tv = 0;
289 #endif
290     m_usemarcon_ini_stage1 = 0;
291     m_usemarcon_ini_stage2 = 0;
292     m_usemarcon = new Yaz_usemarcon();
293     if (!m_parent)
294         low_socket_open();
295     m_my_thread = 0;
296     m_ref_count = 1;
297     m_main_ptr_dec = false;
298     m_peername = 0;
299 }
300
301 void Yaz_Proxy::inc_ref()
302 {
303     m_ref_count++;
304 }
305
306 Yaz_Proxy::~Yaz_Proxy()
307 {
308     yaz_log(YLOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
309             m_bytes_sent, m_bytes_recv);
310     nmem_destroy(m_initRequest_mem);
311     nmem_destroy(m_mem_invalid_session);
312     nmem_destroy(m_referenceId_mem);
313
314     xfree(m_proxyTarget);
315     xfree(m_default_target);
316     xfree(m_proxy_negotiation_charset);
317     xfree(m_proxy_negotiation_lang);
318     xfree(m_proxy_negotiation_default_charset);
319     delete m_charset_converter;
320     xfree(m_optimize);
321
322 #if HAVE_XSLT
323     if (m_stylesheet_xsp)
324         xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
325 #endif
326     xfree (m_time_tv);
327
328     xfree (m_peername);
329     xfree (m_schema);
330     xfree (m_backend_type);
331     xfree (m_backend_charset);
332     xfree (m_usemarcon_ini_stage1);
333     xfree (m_usemarcon_ini_stage2);
334     delete m_usemarcon;
335     if (m_s2z_odr_init)
336         odr_destroy(m_s2z_odr_init);
337     if (m_s2z_odr_search)
338         odr_destroy(m_s2z_odr_search);
339     if (!m_parent)
340         low_socket_close();
341     if (!m_parent)
342         delete m_my_thread;
343     delete m_config;
344 }
345
346 void Yaz_Proxy::set_debug_mode(int mode)
347 {
348     m_debug_mode = mode;
349 }
350
351 int Yaz_Proxy::set_config(const char *config)
352 {
353     delete m_config;
354     m_config = new Yaz_ProxyConfig();
355     xfree(m_config_fname);
356     m_config_fname = xstrdup(config);
357     int r = m_config->read_xml(config);
358     if (!r)
359         m_config->get_generic_info(&m_log_mask, &m_max_clients);
360     return r;
361 }
362
363 void Yaz_Proxy::set_default_target(const char *target)
364 {
365     xfree (m_default_target);
366     m_default_target = 0;
367     if (target)
368         m_default_target = (char *) xstrdup (target);
369 }
370
371 void Yaz_Proxy::set_proxy_negotiation (const char *charset, const char *lang,
372                                        const char *default_charset)
373 {
374     yaz_log(YLOG_LOG, "%sSet the proxy negotiation: charset to '%s', "
375         "default charset to '%s', language to '%s'", m_session_str, 
376         charset?charset:"none",
377         default_charset?default_charset:"none",
378         lang?lang:"none");
379     xfree (m_proxy_negotiation_charset);
380     xfree (m_proxy_negotiation_lang);
381     m_proxy_negotiation_charset = m_proxy_negotiation_lang = 0;
382     if (charset)
383         m_proxy_negotiation_charset = (char *) xstrdup (charset);
384     if (lang)
385         m_proxy_negotiation_lang = (char *) xstrdup (lang);
386     if (default_charset)
387         m_proxy_negotiation_default_charset =
388             (char *) xstrdup (default_charset);
389 }
390
391 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
392 {
393     if (m_parent)
394         return m_parent->check_reconfigure();
395
396     Yaz_ProxyConfig *cfg = m_config;
397     if (m_reconfig_flag)
398     {
399         yaz_log(YLOG_LOG, "reconfigure");
400         yaz_log_reopen();
401         if (m_config_fname && cfg)
402         {
403             yaz_log(YLOG_LOG, "reconfigure config %s", m_config_fname);
404             int r = cfg->read_xml(m_config_fname);
405             if (r)
406                 yaz_log(YLOG_WARN, "reconfigure failed");
407             else
408             {
409                 m_log_mask = 0;
410                 cfg->get_generic_info(&m_log_mask, &m_max_clients);
411             }
412         }
413         else
414             yaz_log(YLOG_LOG, "reconfigure");
415         m_reconfig_flag = 0;
416     }
417     return cfg;
418 }
419
420 IPDU_Observer *Yaz_Proxy::sessionNotify(IPDU_Observable
421                                         *the_PDU_Observable, int fd)
422 {
423     check_reconfigure();
424     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable,
425                                          m_socket_observable, this);
426     new_proxy->m_config = 0;
427     new_proxy->m_config_fname = 0;
428     new_proxy->timeout(m_client_idletime);
429     new_proxy->m_target_idletime = m_target_idletime;
430     new_proxy->set_default_target(m_default_target);
431     new_proxy->m_max_clients = m_max_clients;
432     new_proxy->m_log_mask = m_log_mask;
433     new_proxy->set_APDU_log(get_APDU_log());
434     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
435         new_proxy->set_APDU_yazlog(1);
436     else
437         new_proxy->set_APDU_yazlog(0);
438     sprintf(new_proxy->m_session_str, "%ld:%d ", (long) time(0), m_session_no);
439     m_session_no++;
440     new_proxy->m_peername = xstrdup(the_PDU_Observable->getpeername());
441     yaz_log (YLOG_LOG, "%sNew session %s", new_proxy->m_session_str,
442              new_proxy->m_peername);
443     new_proxy->set_proxy_negotiation(m_proxy_negotiation_charset,
444         m_proxy_negotiation_lang, m_proxy_negotiation_default_charset);
445     // create thread object the first time we get an incoming connection
446     if (!m_my_thread)
447         m_my_thread = new Msg_Thread(m_socket_observable, 1);
448     new_proxy->m_my_thread = m_my_thread;
449     return new_proxy;
450 }
451
452 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
453 {
454     int oid[OID_SIZE];
455     Z_OtherInformationUnit *oi;
456     struct oident ent;
457     ent.proto = PROTO_Z3950;
458     ent.oclass = CLASS_USERINFO;
459     ent.value = (oid_value) VAL_COOKIE;
460     assert (oid_ent_to_oid (&ent, oid));
461
462     if (oid_ent_to_oid (&ent, oid) &&
463         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
464         oi->which == Z_OtherInfo_characterInfo)
465         return oi->information.characterInfo;
466     return 0;
467 }
468 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
469 {
470     int oid[OID_SIZE];
471     Z_OtherInformationUnit *oi;
472     struct oident ent;
473     ent.proto = PROTO_Z3950;
474     ent.oclass = CLASS_USERINFO;
475     ent.value = (oid_value) VAL_PROXY;
476     if (oid_ent_to_oid (&ent, oid) &&
477         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
478         oi->which == Z_OtherInfo_characterInfo)
479         return oi->information.characterInfo;
480     return 0;
481 }
482 const char *Yaz_Proxy::load_balance(const char **url)
483 {
484     int zurl_in_use[MAX_ZURL_PLEX];
485     int zurl_in_spare[MAX_ZURL_PLEX];
486     Yaz_ProxyClient *c;
487     int i;
488
489     for (i = 0; i<MAX_ZURL_PLEX; i++)
490     {
491         zurl_in_use[i] = 0;
492         zurl_in_spare[i] = 0;
493     }
494     for (c = m_parent->m_clientPool; c; c = c->m_next)
495     {
496         for (i = 0; url[i]; i++)
497             if (!strcmp(url[i], c->get_hostname()))
498             {
499                 zurl_in_use[i]++;
500                 if (c->m_cookie == 0 && c->m_server == 0 && c->m_waiting == 0)
501                     zurl_in_spare[i]++;
502             }
503     }
504     int min_use = 100000;
505     int spare_for_min = 0;
506     int max_spare = 0;
507     const char *ret_min = 0;
508     const char *ret_spare = 0;
509     for (i = 0; url[i]; i++)
510     {
511         yaz_log(YLOG_DEBUG, "%szurl=%s use=%d spare=%d",
512                 m_session_str, url[i], zurl_in_use[i], zurl_in_spare[i]);
513         if (min_use > zurl_in_use[i])
514         {
515             ret_min = url[i];
516             min_use = zurl_in_use[i];
517             spare_for_min = zurl_in_spare[i];
518         }
519         if (max_spare < zurl_in_spare[i])
520         {
521             ret_spare = url[i];
522             max_spare = zurl_in_spare[i];
523         }
524     }
525     return ret_min;
526 }
527
528 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
529                                        const char *proxy_host)
530 {
531     assert (m_parent);
532     Yaz_Proxy *parent = m_parent;
533     Yaz_ProxyClient *c = m_client;
534
535     if (!m_proxyTarget)
536     {
537         const char *url[MAX_ZURL_PLEX];
538         Yaz_ProxyConfig *cfg = check_reconfigure();
539         if (proxy_host)
540         {
541             if (parent && parent->m_debug_mode)
542             {
543                 // only to be enabled for debugging...
544                 if (!strcmp(proxy_host, "stop"))
545                     exit(0);
546             }
547             xfree(m_default_target);
548             m_default_target = xstrdup(proxy_host);
549         }
550         proxy_host = m_default_target;
551         int client_idletime = -1;
552         const char *cql2rpn_fname = 0;
553         const char *negotiation_charset = 0;
554         const char *negotiation_lang = 0;
555         const char *query_charset = 0;
556         const char *default_client_query_charset = 0;
557         url[0] = m_default_target;
558         url[1] = 0;
559         if (cfg)
560         {
561             int pre_init = 0;
562             cfg->get_target_info(proxy_host, url, &m_bw_max,
563                                  &m_pdu_max, &m_max_record_retrieve,
564                                  &m_search_max, &m_connect_max,
565                                  &m_target_idletime, &client_idletime,
566                                  &parent->m_max_clients,
567                                  &m_keepalive_limit_bw,
568                                  &m_keepalive_limit_pdu,
569                                  &pre_init,
570                                  &cql2rpn_fname,
571                                  &negotiation_charset,
572                                  &negotiation_lang,
573                                  &query_charset,
574                                  &default_client_query_charset);
575         }
576         if (client_idletime != -1)
577         {
578             m_client_idletime = client_idletime;
579             timeout(m_client_idletime);
580         }
581         if (cql2rpn_fname)
582             m_cql2rpn.set_pqf_file(cql2rpn_fname);
583         if (negotiation_charset || negotiation_lang || default_client_query_charset)
584         {
585             set_proxy_negotiation(negotiation_charset,
586                 negotiation_lang, default_client_query_charset);
587         }
588         m_charset_converter->set_target_query_charset(query_charset);
589         if (!url[0])
590         {
591             yaz_log(YLOG_LOG, "%sNo default target", m_session_str);
592             return 0;
593         }
594         // we don't handle multiplexing for cookie session, so we just
595         // pick the first one in this case (anonymous users will be able
596         // to use any backend)
597         if (cookie && *cookie)
598             m_proxyTarget = (char*) xstrdup(url[0]);
599         else
600             m_proxyTarget = (char*) xstrdup(load_balance(url));
601     }
602     if (cookie && *cookie)
603     {   // search in sessions with a cookie
604         for (c = parent->m_clientPool; c; c = c->m_next)
605         {
606             assert (c->m_prev);
607             assert (*c->m_prev == c);
608             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
609                 !strcmp(m_proxyTarget, c->get_hostname()))
610             {
611                 // Found it in cache
612                 // The following handles "cancel"
613                 // If connection is busy (waiting for PDU) and
614                 // we have an initRequest we can safely do re-open
615                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
616                 {
617                     yaz_log (YLOG_LOG, "%s REOPEN target=%s", m_session_str,
618                              c->get_hostname());
619                     c->close();
620                     c->m_init_flag = 0;
621
622                     c->m_last_ok = 0;
623                     c->m_cache.clear();
624                     c->m_last_resultCount = 0;
625                     c->m_sr_transform = 0;
626                     c->m_waiting = 0;
627                     c->m_resultSetStartPoint = 0;
628                     c->m_target_idletime = m_target_idletime;
629                     if (c->client(m_proxyTarget))
630                     {
631                         delete c;
632                         return 0;
633                     }
634                     c->timeout(30);
635                 }
636                 c->m_seqno = parent->m_seqno;
637                 if (c->m_server && c->m_server != this)
638                     c->m_server->m_client = 0;
639                 c->m_server = this;
640                 (parent->m_seqno)++;
641                 yaz_log (YLOG_DEBUG, "get_client 1 %p %p", this, c);
642                 return c;
643             }
644         }
645     }
646     else if (!c && apdu->which == Z_APDU_initRequest )
647     {
648         // anonymous sessions without cookie.
649         // if authentication is set it is NOT anonymous se we can't share them.
650         // If charset and lang negotiation is use it is NOT anonymous session too.
651         for (c = parent->m_clientPool; c; c = c->m_next)
652         {
653             assert(c->m_prev);
654             assert(*c->m_prev == c);
655             if (c->m_server == 0 && c->m_cookie == 0 &&  c->m_waiting == 0 
656                 && c->compare_idAuthentication(apdu)
657                 && c->compare_charset(apdu)
658                 && !strcmp(m_proxyTarget, c->get_hostname()))
659             {
660                 // found it in cache
661                 yaz_log (YLOG_LOG, "%sREUSE %d %s",
662                          m_session_str, parent->m_seqno, c->get_hostname());
663                 
664                 c->m_seqno = parent->m_seqno;
665                 assert(c->m_server == 0);
666                 c->m_server = this;
667
668                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
669                     c->set_APDU_yazlog(1);
670                 else
671                     c->set_APDU_yazlog(0);
672
673                 (parent->m_seqno)++;
674
675                 parent->pre_init();
676
677                 return c;
678             }
679         }
680     }
681     if (!m_client)
682     {
683         if (apdu->which != Z_APDU_initRequest)
684         {
685             yaz_log (YLOG_LOG, "%sno init request as first PDU", m_session_str);
686             return 0;
687         }
688         // go through list of clients - and find the lowest/oldest one.
689         Yaz_ProxyClient *c_min = 0;
690         int min_seq = -1;
691         int no_of_clients = 0;
692         if (parent->m_clientPool)
693             yaz_log (YLOG_DEBUG, "Existing sessions");
694         for (c = parent->m_clientPool; c; c = c->m_next)
695         {
696             yaz_log (YLOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
697                                c->m_waiting, c->get_hostname(),
698                                c->m_cookie ? c->m_cookie : "");
699             no_of_clients++;
700             if (min_seq < 0 || c->m_seqno < min_seq)
701             {
702                 min_seq = c->m_seqno;
703                 c_min = c;
704             }
705         }
706         if (no_of_clients >= parent->m_max_clients)
707         {
708             c = c_min;
709             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
710             {
711                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Destroy %d",
712                          m_session_str, parent->m_max_clients, c->m_seqno);
713                 if (c->m_server && c->m_server != this)
714                     delete c->m_server;   // PROBLEM: m_ref_count!
715                 c->m_server = 0;
716             }
717             else
718             {
719                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Reuse %d %d %s",
720                          m_session_str, parent->m_max_clients,
721                          c->m_seqno, parent->m_seqno, c->get_hostname());
722                 xfree (c->m_cookie);
723                 c->m_cookie = 0;
724                 if (cookie)
725                     c->m_cookie = xstrdup(cookie);
726                 c->m_seqno = parent->m_seqno;
727                 if (c->m_server && c->m_server != this)
728                 {
729                     c->m_server->m_client = 0;
730                     delete c->m_server;   // PROBLEM: m_ref_count!
731                 }
732                 (parent->m_seqno)++;
733                 c->m_target_idletime = m_target_idletime;
734                 c->timeout(m_target_idletime);
735
736                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
737                     c->set_APDU_yazlog(1);
738                 else
739                     c->set_APDU_yazlog(0);
740
741                 return c;
742             }
743         }
744         else
745         {
746             yaz_log (YLOG_LOG, "%sNEW %d %s",
747                      m_session_str, parent->m_seqno, m_proxyTarget);
748             c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
749             c->m_next = parent->m_clientPool;
750             if (c->m_next)
751                 c->m_next->m_prev = &c->m_next;
752             parent->m_clientPool = c;
753             c->m_prev = &parent->m_clientPool;
754         }
755
756         xfree (c->m_cookie);
757         c->m_cookie = 0;
758         if (cookie)
759             c->m_cookie = xstrdup(cookie);
760
761         c->m_seqno = parent->m_seqno;
762         c->m_init_flag = 0;
763         c->m_last_resultCount = 0;
764         c->m_last_ok = 0;
765         c->m_cache.clear();
766         c->m_sr_transform = 0;
767         c->m_waiting = 0;
768         c->m_resultSetStartPoint = 0;
769         (parent->m_seqno)++;
770         if (c->client(m_proxyTarget))
771         {
772             delete c;
773             return 0;
774         }
775         c->m_target_idletime = m_target_idletime;
776         c->timeout(30);
777
778         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
779             c->set_APDU_yazlog(1);
780         else
781             c->set_APDU_yazlog(0);
782
783         c->set_idAuthentication(apdu);
784     }
785     yaz_log (YLOG_DEBUG, "get_client 3 %p %p", this, c);
786     return c;
787 }
788
789 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
790 {
791     int i;
792     for (i = 0; i<num; i++)
793     {
794         oident *ent;
795         Z_DefaultDiagFormat *r;
796         Z_DiagRec *p = pp[i];
797         if (p->which != Z_DiagRec_defaultFormat)
798         {
799             yaz_log(YLOG_LOG, "%sError no diagnostics", m_session_str);
800             return;
801         }
802         else
803             r = p->u.defaultFormat;
804         if (!(ent = oid_getentbyoid(r->diagnosticSetId)) ||
805             ent->oclass != CLASS_DIAGSET || ent->value != VAL_BIB1)
806             yaz_log(YLOG_LOG, "%sError unknown diagnostic set", m_session_str);
807         switch (r->which)
808         {
809         case Z_DefaultDiagFormat_v2Addinfo:
810             yaz_log(YLOG_LOG, "%sError %d %s:%s",
811                     m_session_str,
812                     *r->condition, diagbib1_str(*r->condition),
813                     r->u.v2Addinfo);
814             break;
815         case Z_DefaultDiagFormat_v3Addinfo:
816             yaz_log(YLOG_LOG, "%sError %d %s:%s",
817                     m_session_str,
818                     *r->condition, diagbib1_str(*r->condition),
819                     r->u.v3Addinfo);
820             break;
821         }
822     }
823 }
824
825 int Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p, Z_APDU *apdu)
826 {
827     if (!m_stylesheet_xsp || p->num_records <= 0)
828     {
829         return 0;  /* no XSLT to be done ... */
830     }
831
832     m_stylesheet_offset = 0;
833     m_stylesheet_nprl = p;
834     m_stylesheet_apdu = apdu;
835     m_timeout_mode = timeout_xsl;
836
837     timeout(0);
838     return 1;
839 }
840
841 void Yaz_Proxy::convert_xsl_delay()
842 {
843 #if HAVE_XSLT
844     Z_NamePlusRecord *npr = m_stylesheet_nprl->records[m_stylesheet_offset];
845     if (npr->which == Z_NamePlusRecord_databaseRecord)
846     {
847         Z_External *r = npr->u.databaseRecord;
848         if (r->which == Z_External_octet)
849         {
850 #if 0
851             fwrite((char*) r->u.octet_aligned->buf, 1, r->u.octet_aligned->len, stdout);
852 #endif
853             xmlDocPtr res, doc = xmlParseMemory(
854                 (char*) r->u.octet_aligned->buf,
855                 r->u.octet_aligned->len);
856
857
858             yaz_log(YLOG_LOG, "%sXSLT convert %d",
859                     m_session_str, m_stylesheet_offset);
860             res = xsltApplyStylesheet((xsltStylesheetPtr) m_stylesheet_xsp,
861                                       doc, 0);
862
863             if (res)
864             {
865                 xmlChar *out_buf;
866                 int out_len;
867                 xmlDocDumpFormatMemory (res, &out_buf, &out_len, 1);
868
869                 m_stylesheet_nprl->records[m_stylesheet_offset]->
870                     u.databaseRecord =
871                     z_ext_record(odr_encode(), VAL_TEXT_XML,
872                                  (char*) out_buf, out_len);
873                 xmlFree(out_buf);
874                 xmlFreeDoc(res);
875             }
876
877             xmlFreeDoc(doc);
878         }
879     }
880 #endif
881     m_stylesheet_offset++;
882     if (m_stylesheet_offset == m_stylesheet_nprl->num_records)
883     {
884         m_timeout_mode = timeout_normal;
885         m_stylesheet_nprl = 0;
886 #if HAVE_XSLT
887         if (m_stylesheet_xsp)
888             xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
889 #endif
890         m_stylesheet_xsp = 0;
891         timeout(m_client_idletime);
892         send_PDU_convert(m_stylesheet_apdu);
893     }
894     else
895         timeout(0);
896 }
897
898 void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p)
899 {
900     if (m_frontend_type != VAL_NONE)
901     {
902         int i;
903         for (i = 0; i < p->num_records; i++)
904         {
905             Z_NamePlusRecord *npr = p->records[i];
906             if (npr->which == Z_NamePlusRecord_databaseRecord)
907             {
908                 Z_External *r = npr->u.databaseRecord;
909                 if (r->which == Z_External_octet)
910                 {
911 #if HAVE_USEMARCON
912                     if (m_usemarcon_ini_stage1 && *m_usemarcon_ini_stage1)
913                     {
914                         if (!m_usemarcon->m_stage1)
915                         {
916                             m_usemarcon->m_stage1 = new CDetails();
917                         }
918                         m_usemarcon->m_stage1->SetIniFileName(m_usemarcon_ini_stage1);
919                         m_usemarcon->m_stage1->SetMarcRecord((char*) r->u.octet_aligned->buf, r->u.octet_aligned->len);
920                         int res = m_usemarcon->m_stage1->Start();
921                         if (res == 0)
922                         {
923                             char *converted;
924                             int convlen;
925                             m_usemarcon->m_stage1->GetMarcRecord(converted, convlen);
926                             if (m_usemarcon_ini_stage2 && *m_usemarcon_ini_stage2)
927                             {
928                                 if (!m_usemarcon->m_stage2)
929                                 {
930                                     m_usemarcon->m_stage2 = new CDetails();
931                                 }
932                                 m_usemarcon->m_stage2->SetIniFileName(m_usemarcon_ini_stage2);
933                                 m_usemarcon->m_stage2->SetMarcRecord(converted, convlen);
934                                 res = m_usemarcon->m_stage2->Start();
935                                 if (res == 0)
936                                 {
937                                     free(converted);
938                                     m_usemarcon->m_stage2->GetMarcRecord(converted, convlen);
939                                 }
940                                 else
941                                 {
942                                     yaz_log(YLOG_LOG, "%sUSEMARCON stage 2 error %d", m_session_str, res);
943                                 }
944                             }
945                             npr->u.databaseRecord =
946                                 z_ext_record(odr_encode(),
947                                              m_frontend_type,
948                                              converted,
949                                              strlen(converted));
950                             free(converted);
951                         }
952                         else
953                         {
954                             yaz_log(YLOG_LOG, "%sUSEMARCON stage 1 error %d", m_session_str, res);
955                         }
956                         continue;
957                     }
958 #endif
959 /* HAVE_USEMARCON */
960                     npr->u.databaseRecord =
961                         z_ext_record(odr_encode(),
962                                      m_frontend_type,
963                                      (char*) r->u.octet_aligned->buf,
964                                      r->u.octet_aligned->len);
965                 }
966             }
967         }
968     }
969 }
970
971 void Yaz_Proxy::convert_records_charset(Z_NamePlusRecordList *p,
972                                         const char *backend_charset)
973 {
974     yaz_log(YLOG_LOG, "%sconvert_to_marc", m_session_str);
975     int sel =   m_charset_converter->get_client_charset_selected();
976     const char *client_record_charset =
977         m_charset_converter->get_client_query_charset();
978     if (sel && backend_charset && client_record_charset &&
979         strcmp(backend_charset, client_record_charset))
980     {
981         int i;
982         yaz_iconv_t cd = yaz_iconv_open(client_record_charset,
983                                         backend_charset);
984         yaz_marc_t mt = yaz_marc_create();
985         yaz_marc_xml(mt, YAZ_MARC_ISO2709);
986         yaz_marc_iconv(mt, cd);
987         for (i = 0; i < p->num_records; i++)
988         {
989             Z_NamePlusRecord *npr = p->records[i];
990             if (npr->which == Z_NamePlusRecord_databaseRecord)
991             {
992                 Z_External *r = npr->u.databaseRecord;
993                 oident *ent = oid_getentbyoid(r->direct_reference);
994                 if (!ent || ent->value == VAL_NONE)
995                     continue;
996
997                 if (ent->value == VAL_SUTRS)
998                 {
999                     WRBUF w = wrbuf_alloc();
1000
1001                     wrbuf_iconv_write(w, cd,  (char*) r->u.octet_aligned->buf,
1002                                       r->u.octet_aligned->len);
1003                     npr->u.databaseRecord =
1004                         z_ext_record(odr_encode(), ent->value, wrbuf_buf(w),
1005                                      wrbuf_len(w));
1006                     wrbuf_free(w, 1);
1007                 }
1008                 else if (ent->value == VAL_TEXT_XML)
1009                 {
1010                     ;
1011                 }
1012                 else if (r->which == Z_External_octet)
1013                 {
1014                     int rlen;
1015                     char *result;
1016                     if (yaz_marc_decode_buf(mt,
1017                                             (char*) r->u.octet_aligned->buf,
1018                                             r->u.octet_aligned->len,
1019                                             &result, &rlen))
1020                     {
1021                         npr->u.databaseRecord =
1022                             z_ext_record(odr_encode(), ent->value, result, rlen);
1023                         yaz_log(YLOG_LOG, "%sRecoding MARC record",
1024                                 m_session_str);
1025                     }
1026                 }
1027             }
1028         }
1029         if (cd)
1030             yaz_iconv_close(cd);
1031         yaz_marc_destroy(mt);
1032     }
1033     else
1034     {
1035         yaz_log(YLOG_LOG, "%sSkipping marc convert", m_session_str);
1036     }
1037 }
1038
1039 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p,
1040                                    const char *backend_charset)
1041 {
1042     int i;
1043     if (!backend_charset)
1044         backend_charset = "MARC-8";
1045     yaz_iconv_t cd = yaz_iconv_open("UTF-8", backend_charset);
1046     yaz_marc_t mt = yaz_marc_create();
1047     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
1048     yaz_marc_iconv(mt, cd);
1049     for (i = 0; i < p->num_records; i++)
1050     {
1051         Z_NamePlusRecord *npr = p->records[i];
1052         if (npr->which == Z_NamePlusRecord_databaseRecord)
1053         {
1054             Z_External *r = npr->u.databaseRecord;
1055             if (r->which == Z_External_OPAC)
1056             {
1057                 WRBUF w = wrbuf_alloc();
1058
1059                 yaz_opac_decode_wrbuf(mt, r->u.opac, w);
1060                 npr->u.databaseRecord = z_ext_record(
1061                     odr_encode(), VAL_TEXT_XML,
1062                     wrbuf_buf(w), wrbuf_len(w)
1063                     );
1064                 wrbuf_free(w, 1);
1065             }
1066             else if (r->which == Z_External_octet)
1067             {
1068                 int rlen;
1069                 char *result;
1070                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
1071                                         r->u.octet_aligned->len,
1072                                         &result, &rlen))
1073                 {
1074                     npr->u.databaseRecord =
1075                         z_ext_record(odr_encode(), VAL_TEXT_XML, result, rlen);
1076                 }
1077             }
1078         }
1079     }
1080     if (cd)
1081         yaz_iconv_close(cd);
1082     yaz_marc_destroy(mt);
1083 }
1084
1085 void Yaz_Proxy::logtime()
1086 {
1087 #if HAVE_GETTIMEOFDAY
1088     struct timeval *tv = (struct timeval*) m_time_tv;
1089     if (tv->tv_sec)
1090     {
1091         struct timeval tv1;
1092         gettimeofday(&tv1, 0);
1093         long diff = (tv1.tv_sec - tv->tv_sec)*1000000 +
1094             (tv1.tv_usec - tv->tv_usec);
1095         if (diff >= 0)
1096             yaz_log(YLOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
1097                     diff/1000000, (diff/1000)%1000);
1098     }
1099     tv->tv_sec = 0;
1100     tv->tv_usec = 0;
1101 #endif
1102 }
1103
1104 int Yaz_Proxy::send_http_response(int code)
1105 {
1106     ODR o = odr_encode();
1107     Z_GDU *gdu = z_get_HTTP_Response(o, code);
1108     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1109     if (m_http_version)
1110         hres->version = odr_strdup(o, m_http_version);
1111     if (m_http_keepalive)
1112         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1113     else
1114         timeout(0);
1115
1116     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1117     {
1118         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1119                  gdu_name(gdu));
1120     }
1121     int len;
1122     int r = send_GDU(gdu, &len);
1123     m_bytes_sent += len;
1124     m_bw_stat.add_bytes(len);
1125     logtime();
1126
1127     recv_GDU_more(true);
1128
1129     return r;
1130 }
1131
1132 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu, int http_code /* = 200 */)
1133 {
1134     ODR o = odr_encode();
1135     const char *ctype = "text/xml";
1136     Z_GDU *gdu = z_get_HTTP_Response(o, http_code);
1137     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1138     if (m_http_version)
1139         hres->version = odr_strdup(o, m_http_version);
1140     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
1141     if (m_http_keepalive)
1142         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1143     else
1144         timeout(0);
1145     if (http_code == 401)
1146         z_HTTP_header_add(o, &hres->headers, "WWW-Authenticate", "Basic realm=\"YAZ Proxy\"");
1147
1148     static Z_SOAP_Handler soap_handlers[2] = {
1149 #if HAVE_XSLT
1150         {"http://www.loc.gov/zing/srw/", 0,
1151          (Z_SOAP_fun) yaz_srw_codec},
1152 #endif
1153         {0, 0, 0}
1154     };
1155
1156     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
1157     soap_package->which = Z_SOAP_generic;
1158     soap_package->u.generic =
1159         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
1160     soap_package->u.generic->no = 0;
1161     soap_package->u.generic->ns = soap_handlers[0].ns;
1162     soap_package->u.generic->p = (void *) srw_pdu;
1163     soap_package->ns = m_soap_ns;
1164     z_soap_codec_enc_xsl(o, &soap_package,
1165                          &hres->content_buf, &hres->content_len,
1166                          soap_handlers, 0, m_s2z_stylesheet);
1167     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1168     {
1169         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1170                  gdu_name(gdu));
1171     }
1172     int len;
1173     int r = send_GDU(gdu, &len);
1174     m_bytes_sent += len;
1175     m_bw_stat.add_bytes(len);
1176     logtime();
1177
1178     recv_GDU_more(true);
1179
1180     return r;
1181 }
1182
1183 int Yaz_Proxy::send_to_srw_client_error(int srw_error, const char *add)
1184 {
1185     ODR o = odr_encode();
1186     Z_SRW_diagnostic *diagnostic = (Z_SRW_diagnostic *)
1187         odr_malloc(o, sizeof(*diagnostic));
1188     int num_diagnostic = 1;
1189     yaz_mk_std_diagnostic(o, diagnostic, srw_error, add);
1190     return send_srw_search_response(diagnostic, num_diagnostic, srw_error == 3 ? 401 : 200);
1191 }
1192
1193 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
1194                              Z_DefaultDiagFormat *ddf)
1195 {
1196     int bib1_code = *ddf->condition;
1197     if (bib1_code == 109)
1198         return 404;
1199     srw_res->num_diagnostics = 1;
1200     srw_res->diagnostics = (Z_SRW_diagnostic *)
1201         odr_malloc(o, sizeof(*srw_res->diagnostics));
1202     yaz_mk_std_diagnostic(o, srw_res->diagnostics,
1203                           yaz_diag_bib1_to_srw(*ddf->condition),
1204                           ddf->u.v2Addinfo);
1205     return 0;
1206 }
1207
1208 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
1209 {
1210     ODR o = odr_encode();
1211     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1212     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1213
1214     srw_res->numberOfRecords = odr_intdup (o, hits);
1215     if (records && records->which == Z_Records_DBOSD)
1216     {
1217         srw_res->num_records =
1218             records->u.databaseOrSurDiagnostics->num_records;
1219         int i;
1220         srw_res->records = (Z_SRW_record *)
1221             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
1222         for (i = 0; i < srw_res->num_records; i++)
1223         {
1224             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
1225             if (npr->which != Z_NamePlusRecord_databaseRecord)
1226             {
1227                 srw_res->records[i].recordSchema = "diagnostic";
1228                 srw_res->records[i].recordPacking = m_s2z_packing;
1229                 srw_res->records[i].recordData_buf = "67";
1230                 srw_res->records[i].recordData_len = 2;
1231                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1232                 continue;
1233             }
1234             Z_External *r = npr->u.databaseRecord;
1235             oident *ent = oid_getentbyoid(r->direct_reference);
1236             if (r->which == Z_External_octet && ent->value == VAL_TEXT_XML)
1237             {
1238                 srw_res->records[i].recordSchema = m_schema;
1239                 srw_res->records[i].recordPacking = m_s2z_packing;
1240                 srw_res->records[i].recordData_buf = (char*)
1241                     r->u.octet_aligned->buf;
1242                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
1243                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1244             }
1245             else
1246             {
1247                 srw_res->records[i].recordSchema = "diagnostic";
1248                 srw_res->records[i].recordPacking = m_s2z_packing;
1249                 srw_res->records[i].recordData_buf = "67";
1250                 srw_res->records[i].recordData_len = 2;
1251                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1252             }
1253         }
1254     }
1255     if (records && records->which == Z_Records_NSD)
1256     {
1257         int http_code;
1258         http_code = z_to_srw_diag(odr_encode(), srw_res,
1259                                    records->u.nonSurrogateDiagnostic);
1260         if (http_code)
1261             return send_http_response(http_code);
1262     }
1263     return send_srw_response(srw_pdu);
1264
1265 }
1266
1267 int Yaz_Proxy::send_srw_search_response(Z_SRW_diagnostic *diagnostics,
1268                                         int num_diagnostics, int http_code /* = 200 */)
1269 {
1270     ODR o = odr_encode();
1271     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1272     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1273
1274     srw_res->num_diagnostics = num_diagnostics;
1275     srw_res->diagnostics = diagnostics;
1276     return send_srw_response(srw_pdu, http_code);
1277 }
1278
1279 int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics,
1280                                         int num_diagnostics)
1281 {
1282     Yaz_ProxyConfig *cfg = check_reconfigure();
1283     if (cfg)
1284     {
1285         int len;
1286         char *b = cfg->get_explain_doc(odr_encode(), 0 /* target */,
1287                                        m_s2z_database, &len);
1288         if (b)
1289         {
1290             Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
1291             Z_SRW_explainResponse *er = res->u.explain_response;
1292
1293             er->record.recordData_buf = b;
1294             er->record.recordData_len = len;
1295             er->record.recordPacking = m_s2z_packing;
1296             er->record.recordSchema = "http://explain.z3950.org/dtd/2.0/";
1297
1298             er->diagnostics = diagnostics;
1299             er->num_diagnostics = num_diagnostics;
1300             return send_srw_response(res);
1301         }
1302     }
1303     return send_http_response(404);
1304 }
1305
1306 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu)
1307 {
1308     if (m_http_version)
1309     {
1310         if (apdu->which == Z_APDU_initResponse)
1311         {
1312             Z_InitResponse *res = apdu->u.initResponse;
1313             if (*res->result == 0)
1314             {
1315                 send_to_srw_client_error(3, 0);
1316             }
1317             else if (!m_s2z_search_apdu)
1318             {
1319                 send_srw_explain_response(0, 0);
1320             }
1321             else
1322             {
1323                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1324             }
1325         }
1326         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
1327         {
1328             m_s2z_search_apdu = 0;
1329             Z_SearchResponse *res = apdu->u.searchResponse;
1330             m_s2z_hit_count = *res->resultCount;
1331             if (res->records && res->records->which == Z_Records_NSD)
1332             {
1333                 send_to_srw_client_ok(0, res->records, 1);
1334             }
1335             else if (m_s2z_present_apdu && m_s2z_hit_count > 0)
1336             {
1337                 // adjust
1338                 Z_PresentRequest *pr = m_s2z_present_apdu->u.presentRequest;
1339
1340                 if (*pr->resultSetStartPoint <= m_s2z_hit_count)
1341                 {
1342                     if (*pr->numberOfRecordsRequested+ *pr->resultSetStartPoint
1343                         > m_s2z_hit_count)
1344                         *pr->numberOfRecordsRequested =
1345                             1 + m_s2z_hit_count - *pr->resultSetStartPoint;
1346                 }
1347                 handle_incoming_Z_PDU(m_s2z_present_apdu);
1348             }
1349             else
1350             {
1351                 m_s2z_present_apdu = 0;
1352                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
1353             }
1354         }
1355         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
1356         {
1357             int start =
1358                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
1359
1360             m_s2z_present_apdu = 0;
1361             Z_PresentResponse *res = apdu->u.presentResponse;
1362             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
1363         }
1364     }
1365     else
1366     {
1367         int len = 0;
1368         if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1369             yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1370                      apdu_name(apdu));
1371         int r = send_Z_PDU(apdu, &len);
1372         m_bytes_sent += len;
1373         m_bw_stat.add_bytes(len);
1374         logtime();
1375         return r;
1376     }
1377     return 0;
1378 }
1379
1380 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
1381 {
1382     int kill_session = 0;
1383     Z_ReferenceId **new_id = get_referenceIdP(apdu);
1384
1385     if (new_id)
1386         *new_id = m_referenceId;
1387
1388     if (apdu->which == Z_APDU_searchResponse)
1389     {
1390         Z_SearchResponse *sr = apdu->u.searchResponse;
1391         Z_Records *p = sr->records;
1392         if (p && p->which == Z_Records_NSD)
1393         {
1394             Z_DiagRec dr, *dr_p = &dr;
1395             dr.which = Z_DiagRec_defaultFormat;
1396             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1397
1398             *sr->searchStatus = 0;
1399             display_diagrecs(&dr_p, 1);
1400         }
1401         else
1402         {
1403             if (p && p->which == Z_Records_DBOSD)
1404             {
1405                 if (m_backend_type
1406 #if HAVE_USEMARCON
1407                     || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1408 #endif
1409                     )
1410                     convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1411                 if (m_marcxml_mode == marcxml)
1412                     convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1413                                        m_backend_charset);
1414                 else
1415                     convert_records_charset(p->u.databaseOrSurDiagnostics,
1416                                             m_backend_charset);
1417                 if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1418                     return 0;
1419
1420             }
1421             if (sr->resultCount)
1422             {
1423                 yaz_log(YLOG_LOG, "%s%d hits", m_session_str,
1424                         *sr->resultCount);
1425                 if (*sr->resultCount < 0)
1426                 {
1427                     m_flag_invalid_session = 1;
1428                     kill_session = 1;
1429
1430                     *sr->searchStatus = 0;
1431                     sr->records =
1432                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
1433                     *sr->resultCount = 0;
1434                 }
1435             }
1436         }
1437     }
1438     else if (apdu->which == Z_APDU_presentResponse)
1439     {
1440         Z_PresentResponse *sr = apdu->u.presentResponse;
1441         Z_Records *p = sr->records;
1442         if (p && p->which == Z_Records_NSD)
1443         {
1444             Z_DiagRec dr, *dr_p = &dr;
1445             dr.which = Z_DiagRec_defaultFormat;
1446             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1447             if (*sr->presentStatus == Z_PresentStatus_success)
1448                 *sr->presentStatus = Z_PresentStatus_failure;
1449             display_diagrecs(&dr_p, 1);
1450         }
1451         if (p && p->which == Z_Records_DBOSD)
1452         {
1453             if (m_backend_type
1454 #if HAVE_USEMARCON
1455                 || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1456 #endif
1457                 )
1458                 convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1459             if (m_marcxml_mode == marcxml)
1460                 convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1461                                    m_backend_charset);
1462             else
1463                 convert_records_charset(p->u.databaseOrSurDiagnostics,
1464                                         m_backend_charset);
1465             if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1466                 return 0;
1467         }
1468     }
1469     else if (apdu->which == Z_APDU_initResponse)
1470     {
1471         //Get and check negotiation record
1472         //from init response.
1473         handle_charset_lang_negotiation(apdu);
1474
1475         if (m_initRequest_options)
1476         {
1477             Z_Options *nopt =
1478                 (Odr_bitmask *)odr_malloc(odr_encode(),
1479                                           sizeof(Odr_bitmask));
1480             ODR_MASK_ZERO(nopt);
1481
1482             int i;
1483             for (i = 0; i<24; i++)
1484                 if (ODR_MASK_GET(m_initRequest_options, i) &&
1485                     ODR_MASK_GET(apdu->u.initResponse->options, i))
1486                     ODR_MASK_SET(nopt, i);
1487             apdu->u.initResponse->options = nopt;
1488         }
1489         if (m_initRequest_version)
1490         {
1491             Z_ProtocolVersion *nopt =
1492                 (Odr_bitmask *)odr_malloc(odr_encode(),
1493                                           sizeof(Odr_bitmask));
1494             ODR_MASK_ZERO(nopt);
1495
1496             int i;
1497             for (i = 0; i<8; i++)
1498                 if (ODR_MASK_GET(m_initRequest_version, i) &&
1499                     ODR_MASK_GET(apdu->u.initResponse->protocolVersion, i))
1500                     ODR_MASK_SET(nopt, i);
1501             apdu->u.initResponse->protocolVersion = nopt;
1502         }
1503         apdu->u.initResponse->preferredMessageSize =
1504             odr_intdup(odr_encode(),
1505                        m_client->m_initResponse_preferredMessageSize >
1506                        m_initRequest_preferredMessageSize ?
1507                        m_initRequest_preferredMessageSize :
1508                        m_client->m_initResponse_preferredMessageSize);
1509         apdu->u.initResponse->maximumRecordSize =
1510             odr_intdup(odr_encode(),
1511                        m_client->m_initResponse_maximumRecordSize >
1512                        m_initRequest_maximumRecordSize ?
1513                        m_initRequest_maximumRecordSize :
1514                        m_client->m_initResponse_maximumRecordSize);
1515     }
1516
1517     int r = send_PDU_convert(apdu);
1518     if (r)
1519         return r;
1520     if (kill_session)
1521     {
1522         delete m_client;
1523         m_client = 0;
1524         m_parent->pre_init();
1525     }
1526     return r;
1527 }
1528
1529 void Yaz_ProxyClient::set_idAuthentication(Z_APDU *apdu)
1530 {
1531     Z_IdAuthentication *t = apdu->u.initRequest->idAuthentication;
1532     
1533     odr_reset(m_idAuthentication_odr);
1534     z_IdAuthentication(m_idAuthentication_odr, &t, 1, 0);
1535     m_idAuthentication_ber_buf =
1536         odr_getbuf(m_idAuthentication_odr, 
1537                    &m_idAuthentication_ber_size, 0);
1538 }
1539
1540 bool Yaz_ProxyClient::compare_charset(Z_APDU *apdu)
1541 {
1542     return true;
1543 }
1544
1545 bool Yaz_ProxyClient::compare_idAuthentication(Z_APDU *apdu)
1546 {
1547     Z_IdAuthentication *t = apdu->u.initRequest->idAuthentication;
1548     ODR odr = odr_createmem(ODR_ENCODE);
1549
1550     z_IdAuthentication(odr, &t, 1, 0);
1551     int sz;
1552     char *buf = odr_getbuf(odr, &sz, 0);
1553     if (buf && m_idAuthentication_ber_buf
1554         && sz == m_idAuthentication_ber_size
1555         && !memcmp(m_idAuthentication_ber_buf, buf, sz))
1556     {
1557         odr_destroy(odr);
1558         return true;
1559     }
1560     odr_destroy(odr);
1561     if (!buf && !m_idAuthentication_ber_buf)
1562         return true;
1563     return false;
1564 }
1565
1566 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
1567 {
1568     int len = 0;
1569     const char *apdu_name_tmp = apdu_name(apdu);
1570     int r = send_Z_PDU(apdu, &len);
1571     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
1572         yaz_log (YLOG_LOG, "%sSending %s to %s %d bytes",
1573                  get_session_str(),
1574                  apdu_name_tmp, get_hostname(), len);
1575     m_bytes_sent += len;
1576     return r;
1577 }
1578
1579 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
1580 {
1581     if (apdu->which == Z_APDU_presentRequest)
1582     {
1583         Z_PresentRequest *pr = apdu->u.presentRequest;
1584         int toget = *pr->numberOfRecordsRequested;
1585         int start = *pr->resultSetStartPoint;
1586
1587         yaz_log(YLOG_LOG, "%sPresent %s %d+%d", m_session_str,
1588                 pr->resultSetId, start, toget);
1589
1590         if (*m_parent->m_optimize == '0')
1591             return apdu;
1592
1593         if (!m_client->m_last_resultSetId)
1594         {
1595             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1596             new_apdu->u.presentResponse->records =
1597                 create_nonSurrogateDiagnostics(odr_encode(), 30,
1598                                                pr->resultSetId);
1599             send_to_client(new_apdu);
1600             return 0;
1601         }
1602         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1603         {
1604             if (start+toget-1 > m_client->m_last_resultCount)
1605             {
1606                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1607                 new_apdu->u.presentResponse->records =
1608                     create_nonSurrogateDiagnostics(odr_encode(), 13, 0);
1609                 send_to_client(new_apdu);
1610                 return 0;
1611             }
1612             Z_NamePlusRecordList *npr;
1613 #if 0
1614             yaz_log(YLOG_LOG, "%sCache lookup %d+%d syntax=%s",
1615                     m_session_str, start, toget, yaz_z3950oid_to_str(
1616                         pr->preferredRecordSyntax, &oclass));
1617 #endif
1618             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1619                                           pr->preferredRecordSyntax,
1620                                           pr->recordComposition))
1621             {
1622                 yaz_log (YLOG_LOG, "%sReturned cached records for present request",
1623                          m_session_str);
1624                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1625                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1626
1627                 new_apdu->u.presentResponse->numberOfRecordsReturned
1628                     = odr_intdup(odr_encode(), toget);
1629
1630                 new_apdu->u.presentResponse->records = (Z_Records*)
1631                     odr_malloc(odr_encode(), sizeof(Z_Records));
1632                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1633                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1634                 new_apdu->u.presentResponse->nextResultSetPosition =
1635                     odr_intdup(odr_encode(), start+toget);
1636
1637                 send_to_client(new_apdu);
1638                 return 0;
1639             }
1640         }
1641     }
1642
1643     if (apdu->which != Z_APDU_searchRequest)
1644         return apdu;
1645     Z_SearchRequest *sr = apdu->u.searchRequest;
1646     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1647     Yaz_Z_Databases this_databases;
1648
1649     this_databases.set(sr->num_databaseNames, (const char **)
1650                        sr->databaseNames);
1651
1652     this_query->set_Z_Query(sr->query);
1653
1654     char query_str[120];
1655     this_query->print(query_str, sizeof(query_str)-1);
1656     yaz_log(YLOG_LOG, "%sSearch %s", m_session_str, query_str);
1657
1658     if (*m_parent->m_optimize != '0' &&
1659         m_client->m_last_ok && m_client->m_last_query &&
1660         m_client->m_last_query->match(this_query) &&
1661         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1662         m_client->m_last_databases.match(this_databases))
1663     {
1664         delete this_query;
1665         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1666             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1667         {
1668             Z_NamePlusRecordList *npr;
1669             int toget = *sr->mediumSetPresentNumber;
1670             Z_RecordComposition *comp = 0;
1671
1672             if (toget > m_client->m_last_resultCount)
1673                 toget = m_client->m_last_resultCount;
1674
1675             if (sr->mediumSetElementSetNames)
1676             {
1677                 comp = (Z_RecordComposition *)
1678                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1679                 comp->which = Z_RecordComp_simple;
1680                 comp->u.simple = sr->mediumSetElementSetNames;
1681             }
1682
1683             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1684                                           sr->preferredRecordSyntax, comp))
1685             {
1686                 yaz_log (YLOG_LOG, "%sReturned cached records for medium set",
1687                          m_session_str);
1688                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1689                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1690                 new_apdu->u.searchResponse->resultCount =
1691                     &m_client->m_last_resultCount;
1692
1693                 new_apdu->u.searchResponse->numberOfRecordsReturned
1694                     = odr_intdup(odr_encode(), toget);
1695
1696                 new_apdu->u.searchResponse->presentStatus =
1697                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1698                 new_apdu->u.searchResponse->records = (Z_Records*)
1699                     odr_malloc(odr_encode(), sizeof(Z_Records));
1700                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1701                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1702                 new_apdu->u.searchResponse->nextResultSetPosition =
1703                     odr_intdup(odr_encode(), toget+1);
1704                 send_to_client(new_apdu);
1705                 return 0;
1706             }
1707             else
1708             {
1709                 // medium Set
1710                 // send present request (medium size)
1711                 yaz_log (YLOG_LOG, "%sOptimizing search for medium set",
1712                          m_session_str);
1713
1714                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1715                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1716                 pr->referenceId = sr->referenceId;
1717                 pr->resultSetId = sr->resultSetName;
1718                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1719                 *pr->numberOfRecordsRequested = toget;
1720                 pr->recordComposition = comp;
1721                 m_client->m_sr_transform = 1;
1722                 return new_apdu;
1723             }
1724         }
1725         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1726             m_client->m_last_resultCount <= 0)
1727         {
1728             // large set. Return pseudo-search response immediately
1729             yaz_log (YLOG_LOG, "%sOptimizing search for large set",
1730                      m_session_str);
1731             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1732             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1733             new_apdu->u.searchResponse->resultCount =
1734                 &m_client->m_last_resultCount;
1735             send_to_client(new_apdu);
1736             return 0;
1737         }
1738         else
1739         {
1740             Z_NamePlusRecordList *npr;
1741             int toget = m_client->m_last_resultCount;
1742             Z_RecordComposition *comp = 0;
1743             // small set
1744             // send a present request (small set)
1745
1746             if (sr->smallSetElementSetNames)
1747             {
1748                 comp = (Z_RecordComposition *)
1749                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1750                 comp->which = Z_RecordComp_simple;
1751                 comp->u.simple = sr->smallSetElementSetNames;
1752             }
1753
1754             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1755                                           sr->preferredRecordSyntax, comp))
1756             {
1757                 yaz_log (YLOG_LOG, "%sReturned cached records for small set",
1758                          m_session_str);
1759                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1760                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1761                 new_apdu->u.searchResponse->resultCount =
1762                     &m_client->m_last_resultCount;
1763
1764                 new_apdu->u.searchResponse->numberOfRecordsReturned
1765                     = odr_intdup(odr_encode(), toget);
1766
1767                 new_apdu->u.searchResponse->presentStatus =
1768                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1769                 new_apdu->u.searchResponse->records = (Z_Records*)
1770                     odr_malloc(odr_encode(), sizeof(Z_Records));
1771                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1772                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1773                 new_apdu->u.searchResponse->nextResultSetPosition =
1774                     odr_intdup(odr_encode(), toget+1);
1775                 send_to_client(new_apdu);
1776                 return 0;
1777             }
1778             else
1779             {
1780                 yaz_log (YLOG_LOG, "%sOptimizing search for small set",
1781                          m_session_str);
1782                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1783                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1784                 pr->referenceId = sr->referenceId;
1785                 pr->resultSetId = sr->resultSetName;
1786                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1787                 *pr->numberOfRecordsRequested = toget;
1788                 pr->recordComposition = comp;
1789                 m_client->m_sr_transform = 1;
1790                 return new_apdu;
1791             }
1792         }
1793     }
1794     else  // query doesn't match
1795     {
1796         delete m_client->m_last_query;
1797         m_client->m_last_query = this_query;
1798         m_client->m_last_ok = 0;
1799         m_client->m_cache.clear();
1800         m_client->m_resultSetStartPoint = 0;
1801
1802         xfree (m_client->m_last_resultSetId);
1803         m_client->m_last_resultSetId = xstrdup (sr->resultSetName);
1804
1805         m_client->m_last_databases.set(sr->num_databaseNames,
1806                                        (const char **) sr->databaseNames);
1807     }
1808     return apdu;
1809 }
1810
1811
1812 void Yaz_Proxy::inc_request_no()
1813 {
1814     char *cp = strchr(m_session_str, ' ');
1815     m_request_no++;
1816     if (cp)
1817         sprintf(cp+1, "%d ", m_request_no);
1818 }
1819
1820 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
1821 {
1822     inc_request_no();
1823
1824     m_bytes_recv += len;
1825
1826     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1827         yaz_log (YLOG_LOG, "%sReceiving %s from client %d bytes",
1828                  m_session_str, gdu_name(apdu), len);
1829
1830 #if HAVE_GETTIMEOFDAY
1831     gettimeofday((struct timeval *) m_time_tv, 0);
1832 #endif
1833     m_bw_stat.add_bytes(len);
1834     m_pdu_stat.add_bytes(1);
1835
1836     GDU *gdu = new GDU(apdu);
1837     m_in_queue.enqueue(gdu);
1838
1839     recv_GDU_more(false);
1840 }
1841
1842 void Yaz_Proxy::recv_GDU_reduce(GDU *gdu)
1843 {
1844     int bw_total = m_bw_stat.get_total();
1845     int pdu_total = m_pdu_stat.get_total();
1846     int reduce = 0;
1847
1848     assert(m_timeout_mode == timeout_busy);
1849     assert(m_timeout_gdu == 0);
1850
1851     if (m_bw_max)
1852     {
1853         if (bw_total > m_bw_max)
1854         {
1855             reduce = (bw_total/m_bw_max);
1856         }
1857     }
1858     if (m_pdu_max)
1859     {
1860         if (pdu_total > m_pdu_max)
1861         {
1862             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
1863             reduce = (reduce > nreduce) ? reduce : nreduce;
1864         }
1865     }
1866     m_http_version = 0;
1867
1868 #if 0
1869     /* uncomment to force a big reduce */
1870     m_timeout_mode = timeout_reduce;
1871     m_timeout_gdu = gdu;
1872     timeout(3);       // call us reduce seconds later
1873     return;
1874 #endif
1875     if (reduce)
1876     {
1877         yaz_log(YLOG_LOG, "%sdelay=%d bw=%d pdu=%d limit-bw=%d limit-pdu=%d",
1878                 m_session_str, reduce, bw_total, pdu_total,
1879                 m_bw_max, m_pdu_max);
1880
1881         m_timeout_mode = timeout_reduce;
1882         m_timeout_gdu = gdu;
1883         timeout(reduce);       // call us reduce seconds later
1884     }
1885     else
1886         recv_GDU_normal(gdu);
1887 }
1888
1889 void Yaz_Proxy::recv_GDU_more(bool normal)
1890 {
1891     GDU *g;
1892     if (normal && m_timeout_mode == timeout_busy)
1893         m_timeout_mode = timeout_normal;
1894     while (m_timeout_mode == timeout_normal && (g = m_in_queue.dequeue()))
1895     {
1896         m_timeout_mode = timeout_busy;
1897         recv_GDU_reduce(g);
1898     }
1899 }
1900
1901 void Yaz_Proxy::recv_GDU_normal(GDU *gdu)
1902 {
1903     Z_GDU *apdu = 0;
1904     gdu->move_away_gdu(odr_decode(), &apdu);
1905     delete gdu;
1906
1907     if (apdu->which == Z_GDU_Z3950)
1908         handle_incoming_Z_PDU(apdu->u.z3950);
1909     else if (apdu->which == Z_GDU_HTTP_Request)
1910         handle_incoming_HTTP(apdu->u.HTTP_Request);
1911 }
1912
1913 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
1914 {
1915     if (m_max_record_retrieve)
1916     {
1917         if (apdu->which == Z_APDU_presentRequest)
1918         {
1919             Z_PresentRequest *pr = apdu->u.presentRequest;
1920             if (pr->numberOfRecordsRequested &&
1921                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
1922                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
1923         }
1924     }
1925 }
1926
1927 void Yaz_Proxy::handle_charset_lang_negotiation(Z_APDU *apdu)
1928 {
1929     if (apdu->which == Z_APDU_initRequest)
1930     {
1931         yaz_log(YLOG_LOG, "%shandle_charset_lang_negotiation",
1932                 m_session_str);
1933         if (m_initRequest_options &&
1934             !ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel) &&
1935             (m_proxy_negotiation_charset || m_proxy_negotiation_lang))
1936         {
1937             // There is no negotiation proposal from
1938             // client's side. OK. The proxy negotiation
1939             // in use, only.
1940             Z_InitRequest *initRequest = apdu->u.initRequest;
1941             Z_OtherInformation **otherInfo;
1942             Z_OtherInformationUnit *oi;
1943             get_otherInfoAPDU(apdu, &otherInfo);
1944             oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
1945             if (oi)
1946             {
1947                 ODR_MASK_SET(initRequest->options,
1948                     Z_Options_negotiationModel);
1949                 oi->which = Z_OtherInfo_externallyDefinedInfo;
1950                 oi->information.externallyDefinedInfo =
1951                 yaz_set_proposal_charneg(odr_encode(),
1952                     (const char**)&m_proxy_negotiation_charset,
1953                     m_proxy_negotiation_charset ? 1:0,
1954                     (const char**)&m_proxy_negotiation_lang,
1955                     m_proxy_negotiation_lang ? 1:0,
1956                     1);
1957             }
1958         }
1959         else if (m_initRequest_options &&
1960                  ODR_MASK_GET(m_initRequest_options,
1961                               Z_Options_negotiationModel) &&
1962                  m_charset_converter->get_target_query_charset())
1963         {
1964             yaz_log(YLOG_LOG, "%sManaged charset negotiation: charset=%s",
1965                     m_session_str,
1966                     m_charset_converter->get_target_query_charset());
1967             Z_InitRequest *initRequest = apdu->u.initRequest;
1968             Z_CharSetandLanguageNegotiation *negotiation =
1969                 yaz_get_charneg_record (initRequest->otherInfo);
1970             if (negotiation &&
1971                 negotiation->which == Z_CharSetandLanguageNegotiation_proposal)
1972             {
1973                 NMEM nmem = nmem_create();
1974                 char **charsets = 0;
1975                 int num_charsets = 0;
1976                 char **langs = 0;
1977                 int num_langs = 0;
1978                 int selected = 0;
1979                 yaz_get_proposal_charneg (nmem, negotiation,
1980                                           &charsets, &num_charsets,
1981                                           &langs, &num_langs, &selected);
1982                 int i;
1983                 for (i = 0; i<num_charsets; i++)
1984                     yaz_log(YLOG_LOG, "%scharset %s", m_session_str,
1985                             charsets[i]);
1986                 for (i = 0; i<num_langs; i++)
1987                     yaz_log(YLOG_LOG, "%slang %s", m_session_str,
1988                             langs[i]);
1989
1990                 const char *t_charset =
1991                     m_charset_converter->get_target_query_charset();
1992                 // sweep through charsets and pick the first supported
1993                 // conversion
1994                 for (i = 0; i<num_charsets; i++)
1995                 {
1996                     const char *c_charset = charsets[i];
1997                     if (!odr_set_charset(odr_decode(), t_charset, c_charset))
1998                         break;
1999                 }
2000                 if (i != num_charsets)
2001                 {
2002                     // got one .. set up ODR for reverse direction
2003                     const char *c_charset = charsets[i];
2004                     odr_set_charset(odr_encode(), c_charset, t_charset);
2005                     m_charset_converter->set_client_query_charset(c_charset);
2006                     m_charset_converter->set_client_charset_selected(selected);
2007                 }
2008                 nmem_destroy(nmem);
2009                 ODR_MASK_CLEAR(m_initRequest_options,
2010                                Z_Options_negotiationModel);
2011                 yaz_del_charneg_record(&initRequest->otherInfo);
2012             }
2013             else
2014             {
2015                 yaz_log(YLOG_WARN, "%sUnable to decode charset package",
2016                         m_session_str);
2017             }
2018         }
2019         else if (m_charset_converter->get_target_query_charset() &&
2020             m_proxy_negotiation_default_charset)
2021         {
2022             m_charset_converter->
2023                 set_client_query_charset(m_proxy_negotiation_default_charset);
2024         }
2025     }
2026     else if (apdu->which == Z_APDU_initResponse)
2027     {
2028         Z_InitResponse *initResponse = apdu->u.initResponse;
2029         Z_OtherInformation **otherInfo;
2030         get_otherInfoAPDU(apdu, &otherInfo);
2031         
2032         Z_CharSetandLanguageNegotiation *charneg = 0;
2033
2034         if (otherInfo && *otherInfo && 
2035             ODR_MASK_GET(initResponse->options, Z_Options_negotiationModel)
2036             && (charneg = yaz_get_charneg_record(*otherInfo)))
2037         {
2038             char *charset = 0;
2039             char *lang = 0;
2040             int selected = 0;
2041
2042             yaz_get_response_charneg(m_referenceId_mem, charneg,
2043                 &charset, &lang, &selected);
2044
2045             yaz_log(YLOG_LOG, "%sAccepted charset - '%s' and lang - '%s'",
2046                 m_session_str, (charset)?charset:"none", (lang)?lang:"none");
2047
2048             if (m_initRequest_options &&
2049                 ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel))
2050             {
2051                 yaz_log(YLOG_LOG, "%sClient's negotiation record in use",
2052                     m_session_str);
2053             }
2054             else if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
2055             {
2056                 // negotiation-charset, negotiation-lang
2057                 // elements of config file in use.
2058
2059                 yaz_log(YLOG_LOG, "%sProxy's negotiation record in use",
2060                     m_session_str);
2061
2062                 // clear negotiation option.
2063                 ODR_MASK_CLEAR(initResponse->options, Z_Options_negotiationModel);
2064
2065                 // Delete negotiation (charneg-3) entry.
2066                 yaz_del_charneg_record(otherInfo);
2067             }
2068         }
2069         else
2070         {
2071             if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
2072             {
2073                 yaz_log(YLOG_LOG, "%sTarget did not honor negotiation",
2074                         m_session_str);
2075             }
2076             else if (m_charset_converter->get_client_query_charset())
2077             {
2078                 Z_OtherInformation **otherInfo;
2079                 Z_OtherInformationUnit *oi;
2080                 get_otherInfoAPDU(apdu, &otherInfo);
2081                 oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
2082                 if (oi)
2083                 {
2084                     ODR_MASK_SET(initResponse->options,
2085                                  Z_Options_negotiationModel);
2086                     if (m_initRequest_options)
2087                         ODR_MASK_SET(m_initRequest_options,
2088                                      Z_Options_negotiationModel);
2089                     
2090                     oi->which = Z_OtherInfo_externallyDefinedInfo;    
2091                     oi->information.externallyDefinedInfo =
2092                         yaz_set_response_charneg(
2093                             odr_encode(),
2094                             m_charset_converter->get_client_query_charset(),
2095                             0 /* no lang */,
2096                             m_charset_converter->get_client_charset_selected());
2097                 }
2098             }
2099         }
2100     }
2101 }
2102
2103 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
2104                                                      int error,
2105                                                      const char *addinfo)
2106 {
2107     Z_Records *rec = (Z_Records *)
2108         odr_malloc (odr, sizeof(*rec));
2109     int *err = (int *)
2110         odr_malloc (odr, sizeof(*err));
2111     Z_DiagRec *drec = (Z_DiagRec *)
2112         odr_malloc (odr, sizeof(*drec));
2113     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
2114         odr_malloc (odr, sizeof(*dr));
2115     *err = error;
2116     rec->which = Z_Records_NSD;
2117     rec->u.nonSurrogateDiagnostic = dr;
2118     dr->diagnosticSetId =
2119         yaz_oidval_to_z3950oid (odr, CLASS_DIAGSET, VAL_BIB1);
2120     dr->condition = err;
2121     dr->which = Z_DefaultDiagFormat_v2Addinfo;
2122     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
2123     return rec;
2124 }
2125
2126 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
2127 {
2128     if (apdu->which == Z_APDU_searchRequest &&
2129         apdu->u.searchRequest->query &&
2130         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
2131         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
2132     {
2133         Z_RPNQuery *rpnquery = 0;
2134         Z_SearchRequest *sr = apdu->u.searchRequest;
2135         char *addinfo = 0;
2136
2137         yaz_log(YLOG_LOG, "%sCQL: %s", m_session_str,
2138                 sr->query->u.type_104->u.cql);
2139
2140         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
2141                                           &rpnquery, odr_encode(),
2142                                           &addinfo);
2143         if (r == -3)
2144             yaz_log(YLOG_LOG, "%sNo CQL to RPN table", m_session_str);
2145         else if (r)
2146         {
2147             yaz_log(YLOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
2148             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2149
2150             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2151             new_apdu->u.searchResponse->records =
2152                 create_nonSurrogateDiagnostics(odr_encode(),
2153                                                yaz_diag_srw_to_bib1(r),
2154                                                addinfo);
2155             *new_apdu->u.searchResponse->searchStatus = 0;
2156
2157             send_to_client(new_apdu);
2158
2159             return 0;
2160         }
2161         else
2162         {
2163             sr->query->which = Z_Query_type_1;
2164             sr->query->u.type_1 = rpnquery;
2165         }
2166         return apdu;
2167     }
2168     return apdu;
2169 }
2170
2171 Z_APDU *Yaz_Proxy::handle_target_charset_conversion(Z_APDU *apdu)
2172 {
2173     if (apdu->which == Z_APDU_searchRequest &&
2174         apdu->u.searchRequest->query)
2175     {
2176         if (apdu->u.searchRequest->query->which == Z_Query_type_1
2177             || apdu->u.searchRequest->query->which == Z_Query_type_101)
2178         {
2179             if (m_http_version)
2180                 m_charset_converter->set_client_query_charset("UTF-8");
2181             Z_RPNQuery *rpnquery = apdu->u.searchRequest->query->u.type_1;
2182             m_charset_converter->convert_type_1(rpnquery, odr_encode());
2183         }
2184     }
2185     return apdu;
2186 }
2187
2188
2189 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
2190 {
2191     if (apdu->which == Z_APDU_searchRequest)
2192     {
2193         Z_SearchRequest *sr = apdu->u.searchRequest;
2194         int err = 0;
2195         char *addinfo = 0;
2196
2197         Yaz_ProxyConfig *cfg = check_reconfigure();
2198         if (cfg)
2199             err = cfg->check_query(odr_encode(), m_default_target,
2200                                    sr->query, &addinfo);
2201         if (err)
2202         {
2203             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2204
2205             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2206             new_apdu->u.searchResponse->records =
2207                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2208             *new_apdu->u.searchResponse->searchStatus = 0;
2209
2210             send_to_client(new_apdu);
2211
2212             return 0;
2213         }
2214     }
2215     return apdu;
2216 }
2217
2218 int Yaz_Proxy::handle_authentication(Z_APDU *apdu)
2219 {
2220     if (apdu->which != Z_APDU_initRequest)
2221         return 1;  // pass if no init request
2222     Z_InitRequest *req = apdu->u.initRequest;
2223
2224     Yaz_ProxyConfig *cfg = check_reconfigure();
2225     if (!cfg)
2226         return 1;  // pass if no config
2227
2228     int ret;
2229     if (req->idAuthentication == 0)
2230     {
2231         ret = cfg->client_authentication(m_default_target, 0, 0, 0,
2232                                          m_peername);
2233     }
2234     else if (req->idAuthentication->which == Z_IdAuthentication_idPass)
2235     {
2236         ret = cfg->client_authentication(
2237             m_default_target,
2238             req->idAuthentication->u.idPass->userId,
2239             req->idAuthentication->u.idPass->groupId,
2240             req->idAuthentication->u.idPass->password,
2241             m_peername);
2242     }
2243     else if (req->idAuthentication->which == Z_IdAuthentication_open)
2244     {
2245         char user[64], pass[64];
2246         *user = '\0';
2247         *pass = '\0';
2248         sscanf(req->idAuthentication->u.open, "%63[^/]/%63s", user, pass);
2249         ret = cfg->client_authentication(m_default_target, user, 0, pass,
2250                                          m_peername);
2251     }
2252     else
2253         ret = cfg->client_authentication(m_default_target, 0, 0, 0,
2254                                          m_peername);
2255     return ret;
2256 }
2257
2258 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
2259 {
2260     m_marcxml_mode = none;
2261     if (apdu->which == Z_APDU_searchRequest)
2262     {
2263         Z_SearchRequest *sr = apdu->u.searchRequest;
2264         int err = 0;
2265         char *addinfo = 0;
2266         Yaz_ProxyConfig *cfg = check_reconfigure();
2267
2268         Z_RecordComposition rc_temp, *rc = 0;
2269         if (sr->smallSetElementSetNames)
2270         {
2271             rc_temp.which = Z_RecordComp_simple;
2272             rc_temp.u.simple = sr->smallSetElementSetNames;
2273             rc = &rc_temp;
2274         }
2275
2276         if (sr->preferredRecordSyntax)
2277         {
2278             struct oident *ent;
2279             ent = oid_getentbyoid(sr->preferredRecordSyntax);
2280             m_frontend_type = ent->value;
2281         }
2282         else
2283             m_frontend_type = VAL_NONE;
2284
2285         char *stylesheet_name = 0;
2286         if (cfg)
2287             err = cfg->check_syntax(odr_encode(),
2288                                     m_default_target,
2289                                     sr->preferredRecordSyntax, rc,
2290                                     &addinfo, &stylesheet_name, &m_schema,
2291                                     &m_backend_type, &m_backend_charset,
2292                                     &m_usemarcon_ini_stage1,
2293                                     &m_usemarcon_ini_stage2);
2294         if (stylesheet_name)
2295         {
2296             m_parent->low_socket_close();
2297
2298 #if HAVE_XSLT
2299             if (m_stylesheet_xsp)
2300                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2301             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2302                                                        stylesheet_name);
2303 #endif
2304             m_stylesheet_offset = 0;
2305             xfree(stylesheet_name);
2306
2307             m_parent->low_socket_open();
2308         }
2309         if (err == -1)
2310         {
2311             sr->smallSetElementSetNames = 0;
2312             sr->mediumSetElementSetNames = 0;
2313             m_marcxml_mode = marcxml;
2314             if (m_backend_type)
2315             {
2316
2317                 sr->preferredRecordSyntax =
2318                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
2319                                         m_backend_type);
2320             }
2321             else
2322                 sr->preferredRecordSyntax =
2323                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
2324                                            VAL_USMARC);
2325         }
2326         else if (err)
2327         {
2328             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2329
2330             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2331             new_apdu->u.searchResponse->records =
2332                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2333             *new_apdu->u.searchResponse->searchStatus = 0;
2334
2335             send_to_client(new_apdu);
2336
2337             return 0;
2338         }
2339         else if (m_backend_type)
2340         {
2341             sr->preferredRecordSyntax =
2342                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
2343         }
2344     }
2345     else if (apdu->which == Z_APDU_presentRequest)
2346     {
2347         Z_PresentRequest *pr = apdu->u.presentRequest;
2348         int err = 0;
2349         char *addinfo = 0;
2350         Yaz_ProxyConfig *cfg = check_reconfigure();
2351
2352         if (pr->preferredRecordSyntax)
2353         {
2354             struct oident *ent;
2355             ent = oid_getentbyoid(pr->preferredRecordSyntax);
2356             m_frontend_type = ent->value;
2357         }
2358         else
2359             m_frontend_type = VAL_NONE;
2360
2361         char *stylesheet_name = 0;
2362         if (cfg)
2363             err = cfg->check_syntax(odr_encode(), m_default_target,
2364                                     pr->preferredRecordSyntax,
2365                                     pr->recordComposition,
2366                                     &addinfo, &stylesheet_name, &m_schema,
2367                                     &m_backend_type, &m_backend_charset,
2368                                     &m_usemarcon_ini_stage1,
2369                                     &m_usemarcon_ini_stage2
2370                                     );
2371         if (stylesheet_name)
2372         {
2373             m_parent->low_socket_close();
2374
2375 #if HAVE_XSLT
2376             if (m_stylesheet_xsp)
2377                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2378             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2379                                                        stylesheet_name);
2380 #endif
2381             m_stylesheet_offset = 0;
2382             xfree(stylesheet_name);
2383
2384             m_parent->low_socket_open();
2385         }
2386         if (err == -1)
2387         {
2388             pr->recordComposition = 0;
2389             m_marcxml_mode = marcxml;
2390             if (m_backend_type)
2391             {
2392
2393                 pr->preferredRecordSyntax =
2394                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
2395                                         m_backend_type);
2396             }
2397             else
2398                 pr->preferredRecordSyntax =
2399                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
2400                                            VAL_USMARC);
2401         }
2402         else if (err)
2403         {
2404             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
2405
2406             new_apdu->u.presentResponse->referenceId = pr->referenceId;
2407             new_apdu->u.presentResponse->records =
2408                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2409             *new_apdu->u.presentResponse->presentStatus =
2410                 Z_PresentStatus_failure;
2411
2412             send_to_client(new_apdu);
2413
2414             return 0;
2415         }
2416         else if (m_backend_type)
2417         {
2418             pr->preferredRecordSyntax =
2419                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
2420         }
2421     }
2422     return apdu;
2423 }
2424
2425 Z_ElementSetNames *Yaz_Proxy::mk_esn_from_schema(ODR o, const char *schema)
2426 {
2427     if (!schema)
2428         return 0;
2429     Z_ElementSetNames *esn = (Z_ElementSetNames *)
2430         odr_malloc(o, sizeof(Z_ElementSetNames));
2431     esn->which = Z_ElementSetNames_generic;
2432     esn->u.generic = odr_strdup(o, schema);
2433     return esn;
2434 }
2435
2436 void Yaz_Proxy::srw_get_client(const char *db, const char **backend_db)
2437 {
2438     const char *t = 0;
2439     Yaz_ProxyConfig *cfg = check_reconfigure();
2440     if (cfg)
2441         t = cfg->get_explain_name(db, backend_db);
2442
2443     if (m_client && m_default_target && t && strcmp(m_default_target, t))
2444     {
2445         releaseClient();
2446     }
2447
2448     if (t)
2449     {
2450         xfree(m_default_target);
2451         m_default_target = xstrdup(t);
2452     }
2453 }
2454
2455 int Yaz_Proxy::file_access(Z_HTTP_Request *hreq)
2456 {
2457     struct stat sbuf;
2458     if (strcmp(hreq->method, "GET"))
2459         return 0;
2460     if (hreq->path[0] != '/')
2461     {
2462         yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
2463         return 0;
2464     }
2465     const char *cp = hreq->path;
2466     while (*cp)
2467     {
2468         if (*cp == '/' && strchr("/.", cp[1]))
2469         {
2470             yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
2471             return 0;
2472         }
2473         cp++;
2474     }
2475     const char *fname = hreq->path+1;
2476     if (stat(fname, &sbuf))
2477     {
2478         yaz_log(YLOG_WARN|YLOG_ERRNO, "%s: stat failed", fname);
2479         return 0;
2480     }
2481     if ((sbuf.st_mode & S_IFMT) != S_IFREG)
2482     {
2483         yaz_log(YLOG_WARN, "%s: not a regular file", fname);
2484         return 0;
2485     }
2486     if (sbuf.st_size > (off_t) 1000000)
2487     {
2488         yaz_log(YLOG_WARN, "%s: too large for transfer", fname);
2489         return 0;
2490     }
2491
2492     ODR o = odr_encode();
2493     Yaz_ProxyConfig *cfg = check_reconfigure();
2494     const char *ctype = cfg->check_mime_type(fname);
2495     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
2496     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
2497     if (m_http_version)
2498         hres->version = odr_strdup(o, m_http_version);
2499     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
2500     if (m_http_keepalive)
2501         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
2502     else
2503         timeout(0);
2504
2505     hres->content_len = sbuf.st_size;
2506     hres->content_buf = (char*) odr_malloc(o, hres->content_len);
2507     FILE *f = fopen(fname, "rb");
2508     if (f)
2509     {
2510         fread(hres->content_buf, 1, hres->content_len, f);
2511         fclose(f);
2512     }
2513     else
2514     {
2515         return 0;
2516     }
2517     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
2518     {
2519         yaz_log (YLOG_LOG, "%sSending file %s to client", m_session_str,
2520                  fname);
2521     }
2522     int len;
2523     send_GDU(gdu, &len);
2524     recv_GDU_more(true);
2525     return 1;
2526 }
2527
2528 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
2529 {
2530     if (m_s2z_odr_init)
2531     {
2532         odr_destroy(m_s2z_odr_init);
2533         m_s2z_odr_init = 0;
2534     }
2535     if (m_s2z_odr_search)
2536     {
2537         odr_destroy(m_s2z_odr_search);
2538         m_s2z_odr_search = 0;
2539     }
2540
2541     m_http_keepalive = 0;
2542     m_http_version = 0;
2543     if (!strcmp(hreq->version, "1.0"))
2544     {
2545         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2546         if (v && !strcmp(v, "Keep-Alive"))
2547             m_http_keepalive = 1;
2548         else
2549             m_http_keepalive = 0;
2550         m_http_version = "1.0";
2551     }
2552     else
2553     {
2554         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2555         if (v && !strcmp(v, "close"))
2556             m_http_keepalive = 0;
2557         else
2558             m_http_keepalive = 1;
2559         m_http_version = "1.1";
2560     }
2561
2562     const char *a = z_HTTP_header_lookup(hreq->headers, "Authorization");
2563     char authorization_str[255];
2564     *authorization_str = '\0';
2565     if (a && strncasecmp(a, "Basic ", 6) == 0)
2566         base64_decode(a + 6, authorization_str, 254);
2567
2568     Z_SRW_PDU *srw_pdu = 0;
2569     Z_SOAP *soap_package = 0;
2570     char *charset = 0;
2571     Z_SRW_diagnostic *diagnostic = 0;
2572     int num_diagnostic = 0;
2573
2574     if (file_access(hreq))
2575     {
2576         return;
2577     }
2578     else if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2579                             &charset) == 0
2580              || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2581                                &charset, &diagnostic, &num_diagnostic) == 0)
2582     {
2583         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
2584         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
2585         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns);
2586         m_s2z_init_apdu = 0;
2587         m_s2z_search_apdu = 0;
2588         m_s2z_present_apdu = 0;
2589
2590         m_s2z_stylesheet = 0;
2591
2592         Z_IdAuthentication *auth = NULL;
2593         if (*authorization_str)
2594         {
2595             auth = (Z_IdAuthentication *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdAuthentication));
2596             auth->which = Z_IdAuthentication_idPass;
2597             auth->u.idPass = (Z_IdPass *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdPass));
2598             auth->u.idPass->groupId = NULL;
2599             char *p = strchr(authorization_str, ':');
2600             if (p)
2601             {
2602                 *p = '\0';
2603                 p++;
2604                 auth->u.idPass->password = odr_strdup(m_s2z_odr_init, p);
2605             }
2606             auth->u.idPass->userId = odr_strdup(m_s2z_odr_init, authorization_str);
2607         }
2608
2609         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
2610         {
2611             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
2612
2613             const char *backend_db = srw_req->database;
2614             srw_get_client(srw_req->database, &backend_db);
2615
2616             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2617             // recordXPath unsupported.
2618             if (srw_req->recordXPath)
2619             {
2620                 yaz_add_srw_diagnostic(odr_decode(),
2621                                        &diagnostic, &num_diagnostic,
2622                                        72, 0);
2623             }
2624             // sort unsupported
2625             if (srw_req->sort_type != Z_SRW_sort_type_none)
2626             {
2627                 yaz_add_srw_diagnostic(odr_decode(),
2628                                        &diagnostic, &num_diagnostic,
2629                                        80, 0);
2630             }
2631             // save stylesheet
2632             if (srw_req->stylesheet)
2633                 m_s2z_stylesheet =
2634                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2635
2636             // set packing for response records ..
2637             if (srw_req->recordPacking &&
2638                 !strcmp(srw_req->recordPacking, "xml"))
2639                 m_s2z_packing = Z_SRW_recordPacking_XML;
2640             else
2641                 m_s2z_packing = Z_SRW_recordPacking_string;
2642
2643             if (num_diagnostic)
2644             {
2645                 Z_SRW_PDU *srw_pdu =
2646                     yaz_srw_get(odr_encode(),
2647                                 Z_SRW_searchRetrieve_response);
2648                 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
2649
2650                 srw_res->diagnostics = diagnostic;
2651                 srw_res->num_diagnostics = num_diagnostic;
2652                 send_srw_response(srw_pdu);
2653                 return;
2654             }
2655
2656             // prepare search PDU
2657             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
2658                                           Z_APDU_searchRequest);
2659             Z_SearchRequest *z_searchRequest =
2660                 m_s2z_search_apdu->u.searchRequest;
2661
2662             z_searchRequest->num_databaseNames = 1;
2663             z_searchRequest->databaseNames = (char**)
2664                 odr_malloc(m_s2z_odr_search, sizeof(char *));
2665             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
2666                                                            backend_db);
2667
2668             // query transformation
2669             Z_Query *query = (Z_Query *)
2670                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
2671             z_searchRequest->query = query;
2672
2673             if (srw_req->query_type == Z_SRW_query_type_cql)
2674             {
2675                 Z_External *ext = (Z_External *)
2676                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
2677                 ext->direct_reference =
2678                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
2679                 ext->indirect_reference = 0;
2680                 ext->descriptor = 0;
2681                 ext->which = Z_External_CQL;
2682                 ext->u.cql = srw_req->query.cql;
2683
2684                 query->which = Z_Query_type_104;
2685                 query->u.type_104 =  ext;
2686             }
2687             else if (srw_req->query_type == Z_SRW_query_type_pqf)
2688             {
2689                 Z_RPNQuery *RPNquery;
2690                 YAZ_PQF_Parser pqf_parser;
2691
2692                 pqf_parser = yaz_pqf_create ();
2693
2694                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
2695                                           srw_req->query.pqf);
2696                 if (!RPNquery)
2697                 {
2698                     const char *pqf_msg;
2699                     size_t off;
2700                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
2701                     int ioff = off;
2702                     yaz_log(YLOG_LOG, "%*s^\n", ioff+4, "");
2703                     yaz_log(YLOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
2704
2705                     send_to_srw_client_error(10, 0);
2706                     return;
2707                 }
2708                 query->which = Z_Query_type_1;
2709                 query->u.type_1 =  RPNquery;
2710
2711                 yaz_pqf_destroy (pqf_parser);
2712             }
2713             else
2714             {
2715                 send_to_srw_client_error(7, "query");
2716                 return;
2717             }
2718
2719             // present
2720             m_s2z_present_apdu = 0;
2721             int max = 0;
2722             if (srw_req->maximumRecords)
2723                 max = *srw_req->maximumRecords;
2724             int start = 1;
2725             if (srw_req->startRecord)
2726                 start = *srw_req->startRecord;
2727             if (max > 0)
2728             {
2729                 // Some backend, such as Voyager doesn't honor piggyback
2730                 // So we use present always (0 &&).
2731                 if (0 && start <= 1)  // Z39.50 piggyback
2732                 {
2733                     *z_searchRequest->smallSetUpperBound = max;
2734                     *z_searchRequest->mediumSetPresentNumber = max;
2735                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
2736
2737                     z_searchRequest->preferredRecordSyntax =
2738                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2739                                                VAL_TEXT_XML);
2740                     if (srw_req->recordSchema)
2741                     {
2742                         z_searchRequest->smallSetElementSetNames =
2743                             z_searchRequest->mediumSetElementSetNames =
2744                             mk_esn_from_schema(m_s2z_odr_search,
2745                                                srw_req->recordSchema);
2746                     }
2747                 }
2748                 else   // Z39.50 present
2749                 {
2750                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
2751                                                    Z_APDU_presentRequest);
2752                     Z_PresentRequest *z_presentRequest =
2753                         m_s2z_present_apdu->u.presentRequest;
2754                     *z_presentRequest->resultSetStartPoint = start;
2755                     *z_presentRequest->numberOfRecordsRequested = max;
2756                     z_presentRequest->preferredRecordSyntax =
2757                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2758                                                VAL_TEXT_XML);
2759                     if (srw_req->recordSchema)
2760                     {
2761                         z_presentRequest->recordComposition =
2762                             (Z_RecordComposition *)
2763                             odr_malloc(m_s2z_odr_search,
2764                                        sizeof(Z_RecordComposition));
2765                         z_presentRequest->recordComposition->which =
2766                             Z_RecordComp_simple;
2767                         z_presentRequest->recordComposition->u.simple =
2768                             mk_esn_from_schema(m_s2z_odr_search,
2769                                                srw_req->recordSchema);
2770                     }
2771                 }
2772             }
2773             if (!m_client)
2774             {
2775                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2776                                             Z_APDU_initRequest);
2777
2778                 m_s2z_init_apdu->u.initRequest->idAuthentication = auth;
2779
2780                 // prevent m_initRequest_apdu memory from being grabbed
2781                 // in Yaz_Proxy::handle_incoming_Z_PDU
2782                 m_initRequest_apdu = m_s2z_init_apdu;
2783                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2784                 return;
2785             }
2786             else
2787             {
2788                 handle_incoming_Z_PDU(m_s2z_search_apdu);
2789                 return;
2790             }
2791         }
2792         else if (srw_pdu->which == Z_SRW_explain_request)
2793         {
2794             Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
2795
2796             const char *backend_db = srw_req->database;
2797             srw_get_client(srw_req->database, &backend_db);
2798
2799             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2800
2801             // save stylesheet
2802             if (srw_req->stylesheet)
2803                 m_s2z_stylesheet =
2804                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2805
2806             if (srw_req->recordPacking &&
2807                 !strcmp(srw_req->recordPacking, "xml"))
2808                 m_s2z_packing = Z_SRW_recordPacking_XML;
2809             else
2810                 m_s2z_packing = Z_SRW_recordPacking_string;
2811
2812             if (num_diagnostic)
2813             {
2814                 send_srw_explain_response(diagnostic, num_diagnostic);
2815                 return;
2816             }
2817
2818             if (!m_client)
2819             {
2820                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2821                                             Z_APDU_initRequest);
2822
2823                 m_s2z_init_apdu->u.initRequest->idAuthentication = auth;
2824                 
2825                 // prevent m_initRequest_apdu memory from being grabbed
2826                 // in Yaz_Proxy::handle_incoming_Z_PDU
2827                 m_initRequest_apdu = m_s2z_init_apdu;
2828                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2829             }
2830             else
2831                 send_srw_explain_response(0, 0);
2832             return;
2833         }
2834         else if (srw_pdu->which == Z_SRW_scan_request)
2835         {
2836             m_s2z_database = odr_strdup(m_s2z_odr_init,
2837                                         srw_pdu->u.scan_request->database);
2838
2839             yaz_add_srw_diagnostic(odr_decode(),
2840                                    &diagnostic, &num_diagnostic,
2841                                    4, "scan");
2842             Z_SRW_PDU *srw_pdu =
2843                 yaz_srw_get(odr_encode(),
2844                             Z_SRW_scan_response);
2845             Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
2846
2847             srw_res->diagnostics = diagnostic;
2848             srw_res->num_diagnostics = num_diagnostic;
2849             send_srw_response(srw_pdu);
2850             return;
2851         }
2852         else
2853         {
2854             m_s2z_database = 0;
2855
2856             send_to_srw_client_error(4, 0);
2857         }
2858     }
2859     send_http_response(400);
2860 }
2861
2862 void Yaz_Proxy::handle_init(Z_APDU *apdu)
2863 {
2864
2865     Z_OtherInformation **oi;
2866     get_otherInfoAPDU(apdu, &oi);
2867
2868     if (apdu->u.initRequest->implementationId)
2869         yaz_log(YLOG_LOG, "%simplementationId: %s",
2870                 m_session_str, apdu->u.initRequest->implementationId);
2871     if (apdu->u.initRequest->implementationName)
2872         yaz_log(YLOG_LOG, "%simplementationName: %s",
2873                 m_session_str, apdu->u.initRequest->implementationName);
2874     if (apdu->u.initRequest->implementationVersion)
2875         yaz_log(YLOG_LOG, "%simplementationVersion: %s",
2876                 m_session_str, apdu->u.initRequest->implementationVersion);
2877     if (m_initRequest_apdu == 0)
2878     {
2879         if (m_initRequest_mem)
2880             nmem_destroy(m_initRequest_mem);
2881
2882         m_initRequest_apdu = apdu;
2883         m_initRequest_mem = odr_extract_mem(odr_decode());
2884
2885         m_initRequest_preferredMessageSize = *apdu->u.initRequest->
2886             preferredMessageSize;
2887         *apdu->u.initRequest->preferredMessageSize = 1024*1024;
2888         m_initRequest_maximumRecordSize = *apdu->u.initRequest->
2889             maximumRecordSize;
2890         *apdu->u.initRequest->maximumRecordSize = 1024*1024;
2891
2892         Z_CharSetandLanguageNegotiation *charSetandLangRecord =
2893             yaz_get_charneg_record(*oi);
2894
2895         // Save proposal charsets and langs.
2896         if (ODR_MASK_GET(apdu->u.initRequest->options,
2897                          Z_Options_negotiationModel)
2898             && charSetandLangRecord)
2899         {
2900
2901             yaz_get_proposal_charneg(m_referenceId_mem,
2902                                      charSetandLangRecord,
2903                                      &m_initRequest_oi_negotiation_charsets,
2904                                      &m_initRequest_oi_negotiation_num_charsets,
2905                                      &m_initRequest_oi_negotiation_langs,
2906                                      &m_initRequest_oi_negotiation_num_langs,
2907                                      &m_initRequest_oi_negotiation_selected);
2908
2909             for (int i = 0; i<m_initRequest_oi_negotiation_num_charsets; i++)
2910             {
2911                 yaz_log(YLOG_LOG, "%scharacters set proposal: %s",
2912                         m_session_str,(m_initRequest_oi_negotiation_charsets[i])?
2913                         m_initRequest_oi_negotiation_charsets[i]:"none");
2914             }
2915             for (int i=0; i<m_initRequest_oi_negotiation_num_langs; i++)
2916             {
2917                 yaz_log(YLOG_LOG, "%slanguages proposal: %s",
2918                         m_session_str, (m_initRequest_oi_negotiation_langs[i])?
2919                         m_initRequest_oi_negotiation_langs[i]:"none");
2920             }
2921             yaz_log(YLOG_LOG, "%sselected proposal: %d (boolean)",
2922                     m_session_str, m_initRequest_oi_negotiation_selected);
2923         }
2924         // save init options for the response..
2925         m_initRequest_options = apdu->u.initRequest->options;
2926
2927         apdu->u.initRequest->options =
2928             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2929                                        sizeof(Odr_bitmask));
2930         ODR_MASK_ZERO(apdu->u.initRequest->options);
2931         int i;
2932         for (i = 0; i<= 24; i++)
2933             ODR_MASK_SET(apdu->u.initRequest->options, i);
2934         // check negotiation option
2935         if (!ODR_MASK_GET(m_initRequest_options,
2936                           Z_Options_negotiationModel))
2937         {
2938             ODR_MASK_CLEAR(apdu->u.initRequest->options,
2939                            Z_Options_negotiationModel);
2940         }
2941         ODR_MASK_CLEAR(apdu->u.initRequest->options,
2942                        Z_Options_concurrentOperations);
2943         // make new version
2944         m_initRequest_version = apdu->u.initRequest->protocolVersion;
2945         apdu->u.initRequest->protocolVersion =
2946             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2947                                        sizeof(Odr_bitmask));
2948         ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
2949
2950         for (i = 0; i<= 8; i++)
2951             ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
2952     }
2953     handle_charset_lang_negotiation(apdu);
2954     if (m_client->m_init_flag)
2955     {
2956         if (handle_init_response_for_invalid_session(apdu))
2957             return;
2958         if (m_client->m_initResponse)
2959         {
2960             Z_APDU *apdu2 = m_client->m_initResponse;
2961             apdu2->u.initResponse->otherInfo = 0;
2962             if (m_client->m_cookie && *m_client->m_cookie)
2963                 set_otherInformationString(apdu2, VAL_COOKIE, 1,
2964                                            m_client->m_cookie);
2965             apdu2->u.initResponse->referenceId =
2966                 apdu->u.initRequest->referenceId;
2967             apdu2->u.initResponse->options = m_client->m_initResponse_options;
2968             apdu2->u.initResponse->protocolVersion =
2969                 m_client->m_initResponse_version;
2970
2971             handle_charset_lang_negotiation(apdu2);
2972
2973             send_to_client(apdu2);
2974             m_timeout_mode = timeout_normal;
2975             return;
2976         }
2977     }
2978     m_client->m_init_flag = 1;
2979
2980 #if USE_AUTH_MSG
2981     yaz_log(YLOG_LOG, "%suse_auth_msg", m_session_str);
2982     Auth_Msg *m = new Auth_Msg;
2983     m->m_proxy = this;
2984     z_APDU(odr_encode(), &apdu, 0, "encode");
2985     char *apdu_buf = odr_getbuf(odr_encode(), &m->m_apdu_len, 0);
2986     m->m_apdu_buf = (char*) nmem_malloc(m->m_nmem, m->m_apdu_len);
2987     memcpy(m->m_apdu_buf, apdu_buf, m->m_apdu_len);
2988     odr_reset(odr_encode());
2989
2990     inc_ref();
2991     m_my_thread->put(m);
2992 #else
2993     int ret = handle_authentication(apdu);
2994     result_authentication(apdu, ret);
2995 #endif
2996 }
2997
2998 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
2999 {
3000     Z_ReferenceId **refid = get_referenceIdP(apdu);
3001     nmem_reset(m_referenceId_mem);
3002     if (refid && *refid)
3003     {
3004         m_referenceId = (Z_ReferenceId *)
3005             nmem_malloc(m_referenceId_mem, sizeof(*m_referenceId));
3006         m_referenceId->len = m_referenceId->size = (*refid)->len;
3007         m_referenceId->buf = (unsigned char *)
3008             nmem_malloc(m_referenceId_mem, (*refid)->len);
3009         memcpy(m_referenceId->buf, (*refid)->buf, (*refid)->len);
3010     }
3011     else
3012         m_referenceId = 0;
3013
3014     if (!m_client && m_flag_invalid_session)
3015     {
3016         // Got request for a session that is invalid..
3017         m_apdu_invalid_session = apdu; // save package
3018         m_mem_invalid_session = odr_extract_mem(odr_decode());
3019         apdu = m_initRequest_apdu;     // but throw an init to the target
3020     }
3021
3022     // Determine our client.
3023     Z_OtherInformation **oi;
3024     get_otherInfoAPDU(apdu, &oi);
3025     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi));
3026     if (!m_client)
3027     {
3028         if (m_http_version)
3029         {   // HTTP. Send not found
3030             send_http_response(404);
3031             return;
3032         }
3033         else
3034         {
3035             // Z39.50 just shutdown
3036             delete this;
3037             return;
3038         }
3039     }
3040
3041     m_client->m_server = this;
3042
3043     if (apdu->which == Z_APDU_initRequest)
3044         handle_init(apdu);
3045     else
3046         handle_incoming_Z_PDU_2(apdu);
3047 }
3048
3049 void Yaz_Proxy::handle_incoming_Z_PDU_2(Z_APDU *apdu)
3050 {
3051     handle_max_record_retrieve(apdu);
3052
3053     if (apdu)
3054         apdu = handle_syntax_validation(apdu);
3055
3056     if (apdu)
3057         apdu = handle_query_transformation(apdu);
3058
3059     if (apdu)
3060         apdu = handle_target_charset_conversion(apdu);
3061
3062     if (apdu)
3063         apdu = handle_query_validation(apdu);
3064
3065     if (apdu)
3066         apdu = result_set_optimize(apdu);
3067
3068     if (!apdu)
3069     {
3070         m_client->timeout(m_target_idletime);  // mark it active even
3071         recv_GDU_more(true);
3072         // though we didn't use it
3073         return;
3074     }
3075
3076     // delete other info construct completely if 0 elements
3077     Z_OtherInformation **oi;
3078     get_otherInfoAPDU(apdu, &oi);
3079     if (oi && *oi && (*oi)->num_elements == 0)
3080         *oi = 0;
3081
3082     if (apdu->which == Z_APDU_presentRequest &&
3083         m_client->m_resultSetStartPoint == 0)
3084     {
3085         Z_PresentRequest *pr = apdu->u.presentRequest;
3086         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
3087         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
3088     } else {
3089         m_client->m_resultSetStartPoint = 0;
3090     }
3091     if (m_client->send_to_target(apdu) < 0)
3092     {
3093         m_client->shutdown();
3094     }
3095     else
3096         m_client->m_waiting = 1;
3097 }
3098
3099 void Yaz_Proxy::connectNotify()
3100 {
3101 }
3102
3103 void Yaz_Proxy::releaseClient()
3104 {
3105     xfree(m_proxyTarget);
3106     m_proxyTarget = 0;
3107     m_flag_invalid_session = 0;
3108     // only keep if keep_alive flag is set...
3109     if (m_client &&
3110         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
3111         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
3112         m_client->m_waiting == 0)
3113     {
3114         yaz_log(YLOG_LOG, "%sShutdown (client to proxy) keepalive %s",
3115                  m_session_str,
3116                  m_client->get_hostname());
3117         yaz_log(YLOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
3118                 m_session_str, m_client->m_pdu_recv,
3119                 m_client->m_bytes_sent + m_client->m_bytes_recv,
3120                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
3121         assert (m_client->m_waiting != 2);
3122         // Tell client (if any) that no server connection is there..
3123         m_client->m_server = 0;
3124         m_client = 0;
3125     }
3126     else if (m_client)
3127     {
3128         yaz_log (YLOG_LOG, "%sShutdown (client to proxy) close %s",
3129                  m_session_str,
3130                  m_client->get_hostname());
3131         assert (m_client->m_waiting != 2);
3132         delete m_client;
3133         m_client = 0;
3134     }
3135     else if (!m_parent)
3136     {
3137         yaz_log (YLOG_LOG, "%sshutdown (client to proxy) bad state",
3138                  m_session_str);
3139         assert (m_parent);
3140     }
3141     else
3142     {
3143         yaz_log (YLOG_LOG, "%sShutdown (client to proxy)",
3144                  m_session_str);
3145     }
3146     if (m_parent)
3147         m_parent->pre_init();
3148 }
3149
3150 bool Yaz_Proxy::dec_ref(bool main_ptr)
3151 {
3152     yaz_log(YLOG_LOG, "%sdec_ref count=%d", m_session_str, m_ref_count);
3153
3154     assert(m_ref_count > 0);
3155     if (main_ptr)
3156     {
3157         if (m_main_ptr_dec)
3158             return false;
3159         m_main_ptr_dec = true;
3160     }
3161
3162     m_http_keepalive = 0;
3163
3164     --m_ref_count;
3165     if (m_ref_count > 0)
3166         return false;
3167
3168     releaseClient();
3169
3170     delete this;
3171     return true;
3172 }
3173
3174 const char *Yaz_ProxyClient::get_session_str()
3175 {
3176     if (!m_server)
3177         return "0 ";
3178     return m_server->get_session_str();
3179 }
3180
3181 void Yaz_ProxyClient::shutdown()
3182 {
3183     yaz_log (YLOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
3184              get_hostname());
3185
3186     if (m_server)
3187     {
3188         m_waiting = 1;      // ensure it's released from Proxy in releaseClient
3189         m_server->dec_ref(true);
3190     }
3191     else
3192         delete this;
3193 }
3194
3195 void Yaz_Proxy::failNotify()
3196 {
3197     inc_request_no();
3198     yaz_log (YLOG_LOG, "%sConnection closed by client",
3199              get_session_str());
3200     dec_ref(true);
3201 }
3202
3203 void Yaz_Proxy::send_response_fail_client(const char *addr)
3204 {
3205     yaz_log(YLOG_LOG, "%ssend_close_response", get_session_str());
3206     if (m_http_version)
3207     {
3208         Z_SRW_diagnostic *diagnostic = 0;
3209         int num_diagnostic = 0;
3210         
3211         yaz_add_srw_diagnostic(odr_encode(),
3212                                &diagnostic, &num_diagnostic,
3213                                YAZ_SRW_SYSTEM_TEMPORARILY_UNAVAILABLE, addr);
3214         if (m_s2z_search_apdu)
3215             send_srw_search_response(diagnostic, num_diagnostic);
3216         else
3217             send_srw_explain_response(diagnostic, num_diagnostic);
3218     }            
3219 }
3220 void Yaz_ProxyClient::failNotify()
3221 {
3222     if (m_server)
3223         m_server->inc_request_no();
3224     yaz_log (YLOG_LOG, "%sConnection closed by target %s",
3225              get_session_str(), get_hostname());
3226
3227     if (m_server)
3228         m_server->send_response_fail_client(get_hostname());
3229     shutdown();
3230 }
3231
3232 void Yaz_ProxyClient::connectNotify()
3233 {
3234     const char *s = get_session_str();
3235     const char *h = get_hostname();
3236     yaz_log (YLOG_LOG, "%sConnection accepted by %s timeout=%d", s, h,
3237              m_target_idletime);
3238     timeout(m_target_idletime);
3239     if (!m_server)
3240         pre_init_client();
3241 }
3242
3243 IPDU_Observer *Yaz_ProxyClient::sessionNotify(IPDU_Observable
3244                                               *the_PDU_Observable, int fd)
3245 {
3246     return new Yaz_ProxyClient(the_PDU_Observable, 0);
3247 }
3248
3249 Yaz_ProxyClient::~Yaz_ProxyClient()
3250 {
3251     if (m_prev)
3252         *m_prev = m_next;
3253     if (m_next)
3254         m_next->m_prev = m_prev;
3255     m_waiting = 2;     // for debugging purposes only.
3256     odr_destroy(m_init_odr);
3257     odr_destroy(m_idAuthentication_odr);
3258     delete m_last_query;
3259     xfree (m_last_resultSetId);
3260     xfree (m_cookie);
3261 }
3262
3263 void Yaz_ProxyClient::pre_init_client()
3264 {
3265     Z_APDU *apdu = create_Z_PDU(Z_APDU_initRequest);
3266     Z_InitRequest *req = apdu->u.initRequest;
3267
3268     int i;
3269     for (i = 0; i<= 24; i++)
3270         ODR_MASK_SET(req->options, i);
3271     ODR_MASK_CLEAR(apdu->u.initRequest->options,
3272                    Z_Options_negotiationModel);
3273     ODR_MASK_CLEAR(apdu->u.initRequest->options,
3274                    Z_Options_concurrentOperations);
3275     for (i = 0; i<= 10; i++)
3276         ODR_MASK_SET(req->protocolVersion, i);
3277
3278     if (send_to_target(apdu) < 0)
3279     {
3280         delete this;
3281     }
3282     else
3283     {
3284         m_waiting = 1;
3285         m_init_flag = 1;
3286     }
3287 }
3288
3289 void Yaz_Proxy::pre_init()
3290 {
3291     int i;
3292     const char *name = 0;
3293     const char *zurl_in_use[MAX_ZURL_PLEX];
3294     int limit_bw, limit_pdu, limit_req, limit_search, limit_connect;
3295     int target_idletime, client_idletime;
3296     int max_clients;
3297     int keepalive_limit_bw, keepalive_limit_pdu;
3298     int pre_init;
3299     const char *cql2rpn = 0;
3300     const char *authentication = 0;
3301     const char *negotiation_charset = 0;
3302     const char *negotiation_lang = 0;
3303
3304     Yaz_ProxyConfig *cfg = check_reconfigure();
3305
3306     zurl_in_use[0] = 0;
3307
3308     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
3309         set_APDU_yazlog(1);
3310     else
3311         set_APDU_yazlog(0);
3312
3313     for (i = 0; cfg && cfg->get_target_no(i, &name, zurl_in_use,
3314                                           &limit_bw, &limit_pdu, &limit_req,
3315                                           &limit_search, &limit_connect,
3316                                           &target_idletime, &client_idletime,
3317                                           &max_clients,
3318                                           &keepalive_limit_bw,
3319                                           &keepalive_limit_pdu,
3320                                           &pre_init,
3321                                           &cql2rpn,
3322                                           &authentication,
3323                                           &negotiation_charset,
3324                                           &negotiation_lang,
3325                                           0,
3326                                           0) ; i++)
3327     {
3328         if (pre_init)
3329         {
3330             int j;
3331             for (j = 0; zurl_in_use[j]; j++)
3332             {
3333                 Yaz_ProxyClient *c;
3334                 int spare = 0;
3335                 int spare_waiting = 0;
3336                 int in_use = 0;
3337                 int other = 0;
3338                 for (c = m_clientPool; c; c = c->m_next)
3339                 {
3340                     if (!strcmp(zurl_in_use[j], c->get_hostname()))
3341                     {
3342                         if (c->m_cookie == 0)
3343                         {
3344                             if (c->m_server == 0)
3345                                 if (c->m_waiting)
3346                                     spare_waiting++;
3347                                 else
3348                                     spare++;
3349                             else
3350                                 in_use++;
3351                         }
3352                         else