Bump version to 1.3.3.1
[yazproxy-moved-to-github.git] / src / yaz-proxy.cpp
1 /* This file is part of YAZ proxy
2    Copyright (C) 1998-2008 Index Data
3
4 YAZ proxy is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
7 version.
8
9 YAZ proxy is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12 for more details.
13
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
17 */
18
19 #ifdef WIN32
20 #define HAVE_SYS_STAT_H 1
21 #define HAVE_SYS_TYPES_H 1
22 #endif
23
24 #if HAVE_UNISTD_H
25 #include <unistd.h>
26 #endif
27 #if HAVE_SYS_TIME_H
28 #include <sys/time.h>
29 #endif
30 #if HAVE_SYS_TYPES_H
31 #include <sys/types.h>
32 #endif
33 #if HAVE_SYS_STAT_H
34 #include <sys/stat.h>
35 #endif
36
37 #include <assert.h>
38 #include <stdlib.h>
39 #include <time.h>
40 #include <fcntl.h>
41
42 #include <yaz/srw.h>
43 #include <yaz/marcdisp.h>
44 #include <yaz/yaz-iconv.h>
45 #include <yaz/log.h>
46 #include <yaz/diagbib1.h>
47 #include "proxyp.h"
48 #include <yaz/pquery.h>
49 #include <yaz/otherinfo.h>
50 #include <yaz/charneg.h>
51 #include <yaz/oid_db.h>
52 #include "msg-thread.h"
53
54 using namespace yazpp_1;
55
56 #ifdef WIN32
57 #define strncasecmp _strnicmp
58 #endif
59
60 class YAZ_EXPORT Auth_Msg : public IMsg_Thread {
61 public:
62     int m_ret;
63     IMsg_Thread *handle();
64     void result();
65     Yaz_Proxy *m_proxy;
66     NMEM m_nmem;
67     char *m_apdu_buf;
68     int m_apdu_len;
69     Auth_Msg();
70     virtual ~Auth_Msg();
71 };
72
73 Auth_Msg::Auth_Msg()
74 {
75     m_nmem = nmem_create();
76 }
77
78 Auth_Msg::~Auth_Msg()
79 {
80     nmem_destroy(m_nmem);
81 }
82
83 IMsg_Thread *Auth_Msg::handle()
84 {
85     ODR decode = odr_createmem(ODR_DECODE);
86     Z_APDU *apdu;
87
88     odr_setbuf(decode, m_apdu_buf, m_apdu_len, 0);
89     int r = z_APDU(decode, &apdu, 0, 0);
90     if (!r)
91     {
92         yaz_log(YLOG_WARN, "decode failed in Auth_Msg::handle");
93     }
94     else
95     {
96         m_ret = m_proxy->handle_authentication(apdu);
97     }
98     odr_destroy(decode);
99     return this;
100 }
101
102 void Auth_Msg::result()
103 {
104     if (m_proxy->dec_ref())
105     {
106         yaz_log(YLOG_LOG, "Auth_Msg::proxy deleted meanwhile");
107     }
108     else
109     {
110         odr_setbuf(m_proxy->odr_decode(), m_apdu_buf, m_apdu_len, 0);
111         Z_APDU *apdu = 0;
112         int r = z_APDU(m_proxy->odr_decode(), &apdu, 0, 0);
113         if (!r)
114             yaz_log(YLOG_LOG, "Auth_Msg::result z_APDU failed");
115         m_proxy->result_authentication(apdu, m_ret);
116     }
117     delete this;
118 }
119
120 void Yaz_Proxy::result_authentication(Z_APDU *apdu, int ret)
121 {
122     if (apdu == 0 || ret == 0)
123     {
124         Z_APDU *apdu_reject = zget_APDU(odr_encode(), Z_APDU_initResponse);
125         *apdu_reject->u.initResponse->result = 0;
126         send_to_client(apdu_reject);
127         dec_ref();
128     }
129     else
130     {
131         if (apdu->which == Z_APDU_initRequest)
132         {
133             Yaz_ProxyConfig *cfg = check_reconfigure();
134             if (cfg)
135                 cfg->target_authentication(m_default_target, odr_encode(), 
136                                            apdu->u.initRequest);
137         }
138         handle_incoming_Z_PDU_2(apdu);
139     }
140 }
141
142 static const char *apdu_name(Z_APDU *apdu)
143 {
144     switch (apdu->which)
145     {
146     case Z_APDU_initRequest:
147         return "initRequest";
148     case Z_APDU_initResponse:
149         return "initResponse";
150     case Z_APDU_searchRequest:
151         return "searchRequest";
152     case Z_APDU_searchResponse:
153         return "searchResponse";
154     case Z_APDU_presentRequest:
155         return "presentRequest";
156     case Z_APDU_presentResponse:
157         return "presentResponse";
158     case Z_APDU_deleteResultSetRequest:
159         return "deleteResultSetRequest";
160     case Z_APDU_deleteResultSetResponse:
161         return "deleteResultSetResponse";
162     case Z_APDU_scanRequest:
163         return "scanRequest";
164     case Z_APDU_scanResponse:
165         return "scanResponse";
166     case Z_APDU_sortRequest:
167         return "sortRequest";
168     case Z_APDU_sortResponse:
169         return "sortResponse";
170     case Z_APDU_extendedServicesRequest:
171         return "extendedServicesRequest";
172     case Z_APDU_extendedServicesResponse:
173         return "extendedServicesResponse";
174     case Z_APDU_close:
175         return "close";
176     }
177     return "other";
178 }
179
180 static const char *gdu_name(Z_GDU *gdu)
181 {
182     switch(gdu->which)
183     {
184     case Z_GDU_Z3950:
185         return apdu_name(gdu->u.z3950);
186     case Z_GDU_HTTP_Request:
187         return "HTTP Request";
188     case Z_GDU_HTTP_Response:
189         return "HTTP Response";
190     }
191     return "Unknown request/response";
192 }
193
194 Yaz_Proxy::Yaz_Proxy(IPDU_Observable *the_PDU_Observable,
195                      ISocketObservable *the_socket_observable,
196                      Yaz_Proxy *parent)
197     :
198     Z_Assoc(the_PDU_Observable),
199     m_bw_stat(60), m_pdu_stat(60), m_search_stat(60)
200 {
201     m_PDU_Observable = the_PDU_Observable;
202     m_socket_observable = the_socket_observable;
203     m_client = 0;
204     m_parent = parent;
205     m_clientPool = 0;
206     m_seqno = 1;
207     m_keepalive_limit_bw = 500000;
208     m_keepalive_limit_pdu = 1000;
209     m_proxyTarget = 0;
210     m_default_target = 0;
211     m_proxy_negotiation_charset = 0;
212     m_proxy_negotiation_lang = 0;
213     m_proxy_negotiation_default_charset = 0;
214     m_charset_converter = new Yaz_CharsetConverter;
215     m_max_clients = 150;
216     m_log_mask = 0;
217     m_seed = time(0);
218     m_client_idletime = 600;
219     m_target_idletime = 600;
220     m_max_sockets = 1024;
221     m_optimize = xstrdup("1");
222     strcpy(m_session_str, "0 ");
223     m_session_no = 0;
224     m_bytes_sent = 0;
225     m_bytes_recv = 0;
226     m_bw_max = 0;
227     m_pdu_max = 0;
228     m_search_max = 0;
229     m_max_connect = 0;
230     m_max_connect_period = 0;
231     m_limit_connect = 0;
232     m_limit_connect_period = 0;
233     m_timeout_mode = timeout_normal;
234     m_timeout_gdu = 0;
235     m_max_record_retrieve = 0;
236     m_reconfig_flag = 0;
237     m_config_fname = 0;
238     m_request_no = 0;
239     m_flag_invalid_session = 0;
240     m_referenceId = 0;
241     m_referenceId_mem = nmem_create();
242     m_config = 0;
243     m_marcxml_mode = none;
244     m_stylesheet_xsp = 0;
245     m_stylesheet_nprl = 0;
246     m_stylesheet_apdu = 0;
247     m_s2z_stylesheet = 0;
248     m_s2z_database = 0;
249     m_schema = 0;
250     m_backend_type = 0;
251     m_backend_charset = 0;
252     m_frontend_type[0] = -1;
253     m_initRequest_apdu = 0;
254     m_initRequest_mem = 0;
255     m_initRequest_preferredMessageSize = 0;
256     m_initRequest_maximumRecordSize = 0;
257     m_initRequest_options = 0;
258     m_initRequest_version = 0;
259     m_initRequest_oi_negotiation_charsets = 0;
260     m_initRequest_oi_negotiation_num_charsets = 0;
261     m_initRequest_oi_negotiation_langs = 0;
262     m_initRequest_oi_negotiation_num_langs = 0;
263     m_initRequest_oi_negotiation_selected = 0;
264     m_apdu_invalid_session = 0;
265     m_mem_invalid_session = 0;
266     m_s2z_odr_init = 0;
267     m_s2z_odr_search = 0;
268     m_s2z_init_apdu = 0;
269     m_s2z_search_apdu = 0;
270     m_s2z_present_apdu = 0;
271     m_http_keepalive = 0;
272     m_http_version = 0;
273     m_soap_ns = 0;
274     m_s2z_packing = Z_SRW_recordPacking_string;
275 #if HAVE_GETTIMEOFDAY
276     m_time_tv = xmalloc(sizeof(struct timeval));
277     struct timeval *tv = (struct timeval *) m_time_tv;
278     tv->tv_sec = 0;
279     tv->tv_usec = 0;
280 #else
281     m_time_tv = 0;
282 #endif
283     m_usemarcon_ini_stage1 = 0;
284     m_usemarcon_ini_stage2 = 0;
285     m_usemarcon = new Yaz_usemarcon();
286     if (!m_parent)
287         low_socket_open();
288     m_my_thread = 0;
289     m_ref_count = 1;
290     m_main_ptr_dec = false;
291     m_peername = 0;
292     m_num_msg_threads = 0;
293 }
294
295 void Yaz_Proxy::inc_ref()
296 {
297     m_ref_count++;
298 }
299
300 Yaz_Proxy::~Yaz_Proxy()
301 {
302     yaz_log(YLOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
303             m_bytes_sent, m_bytes_recv);
304     nmem_destroy(m_initRequest_mem);
305     nmem_destroy(m_mem_invalid_session);
306     nmem_destroy(m_referenceId_mem);
307
308     xfree(m_proxyTarget);
309     xfree(m_default_target);
310     xfree(m_proxy_negotiation_charset);
311     xfree(m_proxy_negotiation_lang);
312     xfree(m_proxy_negotiation_default_charset);
313     delete m_charset_converter;
314     xfree(m_optimize);
315
316 #if YAZ_HAVE_XSLT
317     if (m_stylesheet_xsp)
318         xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
319 #endif
320     xfree(m_time_tv);
321
322     xfree(m_peername);
323     xfree(m_schema);
324     xfree(m_backend_type);
325     xfree(m_backend_charset);
326     xfree(m_usemarcon_ini_stage1);
327     xfree(m_usemarcon_ini_stage2);
328     delete m_usemarcon;
329     if (m_s2z_odr_init)
330         odr_destroy(m_s2z_odr_init);
331     if (m_s2z_odr_search)
332         odr_destroy(m_s2z_odr_search);
333     if (!m_parent)
334         low_socket_close();
335     if (!m_parent)
336         delete m_my_thread;
337     delete m_config;
338 }
339
340 void Yaz_Proxy::set_debug_mode(int mode)
341 {
342     m_debug_mode = mode;
343 }
344
345 int Yaz_Proxy::set_config(const char *config)
346 {
347     delete m_config;
348     m_config = new Yaz_ProxyConfig();
349     xfree(m_config_fname);
350     m_config_fname = xstrdup(config);
351     int r = m_config->read_xml(config);
352     if (!r)
353     {
354         int period = 60;
355         m_config->get_generic_info(&m_log_mask, &m_max_clients,
356                                    &m_max_connect, &m_limit_connect, &period,
357                                    &m_num_msg_threads);
358         m_connect.set_period(period);
359     }
360     return r;
361 }
362
363 void Yaz_Proxy::set_default_target(const char *target)
364 {
365     xfree(m_default_target);
366     m_default_target = 0;
367     if (target)
368         m_default_target = (char *) xstrdup(target);
369 }
370
371 void Yaz_Proxy::set_proxy_negotiation(const char *charset, const char *lang,
372                                        const char *default_charset)
373 {
374     yaz_log(YLOG_DEBUG, "%sSet the proxy negotiation: charset to '%s', "
375         "default charset to '%s', language to '%s'", m_session_str, 
376         charset?charset:"none",
377         default_charset?default_charset:"none",
378         lang?lang:"none");
379     xfree(m_proxy_negotiation_charset);
380     xfree(m_proxy_negotiation_lang);
381     m_proxy_negotiation_charset = m_proxy_negotiation_lang = 0;
382     if (charset)
383         m_proxy_negotiation_charset = (char *) xstrdup(charset);
384     if (lang)
385         m_proxy_negotiation_lang = (char *) xstrdup(lang);
386     if (default_charset)
387         m_proxy_negotiation_default_charset =
388             (char *) xstrdup(default_charset);
389 }
390
391 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
392 {
393     if (m_parent)
394         return m_parent->check_reconfigure();
395
396     Yaz_ProxyConfig *cfg = m_config;
397     if (m_reconfig_flag)
398     {
399         yaz_log(YLOG_LOG, "reconfigure");
400         if (m_config_fname && cfg)
401         {
402             yaz_log(YLOG_LOG, "reconfigure config %s", m_config_fname);
403             int r = cfg->read_xml(m_config_fname);
404             if (r)
405                 yaz_log(YLOG_WARN, "reconfigure failed");
406             else
407             {
408                 m_log_mask = 0;
409                 int period = 60;
410                 cfg->get_generic_info(&m_log_mask, &m_max_clients,
411                                       &m_max_connect, &m_limit_connect,
412                                       &period, &m_num_msg_threads);
413                 m_connect.set_period(period);
414             }
415         }
416         else
417             yaz_log(YLOG_LOG, "reconfigure");
418         m_reconfig_flag = 0;
419     }
420     return cfg;
421 }
422
423 IPDU_Observer *Yaz_Proxy::sessionNotify(IPDU_Observable
424                                         *the_PDU_Observable, int fd)
425 {
426     check_reconfigure();
427
428     char session_str[200];
429     const char *peername = the_PDU_Observable->getpeername();
430     if (!peername)
431         peername = "nullpeer";
432
433     if (m_log_mask & PROXY_LOG_IP_CLIENT)
434         sprintf(session_str, "%ld:%d %.80s %d ",
435                 (long) time(0), m_session_no, peername, 0);
436     else
437         sprintf(session_str, "%ld:%d %d ",
438                 (long) time(0), m_session_no, 0);
439     m_session_no++;
440
441     yaz_log(YLOG_LOG, "%sNew session %s", session_str, peername);
442
443     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable,
444                                          m_socket_observable, this);
445
446     new_proxy->m_config = 0;
447     new_proxy->m_config_fname = 0;
448     new_proxy->timeout(m_client_idletime);
449     new_proxy->m_target_idletime = m_target_idletime;
450     new_proxy->set_default_target(m_default_target);
451     new_proxy->m_max_clients = m_max_clients;
452     new_proxy->m_log_mask = m_log_mask;
453     new_proxy->m_session_no = m_session_no;
454     new_proxy->m_num_msg_threads = m_num_msg_threads;
455
456 #if 0
457     // in case we want to watch a particular client..
458     if (!strcmp(peername, "tcp:163.121.19.82")) // NIS GROUP
459         new_proxy->m_log_mask = 255;
460 #endif
461
462     new_proxy->set_APDU_log(get_APDU_log());
463     if (new_proxy->m_log_mask & PROXY_LOG_APDU_CLIENT)
464         new_proxy->set_APDU_yazlog(1);
465     else
466         new_proxy->set_APDU_yazlog(0);
467     strcpy(new_proxy->m_session_str, session_str);
468     new_proxy->m_peername = xstrdup(peername);
469     new_proxy->set_proxy_negotiation(m_proxy_negotiation_charset,
470         m_proxy_negotiation_lang, m_proxy_negotiation_default_charset);
471     // create thread object the first time we get an incoming connection
472     if (!m_my_thread && m_num_msg_threads > 0)
473     {
474         yaz_log(YLOG_LOG, "%sStarting message thread management. number=%d",
475                  session_str, m_num_msg_threads);
476         m_my_thread = new Msg_Thread(m_socket_observable, m_num_msg_threads);
477     }
478     new_proxy->m_my_thread = m_my_thread;
479     return new_proxy;
480 }
481
482 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
483 {
484     Z_OtherInformationUnit *oi =
485         update_otherInformation(otherInfo, 0, yaz_oid_userinfo_cookie, 1, 1);
486     
487     if (oi && oi->which == Z_OtherInfo_characterInfo)
488         return oi->information.characterInfo;
489     return 0;
490 }
491
492 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
493 {
494     Z_OtherInformationUnit *oi =
495         update_otherInformation(otherInfo, 0, yaz_oid_userinfo_proxy, 1, 1);
496     
497     if (oi && oi->which == Z_OtherInfo_characterInfo)
498         return oi->information.characterInfo;
499     return 0;
500 }
501 const char *Yaz_Proxy::load_balance(const char **url)
502 {
503     int zurl_in_use[MAX_ZURL_PLEX];
504     int zurl_in_spare[MAX_ZURL_PLEX];
505     Yaz_ProxyClient *c;
506     int i;
507
508     for (i = 0; i<MAX_ZURL_PLEX; i++)
509     {
510         zurl_in_use[i] = 0;
511         zurl_in_spare[i] = 0;
512     }
513     for (c = m_parent->m_clientPool; c; c = c->m_next)
514     {
515         for (i = 0; url[i]; i++)
516             if (!strcmp(url[i], c->get_hostname()))
517             {
518                 zurl_in_use[i]++;
519                 if (c->m_cookie == 0 && c->m_server == 0 && c->m_waiting == 0)
520                     zurl_in_spare[i]++;
521             }
522     }
523     int min_use = 100000;
524     int spare_for_min = 0;
525     int max_spare = 0;
526     const char *ret_min = 0;
527     const char *ret_spare = 0;
528     for (i = 0; url[i]; i++)
529     {
530         yaz_log(YLOG_DEBUG, "%szurl=%s use=%d spare=%d",
531                 m_session_str, url[i], zurl_in_use[i], zurl_in_spare[i]);
532         if (min_use > zurl_in_use[i])
533         {
534             ret_min = url[i];
535             min_use = zurl_in_use[i];
536             spare_for_min = zurl_in_spare[i];
537         }
538         if (max_spare < zurl_in_spare[i])
539         {
540             ret_spare = url[i];
541             max_spare = zurl_in_spare[i];
542         }
543     }
544     return ret_min;
545 }
546
547 int Yaz_Proxy::get_number_of_connections()
548 {
549     int no_connections = 0;
550     Yaz_ProxyClient *c;
551
552     for (c = m_parent->m_clientPool; c; c = c->m_next)
553     {
554         assert(c->m_prev);
555         assert(*c->m_prev == c);
556         if (!strcmp(m_proxyTarget, c->get_hostname()))
557         {
558             no_connections++;
559         }
560     }
561     yaz_log(YLOG_LOG, "%sExisting %s connections: %d", m_session_str, m_proxyTarget,
562         no_connections);
563     return no_connections;
564 }
565
566 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
567                                        const char *proxy_host, int *http_code)
568 {
569     assert(m_parent);
570     Yaz_Proxy *parent = m_parent;
571     Yaz_ProxyClient *c = m_client;
572
573     if (!m_proxyTarget)
574     {
575         const char *url[MAX_ZURL_PLEX];
576         Yaz_ProxyConfig *cfg = check_reconfigure();
577         if (proxy_host)
578         {
579             if (parent && parent->m_debug_mode)
580             {
581                 // only to be enabled for debugging...
582                 if (!strcmp(proxy_host, "stop"))
583                     exit(0);
584             }
585             xfree(m_default_target);
586             m_default_target = xstrdup(proxy_host);
587         }
588         proxy_host = m_default_target;
589         int client_idletime = -1;
590         const char *cql2rpn_fname = 0;
591         const char *negotiation_charset = 0;
592         const char *negotiation_lang = 0;
593         const char *query_charset = 0;
594         const char *default_client_query_charset = 0;
595         url[0] = m_default_target;
596         url[1] = 0;
597         if (cfg)
598         {
599             int pre_init = 0;
600             cfg->get_target_info(proxy_host, url, &m_bw_max,
601                                  &m_pdu_max, &m_max_record_retrieve,
602                                  &m_search_max,
603                                  &m_target_idletime, &client_idletime,
604                                  &m_max_sockets,
605                                  &parent->m_max_clients,
606                                  &m_keepalive_limit_bw,
607                                  &m_keepalive_limit_pdu,
608                                  &pre_init,
609                                  &cql2rpn_fname,
610                                  &negotiation_charset,
611                                  &negotiation_lang,
612                                  &query_charset,
613                                  &default_client_query_charset);
614         }
615         if (client_idletime != -1)
616         {
617             m_client_idletime = client_idletime;
618             timeout(m_client_idletime);
619         }
620
621         // get those FILE descriptors available 
622         m_parent->low_socket_close();
623         if (cql2rpn_fname)
624             m_cql2rpn.set_pqf_file(cql2rpn_fname);
625         // reserve them again
626         m_parent->low_socket_open();
627         
628         if (negotiation_charset || negotiation_lang || default_client_query_charset)
629         {
630             set_proxy_negotiation(negotiation_charset,
631                 negotiation_lang, default_client_query_charset);
632         }
633         m_charset_converter->set_target_query_charset(query_charset);
634         if (!url[0])
635         {
636             yaz_log(YLOG_LOG, "%sNo default target", m_session_str);
637             return 0;
638         }
639         // we don't handle multiplexing for cookie session, so we just
640         // pick the first one in this case (anonymous users will be able
641         // to use any backend)
642         if (cookie && *cookie)
643             m_proxyTarget = (char*) xstrdup(url[0]);
644         else
645             m_proxyTarget = (char*) xstrdup(load_balance(url));
646     }
647     if (cookie && *cookie)
648     {   // search in sessions with a cookie
649         for (c = parent->m_clientPool; c; c = c->m_next)
650         {
651             assert(c->m_prev);
652             assert(*c->m_prev == c);
653             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
654                 !strcmp(m_proxyTarget, c->get_hostname()))
655             {
656                 // Found it in cache
657                 // The following handles "cancel"
658                 // If connection is busy (waiting for PDU) and
659                 // we have an initRequest we can safely do re-open
660                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
661                 {
662                     yaz_log(YLOG_LOG, "%s REOPEN target=%s", m_session_str,
663                              c->get_hostname());
664                     c->close();
665                     c->m_init_flag = 0;
666
667                     c->m_last_ok = 0;
668                     c->m_cache.clear();
669                     c->m_last_resultCount = 0;
670                     c->m_sr_transform = 0;
671                     c->m_waiting = 0;
672                     c->m_resultSetStartPoint = 0;
673                     c->m_target_idletime = m_target_idletime;
674                     if (c->client(m_proxyTarget))
675                     {
676                         delete c;
677                         return 0;
678                     }
679                     c->timeout(30);
680                 }
681                 c->m_seqno = parent->m_seqno;
682                 if (c->m_server && c->m_server != this)
683                     c->m_server->m_client = 0;
684                 c->m_server = this;
685                 (parent->m_seqno)++;
686                 yaz_log(YLOG_DEBUG, "get_client 1 %p %p", this, c);
687                 return c;
688             }
689         }
690     }
691     else if (!c && apdu->which == Z_APDU_initRequest )
692     {
693         // anonymous sessions without cookie.
694         // if authentication is set it is NOT anonymous se we can't share them.
695         // If charset and lang negotiation is use it is NOT anonymous session too.
696         for (c = parent->m_clientPool; c; c = c->m_next)
697         {
698             assert(c->m_prev);
699             assert(*c->m_prev == c);
700             if (c->m_server == 0 && c->m_cookie == 0 && c->m_waiting == 0 
701                 && c->compare_idAuthentication(apdu)
702                 && c->compare_charset(apdu)
703                 && !strcmp(m_proxyTarget, c->get_hostname()))
704             {
705                 // found it in cache
706                 yaz_log(YLOG_LOG, "%sREUSE %d %s",
707                          m_session_str, parent->m_seqno, c->get_hostname());
708                 
709                 c->m_seqno = parent->m_seqno;
710                 assert(c->m_server == 0);
711                 c->m_server = this;
712
713                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
714                     c->set_APDU_yazlog(1);
715                 else
716                     c->set_APDU_yazlog(0);
717
718                 (parent->m_seqno)++;
719
720                 parent->pre_init();
721
722                 return c;
723             }
724         }
725     }
726     if (!m_client)
727     {
728         if (apdu->which != Z_APDU_initRequest)
729         {
730             yaz_log(YLOG_LOG, "%sno init request as first PDU", m_session_str);
731             *http_code = 500;
732             return 0;
733         }
734
735         int no_in_use = get_number_of_connections();
736         if (no_in_use >= m_max_sockets)
737         {
738             yaz_log(YLOG_LOG, "%smax sockets reached %d", m_session_str,
739                 m_max_sockets);
740             *http_code = 500;
741             return 0;
742         }
743         // go through list of clients - and find the lowest/oldest one.
744         Yaz_ProxyClient *c_min = 0;
745         int min_seq = -1;
746         int no_of_clients = 0;
747         if (parent->m_clientPool)
748             yaz_log(YLOG_DEBUG, "Existing sessions");
749         for (c = parent->m_clientPool; c; c = c->m_next)
750         {
751             yaz_log(YLOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
752                                c->m_waiting, c->get_hostname(),
753                                c->m_cookie ? c->m_cookie : "");
754             no_of_clients++;
755             if (min_seq < 0 || c->m_seqno < min_seq)
756             {
757                 min_seq = c->m_seqno;
758                 c_min = c;
759             }
760         }
761         if (no_of_clients >= parent->m_max_clients)
762         {
763             c = c_min;
764             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
765             {
766                 yaz_log(YLOG_LOG, "%sMAXCLIENTS %d Destroy %d",
767                          m_session_str, parent->m_max_clients, c->m_seqno);
768                 if (c->m_server && c->m_server != this)
769                     c->m_server->dec_ref();
770             }
771             else
772             {
773                 yaz_log(YLOG_LOG, "%sMAXCLIENTS %d Reuse %d %d %s",
774                          m_session_str, parent->m_max_clients,
775                          c->m_seqno, parent->m_seqno, c->get_hostname());
776                 xfree(c->m_cookie);
777                 c->m_cookie = 0;
778                 if (cookie)
779                     c->m_cookie = xstrdup(cookie);
780                 c->m_seqno = parent->m_seqno;
781                 if (c->m_server && c->m_server != this)
782                 {
783                     c->m_server->m_client = 0;
784                     c->m_server->dec_ref();
785                 }
786                 (parent->m_seqno)++;
787                 c->m_target_idletime = m_target_idletime;
788                 c->timeout(m_target_idletime);
789
790                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
791                     c->set_APDU_yazlog(1);
792                 else
793                     c->set_APDU_yazlog(0);
794
795                 return c;
796             }
797         }
798         else
799         {
800
801             yaz_log(YLOG_LOG, "%sNEW %d %s",
802                      m_session_str, parent->m_seqno, m_proxyTarget);
803             c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
804             c->m_next = parent->m_clientPool;
805             if (c->m_next)
806                 c->m_next->m_prev = &c->m_next;
807             parent->m_clientPool = c;
808             c->m_prev = &parent->m_clientPool;
809         }
810
811         xfree(c->m_cookie);
812         c->m_cookie = 0;
813         if (cookie)
814             c->m_cookie = xstrdup(cookie);
815
816         c->m_seqno = parent->m_seqno;
817         c->m_init_flag = 0;
818         c->m_last_resultCount = 0;
819         c->m_last_ok = 0;
820         c->m_cache.clear();
821         c->m_sr_transform = 0;
822         c->m_waiting = 0;
823         c->m_resultSetStartPoint = 0;
824         (parent->m_seqno)++;
825         if (c->client(m_proxyTarget))
826         {
827             delete c;
828             return 0;
829         }
830         c->m_target_idletime = m_target_idletime;
831         c->timeout(30);
832
833         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
834             c->set_APDU_yazlog(1);
835         else
836             c->set_APDU_yazlog(0);
837
838         c->set_idAuthentication(apdu);
839     }
840     yaz_log(YLOG_DEBUG, "get_client 3 %p %p", this, c);
841     return c;
842 }
843
844 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
845 {
846     int i;
847     for (i = 0; i<num; i++)
848     {
849         Z_DefaultDiagFormat *r;
850         Z_DiagRec *p = pp[i];
851         if (p->which != Z_DiagRec_defaultFormat)
852         {
853             yaz_log(YLOG_LOG, "%sError no diagnostics", m_session_str);
854             return;
855         }
856         else
857             r = p->u.defaultFormat;
858         switch (r->which)
859         {
860         case Z_DefaultDiagFormat_v2Addinfo:
861             yaz_log(YLOG_LOG, "%sError %d %s:%s",
862                     m_session_str,
863                     *r->condition, diagbib1_str(*r->condition),
864                     r->u.v2Addinfo);
865             break;
866         case Z_DefaultDiagFormat_v3Addinfo:
867             yaz_log(YLOG_LOG, "%sError %d %s:%s",
868                     m_session_str,
869                     *r->condition, diagbib1_str(*r->condition),
870                     r->u.v3Addinfo);
871             break;
872         }
873     }
874 }
875
876 int Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p, Z_APDU *apdu)
877 {
878     if (!m_stylesheet_xsp || p->num_records <= 0)
879     {
880         return 0;  /* no XSLT to be done ... */
881     }
882
883     m_stylesheet_offset = 0;
884     m_stylesheet_nprl = p;
885     m_stylesheet_apdu = apdu;
886     m_timeout_mode = timeout_xsl;
887
888     timeout(0);
889     return 1;
890 }
891
892 void Yaz_Proxy::convert_xsl_delay()
893 {
894 #if YAZ_HAVE_XSLT
895     Z_NamePlusRecord *npr = m_stylesheet_nprl->records[m_stylesheet_offset];
896     if (npr->which == Z_NamePlusRecord_databaseRecord)
897     {
898         Z_External *r = npr->u.databaseRecord;
899         if (r->which == Z_External_octet)
900         {
901 #if 0
902             fwrite((char*) r->u.octet_aligned->buf, 1, r->u.octet_aligned->len, stdout);
903 #endif
904             xmlDocPtr res, doc = xmlParseMemory(
905                 (char*) r->u.octet_aligned->buf,
906                 r->u.octet_aligned->len);
907
908
909             yaz_log(YLOG_LOG, "%sXSLT convert %d",
910                     m_session_str, m_stylesheet_offset);
911             res = xsltApplyStylesheet((xsltStylesheetPtr) m_stylesheet_xsp,
912                                       doc, 0);
913
914             if (res)
915             {
916                 xmlChar *out_buf;
917                 int out_len;
918                 xmlDocDumpFormatMemory(res, &out_buf, &out_len, 1);
919                 m_stylesheet_nprl->records[m_stylesheet_offset]->
920                     u.databaseRecord =
921                     z_ext_record_oid(odr_encode(), yaz_oid_recsyn_xml,
922                                      (char*) out_buf, out_len);
923                 xmlFree(out_buf);
924                 xmlFreeDoc(res);
925             }
926
927             xmlFreeDoc(doc);
928         }
929     }
930 #endif
931     m_stylesheet_offset++;
932     if (m_stylesheet_offset == m_stylesheet_nprl->num_records)
933     {
934         m_timeout_mode = timeout_normal;
935         m_stylesheet_nprl = 0;
936 #if YAZ_HAVE_XSLT
937         if (m_stylesheet_xsp)
938             xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
939 #endif
940         m_stylesheet_xsp = 0;
941         timeout(m_client_idletime);
942         send_PDU_convert(m_stylesheet_apdu);
943     }
944     else
945         timeout(0);
946 }
947
948 void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p)
949 {
950     if (m_frontend_type[0] != -1)
951     {
952         int i;
953         for (i = 0; i < p->num_records; i++)
954         {
955             Z_NamePlusRecord *npr = p->records[i];
956             if (npr->which == Z_NamePlusRecord_databaseRecord)
957             {
958                 Z_External *r = npr->u.databaseRecord;
959                 if (r->which == Z_External_octet)
960                 {
961 #if !HAVE_USEMARCON
962                     if (m_usemarcon_ini_stage1 && *m_usemarcon_ini_stage1)
963                         yaz_log(YLOG_LOG, "%sError: USEMARCON requested but not available",
964                             m_session_str);
965 #endif
966 #if HAVE_USEMARCON
967                     yaz_log(YLOG_DEBUG, "%sUSEMARCON stage1=%s stage2=%s",
968                         m_session_str,
969                         m_usemarcon_ini_stage1 ? m_usemarcon_ini_stage1 : "(none)",
970                         m_usemarcon_ini_stage2 ? m_usemarcon_ini_stage2 : "(none)");
971                     char *converted;
972                     int convlen;
973                     if (m_usemarcon->convert(m_usemarcon_ini_stage1, m_usemarcon_ini_stage2,
974                         (char*) r->u.octet_aligned->buf, r->u.octet_aligned->len,
975                         &converted, &convlen))
976                     {
977                         npr->u.databaseRecord =
978                             z_ext_record_oid(odr_encode(),
979                                              m_frontend_type,
980                                              converted,
981                                              strlen(converted));
982                         free(converted);
983                     }
984                     else
985 #endif
986 /* HAVE_USEMARCON */
987                     npr->u.databaseRecord =
988                         z_ext_record_oid(odr_encode(),
989                                          m_frontend_type,
990                                          (char*) r->u.octet_aligned->buf,
991                                          r->u.octet_aligned->len);
992                 }
993             }
994         }
995     }
996 }
997
998 void Yaz_Proxy::convert_records_charset(Z_NamePlusRecordList *p,
999                                         const char *backend_charset)
1000 {
1001     int sel =   m_charset_converter->get_client_charset_selected();
1002     const char *client_record_charset =
1003         m_charset_converter->get_client_query_charset();
1004     if (sel && backend_charset && client_record_charset &&
1005         strcmp(backend_charset, client_record_charset))
1006     {
1007         int i;
1008         yaz_iconv_t cd = yaz_iconv_open(client_record_charset,
1009                                         backend_charset);
1010         yaz_marc_t mt = yaz_marc_create();
1011         yaz_marc_xml(mt, YAZ_MARC_ISO2709);
1012         yaz_marc_iconv(mt, cd);
1013         for (i = 0; i < p->num_records; i++)
1014         {
1015             Z_NamePlusRecord *npr = p->records[i];
1016             if (npr->which == Z_NamePlusRecord_databaseRecord)
1017             {
1018                 Z_External *r = npr->u.databaseRecord;
1019                 const Odr_oid *oid = r->direct_reference;
1020                 if (!oid)
1021                     continue;
1022
1023                 if (!oid_oidcmp(oid, yaz_oid_recsyn_sutrs))
1024                 {
1025                     WRBUF w = wrbuf_alloc();
1026
1027                     wrbuf_iconv_write(w, cd,  (char*) r->u.octet_aligned->buf,
1028                                       r->u.octet_aligned->len);
1029                     npr->u.databaseRecord =
1030                         z_ext_record_oid(odr_encode(), oid, wrbuf_buf(w),
1031                                          wrbuf_len(w));
1032                     wrbuf_destroy(w);
1033                 }
1034                 else if (!oid_oidcmp(oid, yaz_oid_recsyn_xml))
1035                 {
1036                     ;
1037                 }
1038                 else if (r->which == Z_External_octet)
1039                 {
1040                     size_t rlen;
1041                     const char *result;
1042                     if (yaz_marc_decode_buf(mt,
1043                                             (char*) r->u.octet_aligned->buf,
1044                                             r->u.octet_aligned->len,
1045                                             &result, &rlen))
1046                     {
1047                         npr->u.databaseRecord =
1048                             z_ext_record_oid(odr_encode(), oid, result, rlen);
1049                         yaz_log(YLOG_LOG, "%sRecoding MARC record",
1050                                 m_session_str);
1051                     }
1052                 }
1053             }
1054         }
1055         if (cd)
1056             yaz_iconv_close(cd);
1057         yaz_marc_destroy(mt);
1058     }
1059 }
1060
1061 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p,
1062                                    const char *backend_charset)
1063 {
1064     int i;
1065     if (!backend_charset)
1066         backend_charset = "MARC-8";
1067     yaz_iconv_t cd = yaz_iconv_open("UTF-8", backend_charset);
1068     yaz_marc_t mt = yaz_marc_create();
1069     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
1070     yaz_marc_iconv(mt, cd);
1071     for (i = 0; i < p->num_records; i++)
1072     {
1073         Z_NamePlusRecord *npr = p->records[i];
1074         if (npr->which == Z_NamePlusRecord_databaseRecord)
1075         {
1076             Z_External *r = npr->u.databaseRecord;
1077             if (r->which == Z_External_OPAC)
1078             {
1079                 WRBUF w = wrbuf_alloc();
1080
1081                 yaz_opac_decode_wrbuf(mt, r->u.opac, w);
1082                 npr->u.databaseRecord = z_ext_record_oid(
1083                     odr_encode(), yaz_oid_recsyn_xml,
1084                     wrbuf_buf(w), wrbuf_len(w));
1085                 wrbuf_destroy(w);
1086             }
1087             else if (r->which == Z_External_octet)
1088             {
1089                 size_t rlen;
1090                 const char *result;
1091                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
1092                                         r->u.octet_aligned->len,
1093                                         &result, &rlen))
1094                 {
1095                     npr->u.databaseRecord =
1096                         z_ext_record_oid(odr_encode(), yaz_oid_recsyn_xml,
1097                                          result, rlen);
1098                 }
1099             }
1100         }
1101     }
1102     if (cd)
1103         yaz_iconv_close(cd);
1104     yaz_marc_destroy(mt);
1105 }
1106
1107 void Yaz_Proxy::logtime()
1108 {
1109 #if HAVE_GETTIMEOFDAY
1110     struct timeval *tv = (struct timeval*) m_time_tv;
1111     if (tv->tv_sec)
1112     {
1113         struct timeval tv1;
1114         gettimeofday(&tv1, 0);
1115         long diff = (tv1.tv_sec - tv->tv_sec)*1000000 +
1116             (tv1.tv_usec - tv->tv_usec);
1117         if (diff >= 0)
1118             yaz_log(YLOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
1119                     diff/1000000, (diff/1000)%1000);
1120     }
1121     tv->tv_sec = 0;
1122     tv->tv_usec = 0;
1123 #endif
1124 }
1125
1126 int Yaz_Proxy::send_http_response(int code)
1127 {
1128     ODR o = odr_encode();
1129     Z_GDU *gdu = z_get_HTTP_Response(o, code);
1130     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1131     if (m_http_version)
1132         hres->version = odr_strdup(o, m_http_version);
1133     if (m_http_keepalive)
1134         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1135     else
1136         timeout(0);
1137     if (code == 401)
1138         z_HTTP_header_add(o, &hres->headers, "WWW-Authenticate", 
1139                           "Basic realm=\"YAZ Proxy\"");
1140
1141
1142     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1143     {
1144         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1145                  gdu_name(gdu));
1146     }
1147     int len;
1148     int r = send_GDU(gdu, &len);
1149     m_bytes_sent += len;
1150     m_bw_stat.add_bytes(len);
1151     logtime();
1152
1153     recv_GDU_more(true);
1154
1155     return r;
1156 }
1157
1158 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu, int http_code /* = 200 */)
1159 {
1160     ODR o = odr_encode();
1161     const char *ctype = "text/xml";
1162     Z_GDU *gdu = z_get_HTTP_Response(o, http_code);
1163     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1164     if (m_http_version)
1165         hres->version = odr_strdup(o, m_http_version);
1166     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
1167     if (m_http_keepalive)
1168         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1169     else
1170         timeout(0);
1171     if (http_code == 401)
1172         z_HTTP_header_add(o, &hres->headers, "WWW-Authenticate", "Basic realm=\"YAZ Proxy\"");
1173
1174     static Z_SOAP_Handler soap_handlers[2] = {
1175 #if YAZ_HAVE_XSLT
1176         { (char*) "http://www.loc.gov/zing/srw/", 0,
1177          (Z_SOAP_fun) yaz_srw_codec},
1178 #endif
1179         {0, 0, 0}
1180     };
1181
1182     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
1183     soap_package->which = Z_SOAP_generic;
1184     soap_package->u.generic =
1185         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
1186     soap_package->u.generic->no = 0;
1187     soap_package->u.generic->ns = soap_handlers[0].ns;
1188     soap_package->u.generic->p = (void *) srw_pdu;
1189     soap_package->ns = m_soap_ns;
1190     z_soap_codec_enc_xsl(o, &soap_package,
1191                          &hres->content_buf, &hres->content_len,
1192                          soap_handlers, 0, m_s2z_stylesheet);
1193     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1194     {
1195         yaz_log(YLOG_LOG, "%sSending %s to client", m_session_str,
1196                 gdu_name(gdu));
1197     }
1198     int len;
1199     int r = send_GDU(gdu, &len);
1200     m_bytes_sent += len;
1201     m_bw_stat.add_bytes(len);
1202     logtime();
1203
1204     recv_GDU_more(true);
1205
1206     return r;
1207 }
1208
1209 int Yaz_Proxy::send_to_srw_client_error(int srw_error, const char *add)
1210 {
1211     ODR o = odr_encode();
1212     Z_SRW_diagnostic *diagnostic = (Z_SRW_diagnostic *)
1213         odr_malloc(o, sizeof(*diagnostic));
1214     int num_diagnostic = 1;
1215     yaz_mk_std_diagnostic(o, diagnostic, srw_error, add);
1216     return send_srw_search_response(diagnostic, num_diagnostic, srw_error == 3 ? 401 : 200);
1217 }
1218
1219 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
1220                              Z_DefaultDiagFormat *ddf)
1221 {
1222     int bib1_code = *ddf->condition;
1223     if (bib1_code == 109)
1224         return 404;
1225     srw_res->num_diagnostics = 1;
1226     srw_res->diagnostics = (Z_SRW_diagnostic *)
1227         odr_malloc(o, sizeof(*srw_res->diagnostics));
1228     yaz_mk_std_diagnostic(o, srw_res->diagnostics,
1229                           yaz_diag_bib1_to_srw(*ddf->condition),
1230                           ddf->u.v2Addinfo);
1231     return 0;
1232 }
1233
1234 static void yazproxy_encode_sru_surrogate(ODR o, Z_SRW_record *record, int pos,
1235                                           int code, const char *details)
1236 {
1237     const char *message = yaz_diag_srw_str(code);
1238     int len = 200;
1239     if (message)
1240         len += strlen(message);
1241     if (details)
1242         len += strlen(details);
1243     
1244     record->recordData_buf = (char *) odr_malloc(o, len);
1245     
1246     sprintf(record->recordData_buf, "<diagnostic "
1247             "xmlns=\"http://www.loc.gov/zing/srw/diagnostic/\">\n"
1248             " <uri>info:srw/diagnostic/1/%d</uri>\n", code);
1249     if (details)
1250         sprintf(record->recordData_buf + strlen(record->recordData_buf),
1251                 " <details>%s</details>\n", details);
1252     if (message)
1253         sprintf(record->recordData_buf + strlen(record->recordData_buf),
1254                 " <message>%s</message>\n", message);
1255     sprintf(record->recordData_buf + strlen(record->recordData_buf),
1256             "</diagnostic>\n");
1257     record->recordData_len = strlen(record->recordData_buf);
1258     record->recordPosition = odr_intdup(o, pos);
1259     record->recordSchema = odr_strdup(o, "info:srw/schema/1/diagnostics-v1.1");
1260 }
1261
1262
1263 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
1264 {
1265     ODR o = odr_encode();
1266     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1267     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1268
1269     srw_res->numberOfRecords = odr_intdup (o, hits);
1270     if (records && records->which == Z_Records_DBOSD)
1271     {
1272         srw_res->num_records =
1273             records->u.databaseOrSurDiagnostics->num_records;
1274         int i;
1275         srw_res->records = (Z_SRW_record *)
1276             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
1277         for (i = 0; i < srw_res->num_records; i++)
1278         {
1279             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
1280             if (npr->which != Z_NamePlusRecord_databaseRecord)
1281             {
1282                 yaz_log(YLOG_LOG, "Point 1");
1283                 yazproxy_encode_sru_surrogate(
1284                     o, srw_res->records + i, i+start,  
1285                     YAZ_SRW_RECORD_NOT_AVAILABLE_IN_THIS_SCHEMA, 0);
1286                 continue;
1287             }
1288             Z_External *r = npr->u.databaseRecord;
1289
1290             if (r->which == Z_External_octet 
1291                 && !oid_oidcmp(r->direct_reference, yaz_oid_recsyn_xml))
1292             {
1293                 srw_res->records[i].recordSchema = m_schema;
1294                 srw_res->records[i].recordPacking = m_s2z_packing;
1295                 srw_res->records[i].recordData_buf = (char*)
1296                     r->u.octet_aligned->buf;
1297                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
1298                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1299             }
1300             else
1301             {
1302                 yaz_log(YLOG_LOG, "Point 2");
1303                 yazproxy_encode_sru_surrogate(
1304                     o, srw_res->records + i, i+start,  
1305                     YAZ_SRW_RECORD_NOT_AVAILABLE_IN_THIS_SCHEMA, 0);
1306             }
1307         }
1308     }
1309     if (records && records->which == Z_Records_NSD)
1310     {
1311         int http_code;
1312         http_code = z_to_srw_diag(odr_encode(), srw_res,
1313                                    records->u.nonSurrogateDiagnostic);
1314         if (http_code)
1315             return send_http_response(http_code);
1316     }
1317     return send_srw_response(srw_pdu);
1318
1319 }
1320
1321 int Yaz_Proxy::send_srw_search_response(Z_SRW_diagnostic *diagnostics,
1322                                         int num_diagnostics, int http_code /* = 200 */)
1323 {
1324     ODR o = odr_encode();
1325     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1326     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1327
1328     srw_res->num_diagnostics = num_diagnostics;
1329     srw_res->diagnostics = diagnostics;
1330     return send_srw_response(srw_pdu, http_code);
1331 }
1332
1333 int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics,
1334                                         int num_diagnostics)
1335 {
1336     Yaz_ProxyConfig *cfg = check_reconfigure();
1337     if (cfg)
1338     {
1339         int len;
1340         char *b = cfg->get_explain_doc(odr_encode(), 0 /* target */,
1341                                        m_s2z_database, &len);
1342         if (b)
1343         {
1344             Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
1345             Z_SRW_explainResponse *er = res->u.explain_response;
1346
1347             er->record.recordData_buf = b;
1348             er->record.recordData_len = len;
1349             er->record.recordPacking = m_s2z_packing;
1350             er->record.recordSchema = odr_strdup(odr_encode(),
1351                                                  "http://explain.z3950.org/dtd/2.0/");
1352
1353             er->diagnostics = diagnostics;
1354             er->num_diagnostics = num_diagnostics;
1355             return send_srw_response(res);
1356         }
1357     }
1358     return send_http_response(404);
1359 }
1360
1361 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu)
1362 {
1363     if (m_http_version)
1364     {
1365         if (apdu->which == Z_APDU_initResponse)
1366         {
1367             Z_InitResponse *res = apdu->u.initResponse;
1368             if (*res->result == 0)
1369             {
1370                 send_to_srw_client_error(3, 0);
1371             }
1372             else if (!m_s2z_search_apdu)
1373             {
1374                 send_srw_explain_response(0, 0);
1375             }
1376             else
1377             {
1378                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1379             }
1380         }
1381         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
1382         {
1383             m_s2z_search_apdu = 0;
1384             Z_SearchResponse *res = apdu->u.searchResponse;
1385             m_s2z_hit_count = *res->resultCount;
1386             if (res->records && res->records->which == Z_Records_NSD)
1387             {
1388                 send_to_srw_client_ok(0, res->records, 1);
1389             }
1390             else if (m_s2z_present_apdu && m_s2z_hit_count > 0)
1391             {
1392                 // adjust
1393                 Z_PresentRequest *pr = m_s2z_present_apdu->u.presentRequest;
1394
1395                 if (*pr->resultSetStartPoint <= m_s2z_hit_count)
1396                 {
1397                     if (*pr->numberOfRecordsRequested+ *pr->resultSetStartPoint
1398                         > m_s2z_hit_count)
1399                         *pr->numberOfRecordsRequested =
1400                             1 + m_s2z_hit_count - *pr->resultSetStartPoint;
1401                 }
1402                 handle_incoming_Z_PDU(m_s2z_present_apdu);
1403             }
1404             else
1405             {
1406                 m_s2z_present_apdu = 0;
1407                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
1408             }
1409         }
1410         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
1411         {
1412             int start =
1413                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
1414
1415             m_s2z_present_apdu = 0;
1416             Z_PresentResponse *res = apdu->u.presentResponse;
1417             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
1418         }
1419     }
1420     else
1421     {
1422         int len = 0;
1423         if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1424             yaz_log(YLOG_LOG, "%sSending %s to client", m_session_str,
1425                      apdu_name(apdu));
1426         int r = send_Z_PDU(apdu, &len);
1427         m_bytes_sent += len;
1428         m_bw_stat.add_bytes(len);
1429         logtime();
1430         return r;
1431     }
1432     return 0;
1433 }
1434
1435 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
1436 {
1437     int kill_session = 0;
1438     Z_ReferenceId **new_id = get_referenceIdP(apdu);
1439
1440     if (new_id)
1441         *new_id = m_referenceId;
1442
1443     if (apdu->which == Z_APDU_searchResponse)
1444     {
1445         Z_SearchResponse *sr = apdu->u.searchResponse;
1446         Z_Records *p = sr->records;
1447         if (p && p->which == Z_Records_NSD)
1448         {
1449             Z_DiagRec dr, *dr_p = &dr;
1450             dr.which = Z_DiagRec_defaultFormat;
1451             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1452
1453             *sr->searchStatus = 0;
1454             display_diagrecs(&dr_p, 1);
1455         }
1456         else
1457         {
1458             if (p && p->which == Z_Records_DBOSD)
1459             {
1460                 if (m_backend_type
1461 #if HAVE_USEMARCON
1462                     || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1463 #endif
1464                     )
1465                     convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1466                 if (m_marcxml_mode == marcxml)
1467                     convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1468                                        m_backend_charset);
1469                 else
1470                     convert_records_charset(p->u.databaseOrSurDiagnostics,
1471                                             m_backend_charset);
1472                 if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1473                     return 0;
1474
1475             }
1476             if (sr->resultCount)
1477             {
1478                 yaz_log(YLOG_LOG, "%s%d hits", m_session_str,
1479                         *sr->resultCount);
1480                 if (*sr->resultCount < 0)
1481                 {
1482                     m_flag_invalid_session = 1;
1483                     kill_session = 1;
1484
1485                     *sr->searchStatus = 0;
1486                     sr->records =
1487                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
1488                     *sr->resultCount = 0;
1489                 }
1490             }
1491         }
1492     }
1493     else if (apdu->which == Z_APDU_presentResponse)
1494     {
1495         Z_PresentResponse *sr = apdu->u.presentResponse;
1496         Z_Records *p = sr->records;
1497         if (p && p->which == Z_Records_NSD)
1498         {
1499             Z_DiagRec dr, *dr_p = &dr;
1500             dr.which = Z_DiagRec_defaultFormat;
1501             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1502             if (*sr->presentStatus == Z_PresentStatus_success)
1503                 *sr->presentStatus = Z_PresentStatus_failure;
1504             display_diagrecs(&dr_p, 1);
1505         }
1506         if (p && p->which == Z_Records_DBOSD)
1507         {
1508             if (m_backend_type
1509 #if HAVE_USEMARCON
1510                 || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1511 #endif
1512                 )
1513                 convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1514             if (m_marcxml_mode == marcxml)
1515                 convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1516                                    m_backend_charset);
1517             else
1518                 convert_records_charset(p->u.databaseOrSurDiagnostics,
1519                                         m_backend_charset);
1520             if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1521                 return 0;
1522         }
1523     }
1524     else if (apdu->which == Z_APDU_initResponse)
1525     {
1526         //Get and check negotiation record
1527         //from init response.
1528         handle_charset_lang_negotiation(apdu);
1529
1530         if (m_initRequest_options)
1531         {
1532             Z_Options *nopt =
1533                 (Odr_bitmask *)odr_malloc(odr_encode(),
1534                                           sizeof(Odr_bitmask));
1535             ODR_MASK_ZERO(nopt);
1536
1537             int i;
1538             for (i = 0; i<24; i++)
1539                 if (ODR_MASK_GET(m_initRequest_options, i) &&
1540                     ODR_MASK_GET(apdu->u.initResponse->options, i))
1541                     ODR_MASK_SET(nopt, i);
1542             apdu->u.initResponse->options = nopt;
1543         }
1544         if (m_initRequest_version)
1545         {
1546             Z_ProtocolVersion *nopt =
1547                 (Odr_bitmask *)odr_malloc(odr_encode(),
1548                                           sizeof(Odr_bitmask));
1549             ODR_MASK_ZERO(nopt);
1550
1551             int i;
1552             for (i = 0; i<8; i++)
1553                 if (ODR_MASK_GET(m_initRequest_version, i) &&
1554                     ODR_MASK_GET(apdu->u.initResponse->protocolVersion, i))
1555                     ODR_MASK_SET(nopt, i);
1556             apdu->u.initResponse->protocolVersion = nopt;
1557         }
1558         apdu->u.initResponse->preferredMessageSize =
1559             odr_intdup(odr_encode(),
1560                        m_client->m_initResponse_preferredMessageSize >
1561                        m_initRequest_preferredMessageSize ?
1562                        m_initRequest_preferredMessageSize :
1563                        m_client->m_initResponse_preferredMessageSize);
1564         apdu->u.initResponse->maximumRecordSize =
1565             odr_intdup(odr_encode(),
1566                        m_client->m_initResponse_maximumRecordSize >
1567                        m_initRequest_maximumRecordSize ?
1568                        m_initRequest_maximumRecordSize :
1569                        m_client->m_initResponse_maximumRecordSize);
1570     }
1571
1572     int r = send_PDU_convert(apdu);
1573     if (r)
1574         return r;
1575     if (kill_session)
1576     {
1577         delete m_client;
1578         m_client = 0;
1579         m_parent->pre_init();
1580     }
1581     return r;
1582 }
1583
1584 void Yaz_ProxyClient::set_idAuthentication(Z_APDU *apdu)
1585 {
1586     Z_IdAuthentication *t = apdu->u.initRequest->idAuthentication;
1587     
1588     odr_reset(m_idAuthentication_odr);
1589     z_IdAuthentication(m_idAuthentication_odr, &t, 1, 0);
1590     m_idAuthentication_ber_buf =
1591         odr_getbuf(m_idAuthentication_odr, 
1592                    &m_idAuthentication_ber_size, 0);
1593 }
1594
1595 bool Yaz_ProxyClient::compare_charset(Z_APDU *apdu)
1596 {
1597     return true;
1598 }
1599
1600 bool Yaz_ProxyClient::compare_idAuthentication(Z_APDU *apdu)
1601 {
1602     Z_IdAuthentication *t = apdu->u.initRequest->idAuthentication;
1603     ODR odr = odr_createmem(ODR_ENCODE);
1604
1605     z_IdAuthentication(odr, &t, 1, 0);
1606     int sz;
1607     char *buf = odr_getbuf(odr, &sz, 0);
1608     if (buf && m_idAuthentication_ber_buf
1609         && sz == m_idAuthentication_ber_size
1610         && !memcmp(m_idAuthentication_ber_buf, buf, sz))
1611     {
1612         odr_destroy(odr);
1613         return true;
1614     }
1615     odr_destroy(odr);
1616     if (!buf && !m_idAuthentication_ber_buf)
1617         return true;
1618     return false;
1619 }
1620
1621 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
1622 {
1623     int len = 0;
1624     const char *apdu_name_tmp = apdu_name(apdu);
1625     int r = send_Z_PDU(apdu, &len);
1626     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
1627         yaz_log(YLOG_LOG, "%sSending %s to %s %d bytes",
1628                  get_session_str(),
1629                  apdu_name_tmp, get_hostname(), len);
1630     m_bytes_sent += len;
1631     return r;
1632 }
1633
1634 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
1635 {
1636     if (apdu->which == Z_APDU_presentRequest)
1637     {
1638         Z_PresentRequest *pr = apdu->u.presentRequest;
1639         int toget = *pr->numberOfRecordsRequested;
1640         int start = *pr->resultSetStartPoint;
1641
1642         yaz_log(YLOG_LOG, "%sPresent %s %d+%d", m_session_str,
1643                 pr->resultSetId, start, toget);
1644
1645         if (*m_parent->m_optimize == '0')
1646             return apdu;
1647
1648         if (!m_client->m_last_resultSetId)
1649         {
1650             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1651             new_apdu->u.presentResponse->records =
1652                 create_nonSurrogateDiagnostics(
1653                     odr_encode(), 
1654                     YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
1655                     pr->resultSetId);
1656             send_to_client(new_apdu);
1657             return 0;
1658         }
1659         if (start < 1 || toget < 0)
1660         {
1661             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1662             new_apdu->u.presentResponse->records =
1663                 create_nonSurrogateDiagnostics(
1664                     odr_encode(), 
1665                     YAZ_BIB1_PRESENT_REQUEST_OUT_OF_RANGE, 
1666                     0);
1667             send_to_client(new_apdu);
1668             return 0;
1669         }
1670         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1671         {
1672             if (start+toget-1 > m_client->m_last_resultCount)
1673             {
1674                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1675                 new_apdu->u.presentResponse->records =
1676                     create_nonSurrogateDiagnostics(
1677                         odr_encode(), 
1678                         YAZ_BIB1_PRESENT_REQUEST_OUT_OF_RANGE,
1679                         0);
1680                 send_to_client(new_apdu);
1681                 return 0;
1682             }
1683             Z_NamePlusRecordList *npr;
1684 #if 0
1685             yaz_log(YLOG_LOG, "%sCache lookup %d+%d syntax=%s",
1686                     m_session_str, start, toget, yaz_z3950oid_to_str(
1687                         pr->preferredRecordSyntax, &oclass));
1688 #endif
1689             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1690                                           pr->preferredRecordSyntax,
1691                                           pr->recordComposition))
1692             {
1693                 yaz_log(YLOG_LOG, "%sReturned cached records for present request",
1694                          m_session_str);
1695                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1696                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1697
1698                 new_apdu->u.presentResponse->numberOfRecordsReturned
1699                     = odr_intdup(odr_encode(), toget);
1700
1701                 new_apdu->u.presentResponse->records = (Z_Records*)
1702                     odr_malloc(odr_encode(), sizeof(Z_Records));
1703                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1704                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1705                 new_apdu->u.presentResponse->nextResultSetPosition =
1706                     odr_intdup(odr_encode(), start+toget);
1707
1708                 send_to_client(new_apdu);
1709                 return 0;
1710             }
1711         }
1712     }
1713
1714     if (apdu->which != Z_APDU_searchRequest)
1715         return apdu;
1716     Z_SearchRequest *sr = apdu->u.searchRequest;
1717     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1718     Yaz_Z_Databases this_databases;
1719
1720     this_databases.set(sr->num_databaseNames, (const char **)
1721                        sr->databaseNames);
1722
1723     this_query->set_Z_Query(sr->query);
1724
1725     // Check for non-negative piggyback params.
1726     if (*sr->smallSetUpperBound < 0
1727         || *sr->largeSetLowerBound < 0
1728         || *sr->mediumSetPresentNumber < 0)
1729     {
1730         Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1731         // Not a present request.. But can't find better diagnostic
1732         new_apdu->u.searchResponse->records =
1733             create_nonSurrogateDiagnostics(
1734                 odr_encode(), 
1735                 YAZ_BIB1_PRESENT_REQUEST_OUT_OF_RANGE, 0);
1736         send_to_client(new_apdu);
1737         return 0;
1738     }
1739
1740     char query_str[4096];
1741     this_query->print(query_str, sizeof(query_str)-1);
1742     yaz_log(YLOG_LOG, "%sSearch %s", m_session_str, query_str);
1743
1744     if (*m_parent->m_optimize != '0' &&
1745         m_client->m_last_ok && m_client->m_last_query &&
1746         m_client->m_last_query->match(this_query) &&
1747         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1748         m_client->m_last_databases.match(this_databases))
1749     {
1750         delete this_query;
1751         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1752             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1753         {
1754             Z_NamePlusRecordList *npr;
1755             int toget = *sr->mediumSetPresentNumber;
1756             Z_RecordComposition *comp = 0;
1757
1758             if (toget > m_client->m_last_resultCount)
1759                 toget = m_client->m_last_resultCount;
1760             
1761             if (sr->mediumSetElementSetNames)
1762             {
1763                 comp = (Z_RecordComposition *)
1764                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1765                 comp->which = Z_RecordComp_simple;
1766                 comp->u.simple = sr->mediumSetElementSetNames;
1767             }
1768
1769             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1770                                           sr->preferredRecordSyntax, comp))
1771             {
1772                 yaz_log(YLOG_LOG, "%sReturned cached records for medium set",
1773                          m_session_str);
1774                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1775                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1776                 new_apdu->u.searchResponse->resultCount =
1777                     &m_client->m_last_resultCount;
1778
1779                 new_apdu->u.searchResponse->numberOfRecordsReturned
1780                     = odr_intdup(odr_encode(), toget);
1781
1782                 new_apdu->u.searchResponse->presentStatus =
1783                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1784                 new_apdu->u.searchResponse->records = (Z_Records*)
1785                     odr_malloc(odr_encode(), sizeof(Z_Records));
1786                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1787                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1788                 new_apdu->u.searchResponse->nextResultSetPosition =
1789                     odr_intdup(odr_encode(), toget+1);
1790                 send_to_client(new_apdu);
1791                 return 0;
1792             }
1793             else
1794             {
1795                 // medium Set
1796                 // send present request (medium size)
1797                 yaz_log(YLOG_LOG, "%sOptimizing search for medium set",
1798                          m_session_str);
1799
1800                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1801                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1802                 pr->referenceId = sr->referenceId;
1803                 pr->resultSetId = sr->resultSetName;
1804                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1805                 *pr->numberOfRecordsRequested = toget;
1806                 pr->recordComposition = comp;
1807                 m_client->m_sr_transform = 1;
1808                 return new_apdu;
1809             }
1810         }
1811         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1812             m_client->m_last_resultCount <= 0)
1813         {
1814             // large set. Return pseudo-search response immediately
1815             yaz_log(YLOG_LOG, "%sOptimizing search for large set",
1816                      m_session_str);
1817             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1818             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1819             new_apdu->u.searchResponse->resultCount =
1820                 &m_client->m_last_resultCount;
1821             send_to_client(new_apdu);
1822             return 0;
1823         }
1824         else
1825         {
1826             Z_NamePlusRecordList *npr;
1827             int toget = m_client->m_last_resultCount;
1828             Z_RecordComposition *comp = 0;
1829             // small set
1830             // send a present request (small set)
1831
1832             if (sr->smallSetElementSetNames)
1833             {
1834                 comp = (Z_RecordComposition *)
1835                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1836                 comp->which = Z_RecordComp_simple;
1837                 comp->u.simple = sr->smallSetElementSetNames;
1838             }
1839
1840             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1841                                           sr->preferredRecordSyntax, comp))
1842             {
1843                 yaz_log(YLOG_LOG, "%sReturned cached records for small set",
1844                          m_session_str);
1845                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1846                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1847                 new_apdu->u.searchResponse->resultCount =
1848                     &m_client->m_last_resultCount;
1849
1850                 new_apdu->u.searchResponse->numberOfRecordsReturned
1851                     = odr_intdup(odr_encode(), toget);
1852
1853                 new_apdu->u.searchResponse->presentStatus =
1854                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1855                 new_apdu->u.searchResponse->records = (Z_Records*)
1856                     odr_malloc(odr_encode(), sizeof(Z_Records));
1857                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1858                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1859                 new_apdu->u.searchResponse->nextResultSetPosition =
1860                     odr_intdup(odr_encode(), toget+1);
1861                 send_to_client(new_apdu);
1862                 return 0;
1863             }
1864             else
1865             {
1866                 yaz_log(YLOG_LOG, "%sOptimizing search for small set",
1867                          m_session_str);
1868                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1869                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1870                 pr->referenceId = sr->referenceId;
1871                 pr->resultSetId = sr->resultSetName;
1872                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1873                 *pr->numberOfRecordsRequested = toget;
1874                 pr->recordComposition = comp;
1875                 m_client->m_sr_transform = 1;
1876                 return new_apdu;
1877             }
1878         }
1879     }
1880     else  // query doesn't match
1881     {
1882         delete m_client->m_last_query;
1883         m_client->m_last_query = this_query;
1884         m_client->m_last_ok = 0;
1885         m_client->m_cache.clear();
1886         m_client->m_resultSetStartPoint = 0;
1887
1888         xfree(m_client->m_last_resultSetId);
1889         m_client->m_last_resultSetId = xstrdup(sr->resultSetName);
1890
1891         m_client->m_last_databases.set(sr->num_databaseNames,
1892                                        (const char **) sr->databaseNames);
1893     }
1894     return apdu;
1895 }
1896
1897
1898 void Yaz_Proxy::inc_request_no()
1899 {
1900     m_request_no++;
1901     char *cp = m_session_str + strlen(m_session_str)-1;
1902     if (*cp == ' ')
1903         cp--;
1904     while (*cp && *cp != ' ')
1905         cp--;
1906     if (*cp)
1907         sprintf(cp+1, "%d ", m_request_no);
1908 }
1909
1910 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
1911 {
1912     inc_request_no();
1913
1914     m_bytes_recv += len;
1915
1916     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1917         yaz_log(YLOG_LOG, "%sReceiving %s from client %d bytes",
1918                  m_session_str, gdu_name(apdu), len);
1919
1920 #if 0
1921     // try to make a _bad_ attribute set ID .. Don't enable this in prod.
1922     if (apdu->which == Z_GDU_Z3950 
1923         && apdu->u.z3950->which == Z_APDU_searchRequest)
1924     {
1925         Z_SearchRequest *req = apdu->u.z3950->u.searchRequest;
1926         if (req->query && req->query->which == Z_Query_type_1)
1927         {
1928             Z_RPNQuery *rpnquery = req->query->u.type_1;
1929             if (rpnquery->attributeSetId)
1930             {
1931                 rpnquery->attributeSetId[0] = -2;
1932                 rpnquery->attributeSetId[1] = -1;
1933                 yaz_log(YLOG_WARN, "%sBAD FIXUP TEST", m_session_str);
1934             }
1935         }
1936     }
1937 #endif
1938
1939 #if HAVE_GETTIMEOFDAY
1940     gettimeofday((struct timeval *) m_time_tv, 0);
1941 #endif
1942     m_bw_stat.add_bytes(len);
1943     m_pdu_stat.add_bytes(1);
1944
1945     GDU *gdu = new GDU(apdu);
1946
1947     if (gdu->get() == 0)
1948     {
1949         delete gdu;
1950         yaz_log(YLOG_LOG, "%sUnable to encode package", m_session_str);
1951         m_in_queue.clear();
1952         dec_ref();
1953         return;
1954     }
1955     m_in_queue.enqueue(gdu);
1956     recv_GDU_more(false);
1957 }
1958
1959 void Yaz_Proxy::HTTP_Forwarded(Z_GDU *z_gdu)
1960 {
1961     if (z_gdu->which == Z_GDU_HTTP_Request)
1962     {
1963         Z_HTTP_Request *hreq = z_gdu->u.HTTP_Request;
1964         const char *x_forwarded_for =
1965             z_HTTP_header_lookup(hreq->headers, "X-Forwarded-For");
1966         if (x_forwarded_for)
1967         {
1968             xfree(m_peername);
1969             m_peername = (char*) xmalloc(strlen(x_forwarded_for)+5);
1970             sprintf(m_peername, "tcp:%s", x_forwarded_for);
1971             
1972             yaz_log(YLOG_LOG, "%sHTTP Forwarded from %s", m_session_str,
1973                     m_peername);
1974             if (m_log_mask & PROXY_LOG_IP_CLIENT)
1975                 sprintf(m_session_str, "%ld:%d %.80s %d ",
1976                         (long) time(0), m_session_no, m_peername, m_request_no);
1977             else
1978                 sprintf(m_session_str, "%ld:%d %d ",
1979                         (long) time(0), m_session_no, m_request_no);
1980         }
1981     }
1982 }
1983
1984 void Yaz_Proxy::connect_stat(bool &block, int &reduce)
1985 {
1986
1987     m_parent->m_connect.cleanup(false);
1988     m_parent->m_connect.add_connect(m_peername);
1989
1990     int connect_total = m_parent->m_connect.get_total(m_peername);
1991     int max_connect = m_parent->m_max_connect;
1992
1993     if (max_connect && connect_total > max_connect)
1994     {
1995         yaz_log(YLOG_LOG, "%sconnect not accepted total=%d max=%d",
1996                 m_session_str, connect_total, max_connect);
1997         block = true;
1998     }
1999     else 
2000         block = false;
2001     yaz_log(YLOG_LOG, "%sconnect accepted total=%d", m_session_str,
2002             connect_total);
2003     
2004     int limit_connect = m_parent->m_limit_connect;
2005     if (limit_connect)
2006         reduce = connect_total / limit_connect;
2007     else
2008         reduce = 0;
2009 }
2010
2011 void Yaz_Proxy::recv_GDU_reduce(GDU *gdu)
2012 {
2013     HTTP_Forwarded(gdu->get());
2014
2015     int reduce = 0;
2016     
2017     if (m_request_no == 1)
2018     {
2019         bool block = false;
2020         
2021         connect_stat(block, reduce);
2022
2023         if (block)
2024         {
2025             m_timeout_mode = timeout_busy;
2026             timeout(0);
2027             return;
2028         }
2029     }
2030
2031     int bw_total = m_bw_stat.get_total();
2032     int pdu_total = m_pdu_stat.get_total();
2033     int search_total = m_search_stat.get_total();
2034
2035     assert(m_timeout_mode == timeout_busy);
2036     assert(m_timeout_gdu == 0);
2037
2038     if (m_search_max)
2039         reduce += search_total / m_search_max;
2040     if (m_bw_max)
2041         reduce += (bw_total/m_bw_max);
2042     if (m_pdu_max)
2043     {
2044         if (pdu_total > m_pdu_max)
2045         {
2046             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
2047             reduce = (reduce > nreduce) ? reduce : nreduce;
2048         }
2049     }
2050     m_http_version = 0;
2051
2052 #if 0
2053     /* uncomment to force a big reduce */
2054     m_timeout_mode = timeout_reduce;
2055     m_timeout_gdu = gdu;
2056     timeout(3);       // call us reduce seconds later
2057     return;
2058 #endif
2059     if (reduce)
2060     {
2061         yaz_log(YLOG_LOG, "%sdelay=%d bw=%d pdu=%d search=%d limit-bw=%d limit-pdu=%d limit-search=%d",
2062                 m_session_str, reduce, bw_total, pdu_total, search_total,
2063                 m_bw_max, m_pdu_max, m_search_max);
2064
2065         m_timeout_mode = timeout_reduce;
2066         m_timeout_gdu = gdu;
2067         timeout(reduce);       // call us reduce seconds later
2068     }
2069     else
2070         recv_GDU_normal(gdu);
2071 }
2072
2073 void Yaz_Proxy::recv_GDU_more(bool normal)
2074 {
2075     GDU *g;
2076     if (normal && m_timeout_mode == timeout_busy)
2077         m_timeout_mode = timeout_normal;
2078     while (m_timeout_mode == timeout_normal && (g = m_in_queue.dequeue()))
2079     {
2080         m_timeout_mode = timeout_busy;
2081         inc_ref();
2082         recv_GDU_reduce(g);
2083         if (dec_ref())
2084             break;
2085     }
2086 }
2087
2088 void Yaz_Proxy::recv_GDU_normal(GDU *gdu)
2089 {
2090     Z_GDU *apdu = 0;
2091     gdu->move_away_gdu(odr_decode(), &apdu);
2092     delete gdu;
2093
2094     if (apdu->which == Z_GDU_Z3950)
2095         handle_incoming_Z_PDU(apdu->u.z3950);
2096     else if (apdu->which == Z_GDU_HTTP_Request)
2097         handle_incoming_HTTP(apdu->u.HTTP_Request);
2098 }
2099
2100 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
2101 {
2102     if (m_max_record_retrieve)
2103     {
2104         if (apdu->which == Z_APDU_presentRequest)
2105         {
2106             Z_PresentRequest *pr = apdu->u.presentRequest;
2107             if (pr->numberOfRecordsRequested &&
2108                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
2109                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
2110         }
2111     }
2112 }
2113
2114 void Yaz_Proxy::handle_charset_lang_negotiation(Z_APDU *apdu)
2115 {
2116     if (apdu->which == Z_APDU_initRequest)
2117     {
2118         if (m_initRequest_options &&
2119             !ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel) &&
2120             (m_proxy_negotiation_charset || m_proxy_negotiation_lang))
2121         {
2122             // There is no negotiation proposal from
2123             // client's side. OK. The proxy negotiation
2124             // in use, only.
2125             Z_InitRequest *initRequest = apdu->u.initRequest;
2126             Z_OtherInformation **otherInfo;
2127             Z_OtherInformationUnit *oi;
2128             get_otherInfoAPDU(apdu, &otherInfo);
2129             oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
2130             if (oi)
2131             {
2132                 ODR_MASK_SET(initRequest->options,
2133                     Z_Options_negotiationModel);
2134                 oi->which = Z_OtherInfo_externallyDefinedInfo;
2135                 oi->information.externallyDefinedInfo =
2136                 yaz_set_proposal_charneg(odr_encode(),
2137                     (const char**)&m_proxy_negotiation_charset,
2138                     m_proxy_negotiation_charset ? 1:0,
2139                     (const char**)&m_proxy_negotiation_lang,
2140                     m_proxy_negotiation_lang ? 1:0,
2141                     1);
2142             }
2143         }
2144         else if (m_initRequest_options &&
2145                  ODR_MASK_GET(m_initRequest_options,
2146                               Z_Options_negotiationModel) &&
2147                  m_charset_converter->get_target_query_charset())
2148         {
2149             yaz_log(YLOG_LOG, "%sManaged charset negotiation: charset=%s",
2150                     m_session_str,
2151                     m_charset_converter->get_target_query_charset());
2152             Z_InitRequest *initRequest = apdu->u.initRequest;
2153             Z_CharSetandLanguageNegotiation *negotiation =
2154                 yaz_get_charneg_record (initRequest->otherInfo);
2155             if (negotiation &&
2156                 negotiation->which == Z_CharSetandLanguageNegotiation_proposal)
2157             {
2158                 NMEM nmem = nmem_create();
2159                 char **charsets = 0;
2160                 int num_charsets = 0;
2161                 char **langs = 0;
2162                 int num_langs = 0;
2163                 int selected = 0;
2164                 yaz_get_proposal_charneg (nmem, negotiation,
2165                                           &charsets, &num_charsets,
2166                                           &langs, &num_langs, &selected);
2167                 int i;
2168                 for (i = 0; i<num_charsets; i++)
2169                     yaz_log(YLOG_LOG, "%scharset %s", m_session_str,
2170                             charsets[i]);
2171                 for (i = 0; i<num_langs; i++)
2172                     yaz_log(YLOG_LOG, "%slang %s", m_session_str,
2173                             langs[i]);
2174
2175                 const char *t_charset =
2176                     m_charset_converter->get_target_query_charset();
2177                 // sweep through charsets and pick the first supported
2178                 // conversion
2179                 for (i = 0; i<num_charsets; i++)
2180                 {
2181                     const char *c_charset = charsets[i];
2182                     if (!odr_set_charset(odr_decode(), t_charset, c_charset))
2183                         break;
2184                 }
2185                 if (i != num_charsets)
2186                 {
2187                     // got one .. set up ODR for reverse direction
2188                     const char *c_charset = charsets[i];
2189                     odr_set_charset(odr_encode(), c_charset, t_charset);
2190                     m_charset_converter->set_client_query_charset(c_charset);
2191                     m_charset_converter->set_client_charset_selected(selected);
2192                 }
2193                 nmem_destroy(nmem);
2194                 ODR_MASK_CLEAR(m_initRequest_options,
2195                                Z_Options_negotiationModel);
2196                 yaz_del_charneg_record(&initRequest->otherInfo);
2197             }
2198             else
2199             {
2200                 yaz_log(YLOG_WARN, "%sUnable to decode charset package",
2201                         m_session_str);
2202             }
2203         }
2204         else if (m_charset_converter->get_target_query_charset() &&
2205             m_proxy_negotiation_default_charset)
2206         {
2207             m_charset_converter->
2208                 set_client_query_charset(m_proxy_negotiation_default_charset);
2209         }
2210     }
2211     else if (apdu->which == Z_APDU_initResponse)
2212     {
2213         Z_InitResponse *initResponse = apdu->u.initResponse;
2214         Z_OtherInformation **otherInfo;
2215         get_otherInfoAPDU(apdu, &otherInfo);
2216         
2217         Z_CharSetandLanguageNegotiation *charneg = 0;
2218
2219         if (otherInfo && *otherInfo && 
2220             ODR_MASK_GET(initResponse->options, Z_Options_negotiationModel)
2221             && (charneg = yaz_get_charneg_record(*otherInfo)))
2222         {
2223             char *charset = 0;
2224             char *lang = 0;
2225             int selected = 0;
2226
2227             yaz_get_response_charneg(m_referenceId_mem, charneg,
2228                 &charset, &lang, &selected);
2229
2230             yaz_log(YLOG_LOG, "%sAccepted charset - '%s' and lang - '%s'",
2231                 m_session_str, (charset)?charset:"none", (lang)?lang:"none");
2232
2233             if (m_initRequest_options &&
2234                 ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel))
2235             {
2236                 yaz_log(YLOG_LOG, "%sClient's negotiation record in use",
2237                     m_session_str);
2238             }
2239             else if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
2240             {
2241                 // negotiation-charset, negotiation-lang
2242                 // elements of config file in use.
2243
2244                 yaz_log(YLOG_LOG, "%sProxy's negotiation record in use",
2245                     m_session_str);
2246
2247                 // clear negotiation option.
2248                 ODR_MASK_CLEAR(initResponse->options, Z_Options_negotiationModel);
2249
2250                 // Delete negotiation (charneg-3) entry.
2251                 yaz_del_charneg_record(otherInfo);
2252             }
2253         }
2254         else
2255         {
2256             if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
2257             {
2258                 yaz_log(YLOG_LOG, "%sTarget did not honor negotiation",
2259                         m_session_str);
2260             }
2261             else if (m_charset_converter->get_client_query_charset())
2262             {
2263                 Z_OtherInformation **otherInfo;
2264                 Z_OtherInformationUnit *oi;
2265                 get_otherInfoAPDU(apdu, &otherInfo);
2266                 oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
2267                 if (oi)
2268                 {
2269                     ODR_MASK_SET(initResponse->options,
2270                                  Z_Options_negotiationModel);
2271                     if (m_initRequest_options)
2272                         ODR_MASK_SET(m_initRequest_options,
2273                                      Z_Options_negotiationModel);
2274                     
2275                     oi->which = Z_OtherInfo_externallyDefinedInfo;    
2276                     oi->information.externallyDefinedInfo =
2277                         yaz_set_response_charneg(
2278                             odr_encode(),
2279                             m_charset_converter->get_client_query_charset(),
2280                             0 /* no lang */,
2281                             m_charset_converter->get_client_charset_selected());
2282                 }
2283             }
2284         }
2285     }
2286 }
2287
2288 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
2289                                                      int error,
2290                                                      const char *addinfo)
2291 {
2292     Z_Records *rec = (Z_Records *)
2293         odr_malloc (odr, sizeof(*rec));
2294     int *err = (int *)
2295         odr_malloc (odr, sizeof(*err));
2296     Z_DiagRec *drec = (Z_DiagRec *)
2297         odr_malloc (odr, sizeof(*drec));
2298     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
2299         odr_malloc (odr, sizeof(*dr));
2300     *err = error;
2301     rec->which = Z_Records_NSD;
2302     rec->u.nonSurrogateDiagnostic = dr;
2303     dr->diagnosticSetId = odr_oiddup(odr, yaz_oid_diagset_bib_1);
2304     dr->condition = err;
2305     dr->which = Z_DefaultDiagFormat_v2Addinfo;
2306     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
2307     return rec;
2308 }
2309
2310 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
2311 {
2312     if (apdu->which == Z_APDU_searchRequest &&
2313         apdu->u.searchRequest->query &&
2314         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
2315         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
2316     {
2317         Z_RPNQuery *rpnquery = 0;
2318         Z_SearchRequest *sr = apdu->u.searchRequest;
2319         char *addinfo = 0;
2320
2321         yaz_log(YLOG_LOG, "%sCQL: %s", m_session_str,
2322                 sr->query->u.type_104->u.cql);
2323
2324         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
2325                                           &rpnquery, odr_encode(),
2326                                           &addinfo);
2327         if (r == -3)
2328             yaz_log(YLOG_LOG, "%sNo CQL to RPN table", m_session_str);
2329         else if (r)
2330         {
2331             yaz_log(YLOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
2332             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2333
2334             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2335             new_apdu->u.searchResponse->records =
2336                 create_nonSurrogateDiagnostics(odr_encode(),
2337                                                yaz_diag_srw_to_bib1(r),
2338                                                addinfo);
2339             *new_apdu->u.searchResponse->searchStatus = 0;
2340
2341             send_to_client(new_apdu);
2342
2343             return 0;
2344         }
2345         else
2346         {
2347             sr->query->which = Z_Query_type_1;
2348             sr->query->u.type_1 = rpnquery;
2349         }
2350         return apdu;
2351     }
2352     return apdu;
2353 }
2354
2355 Z_APDU *Yaz_Proxy::handle_target_charset_conversion(Z_APDU *apdu)
2356 {
2357     if (apdu->which == Z_APDU_searchRequest &&
2358         apdu->u.searchRequest->query)
2359     {
2360         if (apdu->u.searchRequest->query->which == Z_Query_type_1
2361             || apdu->u.searchRequest->query->which == Z_Query_type_101)
2362         {
2363             if (m_http_version)
2364                 m_charset_converter->set_client_query_charset("UTF-8");
2365             Z_RPNQuery *rpnquery = apdu->u.searchRequest->query->u.type_1;
2366             m_charset_converter->convert_type_1(rpnquery, odr_encode());
2367         }
2368     }
2369     return apdu;
2370 }
2371
2372
2373 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
2374 {
2375     if (apdu->which == Z_APDU_searchRequest)
2376     {
2377         Z_SearchRequest *sr = apdu->u.searchRequest;
2378         int err = 0;
2379         char *addinfo = 0;
2380
2381         Yaz_ProxyConfig *cfg = check_reconfigure();
2382         if (cfg)
2383             err = cfg->check_query(odr_encode(), m_default_target,
2384                                    sr->query, &addinfo);
2385         if (err)
2386         {
2387             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2388
2389             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2390             new_apdu->u.searchResponse->records =
2391                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2392             *new_apdu->u.searchResponse->searchStatus = 0;
2393
2394             send_to_client(new_apdu);
2395
2396             return 0;
2397         }
2398     }
2399     return apdu;
2400 }
2401
2402 int Yaz_Proxy::handle_authentication(Z_APDU *apdu)
2403 {
2404     if (apdu->which != Z_APDU_initRequest)
2405         return 1;  // pass if no init request
2406     Z_InitRequest *req = apdu->u.initRequest;
2407
2408     Yaz_ProxyConfig *cfg = check_reconfigure();
2409     if (!cfg)
2410         return 1;  // pass if no config
2411
2412     int ret;
2413     if (req->idAuthentication == 0)
2414     {
2415         ret = cfg->client_authentication(m_default_target, 0, 0, 0,
2416                                          m_peername);
2417     }
2418     else if (req->idAuthentication->which == Z_IdAuthentication_idPass)
2419     {
2420         ret = cfg->client_authentication(
2421             m_default_target,
2422             req->idAuthentication->u.idPass->userId,
2423             req->idAuthentication->u.idPass->groupId,
2424             req->idAuthentication->u.idPass->password,
2425             m_peername);
2426     }
2427     else if (req->idAuthentication->which == Z_IdAuthentication_open)
2428     {
2429         char user[64], pass[64];
2430         *user = '\0';
2431         *pass = '\0';
2432         sscanf(req->idAuthentication->u.open, "%63[^/]/%63s", user, pass);
2433         ret = cfg->client_authentication(m_default_target, user, 0, pass,
2434                                          m_peername);
2435     }
2436     else
2437         ret = cfg->client_authentication(m_default_target, 0, 0, 0,
2438                                          m_peername);
2439     return ret;
2440 }
2441
2442 int Yaz_Proxy::handle_global_authentication(Z_APDU *apdu)
2443 {
2444     if (apdu->which != Z_APDU_initRequest)
2445         return 1;  // pass if no init request
2446     Z_InitRequest *req = apdu->u.initRequest;
2447
2448     Yaz_ProxyConfig *cfg = check_reconfigure();
2449     if (!cfg)
2450         return 1;  // pass if no config
2451
2452     int ret;
2453     if (req->idAuthentication == 0)
2454     {
2455         ret = cfg->global_client_authentication(0, 0, 0,
2456                                                 m_peername);
2457     }
2458     else if (req->idAuthentication->which == Z_IdAuthentication_idPass)
2459     {
2460         ret = cfg->global_client_authentication(
2461             req->idAuthentication->u.idPass->userId,
2462             req->idAuthentication->u.idPass->groupId,
2463             req->idAuthentication->u.idPass->password,
2464             m_peername);
2465     }
2466     else if (req->idAuthentication->which == Z_IdAuthentication_open)
2467     {
2468         char user[64], pass[64];
2469         *user = '\0';
2470         *pass = '\0';
2471         sscanf(req->idAuthentication->u.open, "%63[^/]/%63s", user, pass);
2472         ret = cfg->global_client_authentication(user, 0, pass,
2473                                                 m_peername);
2474     }
2475     else
2476         ret = cfg->global_client_authentication(0, 0, 0, m_peername);
2477     return ret;
2478 }
2479
2480 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
2481 {
2482     m_marcxml_mode = none;
2483     if (apdu->which == Z_APDU_searchRequest)
2484     {
2485         Z_SearchRequest *sr = apdu->u.searchRequest;
2486         int err = 0;
2487         char *addinfo = 0;
2488         Yaz_ProxyConfig *cfg = check_reconfigure();
2489
2490         Z_RecordComposition rc_temp, *rc = 0;
2491         if (sr->smallSetElementSetNames)
2492         {
2493             rc_temp.which = Z_RecordComp_simple;
2494             rc_temp.u.simple = sr->smallSetElementSetNames;
2495             rc = &rc_temp;
2496         }
2497
2498         if (sr->preferredRecordSyntax)
2499             oid_oidcpy(m_frontend_type, sr->preferredRecordSyntax);
2500         else
2501             m_frontend_type[0] = -1;
2502
2503         char *stylesheet_name = 0;
2504         if (cfg)
2505             err = cfg->check_syntax(odr_encode(),
2506                                     m_default_target,
2507                                     sr->preferredRecordSyntax, rc,
2508                                     &addinfo, &stylesheet_name, &m_schema,
2509                                     &m_backend_type, &m_backend_charset,
2510                                     &m_usemarcon_ini_stage1,
2511                                     &m_usemarcon_ini_stage2);
2512         if (stylesheet_name)
2513         {
2514             m_parent->low_socket_close();
2515
2516 #if YAZ_HAVE_XSLT
2517             if (m_stylesheet_xsp)
2518                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2519             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2520                                                        stylesheet_name);
2521 #endif
2522             m_stylesheet_offset = 0;
2523             xfree(stylesheet_name);
2524
2525             m_parent->low_socket_open();
2526         }
2527         if (err == -1)
2528         {
2529             sr->smallSetElementSetNames = 0;
2530             sr->mediumSetElementSetNames = 0;
2531             m_marcxml_mode = marcxml;
2532             sr->preferredRecordSyntax =
2533                 yaz_string_to_oid_odr(
2534                     yaz_oid_std(), CLASS_RECSYN,
2535                     m_backend_type ? m_backend_type : "usmarc", 
2536                     odr_encode());
2537         }
2538         else if (err)
2539         {
2540             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2541
2542             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2543             new_apdu->u.searchResponse->records =
2544                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2545             *new_apdu->u.searchResponse->searchStatus = 0;
2546
2547             send_to_client(new_apdu);
2548
2549             return 0;
2550         }
2551         else if (m_backend_type)
2552         {
2553             sr->preferredRecordSyntax =
2554                 yaz_string_to_oid_odr(yaz_oid_std(), CLASS_RECSYN, 
2555                                       m_backend_type, odr_encode());
2556         }
2557     }
2558     else if (apdu->which == Z_APDU_presentRequest)
2559     {
2560         Z_PresentRequest *pr = apdu->u.presentRequest;
2561         int err = 0;
2562         char *addinfo = 0;
2563         Yaz_ProxyConfig *cfg = check_reconfigure();
2564
2565         if (pr->preferredRecordSyntax)
2566             oid_oidcpy(m_frontend_type, pr->preferredRecordSyntax);
2567         else
2568             m_frontend_type[0] = -1;
2569
2570         char *stylesheet_name = 0;
2571         if (cfg)
2572             err = cfg->check_syntax(odr_encode(), m_default_target,
2573                                     pr->preferredRecordSyntax,
2574                                     pr->recordComposition,
2575                                     &addinfo, &stylesheet_name, &m_schema,
2576                                     &m_backend_type, &m_backend_charset,
2577                                     &m_usemarcon_ini_stage1,
2578                                     &m_usemarcon_ini_stage2
2579                                     );
2580         if (stylesheet_name)
2581         {
2582             m_parent->low_socket_close();
2583
2584 #if YAZ_HAVE_XSLT
2585             if (m_stylesheet_xsp)
2586                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2587             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2588                                                        stylesheet_name);
2589 #endif
2590             m_stylesheet_offset = 0;
2591             xfree(stylesheet_name);
2592
2593             m_parent->low_socket_open();
2594         }
2595         if (err == -1)
2596         {
2597             pr->recordComposition = 0;
2598             m_marcxml_mode = marcxml;
2599
2600             pr->preferredRecordSyntax =
2601                 yaz_string_to_oid_odr(
2602                     yaz_oid_std(), CLASS_RECSYN,
2603                     m_backend_type ? m_backend_type : "usmarc", 
2604                     odr_encode());
2605         }
2606         else if (err)
2607         {
2608             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
2609
2610             new_apdu->u.presentResponse->referenceId = pr->referenceId;
2611             new_apdu->u.presentResponse->records =
2612                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2613             *new_apdu->u.presentResponse->presentStatus =
2614                 Z_PresentStatus_failure;
2615
2616             send_to_client(new_apdu);
2617
2618             return 0;
2619         }
2620         else if (m_backend_type)
2621         {
2622             pr->preferredRecordSyntax =
2623                 yaz_string_to_oid_odr(yaz_oid_std(),
2624                                       CLASS_RECSYN, m_backend_type, 
2625                                       odr_encode());
2626         }
2627     }
2628     return apdu;
2629 }
2630
2631 Z_ElementSetNames *Yaz_Proxy::mk_esn_from_schema(ODR o, const char *schema)
2632 {
2633     if (!schema)
2634         return 0;
2635     Z_ElementSetNames *esn = (Z_ElementSetNames *)
2636         odr_malloc(o, sizeof(Z_ElementSetNames));
2637     esn->which = Z_ElementSetNames_generic;
2638     esn->u.generic = odr_strdup(o, schema);
2639     return esn;
2640 }
2641
2642 void Yaz_Proxy::srw_get_client(const char *db, const char **backend_db)
2643 {
2644     const char *t = 0;
2645     Yaz_ProxyConfig *cfg = check_reconfigure();
2646     if (cfg)
2647         t = cfg->get_explain_name(db, backend_db);
2648
2649     if (m_client && m_default_target && t && strcmp(m_default_target, t))
2650     {
2651         releaseClient();
2652     }
2653
2654     if (t)
2655     {
2656         xfree(m_default_target);
2657         m_default_target = xstrdup(t);
2658     }
2659 }
2660
2661 int Yaz_Proxy::file_access(Z_HTTP_Request *hreq)
2662 {
2663     struct stat sbuf;
2664     if (strcmp(hreq->method, "GET"))
2665         return 0;
2666     if (hreq->path[0] != '/')
2667         return 0;
2668     const char *cp = hreq->path;
2669     while (*cp)
2670     {
2671         if (*cp == '/' && strchr("/.", cp[1]))
2672             return 0;
2673         cp++;
2674     }
2675
2676     Yaz_ProxyConfig *cfg = check_reconfigure();
2677
2678     if (!cfg->get_file_access_info(hreq->path+1))
2679         return 0;
2680
2681     const char *fname = hreq->path+1;
2682     if (stat(fname, &sbuf))
2683     {
2684         yaz_log(YLOG_LOG|YLOG_ERRNO, "%sstat failed for %s", m_session_str,
2685                 fname);
2686         return 0;
2687     }
2688     if ((sbuf.st_mode & S_IFMT) != S_IFREG)
2689     {
2690         yaz_log(YLOG_LOG, "%sNot a regular file %s", m_session_str, fname);
2691         return 0;
2692     }
2693     if (sbuf.st_size > (off_t) 1000000)
2694     {
2695         yaz_log(YLOG_WARN, "%sFile %s too large for transfer", m_session_str,
2696                 fname);
2697         return 0;
2698     }
2699
2700     ODR o = odr_encode();
2701
2702     const char *ctype = cfg->check_mime_type(fname);
2703     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
2704     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
2705     if (m_http_version)
2706         hres->version = odr_strdup(o, m_http_version);
2707     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
2708     if (m_http_keepalive)
2709         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
2710     else
2711         timeout(0);
2712
2713     hres->content_len = sbuf.st_size;
2714     hres->content_buf = (char*) odr_malloc(o, hres->content_len);
2715     FILE *f = fopen(fname, "rb");
2716     if (f)
2717     {
2718         fread(hres->content_buf, 1, hres->content_len, f);
2719         fclose(f);
2720     }
2721     else
2722     {
2723         return 0;
2724     }
2725     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
2726     {
2727         yaz_log(YLOG_LOG, "%sSending file %s to client", m_session_str,
2728                  fname);
2729     }
2730     int len;
2731     send_GDU(gdu, &len);
2732     recv_GDU_more(true);
2733     return 1;
2734 }
2735
2736 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
2737 {
2738     if (m_s2z_odr_init)
2739     {
2740         odr_destroy(m_s2z_odr_init);
2741         m_s2z_odr_init = 0;
2742     }
2743     if (m_s2z_odr_search)
2744     {
2745         odr_destroy(m_s2z_odr_search);
2746         m_s2z_odr_search = 0;
2747     }
2748
2749     m_http_keepalive = 0;
2750     m_http_version = 0;
2751     if (!strcmp(hreq->version, "1.0"))
2752     {
2753         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2754         if (v && !strcmp(v, "Keep-Alive"))
2755             m_http_keepalive = 1;
2756         else
2757             m_http_keepalive = 0;
2758         m_http_version = "1.0";
2759     }
2760     else
2761     {
2762         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2763         if (v && !strcmp(v, "close"))
2764             m_http_keepalive = 0;
2765         else
2766             m_http_keepalive = 1;
2767         m_http_version = "1.1";
2768     }
2769
2770     const char *a = z_HTTP_header_lookup(hreq->headers, "Authorization");
2771     char authorization_str[255];
2772     *authorization_str = '\0';
2773     if (a && strncasecmp(a, "Basic ", 6) == 0)
2774         base64_decode(a + 6, authorization_str, 254);
2775
2776     Z_SRW_PDU *srw_pdu = 0;
2777     Z_SOAP *soap_package = 0;
2778     char *charset = 0;
2779     Z_SRW_diagnostic *diagnostic = 0;
2780     int num_diagnostic = 0;
2781
2782     yaz_log(YLOG_LOG, "%s%s %s", m_session_str, hreq->method, hreq->path);
2783
2784     if (file_access(hreq))
2785     {
2786         return;
2787     }
2788     else if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2789                             &charset) == 0
2790              || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2791                                &charset, &diagnostic, &num_diagnostic) == 0)
2792     {
2793         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
2794         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
2795         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns);
2796         m_s2z_init_apdu = 0;
2797         m_s2z_search_apdu = 0;
2798         m_s2z_present_apdu = 0;
2799
2800         m_s2z_stylesheet = 0;
2801         
2802         Z_IdAuthentication *auth = NULL;
2803         if (srw_pdu->username && srw_pdu->password)
2804         {
2805             yaz_log(YLOG_LOG, "username/password: %s/%s\n",
2806                     srw_pdu->username, srw_pdu->password);
2807             auth = (Z_IdAuthentication *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdAuthentication));
2808             auth->which = Z_IdAuthentication_idPass;
2809             auth->u.idPass = (Z_IdPass *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdPass));
2810             auth->u.idPass->groupId = NULL;
2811             auth->u.idPass->password = odr_strdup(m_s2z_odr_init, srw_pdu->password);
2812             auth->u.idPass->userId = odr_strdup(m_s2z_odr_init, srw_pdu->username);
2813         }
2814         else
2815         {
2816             if (*authorization_str)
2817             {
2818                 yaz_log(YLOG_LOG, "authorization_str present: %s\n", authorization_str);
2819                 auth = (Z_IdAuthentication *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdAuthentication));
2820                 auth->which = Z_IdAuthentication_idPass;
2821                 auth->u.idPass = (Z_IdPass *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdPass));
2822                 auth->u.idPass->groupId = NULL;
2823                 char *p = strchr(authorization_str, ':');
2824                 if (p)
2825                 {
2826                     *p = '\0';
2827                     p++;
2828                     auth->u.idPass->password = odr_strdup(m_s2z_odr_init, p);
2829                 }
2830                 auth->u.idPass->userId = odr_strdup(m_s2z_odr_init, authorization_str);
2831             }
2832         }               
2833         
2834         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
2835         {
2836
2837             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
2838
2839             const char *backend_db = srw_req->database;
2840             srw_get_client(srw_req->database, &backend_db);
2841
2842             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2843             // recordXPath unsupported.
2844             if (srw_req->recordXPath)
2845             {
2846                 yaz_add_srw_diagnostic(odr_decode(),
2847                                        &diagnostic, &num_diagnostic,
2848                                        72, 0);
2849             }
2850             // sort unsupported
2851             if (srw_req->sort_type != Z_SRW_sort_type_none)
2852             {
2853                 yaz_add_srw_diagnostic(odr_decode(),
2854                                        &diagnostic, &num_diagnostic,
2855                                        80, 0);
2856             }
2857             // save stylesheet
2858             if (srw_req->stylesheet)
2859                 m_s2z_stylesheet =
2860                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2861
2862             // set packing for response records ..
2863             if (srw_req->recordPacking &&
2864                 !strcmp(srw_req->recordPacking, "xml"))
2865                 m_s2z_packing = Z_SRW_recordPacking_XML;
2866             else
2867                 m_s2z_packing = Z_SRW_recordPacking_string;
2868
2869             if (num_diagnostic)
2870             {
2871                 Z_SRW_PDU *srw_pdu =
2872                     yaz_srw_get(odr_encode(),
2873                                 Z_SRW_searchRetrieve_response);
2874                 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
2875
2876                 srw_res->diagnostics = diagnostic;
2877                 srw_res->num_diagnostics = num_diagnostic;
2878                 send_srw_response(srw_pdu);
2879                 return;
2880             }
2881
2882             // prepare search PDU
2883             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
2884                                           Z_APDU_searchRequest);
2885             Z_SearchRequest *z_searchRequest =
2886                 m_s2z_search_apdu->u.searchRequest;
2887
2888             z_searchRequest->num_databaseNames = 1;
2889             z_searchRequest->databaseNames = (char**)
2890                 odr_malloc(m_s2z_odr_search, sizeof(char *));
2891             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
2892                                                            backend_db);
2893
2894             // query transformation
2895             Z_Query *query = (Z_Query *)
2896                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
2897             z_searchRequest->query = query;
2898
2899             if (srw_req->query_type == Z_SRW_query_type_cql)
2900             {
2901                 Z_External *ext = (Z_External *)
2902                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
2903                 ext->direct_reference =
2904                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
2905                 ext->indirect_reference = 0;
2906                 ext->descriptor = 0;
2907                 ext->which = Z_External_CQL;
2908                 ext->u.cql = srw_req->query.cql;
2909
2910                 query->which = Z_Query_type_104;
2911                 query->u.type_104 =  ext;
2912             }
2913             else if (srw_req->query_type == Z_SRW_query_type_pqf)
2914             {
2915                 Z_RPNQuery *RPNquery;
2916                 YAZ_PQF_Parser pqf_parser;
2917
2918                 pqf_parser = yaz_pqf_create ();
2919
2920                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
2921                                           srw_req->query.pqf);
2922                 if (!RPNquery)
2923                 {
2924                     const char *pqf_msg;
2925                     size_t off;
2926                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
2927                     int ioff = off;
2928                     yaz_log(YLOG_LOG, "%*s^\n", ioff+4, "");
2929                     yaz_log(YLOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
2930
2931                     send_to_srw_client_error(10, 0);
2932                     return;
2933                 }
2934                 query->which = Z_Query_type_1;
2935                 query->u.type_1 =  RPNquery;
2936
2937                 yaz_pqf_destroy (pqf_parser);
2938             }
2939             else
2940             {
2941                 send_to_srw_client_error(7, "query");
2942                 return;
2943             }
2944
2945             // present
2946             m_s2z_present_apdu = 0;
2947             int max = 0;
2948             if (srw_req->maximumRecords)
2949                 max = *srw_req->maximumRecords;
2950             int start = 1;
2951             if (srw_req->startRecord)
2952                 start = *srw_req->startRecord;
2953             if (max > 0)
2954             {
2955                 // Some backend, such as Voyager doesn't honor piggyback
2956                 // So we use present always (0 &&).
2957                 if (0 && start <= 1)  // Z39.50 piggyback
2958                 {
2959                     *z_searchRequest->smallSetUpperBound = max;
2960                     *z_searchRequest->mediumSetPresentNumber = max;
2961                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
2962
2963                     z_searchRequest->preferredRecordSyntax =
2964                         odr_oiddup(m_s2z_odr_search, yaz_oid_recsyn_xml);
2965
2966                     if (srw_req->recordSchema)
2967                     {
2968                         z_searchRequest->smallSetElementSetNames =
2969                             z_searchRequest->mediumSetElementSetNames =
2970                             mk_esn_from_schema(m_s2z_odr_search,
2971                                                srw_req->recordSchema);
2972                     }
2973                 }
2974                 else   // Z39.50 present
2975                 {
2976                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
2977                                                    Z_APDU_presentRequest);
2978                     Z_PresentRequest *z_presentRequest =
2979                         m_s2z_present_apdu->u.presentRequest;
2980                     *z_presentRequest->resultSetStartPoint = start;
2981                     *z_presentRequest->numberOfRecordsRequested = max;
2982
2983                     z_presentRequest->preferredRecordSyntax =
2984                         odr_oiddup(m_s2z_odr_search, yaz_oid_recsyn_xml);
2985                     if (srw_req->recordSchema)
2986                     {
2987                         z_presentRequest->recordComposition =
2988                             (Z_RecordComposition *)
2989                             odr_malloc(m_s2z_odr_search,
2990                                        sizeof(Z_RecordComposition));
2991                         z_presentRequest->recordComposition->which =
2992                             Z_RecordComp_simple;
2993                         z_presentRequest->recordComposition->u.simple =
2994                             mk_esn_from_schema(m_s2z_odr_search,
2995                                                srw_req->recordSchema);
2996                     }
2997                 }
2998             }
2999             if (!m_client)
3000             {
3001                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
3002                                             Z_APDU_initRequest);
3003
3004                 m_s2z_init_apdu->u.initRequest->idAuthentication = auth;
3005
3006                 // prevent m_initRequest_apdu memory from being grabbed
3007                 // in Yaz_Proxy::handle_incoming_Z_PDU
3008                 m_initRequest_apdu = m_s2z_init_apdu;
3009                 handle_incoming_Z_PDU(m_s2z_init_apdu);
3010                 return;
3011             }
3012             else
3013             {
3014                 handle_incoming_Z_PDU(m_s2z_search_apdu);
3015                 return;
3016             }
3017         }
3018         else if (srw_pdu->which == Z_SRW_explain_request)
3019         {
3020             Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
3021
3022             const char *backend_db = srw_req->database;
3023             srw_get_client(srw_req->database, &backend_db);
3024
3025             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
3026
3027             // save stylesheet
3028             if (srw_req->stylesheet)
3029                 m_s2z_stylesheet =
3030                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
3031
3032             if (srw_req->recordPacking &&
3033                 !strcmp(srw_req->recordPacking, "xml"))
3034                 m_s2z_packing = Z_SRW_recordPacking_XML;
3035             else
3036                 m_s2z_packing = Z_SRW_recordPacking_string;
3037
3038             if (num_diagnostic)
3039             {
3040                 send_srw_explain_response(diagnostic, num_diagnostic);
3041                 return;
3042             }
3043
3044             if (!m_client)
3045             {
3046                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
3047                                             Z_APDU_initRequest);
3048
3049                 m_s2z_init_apdu->u.initRequest->idAuthentication = auth;
3050                 
3051                 // prevent m_initRequest_apdu memory from being grabbed
3052                 // in Yaz_Proxy::handle_incoming_Z_PDU
3053                 m_initRequest_apdu = m_s2z_init_apdu;
3054                 handle_incoming_Z_PDU(m_s2z_init_apdu);
3055             }
3056             else
3057                 send_srw_explain_response(0, 0);
3058             return;
3059         }
3060         else if (srw_pdu->which == Z_SRW_scan_request)
3061         {
3062             m_s2z_database = odr_strdup(m_s2z_odr_init,
3063                                         srw_pdu->u.scan_request->database);
3064
3065             yaz_add_srw_diagnostic(odr_decode(),
3066                                    &diagnostic, &num_diagnostic,
3067                                    4, "scan");
3068             Z_SRW_PDU *srw_pdu =
3069                 yaz_srw_get(odr_encode(),
3070                             Z_SRW_scan_response);
3071             Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
3072
3073             srw_res->diagnostics = diagnostic;
3074             srw_res->num_diagnostics = num_diagnostic;
3075             send_srw_response(srw_pdu);
3076             return;
3077         }
3078         else
3079         {
3080             m_s2z_database = 0;
3081
3082             send_to_srw_client_error(4, 0);
3083         }
3084     }
3085     send_http_response(400);
3086 }
3087
3088 void Yaz_Proxy::handle_init(Z_APDU *apdu)
3089 {
3090
3091     Z_OtherInformation **oi;
3092     get_otherInfoAPDU(apdu, &oi);
3093
3094     if (apdu->u.initRequest->implementationId)
3095         yaz_log(YLOG_LOG, "%simplementationId: %s",
3096                 m_session_str, apdu->u.initRequest->implementationId);
3097     if (apdu->u.initRequest->implementationName)
3098         yaz_log(YLOG_LOG, "%simplementationName: %s",
3099                 m_session_str, apdu->u.initRequest->implementationName);
3100     if (apdu->u.initRequest->implementationVersion)
3101         yaz_log(YLOG_LOG, "%simplementationVersion: %s",
3102                 m_session_str, apdu->u.initRequest->implementationVersion);
3103     if (m_initRequest_apdu == 0)
3104     {
3105         if (m_initRequest_mem)
3106             nmem_destroy(m_initRequest_mem);
3107
3108         m_initRequest_apdu = apdu;
3109         m_initRequest_mem = odr_extract_mem(odr_decode());
3110
3111         m_initRequest_preferredMessageSize = *apdu->u.initRequest->
3112             preferredMessageSize;
3113         *apdu->u.initRequest->preferredMessageSize = 1024*1024;
3114         m_initRequest_maximumRecordSize = *apdu->u.initRequest->
3115             maximumRecordSize;
3116         *apdu->u.initRequest->maximumRecordSize = 1024*1024;
3117
3118         Z_CharSetandLanguageNegotiation *charSetandLangRecord =
3119             yaz_get_charneg_record(*oi);
3120
3121         // Save proposal charsets and langs.
3122         if (ODR_MASK_GET(apdu->u.initRequest->options,
3123                          Z_Options_negotiationModel)
3124             && charSetandLangRecord)
3125         {
3126
3127             yaz_get_proposal_charneg(m_referenceId_mem,
3128                                      charSetandLangRecord,
3129                                      &m_initRequest_oi_negotiation_charsets,
3130                                      &m_initRequest_oi_negotiation_num_charsets,
3131                                      &m_initRequest_oi_negotiation_langs,
3132                                      &m_initRequest_oi_negotiation_num_langs,
3133                                      &m_initRequest_oi_negotiation_selected);
3134
3135             for (int i = 0; i<m_initRequest_oi_negotiation_num_charsets; i++)
3136             {
3137                 yaz_log(YLOG_LOG, "%scharacters set proposal: %s",
3138                         m_session_str,(m_initRequest_oi_negotiation_charsets[i])?
3139                         m_initRequest_oi_negotiation_charsets[i]:"none");
3140             }
3141             for (int i=0; i<m_initRequest_oi_negotiation_num_langs; i++)
3142             {
3143                 yaz_log(YLOG_LOG, "%slanguages proposal: %s",
3144                         m_session_str, (m_initRequest_oi_negotiation_langs[i])?
3145                         m_initRequest_oi_negotiation_langs[i]:"none");
3146             }
3147             yaz_log(YLOG_LOG, "%sselected proposal: %d (boolean)",
3148                     m_session_str, m_initRequest_oi_negotiation_selected);
3149         }
3150         // save init options for the response..
3151         m_initRequest_options = apdu->u.initRequest->options;
3152
3153         apdu->u.initRequest->options =
3154             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
3155                                        sizeof(Odr_bitmask));
3156         ODR_MASK_ZERO(apdu->u.initRequest->options);
3157         int i;
3158         for (i = 0; i<= 24; i++)
3159             ODR_MASK_SET(apdu->u.initRequest->options, i);
3160         // check negotiation option
3161         if (!ODR_MASK_GET(m_initRequest_options,
3162                           Z_Options_negotiationModel))
3163         {
3164             ODR_MASK_CLEAR(apdu->u.initRequest->options,
3165                            Z_Options_negotiationModel);
3166         }
3167         ODR_MASK_CLEAR(apdu->u.initRequest->options,
3168                        Z_Options_concurrentOperations);
3169         // make new version
3170         m_initRequest_version = apdu->u.initRequest->protocolVersion;
3171         apdu->u.initRequest->protocolVersion =
3172             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
3173                                        sizeof(Odr_bitmask));
3174         ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
3175
3176         for (i = 0; i<= 8; i++)
3177             ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
3178     }
3179     handle_charset_lang_negotiation(apdu);
3180     if (m_client->m_init_flag)
3181     {
3182         if (handle_init_response_for_invalid_session(apdu))
3183             return;
3184         if (m_client->m_initResponse)
3185         {
3186             Z_APDU *apdu2 = m_client->m_initResponse;
3187             apdu2->u.initResponse->otherInfo = 0;
3188             if (m_client->m_cookie && *m_client->m_cookie)
3189                 set_otherInformationString(apdu2, yaz_oid_userinfo_cookie, 
3190                                            1, m_client->m_cookie);
3191             apdu2->u.initResponse->referenceId =
3192                 apdu->u.initRequest->referenceId;
3193             apdu2->u.initResponse->options = m_client->m_initResponse_options;
3194             apdu2->u.initResponse->protocolVersion =
3195                 m_client->m_initResponse_version;
3196
3197             handle_charset_lang_negotiation(apdu2);
3198
3199             if (m_timeout_mode == timeout_busy)
3200                 m_timeout_mode = timeout_normal;
3201             send_to_client(apdu2);
3202             return;
3203         }
3204     }
3205     m_client->m_init_flag = 1;
3206
3207     if (m_num_msg_threads && m_my_thread)
3208     {
3209         Auth_Msg *m = new Auth_Msg;
3210         m->m_proxy = this;
3211         z_APDU(odr_encode(), &apdu, 0, "encode");
3212         char *apdu_buf = odr_getbuf(odr_encode(), &m->m_apdu_len, 0);
3213         m->m_apdu_buf = (char*) nmem_malloc(m->m_nmem, m->m_apdu_len);
3214         memcpy(m->m_apdu_buf, apdu_buf, m->m_apdu_len);
3215         odr_reset(odr_encode());
3216         
3217         inc_ref();
3218         m_my_thread->put(m);
3219     }
3220     else
3221     {
3222         int ret = handle_authentication(apdu);
3223         result_authentication(apdu, ret);
3224     }
3225 }
3226
3227 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
3228 {
3229     Z_ReferenceId **refid = get_referenceIdP(apdu);
3230     nmem_reset(m_referenceId_mem);
3231     if (refid && *refid)
3232     {
3233         m_referenceId = (Z_ReferenceId *)
3234             nmem_malloc(m_referenceId_mem, sizeof(*m_referenceId));
3235         m_referenceId->len = m_referenceId->size = (*refid)->len;
3236         m_referenceId->buf = (unsigned char *)
3237             nmem_malloc(m_referenceId_mem, (*refid)->len);
3238         memcpy(m_referenceId->buf, (*refid)->buf, (*refid)->len);
3239     }
3240     else
3241         m_referenceId = 0;
3242
3243     if (!m_client && m_flag_invalid_session)
3244     {
3245         // Got request for a session that is invalid..
3246         m_apdu_invalid_session = apdu; // save package
3247         m_mem_invalid_session = odr_extract_mem(odr_decode());
3248         apdu = m_initRequest_apdu;     // but throw an init to the target
3249     }
3250
3251     if (apdu->which == Z_APDU_searchRequest)
3252         m_search_stat.add_bytes(1);
3253
3254     // Handle global authentication
3255     if (!handle_global_authentication(apdu))
3256     {
3257         if (m_http_version)
3258         {   // HTTP. Send unauthorized
3259             send_http_response(401);
3260             return;
3261         }
3262         else
3263         {
3264             // Z39.50 just shutdown
3265             timeout(0);
3266             return;
3267         }
3268         return;
3269     }
3270
3271     // Determine our client.
3272     Z_OtherInformation **oi;
3273     get_otherInfoAPDU(apdu, &oi);
3274     int http_code = 404;
3275     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi), &http_code);
3276     if (!m_client)
3277     {
3278         if (m_http_version)
3279         {   // HTTP. Send not found
3280             send_http_response(http_code);
3281             return;
3282         }
3283         else
3284         {
3285             // Z39.50 just shutdown
3286             timeout(0);
3287             return;
3288         }
3289     }
3290
3291     m_client->m_server = this;
3292
3293     if (apdu->which == Z_APDU_initRequest)
3294         handle_init(apdu);
3295     else
3296         handle_incoming_Z_PDU_2(apdu);
3297 }
3298
3299 void Yaz_Proxy::handle_incoming_Z_PDU_2(Z_APDU *apdu)
3300 {
3301     handle_max_record_retrieve(apdu);
3302
3303     if (apdu)
3304         apdu = handle_syntax_validation(apdu);
3305
3306     if (apdu)
3307         apdu = handle_query_transformation(apdu);
3308
3309     if (apdu)
3310         apdu = handle_target_charset_conversion(apdu);
3311
3312     if (apdu)
3313         apdu = handle_query_validation(apdu);
3314
3315     if (apdu)
3316         apdu = result_set_optimize(apdu);
3317
3318     if (!apdu)
3319     {
3320         m_client->timeout(m_target_idletime);  // mark it active even
3321         recv_GDU_more(true);
3322         // though we didn't use it
3323         return;
3324     }
3325
3326     // delete other info construct completely if 0 elements
3327     Z_OtherInformation **oi;
3328     get_otherInfoAPDU(apdu, &oi);
3329     if (oi && *oi && (*oi)->num_elements == 0)
3330         *oi = 0;
3331
3332     if (apdu->which == Z_APDU_presentRequest &&
3333         m_client->m_resultSetStartPoint == 0)
3334     {
3335         Z_PresentRequest *pr = apdu->u.presentRequest;
3336         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
3337         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
3338     } else {
3339         m_client->m_resultSetStartPoint = 0;
3340     }
3341     if (m_client->send_to_target(apdu) < 0)
3342     {
3343         m_client->shutdown();
3344     }
3345     else
3346         m_client->m_waiting = 1;
3347 }
3348
3349 void Yaz_Proxy::connectNotify()
3350 {
3351 }
3352
3353 void Yaz_Proxy::releaseClient()
3354 {
3355     xfree(m_proxyTarget);
3356     m_proxyTarget = 0;
3357     m_flag_invalid_session = 0;
3358     // only keep if keep_alive flag is set...
3359     if (m_client &&
3360         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
3361         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
3362         m_client->m_waiting == 0)
3363     {
3364         yaz_log(YLOG_LOG, "%sShutdown (client to proxy) keepalive %s",