4c176187716eecf8e1649c2b4017ad0ab05b4397
[yazproxy-moved-to-github.git] / src / yaz-proxy.cpp
1 /* $Id: yaz-proxy.cpp,v 1.40 2005-11-29 09:17:37 adam Exp $
2    Copyright (c) 1998-2005, Index Data.
3
4 This file is part of the yaz-proxy.
5
6 YAZ proxy is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 2, or (at your option) any later
9 version.
10
11 YAZ proxy is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
14 for more details.
15
16 You should have received a copy of the GNU General Public License
17 along with YAZ proxy; see the file LICENSE.  If not, write to the
18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
19 02111-1307, USA.
20  */
21
22 #ifdef WIN32
23 #define HAVE_SYS_STAT_H 1
24 #define HAVE_SYS_TYPES_H 1
25 #endif
26
27 #if HAVE_UNISTD_H
28 #include <unistd.h>
29 #endif
30 #if HAVE_SYS_TIME_H
31 #include <sys/time.h>
32 #endif
33 #if HAVE_SYS_TYPES_H
34 #include <sys/types.h>
35 #endif
36 #if HAVE_SYS_STAT_H
37 #include <sys/stat.h>
38 #endif
39
40 #include <assert.h>
41 #include <stdlib.h>
42 #include <time.h>
43 #include <fcntl.h>
44
45 #include <yaz/srw.h>
46 #include <yaz/marcdisp.h>
47 #include <yaz/yaz-iconv.h>
48 #include <yaz/log.h>
49 #include <yaz/diagbib1.h>
50 #include "proxyp.h"
51 #include <yaz/pquery.h>
52 #include <yaz/otherinfo.h>
53 #include <yaz/charneg.h>
54 #include "msg-thread.h"
55
56 using namespace yazpp_1;
57
58 #define USE_AUTH_MSG 1
59
60 #if USE_AUTH_MSG
61 class Auth_Msg : public IMsg_Thread {
62 public:
63     int m_ret;
64     IMsg_Thread *handle();
65     void result();
66     Yaz_Proxy *m_proxy;
67     NMEM m_nmem;
68     char *m_apdu_buf;
69     int m_apdu_len;
70     Auth_Msg();
71     virtual ~Auth_Msg();
72 };
73
74 Auth_Msg::Auth_Msg()
75 {
76     m_nmem = nmem_create();
77 }
78
79 Auth_Msg::~Auth_Msg()
80 {
81     nmem_destroy(m_nmem);
82 }
83     
84 IMsg_Thread *Auth_Msg::handle()
85 {
86     yaz_log(YLOG_LOG, "Auth_Msg:handle begin");
87     ODR decode = odr_createmem(ODR_DECODE);
88     Z_APDU *apdu;
89
90     odr_setbuf(decode, m_apdu_buf, m_apdu_len, 0);
91     int r = z_APDU(decode, &apdu, 0, 0);
92     if (!r)
93     {
94         yaz_log(YLOG_WARN, "decode failed in Auth_Msg::handle");
95     }
96     else
97     {
98         m_ret = m_proxy->handle_authentication(apdu);
99     }
100     yaz_log(YLOG_LOG, "Auth_Msg:handle end");
101     odr_destroy(decode);
102     return this;
103 }
104
105 void Auth_Msg::result()
106 {
107     yaz_log(YLOG_LOG, "Auth_Msg:result proxy ok buf=%p len=%d",
108             m_apdu_buf, m_apdu_len);
109     odr_setbuf(m_proxy->odr_decode(), m_apdu_buf, m_apdu_len, 0);
110     Z_APDU *apdu = 0;
111     int r = z_APDU(m_proxy->odr_decode(), &apdu, 0, 0);
112     if (r)
113         yaz_log(YLOG_LOG, "Auth_Msg::result z_APDU OK");
114     else
115         yaz_log(YLOG_LOG, "Auth_Msg::result z_APDU failed");
116     if (m_proxy->dec_ref())
117         yaz_log(YLOG_LOG, "Auth_Msg::proxy deleted meanwhile");
118     else
119     {
120         yaz_log(YLOG_LOG, "Auth_Msg::proxy still alive");
121         m_proxy->result_authentication(apdu, m_ret);
122     }
123     delete this;
124 }
125
126 #endif
127
128 void Yaz_Proxy::result_authentication(Z_APDU *apdu, int ret)
129 {
130     if (apdu == 0 || ret == 0)
131     {
132         Z_APDU *apdu_reject = zget_APDU(odr_encode(), Z_APDU_initResponse);
133         *apdu_reject->u.initResponse->result = 0;
134         send_to_client(apdu_reject);
135         dec_ref();
136     }
137     else
138         handle_incoming_Z_PDU_2(apdu);
139 }
140
141 static const char *apdu_name(Z_APDU *apdu)
142 {
143     switch (apdu->which)
144     {
145     case Z_APDU_initRequest:
146         return "initRequest";
147     case Z_APDU_initResponse:
148         return "initResponse";
149     case Z_APDU_searchRequest:
150         return "searchRequest";
151     case Z_APDU_searchResponse:
152         return "searchResponse";
153     case Z_APDU_presentRequest:
154         return "presentRequest";
155     case Z_APDU_presentResponse:
156         return "presentResponse";
157     case Z_APDU_deleteResultSetRequest:
158         return "deleteResultSetRequest";
159     case Z_APDU_deleteResultSetResponse:
160         return "deleteResultSetResponse";
161     case Z_APDU_scanRequest:
162         return "scanRequest";
163     case Z_APDU_scanResponse:
164         return "scanResponse";
165     case Z_APDU_sortRequest:
166         return "sortRequest";
167     case Z_APDU_sortResponse:
168         return "sortResponse";
169     case Z_APDU_extendedServicesRequest:
170         return "extendedServicesRequest";
171     case Z_APDU_extendedServicesResponse:
172         return "extendedServicesResponse";
173     case Z_APDU_close:
174         return "close";
175     }
176     return "other";
177 }
178
179 static const char *gdu_name(Z_GDU *gdu)
180 {
181     switch(gdu->which)
182     {
183     case Z_GDU_Z3950:
184         return apdu_name(gdu->u.z3950);
185     case Z_GDU_HTTP_Request:
186         return "HTTP Request";
187     case Z_GDU_HTTP_Response:
188         return "HTTP Response";
189     }
190     return "Unknown request/response";
191 }
192
193 Yaz_Proxy::Yaz_Proxy(IPDU_Observable *the_PDU_Observable,
194                      ISocketObservable *the_socket_observable,
195                      Yaz_Proxy *parent) 
196     :
197     Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60)
198 {
199     m_PDU_Observable = the_PDU_Observable;
200     m_socket_observable = the_socket_observable;
201     m_client = 0;
202     m_parent = parent;
203     m_clientPool = 0;
204     m_seqno = 1;
205     m_keepalive_limit_bw = 500000;
206     m_keepalive_limit_pdu = 1000;
207     m_proxyTarget = 0;
208     m_default_target = 0;
209     m_proxy_negotiation_charset = 0;
210     m_proxy_negotiation_lang = 0;
211     m_charset_converter = new Yaz_CharsetConverter;
212     m_max_clients = 150;
213     m_log_mask = 0;
214     m_seed = time(0);
215     m_client_idletime = 600;
216     m_target_idletime = 600;
217     m_optimize = xstrdup ("1");
218     strcpy(m_session_str, "0 ");
219     m_session_no = 0;
220     m_bytes_sent = 0;
221     m_bytes_recv = 0;
222     m_bw_max = 0;
223     m_pdu_max = 0;
224     m_search_max = 0;
225     m_connect_max = 0;
226     m_timeout_mode = timeout_normal;
227     m_timeout_gdu = 0;
228     m_max_record_retrieve = 0;
229     m_reconfig_flag = 0;
230     m_config_fname = 0;
231     m_request_no = 0;
232     m_flag_invalid_session = 0;
233     m_referenceId = 0;
234     m_referenceId_mem = nmem_create();
235     m_config = 0;
236     m_marcxml_mode = none;
237     m_stylesheet_xsp = 0;
238     m_stylesheet_nprl = 0;
239     m_stylesheet_apdu = 0;
240     m_s2z_stylesheet = 0;
241     m_s2z_database = 0;
242     m_schema = 0;
243     m_backend_type = 0;
244     m_backend_charset = 0;
245     m_frontend_type = 0;
246     m_initRequest_apdu = 0;
247     m_initRequest_mem = 0;
248     m_initRequest_preferredMessageSize = 0;
249     m_initRequest_maximumRecordSize = 0;
250     m_initRequest_options = 0;
251     m_initRequest_version = 0;
252     m_initRequest_oi_negotiation_charsets = 0;
253     m_initRequest_oi_negotiation_num_charsets = 0;
254     m_initRequest_oi_negotiation_langs = 0;
255     m_initRequest_oi_negotiation_num_langs = 0;
256     m_initRequest_oi_negotiation_selected = 0;
257     m_apdu_invalid_session = 0;
258     m_mem_invalid_session = 0;
259     m_s2z_odr_init = 0;
260     m_s2z_odr_search = 0;
261     m_s2z_init_apdu = 0;
262     m_s2z_search_apdu = 0;
263     m_s2z_present_apdu = 0;
264     m_http_keepalive = 0;
265     m_http_version = 0;
266     m_soap_ns = 0;
267     m_s2z_packing = Z_SRW_recordPacking_string;
268 #if HAVE_GETTIMEOFDAY
269     m_time_tv = xmalloc(sizeof(struct timeval));
270     struct timeval *tv = (struct timeval *) m_time_tv;
271     tv->tv_sec = 0;
272     tv->tv_usec = 0;
273 #else
274     m_time_tv = 0;
275 #endif
276     m_usemarcon_ini_stage1 = 0;
277     m_usemarcon_ini_stage2 = 0;
278     m_usemarcon = new Yaz_usemarcon();
279     if (!m_parent)
280         low_socket_open();
281     m_my_thread = 0;
282     m_ref_count = 1;
283     m_peername = 0;
284 }
285
286 void Yaz_Proxy::inc_ref()
287 {
288     m_ref_count++;
289 }
290
291 Yaz_Proxy::~Yaz_Proxy()
292 {
293     yaz_log(YLOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
294             m_bytes_sent, m_bytes_recv);
295     nmem_destroy(m_initRequest_mem);
296     nmem_destroy(m_mem_invalid_session);
297     nmem_destroy(m_referenceId_mem);
298
299     xfree(m_proxyTarget);
300     xfree(m_default_target);
301     xfree(m_proxy_negotiation_charset);
302     xfree(m_proxy_negotiation_lang);
303     delete m_charset_converter;
304     xfree(m_optimize);
305
306 #if HAVE_XSLT
307     if (m_stylesheet_xsp)
308         xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
309 #endif
310     xfree (m_time_tv);
311
312     xfree (m_peername);
313     xfree (m_schema);
314     xfree (m_backend_type);
315     xfree (m_backend_charset);
316     xfree (m_usemarcon_ini_stage1);
317     xfree (m_usemarcon_ini_stage2);
318     delete m_usemarcon;
319     if (m_s2z_odr_init)
320         odr_destroy(m_s2z_odr_init);
321     if (m_s2z_odr_search)
322         odr_destroy(m_s2z_odr_search);
323     if (!m_parent)
324         low_socket_close();
325     if (!m_parent)
326         delete m_my_thread;
327     delete m_config;
328 }
329
330 void Yaz_Proxy::set_debug_mode(int mode)
331 {
332     m_debug_mode = mode;
333 }
334
335 int Yaz_Proxy::set_config(const char *config)
336 {
337     delete m_config;
338     m_config = new Yaz_ProxyConfig();
339     xfree(m_config_fname);
340     m_config_fname = xstrdup(config);
341     int r = m_config->read_xml(config);
342     if (!r)
343         m_config->get_generic_info(&m_log_mask, &m_max_clients);
344     return r;
345 }
346
347 void Yaz_Proxy::set_default_target(const char *target)
348 {
349     xfree (m_default_target);
350     m_default_target = 0;
351     if (target)
352         m_default_target = (char *) xstrdup (target);
353 }
354
355 void Yaz_Proxy::set_proxy_negotiation (const char *charset, const char *lang)
356 {
357     yaz_log(YLOG_LOG, "%sSet the proxy negotiation: charset to '%s', "
358         "language to '%s'", m_session_str, charset?charset:"none",
359         lang?lang:"none");
360     xfree (m_proxy_negotiation_charset);
361     xfree (m_proxy_negotiation_lang);
362     m_proxy_negotiation_charset = m_proxy_negotiation_lang = 0;
363     if (charset)
364         m_proxy_negotiation_charset = (char *) xstrdup (charset);
365     if (lang)
366         m_proxy_negotiation_lang = (char *) xstrdup (lang);
367 }
368
369 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
370 {
371     if (m_parent)
372         return m_parent->check_reconfigure();
373
374     Yaz_ProxyConfig *cfg = m_config;
375     if (m_reconfig_flag)
376     {
377         yaz_log(YLOG_LOG, "reconfigure");
378         yaz_log_reopen();
379         if (m_config_fname && cfg)
380         {
381             yaz_log(YLOG_LOG, "reconfigure config %s", m_config_fname);
382             int r = cfg->read_xml(m_config_fname);
383             if (r)
384                 yaz_log(YLOG_WARN, "reconfigure failed");
385             else
386             {
387                 m_log_mask = 0;
388                 cfg->get_generic_info(&m_log_mask, &m_max_clients);
389             }
390         }
391         else
392             yaz_log(YLOG_LOG, "reconfigure");
393         m_reconfig_flag = 0;
394     }
395     return cfg;
396 }
397
398 IPDU_Observer *Yaz_Proxy::sessionNotify(IPDU_Observable
399                                         *the_PDU_Observable, int fd)
400 {
401     check_reconfigure();
402     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable,
403                                          m_socket_observable, this);
404     new_proxy->m_config = 0;
405     new_proxy->m_config_fname = 0;
406     new_proxy->timeout(m_client_idletime);
407     new_proxy->m_target_idletime = m_target_idletime;
408     new_proxy->set_default_target(m_default_target);
409     new_proxy->m_max_clients = m_max_clients;
410     new_proxy->m_log_mask = m_log_mask;
411     new_proxy->set_APDU_log(get_APDU_log());
412     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
413         new_proxy->set_APDU_yazlog(1);
414     else
415         new_proxy->set_APDU_yazlog(0);
416     sprintf(new_proxy->m_session_str, "%ld:%d ", (long) time(0), m_session_no);
417     m_session_no++;
418     new_proxy->m_peername = xstrdup(the_PDU_Observable->getpeername());
419     yaz_log (YLOG_LOG, "%sNew session %s", new_proxy->m_session_str,
420              new_proxy->m_peername);
421     new_proxy->set_proxy_negotiation(m_proxy_negotiation_charset,
422         m_proxy_negotiation_lang);
423     // create thread object the first time we get an incoming connection
424     if (!m_my_thread)
425         m_my_thread = new Msg_Thread(m_socket_observable, 1);
426     new_proxy->m_my_thread = m_my_thread;
427     return new_proxy;
428 }
429
430 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
431 {
432     int oid[OID_SIZE];
433     Z_OtherInformationUnit *oi;
434     struct oident ent;
435     ent.proto = PROTO_Z3950;
436     ent.oclass = CLASS_USERINFO;
437     ent.value = (oid_value) VAL_COOKIE;
438     assert (oid_ent_to_oid (&ent, oid));
439
440     if (oid_ent_to_oid (&ent, oid) && 
441         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
442         oi->which == Z_OtherInfo_characterInfo)
443         return oi->information.characterInfo;
444     return 0;
445 }
446 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
447 {
448     int oid[OID_SIZE];
449     Z_OtherInformationUnit *oi;
450     struct oident ent;
451     ent.proto = PROTO_Z3950;
452     ent.oclass = CLASS_USERINFO;
453     ent.value = (oid_value) VAL_PROXY;
454     if (oid_ent_to_oid (&ent, oid) &&
455         (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
456         oi->which == Z_OtherInfo_characterInfo)
457         return oi->information.characterInfo;
458     return 0;
459 }
460 const char *Yaz_Proxy::load_balance(const char **url)
461 {
462     int zurl_in_use[MAX_ZURL_PLEX];
463     int zurl_in_spare[MAX_ZURL_PLEX];
464     Yaz_ProxyClient *c;
465     int i;
466
467     for (i = 0; i<MAX_ZURL_PLEX; i++)
468     {
469         zurl_in_use[i] = 0;
470         zurl_in_spare[i] = 0;
471     }
472     for (c = m_parent->m_clientPool; c; c = c->m_next)
473     {
474         for (i = 0; url[i]; i++)
475             if (!strcmp(url[i], c->get_hostname()))
476             {
477                 zurl_in_use[i]++;
478                 if (c->m_cookie == 0 && c->m_server == 0 && c->m_waiting == 0)
479                     zurl_in_spare[i]++;
480             }
481     }
482     int min_use = 100000;
483     int spare_for_min = 0;
484     int max_spare = 0;
485     const char *ret_min = 0;
486     const char *ret_spare = 0;
487     for (i = 0; url[i]; i++)
488     {
489         yaz_log(YLOG_DEBUG, "%szurl=%s use=%d spare=%d",
490                 m_session_str, url[i], zurl_in_use[i], zurl_in_spare[i]);
491         if (min_use > zurl_in_use[i])
492         {
493             ret_min = url[i];
494             min_use = zurl_in_use[i];
495             spare_for_min = zurl_in_spare[i];
496         }
497         if (max_spare < zurl_in_spare[i]) 
498         {
499             ret_spare = url[i];
500             max_spare = zurl_in_spare[i];
501         }
502     }
503     return ret_min;
504 }
505
506 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
507                                        const char *proxy_host)
508 {
509     assert (m_parent);
510     Yaz_Proxy *parent = m_parent;
511     Yaz_ProxyClient *c = m_client;
512     
513     if (!m_proxyTarget)
514     {
515         const char *url[MAX_ZURL_PLEX];
516         Yaz_ProxyConfig *cfg = check_reconfigure();
517         if (proxy_host)
518         {
519             if (parent && parent->m_debug_mode)
520             {
521                 // only to be enabled for debugging...
522                 if (!strcmp(proxy_host, "stop"))
523                     exit(0);
524             }
525             xfree(m_default_target);
526             m_default_target = xstrdup(proxy_host);
527         }
528         proxy_host = m_default_target;
529         int client_idletime = -1;
530         const char *cql2rpn_fname = 0;
531         const char *negotiation_charset = 0;
532         const char *negotiation_lang = 0;
533         const char *query_charset = 0;
534         url[0] = m_default_target;
535         url[1] = 0;
536         if (cfg)
537         {
538             int pre_init = 0;
539             cfg->get_target_info(proxy_host, url, &m_bw_max,
540                                  &m_pdu_max, &m_max_record_retrieve,
541                                  &m_search_max, &m_connect_max,
542                                  &m_target_idletime, &client_idletime,
543                                  &parent->m_max_clients,
544                                  &m_keepalive_limit_bw,
545                                  &m_keepalive_limit_pdu,
546                                  &pre_init,
547                                  &cql2rpn_fname,
548                                  &negotiation_charset,
549                                  &negotiation_lang,
550                                  &query_charset);
551         }
552         if (client_idletime != -1)
553         {
554             m_client_idletime = client_idletime;
555             timeout(m_client_idletime);
556         }
557         if (cql2rpn_fname)
558             m_cql2rpn.set_pqf_file(cql2rpn_fname);
559         if (negotiation_charset || negotiation_lang)
560         {
561             set_proxy_negotiation(negotiation_charset,
562                 negotiation_lang);
563         }
564         m_charset_converter->set_target_query_charset(query_charset);
565         if (!url[0])
566         {
567             yaz_log(YLOG_LOG, "%sNo default target", m_session_str);
568             return 0;
569         }
570         // we don't handle multiplexing for cookie session, so we just
571         // pick the first one in this case (anonymous users will be able
572         // to use any backend)
573         if (cookie && *cookie)
574             m_proxyTarget = (char*) xstrdup(url[0]);
575         else
576             m_proxyTarget = (char*) xstrdup(load_balance(url));
577     }
578     if (cookie && *cookie)
579     {   // search in sessions with a cookie
580         for (c = parent->m_clientPool; c; c = c->m_next)
581         {
582             assert (c->m_prev);
583             assert (*c->m_prev == c);
584             if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
585                 !strcmp(m_proxyTarget, c->get_hostname()))
586             {
587                 // Found it in cache
588                 // The following handles "cancel"
589                 // If connection is busy (waiting for PDU) and
590                 // we have an initRequest we can safely do re-open
591                 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
592                 {
593                     yaz_log (YLOG_LOG, "%s REOPEN target=%s", m_session_str,
594                              c->get_hostname());
595                     c->close();
596                     c->m_init_flag = 0;
597                     
598                     c->m_last_ok = 0;
599                     c->m_cache.clear();
600                     c->m_last_resultCount = 0;
601                     c->m_sr_transform = 0;
602                     c->m_waiting = 0;
603                     c->m_resultSetStartPoint = 0;
604                     c->m_target_idletime = m_target_idletime;
605                     if (c->client(m_proxyTarget))
606                     {
607                         delete c;
608                         return 0;
609                     }
610                     c->timeout(30); 
611                 }
612                 c->m_seqno = parent->m_seqno;
613                 if (c->m_server && c->m_server != this)
614                     c->m_server->m_client = 0;
615                 c->m_server = this;
616                 (parent->m_seqno)++;
617                 yaz_log (YLOG_DEBUG, "get_client 1 %p %p", this, c);
618                 return c;
619             }
620         }
621     }
622     else if (!c &&
623             apdu->which == Z_APDU_initRequest &&
624             apdu->u.initRequest->idAuthentication == 0 &&
625             !ODR_MASK_GET(apdu->u.initRequest->options, Z_Options_negotiationModel))
626     {
627         // anonymous sessions without cookie.
628         // if authentication is set it is NOT anonymous se we can't share them.
629         // If charset and lang negotiation is use it is NOT anonymous session too.
630         for (c = parent->m_clientPool; c; c = c->m_next)
631         {
632             assert(c->m_prev);
633             assert(*c->m_prev == c);
634             if (c->m_server == 0 && c->m_cookie == 0 && 
635                 c->m_waiting == 0 && 
636                 !strcmp(m_proxyTarget, c->get_hostname()))
637             {
638                 // found it in cache
639                 yaz_log (YLOG_LOG, "%sREUSE %d %s",
640                          m_session_str, parent->m_seqno, c->get_hostname());
641                 
642                 c->m_seqno = parent->m_seqno;
643                 assert(c->m_server == 0);
644                 c->m_server = this;
645
646                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
647                     c->set_APDU_yazlog(1);
648                 else
649                     c->set_APDU_yazlog(0);
650
651                 (parent->m_seqno)++;
652                 
653                 parent->pre_init();
654                 
655                 return c;
656             }
657         }
658     }
659     if (!m_client)
660     {
661         if (apdu->which != Z_APDU_initRequest)
662         {
663             yaz_log (YLOG_LOG, "%sno init request as first PDU", m_session_str);
664             return 0;
665         }
666         Z_InitRequest *initRequest = apdu->u.initRequest;
667
668         if (initRequest->idAuthentication)
669         {
670             // the client uses authentication. We set the keepalive PDU
671             // to 0 so we don't cache it in releaseClient
672             m_keepalive_limit_pdu = 0;
673         }
674         // go through list of clients - and find the lowest/oldest one.
675         Yaz_ProxyClient *c_min = 0;
676         int min_seq = -1;
677         int no_of_clients = 0;
678         if (parent->m_clientPool)
679             yaz_log (YLOG_DEBUG, "Existing sessions");
680         for (c = parent->m_clientPool; c; c = c->m_next)
681         {
682             yaz_log (YLOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
683                                c->m_waiting, c->get_hostname(),
684                                c->m_cookie ? c->m_cookie : "");
685             no_of_clients++;
686             if (min_seq < 0 || c->m_seqno < min_seq)
687             {
688                 min_seq = c->m_seqno;
689                 c_min = c;
690             }
691         }
692         if (no_of_clients >= parent->m_max_clients)
693         {
694             c = c_min;
695             if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
696             {
697                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Destroy %d",
698                          m_session_str, parent->m_max_clients, c->m_seqno);
699                 if (c->m_server && c->m_server != this)
700                     delete c->m_server;   // PROBLEM: m_ref_count!
701                 c->m_server = 0;
702             }
703             else
704             {
705                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Reuse %d %d %s",
706                          m_session_str, parent->m_max_clients,
707                          c->m_seqno, parent->m_seqno, c->get_hostname());
708                 xfree (c->m_cookie);
709                 c->m_cookie = 0;
710                 if (cookie)
711                     c->m_cookie = xstrdup(cookie);
712                 c->m_seqno = parent->m_seqno;
713                 if (c->m_server && c->m_server != this)
714                 {
715                     c->m_server->m_client = 0;
716                     delete c->m_server;   // PROBLEM: m_ref_count!
717                 }
718                 (parent->m_seqno)++;
719                 c->m_target_idletime = m_target_idletime;
720                 c->timeout(m_target_idletime);
721                 
722                 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
723                     c->set_APDU_yazlog(1);
724                 else
725                     c->set_APDU_yazlog(0);
726
727                 return c;
728             }
729         }
730         else
731         {
732             yaz_log (YLOG_LOG, "%sNEW %d %s",
733                      m_session_str, parent->m_seqno, m_proxyTarget);
734             c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
735             c->m_next = parent->m_clientPool;
736             if (c->m_next)
737                 c->m_next->m_prev = &c->m_next;
738             parent->m_clientPool = c;
739             c->m_prev = &parent->m_clientPool;
740         }
741
742         xfree (c->m_cookie);
743         c->m_cookie = 0;
744         if (cookie)
745             c->m_cookie = xstrdup(cookie);
746
747         c->m_seqno = parent->m_seqno;
748         c->m_init_flag = 0;
749         c->m_last_resultCount = 0;
750         c->m_last_ok = 0;
751         c->m_cache.clear();
752         c->m_sr_transform = 0;
753         c->m_waiting = 0;
754         c->m_resultSetStartPoint = 0;
755         (parent->m_seqno)++;
756         if (c->client(m_proxyTarget))
757         {
758             delete c;
759             return 0;
760         }
761         c->m_target_idletime = m_target_idletime;
762         c->timeout(30);
763
764         if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
765             c->set_APDU_yazlog(1);
766         else
767             c->set_APDU_yazlog(0);
768     }
769     yaz_log (YLOG_DEBUG, "get_client 3 %p %p", this, c);
770     return c;
771 }
772
773 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
774 {
775     int i;
776     for (i = 0; i<num; i++)
777     {
778         oident *ent;
779         Z_DefaultDiagFormat *r;
780         Z_DiagRec *p = pp[i];
781         if (p->which != Z_DiagRec_defaultFormat)
782         {
783             yaz_log(YLOG_LOG, "%sError no diagnostics", m_session_str);
784             return;
785         }
786         else
787             r = p->u.defaultFormat;
788         if (!(ent = oid_getentbyoid(r->diagnosticSetId)) ||
789             ent->oclass != CLASS_DIAGSET || ent->value != VAL_BIB1)
790             yaz_log(YLOG_LOG, "%sError unknown diagnostic set", m_session_str);
791         switch (r->which)
792         {
793         case Z_DefaultDiagFormat_v2Addinfo:
794             yaz_log(YLOG_LOG, "%sError %d %s:%s",
795                     m_session_str,
796                     *r->condition, diagbib1_str(*r->condition),
797                     r->u.v2Addinfo);
798             break;
799         case Z_DefaultDiagFormat_v3Addinfo:
800             yaz_log(YLOG_LOG, "%sError %d %s:%s",
801                     m_session_str,
802                     *r->condition, diagbib1_str(*r->condition),
803                     r->u.v3Addinfo);
804             break;
805         }
806     }
807 }
808
809 int Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p, Z_APDU *apdu)
810 {
811     if (!m_stylesheet_xsp || p->num_records <= 0)
812     {
813         return 0;  /* no XSLT to be done ... */
814     }
815
816     m_stylesheet_offset = 0;
817     m_stylesheet_nprl = p;
818     m_stylesheet_apdu = apdu;
819     m_timeout_mode = timeout_xsl;
820
821     timeout(0);
822     return 1;
823 }
824
825 void Yaz_Proxy::convert_xsl_delay()
826 {
827 #if HAVE_XSLT
828     Z_NamePlusRecord *npr = m_stylesheet_nprl->records[m_stylesheet_offset];
829     if (npr->which == Z_NamePlusRecord_databaseRecord)
830     {
831         Z_External *r = npr->u.databaseRecord;
832         if (r->which == Z_External_octet)
833         {
834 #if 0
835             fwrite((char*) r->u.octet_aligned->buf, 1, r->u.octet_aligned->len, stdout);
836 #endif
837             xmlDocPtr res, doc = xmlParseMemory(
838                 (char*) r->u.octet_aligned->buf,
839                 r->u.octet_aligned->len);
840
841             
842             yaz_log(YLOG_LOG, "%sXSLT convert %d",
843                     m_session_str, m_stylesheet_offset);
844             res = xsltApplyStylesheet((xsltStylesheetPtr) m_stylesheet_xsp,
845                                       doc, 0);
846
847             if (res)
848             {
849                 xmlChar *out_buf;
850                 int out_len;
851                 xmlDocDumpFormatMemory (res, &out_buf, &out_len, 1);
852                 
853                 m_stylesheet_nprl->records[m_stylesheet_offset]->
854                     u.databaseRecord = 
855                     z_ext_record(odr_encode(), VAL_TEXT_XML,
856                                  (char*) out_buf, out_len);
857                 xmlFree(out_buf);
858                 xmlFreeDoc(res);
859             }
860
861             xmlFreeDoc(doc);
862         }
863     }
864 #endif
865     m_stylesheet_offset++;
866     if (m_stylesheet_offset == m_stylesheet_nprl->num_records)
867     {
868         m_timeout_mode = timeout_normal;
869         m_stylesheet_nprl = 0;
870 #if HAVE_XSLT
871         if (m_stylesheet_xsp)
872             xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
873 #endif
874         m_stylesheet_xsp = 0;
875         timeout(m_client_idletime);
876         send_PDU_convert(m_stylesheet_apdu);
877     }
878     else
879         timeout(0);
880 }
881
882 void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p)
883 {
884     if (m_frontend_type != VAL_NONE)
885     {
886         int i;
887         for (i = 0; i < p->num_records; i++)
888         {
889             Z_NamePlusRecord *npr = p->records[i];
890             if (npr->which == Z_NamePlusRecord_databaseRecord)
891             {
892                 Z_External *r = npr->u.databaseRecord;
893                 if (r->which == Z_External_octet)
894                 {
895 #if HAVE_USEMARCON
896                     if (m_usemarcon_ini_stage1 && *m_usemarcon_ini_stage1)
897                     {
898                         if (!m_usemarcon->m_stage1)
899                         {
900                             m_usemarcon->m_stage1 = new CDetails();
901                         }
902                         m_usemarcon->m_stage1->SetIniFileName(m_usemarcon_ini_stage1);
903                         m_usemarcon->m_stage1->SetMarcRecord((char*) r->u.octet_aligned->buf, r->u.octet_aligned->len);
904                         int res = m_usemarcon->m_stage1->Start();
905                         if (res == 0)
906                         {
907                             char *converted;
908                             int convlen;
909                             m_usemarcon->m_stage1->GetMarcRecord(converted, convlen);
910                             if (m_usemarcon_ini_stage2 && *m_usemarcon_ini_stage2)
911                             {
912                                 if (!m_usemarcon->m_stage2)
913                                 {
914                                     m_usemarcon->m_stage2 = new CDetails();
915                                 }
916                                 m_usemarcon->m_stage2->SetIniFileName(m_usemarcon_ini_stage2);
917                                 m_usemarcon->m_stage2->SetMarcRecord(converted, convlen);
918                                 res = m_usemarcon->m_stage2->Start();
919                                 if (res == 0)
920                                 {
921                                     free(converted);
922                                     m_usemarcon->m_stage2->GetMarcRecord(converted, convlen);
923                                 }
924                                 else
925                                 {
926                                     yaz_log(YLOG_LOG, "%sUSEMARCON stage 2 error %d", m_session_str, res);
927                                 }
928                             }
929                             npr->u.databaseRecord =
930                                 z_ext_record(odr_encode(),
931                                              m_frontend_type,
932                                              converted,
933                                              strlen(converted));
934                             free(converted);
935                         }
936                         else
937                         {
938                             yaz_log(YLOG_LOG, "%sUSEMARCON stage 1 error %d", m_session_str, res);
939                         }
940                         continue;
941                     }
942 #endif
943 /* HAVE_USEMARCON */
944                     npr->u.databaseRecord =
945                         z_ext_record(odr_encode(),
946                                      m_frontend_type,
947                                      (char*) r->u.octet_aligned->buf,
948                                      r->u.octet_aligned->len);
949                 }
950             }
951         }
952     }
953 }
954
955 void Yaz_Proxy::convert_records_charset(Z_NamePlusRecordList *p,
956                                         const char *backend_charset)
957 {
958     yaz_log(YLOG_LOG, "%sconvert_to_marc", m_session_str);
959     int sel =   m_charset_converter->get_client_charset_selected();
960     const char *client_record_charset =
961         m_charset_converter->get_client_query_charset();
962     if (sel && backend_charset && client_record_charset &&
963         strcmp(backend_charset, client_record_charset))
964     {
965         int i;
966         yaz_iconv_t cd = yaz_iconv_open(client_record_charset,
967                                         backend_charset);
968         yaz_marc_t mt = yaz_marc_create();
969         yaz_marc_xml(mt, YAZ_MARC_ISO2709);
970         yaz_marc_iconv(mt, cd);
971         for (i = 0; i < p->num_records; i++)
972         {
973             Z_NamePlusRecord *npr = p->records[i];
974             if (npr->which == Z_NamePlusRecord_databaseRecord)
975             {
976                 Z_External *r = npr->u.databaseRecord;
977                 oident *ent = oid_getentbyoid(r->direct_reference);
978                 if (!ent || ent->value == VAL_NONE)
979                     continue;
980
981                 if (ent->value == VAL_SUTRS)
982                 {
983                     WRBUF w = wrbuf_alloc();
984
985                     wrbuf_iconv_write(w, cd,  (char*) r->u.octet_aligned->buf,
986                                       r->u.octet_aligned->len);
987                     npr->u.databaseRecord =
988                         z_ext_record(odr_encode(), ent->value, wrbuf_buf(w),
989                                      wrbuf_len(w));
990                     wrbuf_free(w, 1);
991                 }
992                 else if (ent->value == VAL_TEXT_XML)
993                 {
994                     ;
995                 }
996                 else if (r->which == Z_External_octet)
997                 {
998                     int rlen;
999                     char *result;
1000                     if (yaz_marc_decode_buf(mt,
1001                                             (char*) r->u.octet_aligned->buf,
1002                                             r->u.octet_aligned->len,
1003                                             &result, &rlen))
1004                     {
1005                         npr->u.databaseRecord =
1006                             z_ext_record(odr_encode(), ent->value, result, rlen);
1007                         yaz_log(YLOG_LOG, "%sRecoding MARC record",
1008                                 m_session_str);
1009                     }
1010                 }
1011             }
1012         }
1013         if (cd)
1014             yaz_iconv_close(cd);
1015         yaz_marc_destroy(mt);
1016     }
1017     else
1018     {
1019         yaz_log(YLOG_LOG, "%sSkipping marc convert", m_session_str);
1020     }
1021 }
1022
1023 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p,
1024                                    const char *backend_charset)
1025 {
1026     int i;
1027     if (!backend_charset)
1028         backend_charset = "MARC-8";
1029     yaz_iconv_t cd = yaz_iconv_open("UTF-8", backend_charset);
1030     yaz_marc_t mt = yaz_marc_create();
1031     yaz_marc_xml(mt, YAZ_MARC_MARCXML);
1032     yaz_marc_iconv(mt, cd);
1033     for (i = 0; i < p->num_records; i++)
1034     {
1035         Z_NamePlusRecord *npr = p->records[i];
1036         if (npr->which == Z_NamePlusRecord_databaseRecord)
1037         {
1038             Z_External *r = npr->u.databaseRecord;
1039             if (r->which == Z_External_OPAC)
1040             {
1041                 WRBUF w = wrbuf_alloc();
1042
1043                 yaz_display_OPAC(w, r->u.opac, 0);
1044                 npr->u.databaseRecord = z_ext_record(
1045                     odr_encode(), VAL_TEXT_XML,
1046                     wrbuf_buf(w), wrbuf_len(w)
1047                     );
1048                 wrbuf_free(w, 1);
1049             }
1050             else if (r->which == Z_External_octet)
1051             {
1052                 int rlen;
1053                 char *result;
1054                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
1055                                         r->u.octet_aligned->len,
1056                                         &result, &rlen))
1057                 {
1058                     npr->u.databaseRecord =
1059                         z_ext_record(odr_encode(), VAL_TEXT_XML, result, rlen);
1060                 }
1061             }
1062         }
1063     }
1064     if (cd)
1065         yaz_iconv_close(cd);
1066     yaz_marc_destroy(mt);
1067 }
1068
1069 void Yaz_Proxy::logtime()
1070 {
1071 #if HAVE_GETTIMEOFDAY
1072     struct timeval *tv = (struct timeval*) m_time_tv;
1073     if (tv->tv_sec)
1074     {
1075         struct timeval tv1;
1076         gettimeofday(&tv1, 0);
1077         long diff = (tv1.tv_sec - tv->tv_sec)*1000000 +
1078             (tv1.tv_usec - tv->tv_usec);
1079         if (diff >= 0)
1080             yaz_log(YLOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
1081                     diff/1000000, (diff/1000)%1000);
1082     }
1083     tv->tv_sec = 0;
1084     tv->tv_usec = 0;
1085 #endif
1086 }
1087
1088 int Yaz_Proxy::send_http_response(int code)
1089 {
1090     ODR o = odr_encode();
1091     Z_GDU *gdu = z_get_HTTP_Response(o, code);
1092     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1093     if (m_http_version)
1094         hres->version = odr_strdup(o, m_http_version);
1095     if (m_http_keepalive)
1096         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1097     else
1098         timeout(0);
1099     
1100     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1101     {
1102         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1103                  gdu_name(gdu));
1104     }
1105     int len;
1106     int r = send_GDU(gdu, &len);
1107     m_bytes_sent += len;
1108     m_bw_stat.add_bytes(len);
1109     logtime();
1110
1111     recv_GDU_more(true);
1112
1113     return r;
1114 }
1115
1116 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu)
1117 {
1118     ODR o = odr_encode();
1119     const char *ctype = "text/xml";
1120     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
1121     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
1122     if (m_http_version)
1123         hres->version = odr_strdup(o, m_http_version);
1124     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
1125     if (m_http_keepalive)
1126         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
1127     else
1128         timeout(0);
1129
1130     static Z_SOAP_Handler soap_handlers[2] = {
1131 #if HAVE_XSLT
1132         {"http://www.loc.gov/zing/srw/", 0,
1133          (Z_SOAP_fun) yaz_srw_codec},
1134 #endif
1135         {0, 0, 0}
1136     };
1137     
1138     Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
1139     soap_package->which = Z_SOAP_generic;
1140     soap_package->u.generic = 
1141         (Z_SOAP_Generic *) odr_malloc(o,  sizeof(*soap_package->u.generic));
1142     soap_package->u.generic->no = 0;
1143     soap_package->u.generic->ns = soap_handlers[0].ns;
1144     soap_package->u.generic->p = (void *) srw_pdu;
1145     soap_package->ns = m_soap_ns;
1146     z_soap_codec_enc_xsl(o, &soap_package,
1147                          &hres->content_buf, &hres->content_len,
1148                          soap_handlers, 0, m_s2z_stylesheet);
1149     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1150     {
1151         yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1152                  gdu_name(gdu));
1153     }
1154     int len;
1155     int r = send_GDU(gdu, &len);
1156     m_bytes_sent += len;
1157     m_bw_stat.add_bytes(len);
1158     logtime();
1159
1160     recv_GDU_more(true);
1161
1162     return r;
1163 }
1164
1165 int Yaz_Proxy::send_to_srw_client_error(int srw_error, const char *add)
1166 {
1167     ODR o = odr_encode();
1168     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1169     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1170
1171     srw_res->num_diagnostics = 1;
1172     srw_res->diagnostics = (Z_SRW_diagnostic *)
1173         odr_malloc(o, sizeof(*srw_res->diagnostics));
1174     yaz_mk_std_diagnostic(o, srw_res->diagnostics, srw_error, add);
1175     return send_srw_response(srw_pdu);
1176 }
1177
1178 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
1179                              Z_DefaultDiagFormat *ddf)
1180 {
1181     int bib1_code = *ddf->condition;
1182     if (bib1_code == 109)
1183         return 404;
1184     srw_res->num_diagnostics = 1;
1185     srw_res->diagnostics = (Z_SRW_diagnostic *)
1186         odr_malloc(o, sizeof(*srw_res->diagnostics));
1187     yaz_mk_std_diagnostic(o, srw_res->diagnostics,
1188                           yaz_diag_bib1_to_srw(*ddf->condition), 
1189                           ddf->u.v2Addinfo);
1190     return 0;
1191 }
1192
1193 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
1194 {
1195     ODR o = odr_encode();
1196     Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
1197     Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1198
1199     srw_res->numberOfRecords = odr_intdup (o, hits);
1200     if (records && records->which == Z_Records_DBOSD)
1201     {
1202         srw_res->num_records =
1203             records->u.databaseOrSurDiagnostics->num_records;
1204         int i;
1205         srw_res->records = (Z_SRW_record *)
1206             odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
1207         for (i = 0; i < srw_res->num_records; i++)
1208         {
1209             Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
1210             if (npr->which != Z_NamePlusRecord_databaseRecord)
1211             {
1212                 srw_res->records[i].recordSchema = "diagnostic";
1213                 srw_res->records[i].recordPacking = m_s2z_packing;
1214                 srw_res->records[i].recordData_buf = "67";
1215                 srw_res->records[i].recordData_len = 2;
1216                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1217                 continue;
1218             }
1219             Z_External *r = npr->u.databaseRecord;
1220             oident *ent = oid_getentbyoid(r->direct_reference);
1221             if (r->which == Z_External_octet && ent->value == VAL_TEXT_XML)
1222             {
1223                 srw_res->records[i].recordSchema = m_schema;
1224                 srw_res->records[i].recordPacking = m_s2z_packing;
1225                 srw_res->records[i].recordData_buf = (char*) 
1226                     r->u.octet_aligned->buf;
1227                 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
1228                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1229             }
1230             else
1231             {
1232                 srw_res->records[i].recordSchema = "diagnostic";
1233                 srw_res->records[i].recordPacking = m_s2z_packing;
1234                 srw_res->records[i].recordData_buf = "67";
1235                 srw_res->records[i].recordData_len = 2;
1236                 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
1237             }
1238         }
1239     }
1240     if (records && records->which == Z_Records_NSD)
1241     {
1242         int http_code;
1243         http_code = z_to_srw_diag(odr_encode(), srw_res,
1244                                    records->u.nonSurrogateDiagnostic);
1245         if (http_code)
1246             return send_http_response(http_code);
1247     }
1248     return send_srw_response(srw_pdu);
1249     
1250 }
1251
1252 int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics,
1253                                         int num_diagnostics)
1254 {
1255     Yaz_ProxyConfig *cfg = check_reconfigure();
1256     if (cfg)
1257     {
1258         int len;
1259         char *b = cfg->get_explain_doc(odr_encode(), 0 /* target */,
1260                                        m_s2z_database, &len);
1261         if (b)
1262         {
1263             Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
1264             Z_SRW_explainResponse *er = res->u.explain_response;
1265
1266             er->record.recordData_buf = b;
1267             er->record.recordData_len = len;
1268             er->record.recordPacking = m_s2z_packing;
1269             er->record.recordSchema = "http://explain.z3950.org/dtd/2.0/";
1270
1271             er->diagnostics = diagnostics;
1272             er->num_diagnostics = num_diagnostics;
1273             return send_srw_response(res);
1274         }
1275     }
1276     return send_http_response(404);
1277 }
1278
1279 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu)
1280 {
1281     if (m_http_version)
1282     {
1283         if (apdu->which == Z_APDU_initResponse)
1284         {
1285             Z_InitResponse *res = apdu->u.initResponse;
1286             if (*res->result == 0)
1287             {
1288                 send_to_srw_client_error(3, 0);
1289             }
1290             else if (!m_s2z_search_apdu)
1291             {
1292                 send_srw_explain_response(0, 0);
1293             }
1294             else
1295             {
1296                 handle_incoming_Z_PDU(m_s2z_search_apdu);
1297             }
1298         }
1299         else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
1300         {
1301             m_s2z_search_apdu = 0;
1302             Z_SearchResponse *res = apdu->u.searchResponse;
1303             m_s2z_hit_count = *res->resultCount;
1304             if (res->records && res->records->which == Z_Records_NSD)
1305             {
1306                 send_to_srw_client_ok(0, res->records, 1);
1307             }
1308             else if (m_s2z_present_apdu && m_s2z_hit_count > 0)
1309             {
1310                 // adjust 
1311                 Z_PresentRequest *pr = m_s2z_present_apdu->u.presentRequest;
1312                 
1313                 if (*pr->resultSetStartPoint <= m_s2z_hit_count)
1314                 {
1315                     if (*pr->numberOfRecordsRequested+ *pr->resultSetStartPoint
1316                         > m_s2z_hit_count)
1317                         *pr->numberOfRecordsRequested =
1318                             1 + m_s2z_hit_count - *pr->resultSetStartPoint;
1319                 }
1320                 handle_incoming_Z_PDU(m_s2z_present_apdu);
1321             }
1322             else
1323             {
1324                 m_s2z_present_apdu = 0;
1325                 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
1326             }
1327         }
1328         else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
1329         {
1330             int start = 
1331                 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
1332
1333             m_s2z_present_apdu = 0;
1334             Z_PresentResponse *res = apdu->u.presentResponse;
1335             send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
1336         }
1337     }
1338     else
1339     {
1340         int len = 0;
1341         if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1342             yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str,
1343                      apdu_name(apdu));
1344         int r = send_Z_PDU(apdu, &len);
1345         m_bytes_sent += len;
1346         m_bw_stat.add_bytes(len);
1347         logtime();
1348         return r;
1349     }
1350     return 0;
1351 }
1352
1353 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
1354 {
1355     int kill_session = 0;
1356     Z_ReferenceId **new_id = get_referenceIdP(apdu);
1357
1358     if (new_id)
1359         *new_id = m_referenceId;
1360     
1361     if (apdu->which == Z_APDU_searchResponse)
1362     {
1363         Z_SearchResponse *sr = apdu->u.searchResponse;
1364         Z_Records *p = sr->records;
1365         if (p && p->which == Z_Records_NSD)
1366         {
1367             Z_DiagRec dr, *dr_p = &dr;
1368             dr.which = Z_DiagRec_defaultFormat;
1369             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1370
1371             *sr->searchStatus = 0;
1372             display_diagrecs(&dr_p, 1);
1373         }
1374         else
1375         {
1376             if (p && p->which == Z_Records_DBOSD)
1377             {
1378                 if (m_backend_type
1379 #if HAVE_USEMARCON
1380                     || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1381 #endif
1382                     )
1383                     convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1384                 if (m_marcxml_mode == marcxml)
1385                     convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1386                                        m_backend_charset);
1387                 else
1388                     convert_records_charset(p->u.databaseOrSurDiagnostics,
1389                                             m_backend_charset);
1390                 if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1391                     return 0;
1392                     
1393             }
1394             if (sr->resultCount)
1395             {
1396                 yaz_log(YLOG_LOG, "%s%d hits", m_session_str,
1397                         *sr->resultCount);
1398                 if (*sr->resultCount < 0)
1399                 {
1400                     m_flag_invalid_session = 1;
1401                     kill_session = 1;
1402
1403                     *sr->searchStatus = 0;
1404                     sr->records =
1405                         create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
1406                     *sr->resultCount = 0;
1407                 }
1408             }
1409         }
1410     }
1411     else if (apdu->which == Z_APDU_presentResponse)
1412     {
1413         Z_PresentResponse *sr = apdu->u.presentResponse;
1414         Z_Records *p = sr->records;
1415         if (p && p->which == Z_Records_NSD)
1416         {
1417             Z_DiagRec dr, *dr_p = &dr;
1418             dr.which = Z_DiagRec_defaultFormat;
1419             dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1420             if (*sr->presentStatus == Z_PresentStatus_success)
1421                 *sr->presentStatus = Z_PresentStatus_failure;
1422             display_diagrecs(&dr_p, 1);
1423         }
1424         if (p && p->which == Z_Records_DBOSD)
1425         {
1426             if (m_backend_type 
1427 #if HAVE_USEMARCON
1428                 || m_usemarcon_ini_stage1 || m_usemarcon_ini_stage2
1429 #endif
1430                 )
1431                 convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
1432             if (m_marcxml_mode == marcxml)
1433                 convert_to_marcxml(p->u.databaseOrSurDiagnostics,
1434                                    m_backend_charset);
1435             else
1436                 convert_records_charset(p->u.databaseOrSurDiagnostics,
1437                                         m_backend_charset);
1438             if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
1439                 return 0;
1440         }
1441     }
1442     else if (apdu->which == Z_APDU_initResponse)
1443     {
1444         //Get and check negotiation record
1445         //from init response.
1446         handle_charset_lang_negotiation(apdu);
1447         
1448         if (m_initRequest_options)
1449         {
1450             Z_Options *nopt = 
1451                 (Odr_bitmask *)odr_malloc(odr_encode(),
1452                                           sizeof(Odr_bitmask));
1453             ODR_MASK_ZERO(nopt);
1454
1455             int i;
1456             for (i = 0; i<24; i++)
1457                 if (ODR_MASK_GET(m_initRequest_options, i) &&
1458                     ODR_MASK_GET(apdu->u.initResponse->options, i))
1459                     ODR_MASK_SET(nopt, i);
1460             apdu->u.initResponse->options = nopt;           
1461         }
1462         if (m_initRequest_version)
1463         {
1464             Z_ProtocolVersion *nopt = 
1465                 (Odr_bitmask *)odr_malloc(odr_encode(),
1466                                           sizeof(Odr_bitmask));
1467             ODR_MASK_ZERO(nopt);
1468
1469             int i;
1470             for (i = 0; i<8; i++)
1471                 if (ODR_MASK_GET(m_initRequest_version, i) &&
1472                     ODR_MASK_GET(apdu->u.initResponse->protocolVersion, i))
1473                     ODR_MASK_SET(nopt, i);
1474             apdu->u.initResponse->protocolVersion = nopt;           
1475         }
1476         apdu->u.initResponse->preferredMessageSize =
1477             odr_intdup(odr_encode(),
1478                        m_client->m_initResponse_preferredMessageSize >
1479                        m_initRequest_preferredMessageSize ?
1480                        m_initRequest_preferredMessageSize :
1481                        m_client->m_initResponse_preferredMessageSize);
1482         apdu->u.initResponse->maximumRecordSize =
1483             odr_intdup(odr_encode(),
1484                        m_client->m_initResponse_maximumRecordSize >
1485                        m_initRequest_maximumRecordSize ?
1486                        m_initRequest_maximumRecordSize :
1487                        m_client->m_initResponse_maximumRecordSize);
1488     }
1489     
1490     int r = send_PDU_convert(apdu);
1491     if (r)
1492         return r;
1493     if (kill_session)
1494     {
1495         delete m_client;
1496         m_client = 0;
1497         m_parent->pre_init();
1498     }
1499     return r;
1500 }
1501
1502 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
1503 {
1504     int len = 0;
1505     const char *apdu_name_tmp = apdu_name(apdu);
1506     int r = send_Z_PDU(apdu, &len);
1507     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
1508         yaz_log (YLOG_LOG, "%sSending %s to %s %d bytes",
1509                  get_session_str(),
1510                  apdu_name_tmp, get_hostname(), len);
1511     m_bytes_sent += len;
1512     return r;
1513 }
1514
1515 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
1516 {
1517     if (apdu->which == Z_APDU_presentRequest)
1518     {
1519         Z_PresentRequest *pr = apdu->u.presentRequest;
1520         int toget = *pr->numberOfRecordsRequested;
1521         int start = *pr->resultSetStartPoint;
1522
1523         yaz_log(YLOG_LOG, "%sPresent %s %d+%d", m_session_str,
1524                 pr->resultSetId, start, toget);
1525
1526         if (*m_parent->m_optimize == '0')
1527             return apdu;
1528
1529         if (!m_client->m_last_resultSetId)
1530         {
1531             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1532             new_apdu->u.presentResponse->records =
1533                 create_nonSurrogateDiagnostics(odr_encode(), 30,
1534                                                pr->resultSetId);
1535             send_to_client(new_apdu);
1536             return 0;
1537         }
1538         if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1539         {
1540             if (start+toget-1 > m_client->m_last_resultCount)
1541             {
1542                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1543                 new_apdu->u.presentResponse->records =
1544                     create_nonSurrogateDiagnostics(odr_encode(), 13, 0);
1545                 send_to_client(new_apdu);
1546                 return 0;
1547             }
1548             Z_NamePlusRecordList *npr;
1549 #if 0
1550             yaz_log(YLOG_LOG, "%sCache lookup %d+%d syntax=%s",
1551                     m_session_str, start, toget, yaz_z3950oid_to_str(
1552                         pr->preferredRecordSyntax, &oclass));
1553 #endif
1554             if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1555                                           pr->preferredRecordSyntax,
1556                                           pr->recordComposition))
1557             {
1558                 yaz_log (YLOG_LOG, "%sReturned cached records for present request", 
1559                          m_session_str);
1560                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1561                 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1562                 
1563                 new_apdu->u.presentResponse->numberOfRecordsReturned
1564                     = odr_intdup(odr_encode(), toget);
1565                                                                  
1566                 new_apdu->u.presentResponse->records = (Z_Records*)
1567                     odr_malloc(odr_encode(), sizeof(Z_Records));
1568                 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1569                 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1570                 new_apdu->u.presentResponse->nextResultSetPosition =
1571                     odr_intdup(odr_encode(), start+toget);
1572
1573                 send_to_client(new_apdu);
1574                 return 0;
1575             }
1576         }
1577     }
1578
1579     if (apdu->which != Z_APDU_searchRequest)
1580         return apdu;
1581     Z_SearchRequest *sr = apdu->u.searchRequest;
1582     Yaz_Z_Query *this_query = new Yaz_Z_Query;
1583     Yaz_Z_Databases this_databases;
1584
1585     this_databases.set(sr->num_databaseNames, (const char **)
1586                        sr->databaseNames);
1587     
1588     this_query->set_Z_Query(sr->query);
1589
1590     char query_str[120];
1591     this_query->print(query_str, sizeof(query_str)-1);
1592     yaz_log(YLOG_LOG, "%sSearch %s", m_session_str, query_str);
1593
1594     if (*m_parent->m_optimize != '0' &&
1595         m_client->m_last_ok && m_client->m_last_query &&
1596         m_client->m_last_query->match(this_query) &&
1597         !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1598         m_client->m_last_databases.match(this_databases))
1599     {
1600         delete this_query;
1601         if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1602             m_client->m_last_resultCount < *sr->largeSetLowerBound)
1603         {
1604             Z_NamePlusRecordList *npr;
1605             int toget = *sr->mediumSetPresentNumber;
1606             Z_RecordComposition *comp = 0;
1607
1608             if (toget > m_client->m_last_resultCount)
1609                 toget = m_client->m_last_resultCount;
1610             
1611             if (sr->mediumSetElementSetNames)
1612             {
1613                 comp = (Z_RecordComposition *)
1614                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1615                 comp->which = Z_RecordComp_simple;
1616                 comp->u.simple = sr->mediumSetElementSetNames;
1617             }
1618  
1619             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1620                                           sr->preferredRecordSyntax, comp))
1621             {
1622                 yaz_log (YLOG_LOG, "%sReturned cached records for medium set",
1623                          m_session_str);
1624                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1625                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1626                 new_apdu->u.searchResponse->resultCount =
1627                     &m_client->m_last_resultCount;
1628                 
1629                 new_apdu->u.searchResponse->numberOfRecordsReturned
1630                     = odr_intdup(odr_encode(), toget);
1631                                                         
1632                 new_apdu->u.searchResponse->presentStatus =
1633                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1634                 new_apdu->u.searchResponse->records = (Z_Records*)
1635                     odr_malloc(odr_encode(), sizeof(Z_Records));
1636                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1637                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1638                 new_apdu->u.searchResponse->nextResultSetPosition =
1639                     odr_intdup(odr_encode(), toget+1);
1640                 send_to_client(new_apdu);
1641                 return 0;
1642             }
1643             else
1644             {
1645                 // medium Set
1646                 // send present request (medium size)
1647                 yaz_log (YLOG_LOG, "%sOptimizing search for medium set",
1648                          m_session_str);
1649
1650                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1651                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1652                 pr->referenceId = sr->referenceId;
1653                 pr->resultSetId = sr->resultSetName;
1654                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1655                 *pr->numberOfRecordsRequested = toget;
1656                 pr->recordComposition = comp;
1657                 m_client->m_sr_transform = 1;
1658                 return new_apdu;
1659             }
1660         }
1661         else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1662             m_client->m_last_resultCount <= 0)
1663         {
1664             // large set. Return pseudo-search response immediately
1665             yaz_log (YLOG_LOG, "%sOptimizing search for large set",
1666                      m_session_str);
1667             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1668             new_apdu->u.searchResponse->referenceId = sr->referenceId;
1669             new_apdu->u.searchResponse->resultCount =
1670                 &m_client->m_last_resultCount;
1671             send_to_client(new_apdu);
1672             return 0;
1673         }
1674         else
1675         {
1676             Z_NamePlusRecordList *npr;
1677             int toget = m_client->m_last_resultCount;
1678             Z_RecordComposition *comp = 0;
1679             // small set
1680             // send a present request (small set)
1681             
1682             if (sr->smallSetElementSetNames)
1683             {
1684                 comp = (Z_RecordComposition *)
1685                     odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1686                 comp->which = Z_RecordComp_simple;
1687                 comp->u.simple = sr->smallSetElementSetNames;
1688             }
1689
1690             if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1691                                           sr->preferredRecordSyntax, comp))
1692             {
1693                 yaz_log (YLOG_LOG, "%sReturned cached records for small set",
1694                          m_session_str);
1695                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1696                 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1697                 new_apdu->u.searchResponse->resultCount =
1698                     &m_client->m_last_resultCount;
1699                 
1700                 new_apdu->u.searchResponse->numberOfRecordsReturned
1701                     = odr_intdup(odr_encode(), toget);
1702                                                                  
1703                 new_apdu->u.searchResponse->presentStatus =
1704                     odr_intdup(odr_encode(), Z_PresentStatus_success);
1705                 new_apdu->u.searchResponse->records = (Z_Records*)
1706                     odr_malloc(odr_encode(), sizeof(Z_Records));
1707                 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1708                 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1709                 new_apdu->u.searchResponse->nextResultSetPosition =
1710                     odr_intdup(odr_encode(), toget+1);
1711                 send_to_client(new_apdu);
1712                 return 0;
1713             }
1714             else
1715             {
1716                 yaz_log (YLOG_LOG, "%sOptimizing search for small set",
1717                          m_session_str);
1718                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1719                 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1720                 pr->referenceId = sr->referenceId;
1721                 pr->resultSetId = sr->resultSetName;
1722                 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1723                 *pr->numberOfRecordsRequested = toget;
1724                 pr->recordComposition = comp;
1725                 m_client->m_sr_transform = 1;
1726                 return new_apdu;
1727             }
1728         }
1729     }
1730     else  // query doesn't match
1731     {
1732         delete m_client->m_last_query;
1733         m_client->m_last_query = this_query;
1734         m_client->m_last_ok = 0;
1735         m_client->m_cache.clear();
1736         m_client->m_resultSetStartPoint = 0;
1737
1738         xfree (m_client->m_last_resultSetId);
1739         m_client->m_last_resultSetId = xstrdup (sr->resultSetName);
1740
1741         m_client->m_last_databases.set(sr->num_databaseNames,
1742                                        (const char **) sr->databaseNames);
1743     }
1744     return apdu;
1745 }
1746
1747
1748 void Yaz_Proxy::inc_request_no()
1749 {
1750     char *cp = strchr(m_session_str, ' ');
1751     m_request_no++;
1752     if (cp)
1753         sprintf(cp+1, "%d ", m_request_no);
1754 }
1755
1756 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
1757 {
1758     inc_request_no();
1759
1760     m_bytes_recv += len;
1761     
1762     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
1763         yaz_log (YLOG_LOG, "%sReceiving %s from client %d bytes",
1764                  m_session_str, gdu_name(apdu), len);
1765     
1766 #if HAVE_GETTIMEOFDAY
1767     gettimeofday((struct timeval *) m_time_tv, 0);
1768 #endif
1769     m_bw_stat.add_bytes(len);
1770     m_pdu_stat.add_bytes(1);
1771     
1772     GDU *gdu = new GDU(apdu);
1773     m_in_queue.enqueue(gdu);
1774
1775     recv_GDU_more(false);
1776 }
1777
1778 void Yaz_Proxy::recv_GDU_reduce(GDU *gdu)
1779 {
1780     int bw_total = m_bw_stat.get_total();
1781     int pdu_total = m_pdu_stat.get_total();
1782     int reduce = 0;
1783
1784     assert(m_timeout_mode == timeout_busy);
1785     assert(m_timeout_gdu == 0);
1786     
1787     if (m_bw_max)
1788     {
1789         if (bw_total > m_bw_max)
1790         {
1791             reduce = (bw_total/m_bw_max);
1792         }
1793     }
1794     if (m_pdu_max)
1795     {
1796         if (pdu_total > m_pdu_max)
1797         {
1798             int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
1799             reduce = (reduce > nreduce) ? reduce : nreduce;
1800         }
1801     }
1802     m_http_version = 0;
1803
1804 #if 0
1805     /* uncomment to force a big reduce */
1806     m_timeout_mode = timeout_reduce;
1807     m_timeout_gdu = gdu;
1808     timeout(3);       // call us reduce seconds later
1809     return;
1810 #endif    
1811     if (reduce)  
1812     {
1813         yaz_log(YLOG_LOG, "%sdelay=%d bw=%d pdu=%d limit-bw=%d limit-pdu=%d",
1814                 m_session_str, reduce, bw_total, pdu_total,
1815                 m_bw_max, m_pdu_max);
1816         
1817         m_timeout_mode = timeout_reduce;
1818         m_timeout_gdu = gdu;
1819         timeout(reduce);       // call us reduce seconds later
1820     }
1821     else
1822         recv_GDU_normal(gdu);
1823 }
1824
1825 void Yaz_Proxy::recv_GDU_more(bool normal)
1826 {
1827     GDU *g;
1828     if (normal && m_timeout_mode == timeout_busy)
1829         m_timeout_mode = timeout_normal;
1830     while (m_timeout_mode == timeout_normal && (g = m_in_queue.dequeue()))
1831     {
1832         m_timeout_mode = timeout_busy;
1833         recv_GDU_reduce(g);
1834     }
1835 }
1836
1837 void Yaz_Proxy::recv_GDU_normal(GDU *gdu)
1838 {
1839     Z_GDU *apdu = 0;
1840     gdu->move_away_gdu(odr_decode(), &apdu);
1841     delete gdu;
1842
1843     if (apdu->which == Z_GDU_Z3950)
1844         handle_incoming_Z_PDU(apdu->u.z3950);
1845     else if (apdu->which == Z_GDU_HTTP_Request)
1846         handle_incoming_HTTP(apdu->u.HTTP_Request);
1847 }
1848
1849 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
1850 {
1851     if (m_max_record_retrieve)
1852     {
1853         if (apdu->which == Z_APDU_presentRequest)
1854         {
1855             Z_PresentRequest *pr = apdu->u.presentRequest;
1856             if (pr->numberOfRecordsRequested && 
1857                 *pr->numberOfRecordsRequested > m_max_record_retrieve)
1858                 *pr->numberOfRecordsRequested = m_max_record_retrieve;
1859         }
1860     }
1861 }
1862
1863 void Yaz_Proxy::handle_charset_lang_negotiation(Z_APDU *apdu)
1864 {
1865     if (apdu->which == Z_APDU_initRequest)
1866     {
1867         yaz_log(YLOG_LOG, "%shandle_charset_lang_negotiation", 
1868                 m_session_str);
1869         if (m_initRequest_options &&
1870             !ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel) &&
1871             (m_proxy_negotiation_charset || m_proxy_negotiation_lang))
1872         {
1873             // There is no negotiation proposal from
1874             // client's side. OK. The proxy negotiation
1875             // in use, only.
1876             Z_InitRequest *initRequest = apdu->u.initRequest;
1877             Z_OtherInformation **otherInfo;  
1878             Z_OtherInformationUnit *oi;
1879             get_otherInfoAPDU(apdu, &otherInfo);
1880             oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
1881             if (oi)
1882             {
1883                 ODR_MASK_SET(initRequest->options,
1884                     Z_Options_negotiationModel);
1885                 oi->which = Z_OtherInfo_externallyDefinedInfo;    
1886                 oi->information.externallyDefinedInfo =
1887                 yaz_set_proposal_charneg(odr_encode(),
1888                     (const char**)&m_proxy_negotiation_charset,
1889                     m_proxy_negotiation_charset ? 1:0,
1890                     (const char**)&m_proxy_negotiation_lang,
1891                     m_proxy_negotiation_lang ? 1:0,
1892                     1);
1893             }
1894         }
1895         else if (m_initRequest_options &&
1896                  ODR_MASK_GET(m_initRequest_options,
1897                               Z_Options_negotiationModel) &&
1898                  m_charset_converter->get_target_query_charset())
1899         {
1900             yaz_log(YLOG_LOG, "%sManaged charset negotiation: charset=%s",
1901                     m_session_str,
1902                     m_charset_converter->get_target_query_charset());
1903             Z_InitRequest *initRequest = apdu->u.initRequest;
1904             Z_CharSetandLanguageNegotiation *negotiation =
1905                 yaz_get_charneg_record (initRequest->otherInfo);        
1906             if (negotiation &&
1907                 negotiation->which == Z_CharSetandLanguageNegotiation_proposal)
1908             {
1909                 NMEM nmem = nmem_create();
1910                 char **charsets = 0;
1911                 int num_charsets = 0;
1912                 char **langs = 0;
1913                 int num_langs = 0;
1914                 int selected = 0;
1915                 yaz_get_proposal_charneg (nmem, negotiation,
1916                                           &charsets, &num_charsets,
1917                                           &langs, &num_langs, &selected);
1918                 int i;
1919                 for (i = 0; i<num_charsets; i++)
1920                     yaz_log(YLOG_LOG, "%scharset %s", m_session_str,
1921                             charsets[i]);
1922                 for (i = 0; i<num_langs; i++)
1923                     yaz_log(YLOG_LOG, "%slang %s", m_session_str,
1924                             langs[i]);
1925
1926                 const char *t_charset =
1927                     m_charset_converter->get_target_query_charset();
1928                 // sweep through charsets and pick the first supported
1929                 // conversion
1930                 for (i = 0; i<num_charsets; i++)
1931                 {
1932                     const char *c_charset = charsets[i];
1933                     if (!odr_set_charset(odr_decode(), t_charset, c_charset))
1934                         break;
1935                 }
1936                 if (i != num_charsets)
1937                 {
1938                     // got one .. set up ODR for reverse direction
1939                     const char *c_charset = charsets[i];
1940                     odr_set_charset(odr_encode(), c_charset, t_charset);
1941                     m_charset_converter->set_client_query_charset(c_charset);
1942                     m_charset_converter->set_client_charset_selected(selected);
1943                 }
1944                 nmem_destroy(nmem);
1945                 ODR_MASK_CLEAR(m_initRequest_options, 
1946                                Z_Options_negotiationModel);
1947                 yaz_del_charneg_record(&initRequest->otherInfo);
1948             }
1949             else
1950             {
1951                 yaz_log(YLOG_WARN, "%sUnable to decode charset package",
1952                         m_session_str);
1953             }
1954         }
1955     }
1956     else if (apdu->which == Z_APDU_initResponse)
1957     {
1958         Z_InitResponse *initResponse = apdu->u.initResponse;    
1959         Z_OtherInformation **otherInfo;  
1960         
1961         if (ODR_MASK_GET(initResponse->options, Z_Options_negotiationModel))
1962         {
1963             char *charset = 0;
1964             char *lang = 0;
1965             int selected = 0;
1966             
1967             get_otherInfoAPDU(apdu, &otherInfo);
1968             
1969             if (!otherInfo && !(*otherInfo))
1970                 return;
1971             
1972             Z_CharSetandLanguageNegotiation *charneg =
1973                 yaz_get_charneg_record(*otherInfo);
1974                 
1975             if (!charneg)
1976                 return;
1977                                 
1978             yaz_get_response_charneg(m_referenceId_mem, charneg,
1979                 &charset, &lang, &selected);
1980                                 
1981             yaz_log(YLOG_LOG, "%sAccepted charset - '%s' and lang - '%s'",
1982                 m_session_str, (charset)?charset:"none", (lang)?lang:"none");
1983                 
1984             if (m_initRequest_options &&
1985                 ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel))
1986             {
1987                 yaz_log(YLOG_LOG, "%sClient's negotiation record in use",
1988                     m_session_str);          
1989             }
1990             else if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
1991             {
1992                 // negotiation-charset, negotiation-lang
1993                 // elements of config file in use.
1994
1995                 yaz_log(YLOG_LOG, "%sProxy's negotiation record in use",
1996                     m_session_str);
1997
1998                 // clear negotiation option.
1999                 ODR_MASK_CLEAR(initResponse->options, Z_Options_negotiationModel);
2000                 
2001                 // Delete negotiation (charneg-3) entry.
2002                 yaz_del_charneg_record(otherInfo);
2003             }
2004         }
2005         else
2006         {
2007             if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
2008             {
2009                 yaz_log(YLOG_LOG, "%sTarget did not honor negotiation",
2010                         m_session_str);
2011             }
2012             else if (m_charset_converter->get_client_query_charset())
2013             {
2014                 Z_OtherInformation **otherInfo;  
2015                 Z_OtherInformationUnit *oi;
2016                 get_otherInfoAPDU(apdu, &otherInfo);
2017                 oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
2018                 if (oi)
2019                 {
2020                     ODR_MASK_SET(initResponse->options,
2021                                  Z_Options_negotiationModel);
2022                     if (m_initRequest_options)
2023                         ODR_MASK_SET(m_initRequest_options,
2024                                      Z_Options_negotiationModel);
2025                     
2026                     oi->which = Z_OtherInfo_externallyDefinedInfo;    
2027                     oi->information.externallyDefinedInfo =
2028                         yaz_set_response_charneg(
2029                             odr_encode(),
2030                             m_charset_converter->get_client_query_charset(),
2031                             0 /* no lang */,
2032                             m_charset_converter->get_client_charset_selected());
2033                 }
2034             }
2035         }
2036     }
2037 }
2038
2039 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
2040                                                      int error,
2041                                                      const char *addinfo)
2042 {
2043     Z_Records *rec = (Z_Records *)
2044         odr_malloc (odr, sizeof(*rec));
2045     int *err = (int *)
2046         odr_malloc (odr, sizeof(*err));
2047     Z_DiagRec *drec = (Z_DiagRec *)
2048         odr_malloc (odr, sizeof(*drec));
2049     Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
2050         odr_malloc (odr, sizeof(*dr));
2051     *err = error;
2052     rec->which = Z_Records_NSD;
2053     rec->u.nonSurrogateDiagnostic = dr;
2054     dr->diagnosticSetId =
2055         yaz_oidval_to_z3950oid (odr, CLASS_DIAGSET, VAL_BIB1);
2056     dr->condition = err;
2057     dr->which = Z_DefaultDiagFormat_v2Addinfo;
2058     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
2059     return rec;
2060 }
2061
2062 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
2063 {
2064     if (apdu->which == Z_APDU_searchRequest &&
2065         apdu->u.searchRequest->query &&
2066         apdu->u.searchRequest->query->which == Z_Query_type_104 &&
2067         apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
2068     {
2069         Z_RPNQuery *rpnquery = 0;
2070         Z_SearchRequest *sr = apdu->u.searchRequest;
2071         char *addinfo = 0;
2072         
2073         yaz_log(YLOG_LOG, "%sCQL: %s", m_session_str,
2074                 sr->query->u.type_104->u.cql);
2075
2076         int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
2077                                           &rpnquery, odr_encode(),
2078                                           &addinfo);
2079         if (r == -3)
2080             yaz_log(YLOG_LOG, "%sNo CQL to RPN table", m_session_str);
2081         else if (r)
2082         {
2083             yaz_log(YLOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
2084             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2085
2086             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2087             new_apdu->u.searchResponse->records =
2088                 create_nonSurrogateDiagnostics(odr_encode(),
2089                                                yaz_diag_srw_to_bib1(r),
2090                                                addinfo);
2091             *new_apdu->u.searchResponse->searchStatus = 0;
2092
2093             send_to_client(new_apdu);
2094
2095             return 0;
2096         }
2097         else
2098         {
2099             sr->query->which = Z_Query_type_1;
2100             sr->query->u.type_1 = rpnquery;
2101         }
2102         return apdu;
2103     }
2104     return apdu;
2105 }
2106
2107 Z_APDU *Yaz_Proxy::handle_target_charset_conversion(Z_APDU *apdu)
2108 {
2109     if (apdu->which == Z_APDU_searchRequest &&
2110         apdu->u.searchRequest->query)
2111     {
2112         if (apdu->u.searchRequest->query->which == Z_Query_type_1
2113             || apdu->u.searchRequest->query->which == Z_Query_type_101)
2114         {
2115             if (m_http_version)
2116                 m_charset_converter->set_client_query_charset("UTF-8");
2117             Z_RPNQuery *rpnquery = apdu->u.searchRequest->query->u.type_1;
2118             m_charset_converter->convert_type_1(rpnquery, odr_encode());
2119         }
2120     }
2121     return apdu;
2122 }
2123
2124
2125 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
2126 {
2127     if (apdu->which == Z_APDU_searchRequest)
2128     {
2129         Z_SearchRequest *sr = apdu->u.searchRequest;
2130         int err = 0;
2131         char *addinfo = 0;
2132
2133         Yaz_ProxyConfig *cfg = check_reconfigure();
2134         if (cfg)
2135             err = cfg->check_query(odr_encode(), m_default_target,
2136                                    sr->query, &addinfo);
2137         if (err)
2138         {
2139             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2140
2141             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2142             new_apdu->u.searchResponse->records =
2143                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2144             *new_apdu->u.searchResponse->searchStatus = 0;
2145
2146             send_to_client(new_apdu);
2147
2148             return 0;
2149         }
2150     }
2151     return apdu;
2152 }
2153
2154 int Yaz_Proxy::handle_authentication(Z_APDU *apdu)
2155 {
2156     if (apdu->which != Z_APDU_initRequest)
2157         return 1;  // pass if no init request
2158     Z_InitRequest *req = apdu->u.initRequest;
2159
2160     Yaz_ProxyConfig *cfg = check_reconfigure();
2161     if (!cfg)
2162         return 1;  // pass if no config
2163
2164     int ret;
2165     if (req->idAuthentication == 0)
2166     {
2167         ret = cfg->client_authentication(m_default_target, 0, 0, 0,
2168                                          m_peername);
2169     }
2170     else if (req->idAuthentication->which == Z_IdAuthentication_idPass)
2171     {
2172         ret = cfg->client_authentication(
2173             m_default_target,
2174             req->idAuthentication->u.idPass->userId,
2175             req->idAuthentication->u.idPass->groupId,
2176             req->idAuthentication->u.idPass->password,
2177             m_peername);
2178     }
2179     else if (req->idAuthentication->which == Z_IdAuthentication_open)
2180     {
2181         char user[64], pass[64];
2182         *user = '\0';
2183         *pass = '\0';
2184         sscanf(req->idAuthentication->u.open, "%63[^/]/%63s", user, pass);
2185         ret = cfg->client_authentication(m_default_target, user, 0, pass,
2186                                          m_peername);
2187     }
2188     else
2189         ret = cfg->client_authentication(m_default_target, 0, 0, 0,
2190                                          m_peername);
2191
2192     cfg->target_authentication(m_default_target, odr_encode(), req);
2193
2194     return ret;
2195 }
2196
2197 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
2198 {
2199     m_marcxml_mode = none;
2200     if (apdu->which == Z_APDU_searchRequest)
2201     {
2202         Z_SearchRequest *sr = apdu->u.searchRequest;
2203         int err = 0;
2204         char *addinfo = 0;
2205         Yaz_ProxyConfig *cfg = check_reconfigure();
2206
2207         Z_RecordComposition rc_temp, *rc = 0;
2208         if (sr->smallSetElementSetNames)
2209         {
2210             rc_temp.which = Z_RecordComp_simple;
2211             rc_temp.u.simple = sr->smallSetElementSetNames;
2212             rc = &rc_temp;
2213         }
2214
2215         if (sr->preferredRecordSyntax)
2216         {
2217             struct oident *ent;
2218             ent = oid_getentbyoid(sr->preferredRecordSyntax);
2219             m_frontend_type = ent->value;
2220         }
2221         else
2222             m_frontend_type = VAL_NONE;
2223
2224         char *stylesheet_name = 0;
2225         if (cfg)
2226             err = cfg->check_syntax(odr_encode(),
2227                                     m_default_target,
2228                                     sr->preferredRecordSyntax, rc,
2229                                     &addinfo, &stylesheet_name, &m_schema,
2230                                     &m_backend_type, &m_backend_charset,
2231                                     &m_usemarcon_ini_stage1,
2232                                     &m_usemarcon_ini_stage2);
2233         if (stylesheet_name)
2234         {
2235             m_parent->low_socket_close();
2236
2237 #if HAVE_XSLT
2238             if (m_stylesheet_xsp)
2239                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2240             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2241                                                        stylesheet_name);
2242 #endif
2243             m_stylesheet_offset = 0;
2244             xfree(stylesheet_name);
2245
2246             m_parent->low_socket_open();
2247         }
2248         if (err == -1)
2249         {
2250             sr->smallSetElementSetNames = 0;
2251             sr->mediumSetElementSetNames = 0;
2252             m_marcxml_mode = marcxml;
2253             if (m_backend_type)
2254             {
2255                 
2256                 sr->preferredRecordSyntax =
2257                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
2258                                         m_backend_type);
2259             }
2260             else
2261                 sr->preferredRecordSyntax =
2262                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
2263                                            VAL_USMARC);
2264         }
2265         else if (err)
2266         {
2267             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2268             
2269             new_apdu->u.searchResponse->referenceId = sr->referenceId;
2270             new_apdu->u.searchResponse->records =
2271                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2272             *new_apdu->u.searchResponse->searchStatus = 0;
2273             
2274             send_to_client(new_apdu);
2275             
2276             return 0;
2277         }
2278         else if (m_backend_type)
2279         {
2280             sr->preferredRecordSyntax =
2281                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
2282         }
2283     }
2284     else if (apdu->which == Z_APDU_presentRequest)
2285     {
2286         Z_PresentRequest *pr = apdu->u.presentRequest;
2287         int err = 0;
2288         char *addinfo = 0;
2289         Yaz_ProxyConfig *cfg = check_reconfigure();
2290
2291         if (pr->preferredRecordSyntax)
2292         {
2293             struct oident *ent;
2294             ent = oid_getentbyoid(pr->preferredRecordSyntax);
2295             m_frontend_type = ent->value;
2296         }
2297         else
2298             m_frontend_type = VAL_NONE;
2299
2300         char *stylesheet_name = 0;
2301         if (cfg)
2302             err = cfg->check_syntax(odr_encode(), m_default_target,
2303                                     pr->preferredRecordSyntax,
2304                                     pr->recordComposition,
2305                                     &addinfo, &stylesheet_name, &m_schema,
2306                                     &m_backend_type, &m_backend_charset,
2307                                     &m_usemarcon_ini_stage1,
2308                                     &m_usemarcon_ini_stage2
2309                                     );
2310         if (stylesheet_name)
2311         {
2312             m_parent->low_socket_close();
2313
2314 #if HAVE_XSLT
2315             if (m_stylesheet_xsp)
2316                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
2317             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
2318                                                        stylesheet_name);
2319 #endif
2320             m_stylesheet_offset = 0;
2321             xfree(stylesheet_name);
2322
2323             m_parent->low_socket_open();
2324         }
2325         if (err == -1)
2326         {
2327             pr->recordComposition = 0;
2328             m_marcxml_mode = marcxml;
2329             if (m_backend_type)
2330             {
2331                 
2332                 pr->preferredRecordSyntax =
2333                     yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
2334                                         m_backend_type);
2335             }
2336             else
2337                 pr->preferredRecordSyntax =
2338                     yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
2339                                            VAL_USMARC);
2340         }
2341         else if (err)
2342         {
2343             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
2344             
2345             new_apdu->u.presentResponse->referenceId = pr->referenceId;
2346             new_apdu->u.presentResponse->records =
2347                 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
2348             *new_apdu->u.presentResponse->presentStatus =
2349                 Z_PresentStatus_failure;
2350             
2351             send_to_client(new_apdu);
2352             
2353             return 0;
2354         }
2355         else if (m_backend_type)
2356         {
2357             pr->preferredRecordSyntax =
2358                 yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
2359         }
2360     }
2361     return apdu;
2362 }
2363
2364 Z_ElementSetNames *Yaz_Proxy::mk_esn_from_schema(ODR o, const char *schema)
2365 {
2366     if (!schema)
2367         return 0;
2368     Z_ElementSetNames *esn = (Z_ElementSetNames *)
2369         odr_malloc(o, sizeof(Z_ElementSetNames));
2370     esn->which = Z_ElementSetNames_generic;
2371     esn->u.generic = odr_strdup(o, schema);
2372     return esn;
2373 }
2374
2375 void Yaz_Proxy::srw_get_client(const char *db, const char **backend_db)
2376 {
2377     const char *t = 0;
2378     Yaz_ProxyConfig *cfg = check_reconfigure();
2379     if (cfg)
2380         t = cfg->get_explain_name(db, backend_db);
2381
2382     if (m_client && m_default_target && t && strcmp(m_default_target, t))
2383     {
2384         releaseClient();
2385     }
2386     
2387     if (t)
2388     {
2389         xfree(m_default_target);
2390         m_default_target = xstrdup(t);
2391     }
2392 }
2393
2394 int Yaz_Proxy::file_access(Z_HTTP_Request *hreq)
2395 {
2396     struct stat sbuf;
2397     if (strcmp(hreq->method, "GET"))
2398         return 0;
2399     if (hreq->path[0] != '/')
2400     {
2401         yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
2402         return 0;
2403     }
2404     const char *cp = hreq->path;
2405     while (*cp)
2406     {
2407         if (*cp == '/' && strchr("/.", cp[1]))
2408         {
2409             yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
2410             return 0;
2411         }
2412         cp++;
2413     }
2414     const char *fname = hreq->path+1;
2415     if (stat(fname, &sbuf))
2416     {
2417         yaz_log(YLOG_WARN|YLOG_ERRNO, "%s: stat failed", fname);
2418         return 0;
2419     }
2420     if ((sbuf.st_mode & S_IFMT) != S_IFREG)
2421     {
2422         yaz_log(YLOG_WARN, "%s: not a regular file", fname);
2423         return 0;
2424     }
2425     if (sbuf.st_size > (off_t) 1000000)
2426     {
2427         yaz_log(YLOG_WARN, "%s: too large for transfer", fname);
2428         return 0;
2429     }
2430     
2431     ODR o = odr_encode();
2432     Yaz_ProxyConfig *cfg = check_reconfigure();
2433     const char *ctype = cfg->check_mime_type(fname);
2434     Z_GDU *gdu = z_get_HTTP_Response(o, 200);
2435     Z_HTTP_Response *hres = gdu->u.HTTP_Response;
2436     if (m_http_version)
2437         hres->version = odr_strdup(o, m_http_version);
2438     z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
2439     if (m_http_keepalive)
2440         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
2441     else
2442         timeout(0);
2443
2444     hres->content_len = sbuf.st_size;
2445     hres->content_buf = (char*) odr_malloc(o, hres->content_len);
2446     FILE *f = fopen(fname, "rb");
2447     if (f)
2448     {
2449         fread(hres->content_buf, 1, hres->content_len, f);
2450         fclose(f);
2451     }
2452     else
2453     {
2454         return 0;
2455     }
2456     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
2457     {
2458         yaz_log (YLOG_LOG, "%sSending file %s to client", m_session_str,
2459                  fname);
2460     }
2461     int len;
2462     send_GDU(gdu, &len);
2463     recv_GDU_more(true);
2464     return 1;
2465 }
2466         
2467 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
2468 {
2469     if (m_s2z_odr_init)
2470     {
2471         odr_destroy(m_s2z_odr_init);
2472         m_s2z_odr_init = 0;
2473     }
2474     if (m_s2z_odr_search)
2475     {
2476         odr_destroy(m_s2z_odr_search);
2477         m_s2z_odr_search = 0;
2478     }
2479
2480     m_http_keepalive = 0;
2481     m_http_version = 0;
2482     if (!strcmp(hreq->version, "1.0")) 
2483     {
2484         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2485         if (v && !strcmp(v, "Keep-Alive"))
2486             m_http_keepalive = 1;
2487         else
2488             m_http_keepalive = 0;
2489         m_http_version = "1.0";
2490     }
2491     else
2492     {
2493         const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
2494         if (v && !strcmp(v, "close"))
2495             m_http_keepalive = 0;
2496         else
2497             m_http_keepalive = 1;
2498         m_http_version = "1.1";
2499     }
2500
2501     Z_SRW_PDU *srw_pdu = 0;
2502     Z_SOAP *soap_package = 0;
2503     char *charset = 0;
2504     Z_SRW_diagnostic *diagnostic = 0;
2505     int num_diagnostic = 0;
2506
2507     if (file_access(hreq))
2508     {
2509         return;
2510     }
2511     else if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2512                             &charset) == 0
2513              || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
2514                                &charset, &diagnostic, &num_diagnostic) == 0)
2515     {
2516         m_s2z_odr_init = odr_createmem(ODR_ENCODE);
2517         m_s2z_odr_search = odr_createmem(ODR_ENCODE);
2518         m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns);
2519         m_s2z_init_apdu = 0;
2520         m_s2z_search_apdu = 0;
2521         m_s2z_present_apdu = 0;
2522
2523         m_s2z_stylesheet = 0;
2524         
2525         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
2526         {
2527             Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
2528
2529             const char *backend_db = srw_req->database;
2530             srw_get_client(srw_req->database, &backend_db);
2531
2532             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2533             // recordXPath unsupported.
2534             if (srw_req->recordXPath)
2535             {
2536                 yaz_add_srw_diagnostic(odr_decode(),
2537                                        &diagnostic, &num_diagnostic,
2538                                        72, 0);
2539             }
2540             // sort unsupported
2541             if (srw_req->sort_type != Z_SRW_sort_type_none)
2542             {
2543                 yaz_add_srw_diagnostic(odr_decode(),
2544                                        &diagnostic, &num_diagnostic,
2545                                        80, 0);
2546             }
2547             // save stylesheet
2548             if (srw_req->stylesheet)
2549                 m_s2z_stylesheet =
2550                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2551                                               
2552             // set packing for response records ..
2553             if (srw_req->recordPacking &&
2554                 !strcmp(srw_req->recordPacking, "xml"))
2555                 m_s2z_packing = Z_SRW_recordPacking_XML;
2556             else
2557                 m_s2z_packing = Z_SRW_recordPacking_string;
2558
2559             if (num_diagnostic)
2560             {
2561                 Z_SRW_PDU *srw_pdu =
2562                     yaz_srw_get(odr_encode(),
2563                                 Z_SRW_searchRetrieve_response);
2564                 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
2565                 
2566                 srw_res->diagnostics = diagnostic;
2567                 srw_res->num_diagnostics = num_diagnostic;
2568                 send_srw_response(srw_pdu);
2569                 return;
2570             }
2571
2572             // prepare search PDU
2573             m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
2574                                           Z_APDU_searchRequest);
2575             Z_SearchRequest *z_searchRequest =
2576                 m_s2z_search_apdu->u.searchRequest;
2577
2578             z_searchRequest->num_databaseNames = 1;
2579             z_searchRequest->databaseNames = (char**)
2580                 odr_malloc(m_s2z_odr_search, sizeof(char *));
2581             z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
2582                                                            backend_db);
2583             
2584             // query transformation
2585             Z_Query *query = (Z_Query *)
2586                 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
2587             z_searchRequest->query = query;
2588             
2589             if (srw_req->query_type == Z_SRW_query_type_cql)
2590             {
2591                 Z_External *ext = (Z_External *) 
2592                     odr_malloc(m_s2z_odr_search, sizeof(*ext));
2593                 ext->direct_reference = 
2594                     odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
2595                 ext->indirect_reference = 0;
2596                 ext->descriptor = 0;
2597                 ext->which = Z_External_CQL;
2598                 ext->u.cql = srw_req->query.cql;
2599                 
2600                 query->which = Z_Query_type_104;
2601                 query->u.type_104 =  ext;
2602             }
2603             else if (srw_req->query_type == Z_SRW_query_type_pqf)
2604             {
2605                 Z_RPNQuery *RPNquery;
2606                 YAZ_PQF_Parser pqf_parser;
2607                 
2608                 pqf_parser = yaz_pqf_create ();
2609                 
2610                 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
2611                                           srw_req->query.pqf);
2612                 if (!RPNquery)
2613                 {
2614                     const char *pqf_msg;
2615                     size_t off;
2616                     int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
2617                     int ioff = off;
2618                     yaz_log(YLOG_LOG, "%*s^\n", ioff+4, "");
2619                     yaz_log(YLOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
2620                     
2621                     send_to_srw_client_error(10, 0);
2622                     return;
2623                 }
2624                 query->which = Z_Query_type_1;
2625                 query->u.type_1 =  RPNquery;
2626                 
2627                 yaz_pqf_destroy (pqf_parser);
2628             }
2629             else
2630             {
2631                 send_to_srw_client_error(7, "query");
2632                 return;
2633             }
2634
2635             // present
2636             m_s2z_present_apdu = 0;
2637             int max = 0;
2638             if (srw_req->maximumRecords)
2639                 max = *srw_req->maximumRecords;
2640             int start = 1;
2641             if (srw_req->startRecord)
2642                 start = *srw_req->startRecord;
2643             if (max > 0)
2644             {
2645                 // Some backend, such as Voyager doesn't honor piggyback
2646                 // So we use present always (0 &&).
2647                 if (0 && start <= 1)  // Z39.50 piggyback
2648                 {
2649                     *z_searchRequest->smallSetUpperBound = max;
2650                     *z_searchRequest->mediumSetPresentNumber = max;
2651                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
2652
2653                     z_searchRequest->preferredRecordSyntax =
2654                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2655                                                VAL_TEXT_XML);
2656                     if (srw_req->recordSchema)
2657                     {
2658                         z_searchRequest->smallSetElementSetNames =
2659                             z_searchRequest->mediumSetElementSetNames =
2660                             mk_esn_from_schema(m_s2z_odr_search,
2661                                                srw_req->recordSchema);
2662                     }
2663                 }
2664                 else   // Z39.50 present
2665                 {
2666                     m_s2z_present_apdu = zget_APDU(m_s2z_odr_search, 
2667                                                    Z_APDU_presentRequest);
2668                     Z_PresentRequest *z_presentRequest = 
2669                         m_s2z_present_apdu->u.presentRequest;
2670                     *z_presentRequest->resultSetStartPoint = start;
2671                     *z_presentRequest->numberOfRecordsRequested = max;
2672                     z_presentRequest->preferredRecordSyntax =
2673                         yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
2674                                                VAL_TEXT_XML);
2675                     if (srw_req->recordSchema)
2676                     {
2677                         z_presentRequest->recordComposition =
2678                             (Z_RecordComposition *)
2679                             odr_malloc(m_s2z_odr_search,
2680                                        sizeof(Z_RecordComposition));
2681                         z_presentRequest->recordComposition->which = 
2682                             Z_RecordComp_simple;                    
2683                         z_presentRequest->recordComposition->u.simple =
2684                             mk_esn_from_schema(m_s2z_odr_search,
2685                                                srw_req->recordSchema);
2686                     }
2687                 }
2688             }
2689             if (!m_client)
2690             {
2691                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2692                                             Z_APDU_initRequest);
2693                 
2694                 // prevent m_initRequest_apdu memory from being grabbed
2695                 // in Yaz_Proxy::handle_incoming_Z_PDU
2696                 m_initRequest_apdu = m_s2z_init_apdu;
2697                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2698                 return;
2699             }
2700             else
2701             {
2702                 handle_incoming_Z_PDU(m_s2z_search_apdu);
2703                 return;
2704             }
2705         }
2706         else if (srw_pdu->which == Z_SRW_explain_request)
2707         {
2708             Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
2709
2710             const char *backend_db = srw_req->database;
2711             srw_get_client(srw_req->database, &backend_db);
2712
2713             m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
2714
2715             // save stylesheet
2716             if (srw_req->stylesheet)
2717                 m_s2z_stylesheet =
2718                     odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
2719
2720             if (srw_req->recordPacking &&
2721                 !strcmp(srw_req->recordPacking, "xml"))
2722                 m_s2z_packing = Z_SRW_recordPacking_XML;
2723             else
2724                 m_s2z_packing = Z_SRW_recordPacking_string;
2725
2726             if (num_diagnostic)
2727             {
2728                 send_srw_explain_response(diagnostic, num_diagnostic);
2729                 return;
2730             }
2731
2732             if (!m_client)
2733             {
2734                 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
2735                                             Z_APDU_initRequest);
2736                 
2737                 // prevent m_initRequest_apdu memory from being grabbed
2738                 // in Yaz_Proxy::handle_incoming_Z_PDU
2739                 m_initRequest_apdu = m_s2z_init_apdu;
2740                 handle_incoming_Z_PDU(m_s2z_init_apdu);
2741             }
2742             else
2743                 send_srw_explain_response(0, 0);
2744             return;
2745         }
2746         else if (srw_pdu->which == Z_SRW_scan_request)
2747         {
2748             m_s2z_database = odr_strdup(m_s2z_odr_init,
2749                                         srw_pdu->u.scan_request->database);
2750
2751             yaz_add_srw_diagnostic(odr_decode(),
2752                                    &diagnostic, &num_diagnostic,
2753                                    4, "scan");
2754             Z_SRW_PDU *srw_pdu =
2755                 yaz_srw_get(odr_encode(),
2756                             Z_SRW_scan_response);
2757             Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
2758             
2759             srw_res->diagnostics = diagnostic;
2760             srw_res->num_diagnostics = num_diagnostic;
2761             send_srw_response(srw_pdu);
2762             return;
2763         }
2764         else
2765         {
2766             m_s2z_database = 0;
2767
2768             send_to_srw_client_error(4, 0);
2769         }
2770     }
2771     send_http_response(400);
2772 }
2773
2774 void Yaz_Proxy::handle_init(Z_APDU *apdu)
2775 {
2776     Z_OtherInformation **oi;
2777     get_otherInfoAPDU(apdu, &oi);
2778
2779     if (apdu->u.initRequest->implementationId)
2780         yaz_log(YLOG_LOG, "%simplementationId: %s",
2781                 m_session_str, apdu->u.initRequest->implementationId);
2782     if (apdu->u.initRequest->implementationName)
2783         yaz_log(YLOG_LOG, "%simplementationName: %s",
2784                 m_session_str, apdu->u.initRequest->implementationName);
2785     if (apdu->u.initRequest->implementationVersion)
2786         yaz_log(YLOG_LOG, "%simplementationVersion: %s",
2787                 m_session_str, apdu->u.initRequest->implementationVersion);
2788     if (m_initRequest_apdu == 0)
2789     {
2790         if (m_initRequest_mem)
2791             nmem_destroy(m_initRequest_mem);
2792         
2793         m_initRequest_apdu = apdu;
2794         m_initRequest_mem = odr_extract_mem(odr_decode());
2795         
2796         m_initRequest_preferredMessageSize = *apdu->u.initRequest->
2797             preferredMessageSize;
2798         *apdu->u.initRequest->preferredMessageSize = 1024*1024;
2799         m_initRequest_maximumRecordSize = *apdu->u.initRequest->
2800             maximumRecordSize;
2801         *apdu->u.initRequest->maximumRecordSize = 1024*1024;
2802
2803         Z_CharSetandLanguageNegotiation *charSetandLangRecord =
2804             yaz_get_charneg_record(*oi);
2805         
2806         // Save proposal charsets and langs.
2807         if (ODR_MASK_GET(apdu->u.initRequest->options,
2808                          Z_Options_negotiationModel) 
2809             && charSetandLangRecord)
2810         {
2811             
2812             yaz_get_proposal_charneg(m_referenceId_mem,
2813                                      charSetandLangRecord,
2814                                      &m_initRequest_oi_negotiation_charsets,
2815                                      &m_initRequest_oi_negotiation_num_charsets,
2816                                      &m_initRequest_oi_negotiation_langs,
2817                                      &m_initRequest_oi_negotiation_num_langs,
2818                                      &m_initRequest_oi_negotiation_selected);
2819             
2820             for (int i = 0; i<m_initRequest_oi_negotiation_num_charsets; i++)
2821             {
2822                 yaz_log(YLOG_LOG, "%scharacters set proposal: %s",
2823                         m_session_str,(m_initRequest_oi_negotiation_charsets[i])?
2824                         m_initRequest_oi_negotiation_charsets[i]:"none");
2825             }
2826             for (int i=0; i<m_initRequest_oi_negotiation_num_langs; i++)
2827             {
2828                 yaz_log(YLOG_LOG, "%slanguages proposal: %s",
2829                         m_session_str, (m_initRequest_oi_negotiation_langs[i])?
2830                         m_initRequest_oi_negotiation_langs[i]:"none");
2831             }
2832             yaz_log(YLOG_LOG, "%sselected proposal: %d (boolean)",
2833                     m_session_str, m_initRequest_oi_negotiation_selected);
2834         }       
2835         // save init options for the response..
2836         m_initRequest_options = apdu->u.initRequest->options;
2837         
2838         apdu->u.initRequest->options = 
2839             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2840                                        sizeof(Odr_bitmask));
2841         ODR_MASK_ZERO(apdu->u.initRequest->options);
2842         int i;
2843         for (i = 0; i<= 24; i++)
2844             ODR_MASK_SET(apdu->u.initRequest->options, i);
2845         // check negotiation option
2846         if (!ODR_MASK_GET(m_initRequest_options,
2847                           Z_Options_negotiationModel))
2848         {
2849             ODR_MASK_CLEAR(apdu->u.initRequest->options,
2850                            Z_Options_negotiationModel);
2851         }
2852         ODR_MASK_CLEAR(apdu->u.initRequest->options,
2853                        Z_Options_concurrentOperations);
2854         // make new version
2855         m_initRequest_version = apdu->u.initRequest->protocolVersion;
2856         apdu->u.initRequest->protocolVersion = 
2857             (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
2858                                        sizeof(Odr_bitmask));
2859         ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
2860         
2861         for (i = 0; i<= 8; i++)
2862             ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
2863     }
2864     if (m_client->m_init_flag)
2865     {
2866         if (handle_init_response_for_invalid_session(apdu))
2867             return;
2868         if (m_client->m_initResponse)
2869         {
2870             Z_APDU *apdu2 = m_client->m_initResponse;
2871             apdu2->u.initResponse->otherInfo = 0;
2872             if (m_client->m_cookie && *m_client->m_cookie)
2873                 set_otherInformationString(apdu2, VAL_COOKIE, 1,
2874                                            m_client->m_cookie);
2875             apdu2->u.initResponse->referenceId =
2876                 apdu->u.initRequest->referenceId;
2877             apdu2->u.initResponse->options = m_client->m_initResponse_options;
2878             apdu2->u.initResponse->protocolVersion = 
2879                 m_client->m_initResponse_version;
2880             
2881             send_to_client(apdu2);
2882             m_timeout_mode = timeout_normal;
2883             return;
2884         }
2885     }
2886     m_client->m_init_flag = 1;
2887     
2888 #if USE_AUTH_MSG
2889     yaz_log(YLOG_LOG, "%suse_auth_msg", m_session_str);
2890     Auth_Msg *m = new Auth_Msg;
2891     m->m_proxy = this;
2892     z_APDU(odr_encode(), &apdu, 0, "encode");
2893     char *apdu_buf = odr_getbuf(odr_encode(), &m->m_apdu_len, 0);
2894     m->m_apdu_buf = (char*) nmem_malloc(m->m_nmem, m->m_apdu_len);
2895     memcpy(m->m_apdu_buf, apdu_buf, m->m_apdu_len);
2896     odr_reset(odr_encode());
2897
2898     inc_ref();
2899     m_my_thread->put(m);
2900 #else
2901     int ret = handle_authentication(apdu);
2902     result_authentication(apdu, ret);
2903 #endif
2904 }
2905
2906 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
2907 {
2908     Z_ReferenceId **refid = get_referenceIdP(apdu);
2909     nmem_reset(m_referenceId_mem);
2910     if (refid && *refid)
2911     {
2912         m_referenceId = (Z_ReferenceId *)
2913             nmem_malloc(m_referenceId_mem, sizeof(*m_referenceId));
2914         m_referenceId->len = m_referenceId->size = (*refid)->len;
2915         m_referenceId->buf = (unsigned char *)
2916             nmem_malloc(m_referenceId_mem, (*refid)->len);
2917         memcpy(m_referenceId->buf, (*refid)->buf, (*refid)->len);
2918     }
2919     else
2920         m_referenceId = 0;
2921
2922     if (!m_client && m_flag_invalid_session)
2923     {
2924         // Got request for a session that is invalid..
2925         m_apdu_invalid_session = apdu; // save package
2926         m_mem_invalid_session = odr_extract_mem(odr_decode());
2927         apdu = m_initRequest_apdu;     // but throw an init to the target
2928     }
2929     
2930     // Determine our client.
2931     Z_OtherInformation **oi;
2932     get_otherInfoAPDU(apdu, &oi);
2933     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi));
2934     if (!m_client)
2935     {
2936         if (m_http_version)
2937         {   // HTTP. Send not found
2938             send_http_response(404);
2939             return;
2940         }
2941         else
2942         {
2943             // Z39.50 just shutdown
2944             delete this;
2945             return;
2946         }
2947     }
2948     
2949     m_client->m_server = this;
2950
2951     if (apdu->which == Z_APDU_initRequest)
2952         handle_init(apdu);
2953     else
2954         handle_incoming_Z_PDU_2(apdu);
2955 }
2956
2957 void Yaz_Proxy::handle_incoming_Z_PDU_2(Z_APDU *apdu)
2958 {
2959     handle_max_record_retrieve(apdu);
2960     
2961     if (apdu)
2962         apdu = handle_syntax_validation(apdu);
2963
2964     if (apdu)
2965         apdu = handle_query_transformation(apdu);
2966
2967     if (apdu)
2968         apdu = handle_target_charset_conversion(apdu);
2969
2970     if (apdu)
2971         apdu = handle_query_validation(apdu);
2972
2973     if (apdu)
2974         apdu = result_set_optimize(apdu);
2975
2976     if (!apdu)
2977     {
2978         m_client->timeout(m_target_idletime);  // mark it active even 
2979         recv_GDU_more(true);
2980         // though we didn't use it
2981         return;
2982     }
2983     // Add otherInformation entry in APDU if
2984     // negotiation is in use.
2985     if (apdu)
2986         handle_charset_lang_negotiation(apdu);
2987
2988     // delete other info construct completely if 0 elements
2989     Z_OtherInformation **oi;
2990     get_otherInfoAPDU(apdu, &oi);
2991     if (oi && *oi && (*oi)->num_elements == 0)
2992         *oi = 0;
2993
2994     if (apdu->which == Z_APDU_presentRequest &&
2995         m_client->m_resultSetStartPoint == 0)
2996     {
2997         Z_PresentRequest *pr = apdu->u.presentRequest;
2998         m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
2999         m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
3000     } else {
3001         m_client->m_resultSetStartPoint = 0;
3002     }
3003     if (m_client->send_to_target(apdu) < 0)
3004     {
3005         m_client->shutdown();
3006     }
3007     else
3008         m_client->m_waiting = 1;
3009 }
3010
3011 void Yaz_Proxy::connectNotify()
3012 {
3013 }
3014
3015 void Yaz_Proxy::releaseClient()
3016 {
3017     xfree(m_proxyTarget);
3018     m_proxyTarget = 0;
3019     m_flag_invalid_session = 0;
3020     // only keep if keep_alive flag is set...
3021     if (m_client && 
3022         m_client->m_pdu_recv < m_keepalive_limit_pdu &&
3023         m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
3024         m_client->m_waiting == 0)
3025     {
3026         yaz_log(YLOG_LOG, "%sShutdown (client to proxy) keepalive %s",
3027                  m_session_str,
3028                  m_client->get_hostname());
3029         yaz_log(YLOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
3030                 m_session_str, m_client->m_pdu_recv,
3031                 m_client->m_bytes_sent + m_client->m_bytes_recv,
3032                 m_keepalive_limit_bw, m_keepalive_limit_pdu);
3033         assert (m_client->m_waiting != 2);
3034         // Tell client (if any) that no server connection is there..
3035         m_client->m_server = 0;
3036         m_client = 0;
3037     }
3038     else if (m_client)
3039     {
3040         yaz_log (YLOG_LOG, "%sShutdown (client to proxy) close %s",
3041                  m_session_str,
3042                  m_client->get_hostname());
3043         assert (m_client->m_waiting != 2);
3044         delete m_client;
3045         m_client = 0;
3046     }
3047     else if (!m_parent)
3048     {
3049         yaz_log (YLOG_LOG, "%sshutdown (client to proxy) bad state",
3050                  m_session_str);
3051         assert (m_parent);
3052     }
3053     else 
3054     {
3055         yaz_log (YLOG_LOG, "%sShutdown (client to proxy)",
3056                  m_session_str);
3057     }
3058     if (m_parent)
3059         m_parent->pre_init();
3060 }
3061
3062 bool Yaz_Proxy::dec_ref()
3063 {
3064     yaz_log(YLOG_LOG, "%sdec_ref count=%d", m_session_str, m_ref_count);
3065     assert(m_ref_count > 0);
3066
3067     --m_ref_count;
3068     if (m_ref_count > 0)
3069         return false;
3070
3071     releaseClient();
3072
3073     delete this;
3074     return true;
3075 }
3076
3077 const char *Yaz_ProxyClient::get_session_str() 
3078 {
3079     if (!m_server)
3080         return "0 ";
3081     return m_server->get_session_str();
3082 }
3083
3084 void Yaz_ProxyClient::shutdown()
3085 {
3086     yaz_log (YLOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
3087              get_hostname());
3088
3089     if (m_server)
3090     {
3091         m_waiting = 1;      // ensure it's released from Proxy in releaseClient
3092         m_server->dec_ref();
3093     }
3094     else
3095         delete this;
3096 }
3097
3098 void Yaz_Proxy::failNotify()
3099 {
3100     inc_request_no();
3101     yaz_log (YLOG_LOG, "%sConnection closed by client",
3102              get_session_str());
3103     dec_ref();
3104 }
3105
3106 void Yaz_ProxyClient::failNotify()
3107 {
3108     if (m_server)
3109         m_server->inc_request_no();
3110     yaz_log (YLOG_LOG, "%sConnection closed by target %s", 
3111              get_session_str(), get_hostname());
3112     shutdown();
3113 }
3114
3115 void Yaz_ProxyClient::connectNotify()
3116 {
3117     const char *s = get_session_str();
3118     const char *h = get_hostname();
3119     yaz_log (YLOG_LOG, "%sConnection accepted by %s timeout=%d", s, h,
3120              m_target_idletime);
3121     timeout(m_target_idletime);
3122     if (!m_server)
3123         pre_init_client();
3124 }
3125
3126 IPDU_Observer *Yaz_ProxyClient::sessionNotify(IPDU_Observable
3127                                               *the_PDU_Observable, int fd)
3128 {
3129     return new Yaz_ProxyClient(the_PDU_Observable, 0);
3130 }
3131
3132 Yaz_ProxyClient::~Yaz_ProxyClient()
3133 {
3134     if (m_prev)
3135         *m_prev = m_next;
3136     if (m_next)
3137         m_next->m_prev = m_prev;
3138     m_waiting = 2;     // for debugging purposes only.
3139     odr_destroy(m_init_odr);
3140     delete m_last_query;
3141     xfree (m_last_resultSetId);
3142     xfree (m_cookie);
3143 }
3144
3145 void Yaz_ProxyClient::pre_init_client()
3146 {
3147     Z_APDU *apdu = create_Z_PDU(Z_APDU_initRequest);
3148     Z_InitRequest *req = apdu->u.initRequest;
3149     
3150     int i;
3151     for (i = 0; i<= 24; i++)
3152         ODR_MASK_SET(req->options, i);
3153     ODR_MASK_CLEAR(apdu->u.initRequest->options,
3154                    Z_Options_negotiationModel);
3155     ODR_MASK_CLEAR(apdu->u.initRequest->options,
3156                    Z_Options_concurrentOperations);
3157     for (i = 0; i<= 10; i++)
3158         ODR_MASK_SET(req->protocolVersion, i);
3159
3160     if (send_to_target(apdu) < 0)
3161     {
3162         delete this;
3163     }
3164     else
3165     {
3166         m_waiting = 1;
3167         m_init_flag = 1;
3168     }
3169 }
3170
3171 void Yaz_Proxy::pre_init()
3172 {
3173     int i;
3174     const char *name = 0;
3175     const char *zurl_in_use[MAX_ZURL_PLEX];
3176     int limit_bw, limit_pdu, limit_req, limit_search, limit_connect;
3177     int target_idletime, client_idletime;
3178     int max_clients;
3179     int keepalive_limit_bw, keepalive_limit_pdu;
3180     int pre_init;
3181     const char *cql2rpn = 0;
3182     const char *authentication = 0;
3183     const char *negotiation_charset = 0;
3184     const char *negotiation_lang = 0;
3185
3186     Yaz_ProxyConfig *cfg = check_reconfigure();
3187
3188     zurl_in_use[0] = 0;
3189
3190     if (m_log_mask & PROXY_LOG_APDU_CLIENT)
3191         set_APDU_yazlog(1);
3192     else
3193         set_APDU_yazlog(0);
3194
3195     for (i = 0; cfg && cfg->get_target_no(i, &name, zurl_in_use,
3196                                           &limit_bw, &limit_pdu, &limit_req,
3197                                           &limit_search, &limit_connect,
3198                                           &target_idletime, &client_idletime,
3199                                           &max_clients, 
3200                                           &keepalive_limit_bw,
3201                                           &keepalive_limit_pdu,
3202                                           &pre_init,
3203                                           &cql2rpn,
3204                                           &authentication,
3205                                           &negotiation_charset,
3206                                           &negotiation_lang,
3207                                           0) ; i++)
3208     {
3209         if (pre_init)
3210         {
3211             int j;
3212             for (j = 0; zurl_in_use[j]; j++)
3213             {
3214                 Yaz_ProxyClient *c;
3215                 int spare = 0;
3216                 int spare_waiting = 0;
3217                 int in_use = 0;
3218                 int other = 0;
3219                 for (c = m_clientPool; c; c = c->m_next)
3220                 {
3221                     if (!strcmp(zurl_in_use[j], c->get_hostname()))
3222                     {
3223                         if (c->m_cookie == 0)
3224                         {
3225                             if (c->m_server == 0)
3226                                 if (c->m_waiting)
3227                                     spare_waiting++;
3228                                 else
3229                                     spare++;
3230                             else
3231                                 in_use++;
3232                         }
3233                         else
3234                             other++;
3235                     }
3236                 }
3237                 yaz_log(YLOG_LOG, "%spre-init %s %s use=%d other=%d spare=%d "
3238                         "sparew=%d preinit=%d",m_session_str,
3239                         name, zurl_in_use[j], in_use, other,
3240                         spare, spare_waiting, pre_init);
3241                 if (spare + spare_waiting < pre_init)
3242                 {
3243                     c = new Yaz_ProxyClient(m_PDU_Observable->clone(), this);
3244                     c->m_next = m_clientPool;
3245                     if (c->m_next)
3246                         c->m_next->m_prev = &c->m_next;
3247                     m_clientPool = c;
3248                     c->m_prev = &m_clientPool;
3249                     
3250                     if (m_log_mask & PROXY_LOG_APDU_SERVER)
3251                         c->set_APDU_yazlog(1);
3252                     else
3253                         c->set_APDU_yazlog(0);
3254
3255                     if (c->client(zurl_in_use[j]))
3256                     {
3257                         timeout(60);
3258                         delete c;
3259                         return;
3260                     }
3261                     c->timeout(30);
3262                     c->m_waiting = 1;
3263                     c->m_target_idletime = target_idletime;
3264                     c->m_seqno = m_seqno++;
3265                 }
3266             }
3267         }
3268     }
3269 }
3270
3271 void Yaz_Proxy::timeoutNotify()
3272 {
3273     if (m_parent)
3274     {
3275         GDU *gdu;
3276         switch(m_timeout_mode)
3277         {
3278         case timeout_normal:
3279         case timeout_busy:
3280             inc_request_no();
3281             m_in_queue.clear();
3282             yaz_log (YLOG_LOG, "%sTimeout (client to proxy)", m_session_str);
3283             dec_ref();
3284             break;
3285         case timeout_reduce:
3286             timeout(m_client_idletime);
3287             m_timeout_mode = timeout_busy;
3288             gdu = m_timeout_gdu;
3289             m_timeout_gdu = 0;
3290             recv_GDU_normal(gdu);
3291             break;
3292         case timeout_xsl:
3293             assert(m_stylesheet_nprl);
3294             convert_xsl_delay();
3295             recv_GDU_more(true);
3296         }
3297     }
3298     else
3299     {
3300         timeout(600);
3301         pre_init();
3302     }
3303 }
3304
3305 void Yaz_Proxy::markInvalid()
3306 {
3307     m_client = 0;
3308     m_flag_invalid_session = 1;
3309 }
3310
3311 void Yaz_ProxyClient::timeoutNotify()
3312 {
3313     if (m_server)
3314         m_server->inc_request_no();
3315
3316     yaz_log (YLOG_LOG, "%sTimeout (proxy to target) %s", get_session_str(),
3317              get_hostname());
3318     m_waiting = 1;
3319     m_root->pre_init();
3320     if (m_server && m_init_flag)
3321     {
3322         // target timed out in a session that was properly initialized
3323         // server object stay alive but we mark it as invalid so it
3324         // gets initialized again
3325         m_server->markInvalid();
3326         m_server = 0;
3327     }
3328     shutdown();
3329 }
3330
3331 Yaz_ProxyClient::Yaz_ProxyClient(IPDU_Observable *the_PDU_Observable,
3332                                  Yaz_Proxy *parent) :
3333     Z_Assoc (the_PDU_Observable)
3334 {
3335     m_cookie = 0;
3336     m_next = 0;
3337     m_prev = 0;
3338     m_init_flag = 0;
3339     m_last_query = 0;
3340     m_last_resultSetId = 0;
3341     m_last_resultCount = 0;
3342     m_last_ok = 0;
3343     m_sr_transform = 0;
3344     m_waiting = 0;
3345     m_init_odr = odr_createmem (ODR_DECODE);
3346     m_initResponse = 0;
3347     m_initResponse_options = 0;
3348     m_initResponse_version = 0;
3349     m_initResponse_preferredMessageSize = 0;
3350     m_initResponse_maximumRecordSize = 0;
3351     m_resultSetStartPoint = 0;
3352     m_bytes_sent = m_bytes_recv = 0;
3353     m_pdu_recv = 0;
3354     m_server = 0;
3355     m_seqno = 0;
3356     m_target_idletime = 600;
3357     m_root = parent;
3358 }
3359
3360 const char *Yaz_Proxy::option(const char *name, const char *value)
3361 {
3362     if (!strcmp (name, "optimize")) {
3363         if (value) {
3364             xfree (m_optimize); 
3365             m_optimize = xstrdup (value);
3366         }
3367         return m_optimize;
3368     }
3369     return 0;
3370 }
3371
3372 void Yaz_ProxyClient::recv_HTTP_response(Z_HTTP_Response *apdu, int len)
3373 {
3374
3375 }
3376
3377 void Yaz_ProxyClient::recv_GDU(Z_GDU *apdu, int len)
3378 {
3379     if (apdu->which == Z_GDU_Z3950)
3380         recv_Z_PDU(apdu->u.z3950, len);
3381     else if (apdu->which == Z_GDU_HTTP_Response)
3382         recv_HTTP_response(apdu->u.HTTP_Response, len);
3383     else
3384         shutdown();
3385 }
3386
3387 int Yaz_Proxy::handle_init_response_for_invalid_session(Z_APDU *apdu)
3388 {
3389     if (!m_flag_invalid_session)
3390         return 0;
3391     m_flag_invalid_session = 0;
3392     handle_incoming_Z_PDU(m_apdu_invalid_session);
3393     assert (m_mem_invalid_session);
3394     nmem_destroy(m_mem_invalid_session);
3395     m_mem_invalid_session = 0;
3396     return 1;
3397 }
3398
3399 void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
3400 {
3401     m_bytes_recv += len;
3402
3403     m_pdu_recv++;
3404     m_waiting = 0;
3405     if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
3406         yaz_log (YLOG_LOG, "%sReceiving %s from %s %d bytes", get_session_str(),
3407                  apdu_name(apdu), get_hostname(), len);
3408     if (apdu->which == Z_APDU_initResponse)
3409     {
3410         if (!m_server)  // if this is a pre init session , check for more
3411             m_root->pre_init();
3412         NMEM nmem = odr_extract_mem (odr_decode());
3413         odr_reset (m_init_odr);
3414         nmem_transfer (m_init_odr->mem, nmem);
3415         m_initResponse = apdu;
3416         m_initResponse_options = apdu->u.initResponse->options;
3417         m_initResponse_version = apdu->u.initResponse->protocolVersion;
3418         m_initResponse_preferredMessageSize = 
3419             *apdu->u.initResponse->preferredMessageSize;
3420         m_initResponse_maximumRecordSize = 
3421             *apdu->u.initResponse->maximumRecordSize;
3422
3423         Z_InitResponse *ir = apdu->u.initResponse;
3424         char *im0 = ir->implementationName;
3425         
3426         char *im1 = (char*) 
3427             odr_malloc(m_init_odr, 20 + (im0 ? strlen(im0) : 0));
3428         *im1 = '\0';
3429         if (im0)
3430         {
3431             strcat(im1, im0);
3432             strcat(im1, " ");
3433         }
3434         strcat(im1, "(YAZ Proxy)");
3435         ir->implementationName = im1;
3436
3437         nmem_destroy (nmem);
3438
3439         if (m_server && m_server->handle_init_response_for_invalid_session(apdu))
3440             return;
3441     }
3442     if (apdu->which == Z_APDU_searchResponse)
3443     {
3444         Z_SearchResponse *sr = apdu->u.searchResponse;
3445         m_last_resultCount = *sr->resultCount;
3446         int status = *sr->searchStatus;
3447         if (status && (!sr->records || sr->records->which == Z_Records_DBOSD))
3448         {
3449             m_last_ok = 1;
3450             
3451             if (sr->records && sr->records->which == Z_Records_DBOSD)
3452             {
3453                 m_cache.add(odr_decode(),
3454                             sr->records->u.databaseOrSurDiagnostics, 1,
3455                             *sr->resultCount);
3456             }
3457         }
3458     }
3459     if (apdu->which == Z_APDU_presentResponse)
3460     {
3461         Z_PresentResponse *pr = apdu->u.presentResponse;
3462         if (m_sr_transform)
3463         {
3464             m_sr_transform = 0;
3465             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
3466             Z_SearchResponse *sr = new_apdu->u.searchResponse;
3467             sr->referenceId = pr->referenceId;
3468             *sr->resultCount = m_last_resultCount;
3469             sr->records = pr->records;
3470             sr->nextResultSetPosition = pr->nextResultSetPosition;
3471             sr->numberOfRecordsReturned = pr->numberOfRecordsReturned;
3472             apdu = new_apdu;
3473         }
3474         if (pr->records && 
3475             pr->records->which == Z_Records_DBOSD && m_resultSetStartPoint)
3476         {
3477             m_cache.add(odr_decode(),
3478                         pr->records->u.databaseOrSurDiagnostics,
3479                         m_resultSetStartPoint, -1);
3480             m_resultSetStartPoint = 0;
3481         }
3482     }
3483     if (m_cookie)
3484         set_otherInformationString (apdu, VAL_COOKIE, 1, m_cookie);
3485
3486     Yaz_Proxy *server = m_server; // save it. send_to_client may destroy us
3487
3488     if (server)
3489         server->send_to_client(apdu);
3490     if (apdu->which == Z_APDU_close)
3491         shutdown();
3492     else if (server)
3493         server->recv_GDU_more(true);
3494 }
3495
3496 void Yaz_Proxy::low_socket_close()
3497 {
3498 #if WIN32
3499 #else
3500     int i;
3501     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
3502         if  (m_lo_fd[i] >= 0)
3503             ::close(m_lo_fd[i]);
3504 #endif
3505 }
3506
3507 void Yaz_Proxy::low_socket_open()
3508 {
3509 #if WIN32
3510 #else
3511     int i;
3512     for (i = 0; i<NO_SPARE_SOLARIS_FD; i++)
3513         m_lo_fd[i] = open("/dev/null", O_RDONLY);
3514 #endif
3515 }
3516
3517 int Yaz_Proxy::server(const char *addr)
3518 {
3519     int r = Z_Assoc::server(addr);
3520     if (!r)
3521     {
3522         yaz_log(YLOG_LOG, "%sStarted proxy " 
3523 #ifdef VERSION
3524             VERSION 
3525 #endif
3526             " on %s", m_session_str, addr);
3527         timeout(1);
3528     }
3529     return r;
3530 }
3531
3532 /*
3533  * Local variables:
3534  * c-basic-offset: 4
3535  * indent-tabs-mode: nil
3536  * End:
3537  * vim: shiftwidth=4 tabstop=8 expandtab
3538  */
3539