Handle multiple input PDU packages using GDU class of YAZ++ 1.0.
[yazproxy-moved-to-github.git] / src / yaz-proxy.cpp
index 6857d0c..d4693c6 100644 (file)
@@ -1,5 +1,5 @@
-/* $Id: yaz-proxy.cpp,v 1.13 2004-12-03 14:28:18 adam Exp $
-   Copyright (c) 1998-2004, Index Data.
+/* $Id: yaz-proxy.cpp,v 1.33 2005-06-21 18:46:04 adam Exp $
+   Copyright (c) 1998-2005, Index Data.
 
 This file is part of the yaz-proxy.
 
@@ -20,27 +20,118 @@ Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  */
 
 #ifdef WIN32
-#else
-#include <unistd.h>
+#define HAVE_SYS_STAT_H 1
+#define HAVE_SYS_TYPES_H 1
 #endif
 
-#if HAVE_GETTIMEOFDAY
+#if HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#if HAVE_SYS_TIME_H
 #include <sys/time.h>
 #endif
+#if HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#if HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
 
 #include <assert.h>
+#include <stdlib.h>
 #include <time.h>
-#include <sys/types.h>
 #include <fcntl.h>
 
 #include <yaz/srw.h>
 #include <yaz/marcdisp.h>
 #include <yaz/yaz-iconv.h>
-#include <yaz/ylog.h>
+#include <yaz/log.h>
 #include <yaz/diagbib1.h>
 #include "proxyp.h"
 #include <yaz/pquery.h>
 #include <yaz/otherinfo.h>
+#include <yaz/charneg.h>
+#include "msg-thread.h"
+
+using namespace yazpp_1;
+
+#define USE_AUTH_MSG 1
+
+#if USE_AUTH_MSG
+class Auth_Msg : public IMsg_Thread {
+public:
+    int m_ret;
+    IMsg_Thread *handle();
+    void result();
+    Yaz_Proxy *m_proxy;
+    NMEM m_nmem;
+    char *m_apdu_buf;
+    int m_apdu_len;
+    Auth_Msg();
+    virtual ~Auth_Msg();
+};
+
+Auth_Msg::Auth_Msg()
+{
+    m_nmem = nmem_create();
+}
+
+Auth_Msg::~Auth_Msg()
+{
+    nmem_destroy(m_nmem);
+}
+    
+IMsg_Thread *Auth_Msg::handle()
+{
+    yaz_log(YLOG_LOG, "Auth_Msg:handle begin");
+    ODR decode = odr_createmem(ODR_DECODE);
+    Z_APDU *apdu;
+
+    odr_setbuf(decode, m_apdu_buf, m_apdu_len, 0);
+    int r = z_APDU(decode, &apdu, 0, 0);
+    if (!r)
+    {
+       yaz_log(YLOG_WARN, "decode failed in Auth_Msg::handle");
+    }
+    else
+    {
+       m_ret = m_proxy->handle_authentication(apdu);
+    }
+    yaz_log(YLOG_LOG, "Auth_Msg:handle end");
+    odr_destroy(decode);
+    return this;
+}
+
+void Auth_Msg::result()
+{
+    yaz_log(YLOG_LOG, "Auth_Msg:result proxy ok buf=%p len=%d",
+           m_apdu_buf, m_apdu_len);
+    odr_reset(m_proxy->odr_decode());  
+    odr_setbuf(m_proxy->odr_decode(), m_apdu_buf, m_apdu_len, 0);
+    Z_APDU *apdu = 0;
+    int r = z_APDU(m_proxy->odr_decode(), &apdu, 0, 0);
+    if (r)
+       yaz_log(YLOG_LOG, "Auth_Msg::result z_APDU OK");
+    else
+       yaz_log(YLOG_LOG, "Auth_Msg::result z_APDU failed");
+    m_proxy->result_authentication(apdu, m_ret);
+    delete this;
+}
+
+#endif
+
+void Yaz_Proxy::result_authentication(Z_APDU *apdu, int ret)
+{
+    if (ret == 0)
+    {
+       Z_APDU *apdu_reject = zget_APDU(odr_encode(), Z_APDU_initResponse);
+       *apdu_reject->u.initResponse->result = 0;
+       send_to_client(apdu_reject);
+       dec_ref();
+    }
+    else
+       handle_incoming_Z_PDU_2(apdu);
+}
 
 static const char *apdu_name(Z_APDU *apdu)
 {
@@ -94,11 +185,14 @@ static const char *gdu_name(Z_GDU *gdu)
     return "Unknown request/response";
 }
 
-Yaz_Proxy::Yaz_Proxy(IYaz_PDU_Observable *the_PDU_Observable,
-                    Yaz_Proxy *parent) :
-    Yaz_Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60)
+Yaz_Proxy::Yaz_Proxy(IPDU_Observable *the_PDU_Observable,
+                    ISocketObservable *the_socket_observable,
+                    Yaz_Proxy *parent) 
+    :
+    Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60)
 {
     m_PDU_Observable = the_PDU_Observable;
+    m_socket_observable = the_socket_observable;
     m_client = 0;
     m_parent = parent;
     m_clientPool = 0;
@@ -107,7 +201,9 @@ Yaz_Proxy::Yaz_Proxy(IYaz_PDU_Observable *the_PDU_Observable,
     m_keepalive_limit_pdu = 1000;
     m_proxyTarget = 0;
     m_default_target = 0;
-    m_proxy_authentication = 0;
+    m_proxy_negotiation_charset = 0;
+    m_proxy_negotiation_lang = 0;
+    m_charset_converter = new Yaz_CharsetConverter;
     m_max_clients = 150;
     m_log_mask = 0;
     m_seed = time(0);
@@ -115,23 +211,26 @@ Yaz_Proxy::Yaz_Proxy(IYaz_PDU_Observable *the_PDU_Observable,
     m_target_idletime = 600;
     m_optimize = xstrdup ("1");
     strcpy(m_session_str, "0 ");
-    m_session_no=0;
+    m_session_no = 0;
     m_bytes_sent = 0;
     m_bytes_recv = 0;
     m_bw_hold_PDU = 0;
     m_bw_max = 0;
     m_pdu_max = 0;
+    m_timeout_mode = timeout_normal;
+    m_timeout_gdu = 0;
     m_max_record_retrieve = 0;
     m_reconfig_flag = 0;
     m_config_fname = 0;
     m_request_no = 0;
-    m_invalid_session = 0;
+    m_flag_invalid_session = 0;
     m_referenceId = 0;
     m_referenceId_mem = nmem_create();
     m_config = 0;
-    m_marcxml_flag = 0;
+    m_marcxml_mode = none;
     m_stylesheet_xsp = 0;
     m_stylesheet_nprl = 0;
+    m_stylesheet_apdu = 0;
     m_s2z_stylesheet = 0;
     m_s2z_database = 0;
     m_schema = 0;
@@ -144,6 +243,11 @@ Yaz_Proxy::Yaz_Proxy(IYaz_PDU_Observable *the_PDU_Observable,
     m_initRequest_maximumRecordSize = 0;
     m_initRequest_options = 0;
     m_initRequest_version = 0;
+    m_initRequest_oi_negotiation_charsets = 0;
+    m_initRequest_oi_negotiation_num_charsets = 0;
+    m_initRequest_oi_negotiation_langs = 0;
+    m_initRequest_oi_negotiation_num_langs = 0;
+    m_initRequest_oi_negotiation_selected = 0;
     m_apdu_invalid_session = 0;
     m_mem_invalid_session = 0;
     m_s2z_odr_init = 0;
@@ -168,6 +272,14 @@ Yaz_Proxy::Yaz_Proxy(IYaz_PDU_Observable *the_PDU_Observable,
     m_usemarcon = new Yaz_usemarcon();
     if (!m_parent)
        low_socket_open();
+    m_my_thread = 0;
+    m_ref_count = 1;
+    m_peername = 0;
+}
+
+void Yaz_Proxy::inc_ref()
+{
+//    m_ref_count++;
 }
 
 Yaz_Proxy::~Yaz_Proxy()
@@ -180,7 +292,9 @@ Yaz_Proxy::~Yaz_Proxy()
 
     xfree(m_proxyTarget);
     xfree(m_default_target);
-    xfree(m_proxy_authentication);
+    xfree(m_proxy_negotiation_charset);
+    xfree(m_proxy_negotiation_lang);
+    delete m_charset_converter;
     xfree(m_optimize);
 
 #if HAVE_XSLT
@@ -189,6 +303,7 @@ Yaz_Proxy::~Yaz_Proxy()
 #endif
     xfree (m_time_tv);
 
+    xfree (m_peername);
     xfree (m_schema);
     xfree (m_backend_type);
     xfree (m_backend_charset);
@@ -201,9 +316,16 @@ Yaz_Proxy::~Yaz_Proxy()
        odr_destroy(m_s2z_odr_search);
     if (!m_parent)
        low_socket_close();
+    if (!m_parent)
+       delete m_my_thread;
     delete m_config;
 }
 
+void Yaz_Proxy::set_debug_mode(int mode)
+{
+    m_debug_mode = mode;
+}
+
 int Yaz_Proxy::set_config(const char *config)
 {
     delete m_config;
@@ -224,12 +346,18 @@ void Yaz_Proxy::set_default_target(const char *target)
        m_default_target = (char *) xstrdup (target);
 }
 
-void Yaz_Proxy::set_proxy_authentication (const char *auth)
+void Yaz_Proxy::set_proxy_negotiation (const char *charset, const char *lang)
 {
-    xfree (m_proxy_authentication);
-    m_proxy_authentication = 0;
-    if (auth)
-       m_proxy_authentication = (char *) xstrdup (auth);
+    yaz_log(YLOG_LOG, "%sSet the proxy negotiation: charset to '%s', "
+       "language to '%s'", m_session_str, charset?charset:"none",
+       lang?lang:"none");
+    xfree (m_proxy_negotiation_charset);
+    xfree (m_proxy_negotiation_lang);
+    m_proxy_negotiation_charset = m_proxy_negotiation_lang = 0;
+    if (charset)
+       m_proxy_negotiation_charset = (char *) xstrdup (charset);
+    if (lang)
+       m_proxy_negotiation_lang = (char *) xstrdup (lang);
 }
 
 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
@@ -261,11 +389,12 @@ Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
     return cfg;
 }
 
-IYaz_PDU_Observer *Yaz_Proxy::sessionNotify(IYaz_PDU_Observable
-                                           *the_PDU_Observable, int fd)
+IPDU_Observer *Yaz_Proxy::sessionNotify(IPDU_Observable
+                                       *the_PDU_Observable, int fd)
 {
     check_reconfigure();
-    Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable, this);
+    Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable,
+                                        m_socket_observable, this);
     new_proxy->m_config = 0;
     new_proxy->m_config_fname = 0;
     new_proxy->timeout(m_client_idletime);
@@ -278,11 +407,17 @@ IYaz_PDU_Observer *Yaz_Proxy::sessionNotify(IYaz_PDU_Observable
        new_proxy->set_APDU_yazlog(1);
     else
        new_proxy->set_APDU_yazlog(0);
-    new_proxy->set_proxy_authentication(m_proxy_authentication);
     sprintf(new_proxy->m_session_str, "%ld:%d ", (long) time(0), m_session_no);
     m_session_no++;
+    new_proxy->m_peername = xstrdup(the_PDU_Observable->getpeername());
     yaz_log (YLOG_LOG, "%sNew session %s", new_proxy->m_session_str,
-            the_PDU_Observable->getpeername());
+            new_proxy->m_peername);
+    new_proxy->set_proxy_negotiation(m_proxy_negotiation_charset,
+       m_proxy_negotiation_lang);
+    // create thread object the first time we get an incoming connection
+    if (!m_my_thread)
+       m_my_thread = new Msg_Thread(m_socket_observable);
+    new_proxy->m_my_thread = m_my_thread;
     return new_proxy;
 }
 
@@ -302,7 +437,6 @@ char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
        return oi->information.characterInfo;
     return 0;
 }
-
 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
 {
     int oid[OID_SIZE];
@@ -317,7 +451,6 @@ char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
        return oi->information.characterInfo;
     return 0;
 }
-
 const char *Yaz_Proxy::load_balance(const char **url)
 {
     int zurl_in_use[MAX_ZURL_PLEX];
@@ -361,12 +494,6 @@ const char *Yaz_Proxy::load_balance(const char **url)
            max_spare = zurl_in_spare[i];
        }
     }
-    // use the one with minimum connections if spare is > 3
-    if (spare_for_min > 3)
-       return ret_min;
-    // use one with most spares (if any)
-    if (max_spare > 0)
-       return ret_spare;
     return ret_min;
 }
 
@@ -383,18 +510,21 @@ Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
        Yaz_ProxyConfig *cfg = check_reconfigure();
        if (proxy_host)
        {
-#if 0
-/* only to be enabled for debugging... */
-           if (!strcmp(proxy_host, "stop"))
-               exit(0);
-#endif
+           if (parent && parent->m_debug_mode)
+           {
+                // only to be enabled for debugging...
+               if (!strcmp(proxy_host, "stop"))
+                   exit(0);
+           }
            xfree(m_default_target);
            m_default_target = xstrdup(proxy_host);
        }
        proxy_host = m_default_target;
        int client_idletime = -1;
        const char *cql2rpn_fname = 0;
-       const char *authentication = 0;
+       const char *negotiation_charset = 0;
+       const char *negotiation_lang = 0;
+       const char *query_charset = 0;
        url[0] = m_default_target;
        url[1] = 0;
        if (cfg)
@@ -408,7 +538,9 @@ Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
                                 &m_keepalive_limit_pdu,
                                 &pre_init,
                                 &cql2rpn_fname,
-                                &authentication);
+                                &negotiation_charset,
+                                &negotiation_lang,
+                                &query_charset);
        }
        if (client_idletime != -1)
        {
@@ -417,8 +549,12 @@ Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
        }
        if (cql2rpn_fname)
            m_cql2rpn.set_pqf_file(cql2rpn_fname);
-       if (authentication)
-           set_proxy_authentication(authentication);
+       if (negotiation_charset || negotiation_lang)
+       {
+           set_proxy_negotiation(negotiation_charset,
+               negotiation_lang);
+       }
+       m_charset_converter->set_target_query_charset(query_charset);
        if (!url[0])
        {
            yaz_log(YLOG_LOG, "%sNo default target", m_session_str);
@@ -476,11 +612,14 @@ Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
            }
        }
     }
-    else if (!c && apdu->which == Z_APDU_initRequest
-            && apdu->u.initRequest->idAuthentication == 0)
+    else if (!c &&
+           apdu->which == Z_APDU_initRequest &&
+           apdu->u.initRequest->idAuthentication == 0 &&
+           !ODR_MASK_GET(apdu->u.initRequest->options, Z_Options_negotiationModel))
     {
        // anonymous sessions without cookie.
        // if authentication is set it is NOT anonymous se we can't share them.
+       // If charset and lang negotiation is use it is NOT anonymous session too.
        for (c = parent->m_clientPool; c; c = c->m_next)
        {
            assert(c->m_prev);
@@ -519,23 +658,9 @@ Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
        }
         Z_InitRequest *initRequest = apdu->u.initRequest;
 
-        if (!initRequest->idAuthentication)
-        {
-            if (m_proxy_authentication)
-            {
-                initRequest->idAuthentication =
-                    (Z_IdAuthentication *)
-                    odr_malloc (odr_encode(),
-                                sizeof(*initRequest->idAuthentication));
-                initRequest->idAuthentication->which =
-                    Z_IdAuthentication_open;
-                initRequest->idAuthentication->u.open =
-                    odr_strdup (odr_encode(), m_proxy_authentication);
-            }
-        }
-       else
+        if (initRequest->idAuthentication)
        {
-           // the client use authentication. We set the keepalive PDU
+           // the client uses authentication. We set the keepalive PDU
            // to 0 so we don't cache it in releaseClient
            m_keepalive_limit_pdu = 0;
        }
@@ -682,14 +807,16 @@ int Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p, Z_APDU *apdu)
     m_stylesheet_offset = 0;
     m_stylesheet_nprl = p;
     m_stylesheet_apdu = apdu;
+    m_timeout_mode = timeout_xsl;
+
     timeout(0);
     return 1;
 }
 
 void Yaz_Proxy::convert_xsl_delay()
 {
-    Z_NamePlusRecord *npr = m_stylesheet_nprl->records[m_stylesheet_offset];
 #if HAVE_XSLT
+    Z_NamePlusRecord *npr = m_stylesheet_nprl->records[m_stylesheet_offset];
     if (npr->which == Z_NamePlusRecord_databaseRecord)
     {
        Z_External *r = npr->u.databaseRecord;
@@ -729,6 +856,7 @@ void Yaz_Proxy::convert_xsl_delay()
     m_stylesheet_offset++;
     if (m_stylesheet_offset == m_stylesheet_nprl->num_records)
     {
+       m_timeout_mode = timeout_normal;
        m_stylesheet_nprl = 0;
 #if HAVE_XSLT
        if (m_stylesheet_xsp)
@@ -803,6 +931,7 @@ void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p)
                        continue;
                    }
 #endif
+/* HAVE_USEMARCON */
                    npr->u.databaseRecord =
                        z_ext_record(odr_encode(),
                                     m_frontend_type,
@@ -814,11 +943,78 @@ void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p)
     }
 }
 
+void Yaz_Proxy::convert_records_charset(Z_NamePlusRecordList *p,
+                                       const char *backend_charset)
+{
+    yaz_log(YLOG_LOG, "%sconvert_to_marc", m_session_str);
+    int sel =  m_charset_converter->get_client_charset_selected();
+    const char *client_record_charset =
+       m_charset_converter->get_client_query_charset();
+    if (sel && backend_charset && client_record_charset &&
+       strcmp(backend_charset, client_record_charset))
+    {
+       int i;
+       yaz_iconv_t cd = yaz_iconv_open(client_record_charset,
+                                       backend_charset);
+       yaz_marc_t mt = yaz_marc_create();
+       yaz_marc_xml(mt, YAZ_MARC_ISO2709);
+       yaz_marc_iconv(mt, cd);
+       for (i = 0; i < p->num_records; i++)
+       {
+           Z_NamePlusRecord *npr = p->records[i];
+           if (npr->which == Z_NamePlusRecord_databaseRecord)
+           {
+               Z_External *r = npr->u.databaseRecord;
+               oident *ent = oid_getentbyoid(r->direct_reference);
+               if (!ent || ent->value == VAL_NONE)
+                   continue;
+
+               if (ent->value == VAL_SUTRS)
+               {
+                   WRBUF w = wrbuf_alloc();
+
+                   wrbuf_iconv_write(w, cd,  (char*) r->u.octet_aligned->buf,
+                                     r->u.octet_aligned->len);
+                   npr->u.databaseRecord =
+                       z_ext_record(odr_encode(), ent->value, wrbuf_buf(w),
+                                    wrbuf_len(w));
+                   wrbuf_free(w, 1);
+               }
+               else if (ent->value == VAL_TEXT_XML)
+               {
+                   ;
+               }
+               else if (r->which == Z_External_octet)
+               {
+                   int rlen;
+                   char *result;
+                   if (yaz_marc_decode_buf(mt,
+                                           (char*) r->u.octet_aligned->buf,
+                                           r->u.octet_aligned->len,
+                                           &result, &rlen))
+                   {
+                       npr->u.databaseRecord =
+                           z_ext_record(odr_encode(), ent->value, result, rlen);
+                       yaz_log(YLOG_LOG, "%sRecoding MARC record",
+                               m_session_str);
+                   }
+               }
+           }
+       }
+       if (cd)
+           yaz_iconv_close(cd);
+       yaz_marc_destroy(mt);
+    }
+    else
+    {
+       yaz_log(YLOG_LOG, "%sSkipping marc convert", m_session_str);
+    }
+}
+
 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p,
                                   const char *backend_charset)
 {
     int i;
-
     if (!backend_charset)
        backend_charset = "MARC-8";
     yaz_iconv_t cd = yaz_iconv_open("UTF-8", backend_charset);
@@ -831,7 +1027,18 @@ void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p,
        if (npr->which == Z_NamePlusRecord_databaseRecord)
        {
            Z_External *r = npr->u.databaseRecord;
-           if (r->which == Z_External_octet)
+           if (r->which == Z_External_OPAC)
+           {
+               WRBUF w = wrbuf_alloc();
+
+               yaz_display_OPAC(w, r->u.opac, 0);
+               npr->u.databaseRecord = z_ext_record(
+                   odr_encode(), VAL_TEXT_XML,
+                   wrbuf_buf(w), wrbuf_len(w)
+                   );
+               wrbuf_free(w, 1);
+           }
+           else if (r->which == Z_External_octet)
            {
                int rlen;
                char *result;
@@ -839,9 +1046,8 @@ void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p,
                                        r->u.octet_aligned->len,
                                        &result, &rlen))
                {
-                   npr->u.databaseRecord = z_ext_record(odr_encode(),
-                                                        VAL_TEXT_XML,
-                                                        result, rlen);
+                   npr->u.databaseRecord =
+                       z_ext_record(odr_encode(), VAL_TEXT_XML, result, rlen);
                }
            }
        }
@@ -1160,9 +1366,12 @@ int Yaz_Proxy::send_to_client(Z_APDU *apdu)
 #endif
                    )
                    convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
-               if (m_marcxml_flag)
+               if (m_marcxml_mode == marcxml)
                    convert_to_marcxml(p->u.databaseOrSurDiagnostics,
                                       m_backend_charset);
+               else
+                   convert_records_charset(p->u.databaseOrSurDiagnostics,
+                                           m_backend_charset);
                if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
                    return 0;
                    
@@ -1173,7 +1382,7 @@ int Yaz_Proxy::send_to_client(Z_APDU *apdu)
                        *sr->resultCount);
                if (*sr->resultCount < 0)
                {
-                   m_invalid_session = 1;
+                   m_flag_invalid_session = 1;
                    kill_session = 1;
 
                    *sr->searchStatus = 0;
@@ -1205,15 +1414,22 @@ int Yaz_Proxy::send_to_client(Z_APDU *apdu)
 #endif
                )
                convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
-           if (m_marcxml_flag)
+           if (m_marcxml_mode == marcxml)
                convert_to_marcxml(p->u.databaseOrSurDiagnostics,
                                   m_backend_charset);
+           else
+               convert_records_charset(p->u.databaseOrSurDiagnostics,
+                                       m_backend_charset);
            if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
                return 0;
        }
     }
     else if (apdu->which == Z_APDU_initResponse)
     {
+       //Get and check negotiation record
+       //from init response.
+       handle_charset_lang_negotiation(apdu);
+       
        if (m_initRequest_options)
        {
            Z_Options *nopt = 
@@ -1255,6 +1471,7 @@ int Yaz_Proxy::send_to_client(Z_APDU *apdu)
                       m_initRequest_maximumRecordSize :
                       m_client->m_initResponse_maximumRecordSize);
     }
+    
     int r = send_PDU_convert(apdu);
     if (r)
        return r;
@@ -1530,21 +1747,37 @@ void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
        yaz_log (YLOG_LOG, "%sReceiving %s from client %d bytes",
                 m_session_str, gdu_name(apdu), len);
-
-    if (m_bw_hold_PDU)     // double incoming PDU. shutdown now.
-       shutdown();
-
-    m_bw_stat.add_bytes(len);
-    m_pdu_stat.add_bytes(1);
-
+    
 #if HAVE_GETTIMEOFDAY
     gettimeofday((struct timeval *) m_time_tv, 0);
 #endif
+    m_bw_stat.add_bytes(len);
+    m_pdu_stat.add_bytes(1);
+    
+    GDU *gdu = new GDU(apdu);
+    int qsize = m_in_queue.size();
+    if (m_timeout_mode != timeout_normal)
+    {
+       yaz_log(YLOG_LOG, "%sAdded gdu in queue of size %d", m_session_str,
+               qsize);
+       m_in_queue.enqueue(gdu);
+    }
+    else
+    {
+       recv_GDU_reduce(gdu);
+       recv_GDU_more();
+    }
+}
 
+void Yaz_Proxy::recv_GDU_reduce(GDU *gdu)
+{
     int bw_total = m_bw_stat.get_total();
     int pdu_total = m_pdu_stat.get_total();
-
     int reduce = 0;
+
+    assert(m_timeout_mode == timeout_normal);
+    assert(m_timeout_gdu == 0);
+    
     if (m_bw_max)
     {
        if (bw_total > m_bw_max)
@@ -1560,16 +1793,37 @@ void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
            reduce = (reduce > nreduce) ? reduce : nreduce;
        }
     }
+    m_http_version = 0;
+
     if (reduce)  
     {
        yaz_log(YLOG_LOG, "%sdelay=%d bw=%d pdu=%d limit-bw=%d limit-pdu=%d",
                m_session_str, reduce, bw_total, pdu_total,
                m_bw_max, m_pdu_max);
        
-       m_bw_hold_PDU = apdu;  // save PDU and signal "on hold"
+       m_timeout_mode = timeout_reduce;
+       m_timeout_gdu = gdu;
+       // m_bw_hold_PDU = apdu;  // save PDU and signal "on hold"
        timeout(reduce);       // call us reduce seconds later
     }
-    else if (apdu->which == Z_GDU_Z3950)
+    else
+       recv_GDU_normal(gdu);
+}
+
+void Yaz_Proxy::recv_GDU_more()
+{
+    GDU *g;
+    while (m_timeout_mode == timeout_normal && (g = m_in_queue.dequeue()))
+       recv_GDU_reduce(g);
+}
+
+void Yaz_Proxy::recv_GDU_normal(GDU *gdu)
+{
+    Z_GDU *apdu = gdu->get();
+    gdu->extract_odr_to(odr_decode());
+    delete gdu;
+
+    if (apdu->which == Z_GDU_Z3950)
        handle_incoming_Z_PDU(apdu->u.z3950);
     else if (apdu->which == Z_GDU_HTTP_Request)
        handle_incoming_HTTP(apdu->u.HTTP_Request);
@@ -1589,6 +1843,181 @@ void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
     }
 }
 
+void Yaz_Proxy::handle_charset_lang_negotiation(Z_APDU *apdu)
+{
+    if (apdu->which == Z_APDU_initRequest)
+    {
+       yaz_log(YLOG_LOG, "%shandle_charset_lang_negotiation", 
+               m_session_str);
+       if (m_initRequest_options &&
+           !ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel) &&
+           (m_proxy_negotiation_charset || m_proxy_negotiation_lang))
+       {
+           // There is no negotiation proposal from
+           // client's side. OK. The proxy negotiation
+           // in use, only.
+           Z_InitRequest *initRequest = apdu->u.initRequest;
+           Z_OtherInformation **otherInfo;  
+           Z_OtherInformationUnit *oi;
+           get_otherInfoAPDU(apdu, &otherInfo);
+           oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
+           if (oi)
+           {
+               ODR_MASK_SET(initRequest->options,
+                   Z_Options_negotiationModel);
+               oi->which = Z_OtherInfo_externallyDefinedInfo;    
+               oi->information.externallyDefinedInfo =
+               yaz_set_proposal_charneg(odr_encode(),
+                   (const char**)&m_proxy_negotiation_charset,
+                   m_proxy_negotiation_charset ? 1:0,
+                   (const char**)&m_proxy_negotiation_lang,
+                   m_proxy_negotiation_lang ? 1:0,
+                   1);
+           }
+       }
+       else if (m_initRequest_options &&
+                ODR_MASK_GET(m_initRequest_options,
+                             Z_Options_negotiationModel) &&
+                m_charset_converter->get_target_query_charset())
+       {
+           yaz_log(YLOG_LOG, "%sManaged charset negotiation: charset=%s",
+                   m_session_str,
+                   m_charset_converter->get_target_query_charset());
+           Z_InitRequest *initRequest = apdu->u.initRequest;
+           Z_CharSetandLanguageNegotiation *negotiation =
+               yaz_get_charneg_record (initRequest->otherInfo);        
+           if (negotiation &&
+               negotiation->which == Z_CharSetandLanguageNegotiation_proposal)
+           {
+               NMEM nmem = nmem_create();
+               char **charsets = 0;
+               int num_charsets = 0;
+               char **langs = 0;
+               int num_langs = 0;
+               int selected = 0;
+               yaz_get_proposal_charneg (nmem, negotiation,
+                                         &charsets, &num_charsets,
+                                         &langs, &num_langs, &selected);
+               int i;
+               for (i = 0; i<num_charsets; i++)
+                   yaz_log(YLOG_LOG, "%scharset %s", m_session_str,
+                           charsets[i]);
+               for (i = 0; i<num_langs; i++)
+                   yaz_log(YLOG_LOG, "%slang %s", m_session_str,
+                           langs[i]);
+
+               const char *t_charset =
+                   m_charset_converter->get_target_query_charset();
+               // sweep through charsets and pick the first supported
+               // conversion
+               for (i = 0; i<num_charsets; i++)
+               {
+                   const char *c_charset = charsets[i];
+                   if (!odr_set_charset(odr_decode(), t_charset, c_charset))
+                       break;
+               }
+               if (i != num_charsets)
+               {
+                   // got one .. set up ODR for reverse direction
+                   const char *c_charset = charsets[i];
+                   odr_set_charset(odr_encode(), c_charset, t_charset);
+                   m_charset_converter->set_client_query_charset(c_charset);
+                   m_charset_converter->set_client_charset_selected(selected);
+               }
+               nmem_destroy(nmem);
+               ODR_MASK_CLEAR(m_initRequest_options, 
+                              Z_Options_negotiationModel);
+               yaz_del_charneg_record(&initRequest->otherInfo);
+           }
+           else
+           {
+               yaz_log(YLOG_WARN, "%sUnable to decode charset package",
+                       m_session_str);
+           }
+       }
+    }
+    else if (apdu->which == Z_APDU_initResponse)
+    {
+       Z_InitResponse *initResponse = apdu->u.initResponse;    
+       Z_OtherInformation **otherInfo;  
+       
+       if (ODR_MASK_GET(initResponse->options, Z_Options_negotiationModel))
+       {
+           char *charset = 0;
+           char *lang = 0;
+           int selected = 0;
+           
+           get_otherInfoAPDU(apdu, &otherInfo);
+           
+           if (!otherInfo && !(*otherInfo))
+               return;
+           
+           Z_CharSetandLanguageNegotiation *charneg =
+               yaz_get_charneg_record(*otherInfo);
+               
+           if (!charneg)
+               return;
+                               
+           yaz_get_response_charneg(m_referenceId_mem, charneg,
+               &charset, &lang, &selected);
+                               
+           yaz_log(YLOG_LOG, "%sAccepted charset - '%s' and lang - '%s'",
+               m_session_str, (charset)?charset:"none", (lang)?lang:"none");
+               
+           if (m_initRequest_options &&
+               ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel))
+           {
+               yaz_log(YLOG_LOG, "%sClient's negotiation record in use",
+                   m_session_str);          
+           }
+           else if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
+           {
+               // negotiation-charset, negotiation-lang
+               // elements of config file in use.
+
+               yaz_log(YLOG_LOG, "%sProxy's negotiation record in use",
+                   m_session_str);
+
+               // clear negotiation option.
+               ODR_MASK_CLEAR(initResponse->options, Z_Options_negotiationModel);
+               
+               // Delete negotiation (charneg-3) entry.
+               yaz_del_charneg_record(otherInfo);
+           }
+       }
+       else
+       {
+           if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
+           {
+               yaz_log(YLOG_LOG, "%sTarget did not honor negotiation",
+                       m_session_str);
+           }
+           else if (m_charset_converter->get_client_query_charset())
+           {
+               Z_OtherInformation **otherInfo;  
+               Z_OtherInformationUnit *oi;
+               get_otherInfoAPDU(apdu, &otherInfo);
+               oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
+               if (oi)
+               {
+                   ODR_MASK_SET(initResponse->options,
+                                Z_Options_negotiationModel);
+                   ODR_MASK_SET(m_initRequest_options,
+                                Z_Options_negotiationModel);
+
+                   oi->which = Z_OtherInfo_externallyDefinedInfo;    
+                   oi->information.externallyDefinedInfo =
+                       yaz_set_response_charneg(
+                           odr_encode(),
+                           m_charset_converter->get_client_query_charset(),
+                           0 /* no lang */,
+                           m_charset_converter->get_client_charset_selected());
+               }
+           }
+       }
+    }
+}
+
 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
                                                     int error,
                                                     const char *addinfo)
@@ -1657,6 +2086,24 @@ Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
     return apdu;
 }
 
+Z_APDU *Yaz_Proxy::handle_target_charset_conversion(Z_APDU *apdu)
+{
+    if (apdu->which == Z_APDU_searchRequest &&
+       apdu->u.searchRequest->query)
+    {
+       if (apdu->u.searchRequest->query->which == Z_Query_type_1
+           || apdu->u.searchRequest->query->which == Z_Query_type_101)
+       {
+           if (m_http_version)
+               m_charset_converter->set_client_query_charset("UTF-8");
+           Z_RPNQuery *rpnquery = apdu->u.searchRequest->query->u.type_1;
+           m_charset_converter->convert_type_1(rpnquery, odr_encode());
+       }
+    }
+    return apdu;
+}
+
+
 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
 {
     if (apdu->which == Z_APDU_searchRequest)
@@ -1686,9 +2133,52 @@ Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
     return apdu;
 }
 
+int Yaz_Proxy::handle_authentication(Z_APDU *apdu)
+{
+    if (apdu->which != Z_APDU_initRequest)
+       return 1;  // pass if no init request
+    Z_InitRequest *req = apdu->u.initRequest;
+
+    Yaz_ProxyConfig *cfg = check_reconfigure();
+    if (!cfg)
+       return 1;  // pass if no config
+
+    int ret;
+    if (req->idAuthentication == 0)
+    {
+       ret = cfg->client_authentication(m_default_target, 0, 0, 0,
+                                        m_peername);
+    }
+    else if (req->idAuthentication->which == Z_IdAuthentication_idPass)
+    {
+       ret = cfg->client_authentication(
+           m_default_target,
+           req->idAuthentication->u.idPass->userId,
+           req->idAuthentication->u.idPass->groupId,
+           req->idAuthentication->u.idPass->password,
+           m_peername);
+    }
+    else if (req->idAuthentication->which == Z_IdAuthentication_open)
+    {
+       char user[64], pass[64];
+       *user = '\0';
+       *pass = '\0';
+       sscanf(req->idAuthentication->u.open, "%63[^/]/%63s", user, pass);
+       ret = cfg->client_authentication(m_default_target, user, 0, pass,
+                                        m_peername);
+    }
+    else
+       ret = cfg->client_authentication(m_default_target, 0, 0, 0,
+                                        m_peername);
+
+    cfg->target_authentication(m_default_target, odr_encode(), req);
+
+    return ret;
+}
+
 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
 {
-    m_marcxml_flag = 0;
+    m_marcxml_mode = none;
     if (apdu->which == Z_APDU_searchRequest)
     {
        Z_SearchRequest *sr = apdu->u.searchRequest;
@@ -1741,6 +2231,7 @@ Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
        {
            sr->smallSetElementSetNames = 0;
            sr->mediumSetElementSetNames = 0;
+           m_marcxml_mode = marcxml;
            if (m_backend_type)
            {
                
@@ -1752,7 +2243,6 @@ Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
                sr->preferredRecordSyntax =
                    yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
                                           VAL_USMARC);
-           m_marcxml_flag = 1;
        }
        else if (err)
        {
@@ -1817,6 +2307,7 @@ Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
        if (err == -1)
        {
            pr->recordComposition = 0;
+           m_marcxml_mode = marcxml;
            if (m_backend_type)
            {
                
@@ -1828,7 +2319,6 @@ Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
                pr->preferredRecordSyntax =
                    yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
                                           VAL_USMARC);
-           m_marcxml_flag = 1;
        }
        else if (err)
        {
@@ -1882,6 +2372,78 @@ void Yaz_Proxy::srw_get_client(const char *db, const char **backend_db)
        m_default_target = xstrdup(t);
     }
 }
+
+int Yaz_Proxy::file_access(Z_HTTP_Request *hreq)
+{
+    struct stat sbuf;
+    if (strcmp(hreq->method, "GET"))
+       return 0;
+    if (hreq->path[0] != '/')
+    {
+       yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
+       return 0;
+    }
+    const char *cp = hreq->path;
+    while (*cp)
+    {
+       if (*cp == '/' && strchr("/.", cp[1]))
+       {
+           yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
+           return 0;
+       }
+       cp++;
+    }
+    const char *fname = hreq->path+1;
+    if (stat(fname, &sbuf))
+    {
+       yaz_log(YLOG_WARN|YLOG_ERRNO, "%s: stat failed", fname);
+       return 0;
+    }
+    if ((sbuf.st_mode & S_IFMT) != S_IFREG)
+    {
+       yaz_log(YLOG_WARN, "%s: not a regular file", fname);
+       return 0;
+    }
+    if (sbuf.st_size > (off_t) 1000000)
+    {
+       yaz_log(YLOG_WARN, "%s: too large for transfer", fname);
+       return 0;
+    }
+    
+    ODR o = odr_encode();
+    Yaz_ProxyConfig *cfg = check_reconfigure();
+    const char *ctype = cfg->check_mime_type(fname);
+    Z_GDU *gdu = z_get_HTTP_Response(o, 200);
+    Z_HTTP_Response *hres = gdu->u.HTTP_Response;
+    if (m_http_version)
+       hres->version = odr_strdup(o, m_http_version);
+    z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
+    if (m_http_keepalive)
+        z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
+    else
+       timeout(0);
+
+    hres->content_len = sbuf.st_size;
+    hres->content_buf = (char*) odr_malloc(o, hres->content_len);
+    FILE *f = fopen(fname, "rb");
+    if (f)
+    {
+       fread(hres->content_buf, 1, hres->content_len, f);
+       fclose(f);
+    }
+    else
+    {
+       return 0;
+    }
+    if (m_log_mask & PROXY_LOG_REQ_CLIENT)
+    {
+       yaz_log (YLOG_LOG, "%sSending file %s to client", m_session_str,
+                fname);
+    }
+    int len;
+    send_GDU(gdu, &len);
+    return 1;
+}
        
 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
 {
@@ -1922,10 +2484,15 @@ void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
     char *charset = 0;
     Z_SRW_diagnostic *diagnostic = 0;
     int num_diagnostic = 0;
-    if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
-                      &charset) == 0
-       || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
-                         &charset, &diagnostic, &num_diagnostic) == 0)
+
+    if (file_access(hreq))
+    {
+       return;
+    }
+    else if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
+                           &charset) == 0
+            || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
+                              &charset, &diagnostic, &num_diagnostic) == 0)
     {
        m_s2z_odr_init = odr_createmem(ODR_ENCODE);
        m_s2z_odr_search = odr_createmem(ODR_ENCODE);
@@ -2181,10 +2748,137 @@ void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
            send_to_srw_client_error(4, 0);
         }
     }
-    int len = 0;
-    Z_GDU *p = z_get_HTTP_Response(odr_encode(), 400);
-    timeout(0);
-    send_GDU(p, &len);
+    send_http_response(400);
+}
+
+void Yaz_Proxy::handle_init(Z_APDU *apdu)
+{
+    Z_OtherInformation **oi;
+    get_otherInfoAPDU(apdu, &oi);
+
+    if (apdu->u.initRequest->implementationId)
+       yaz_log(YLOG_LOG, "%simplementationId: %s",
+               m_session_str, apdu->u.initRequest->implementationId);
+    if (apdu->u.initRequest->implementationName)
+       yaz_log(YLOG_LOG, "%simplementationName: %s",
+               m_session_str, apdu->u.initRequest->implementationName);
+    if (apdu->u.initRequest->implementationVersion)
+       yaz_log(YLOG_LOG, "%simplementationVersion: %s",
+               m_session_str, apdu->u.initRequest->implementationVersion);
+    if (m_initRequest_apdu == 0)
+    {
+       if (m_initRequest_mem)
+           nmem_destroy(m_initRequest_mem);
+       
+       m_initRequest_apdu = apdu;
+       m_initRequest_mem = odr_extract_mem(odr_decode());
+       
+       m_initRequest_preferredMessageSize = *apdu->u.initRequest->
+           preferredMessageSize;
+       *apdu->u.initRequest->preferredMessageSize = 1024*1024;
+       m_initRequest_maximumRecordSize = *apdu->u.initRequest->
+           maximumRecordSize;
+       *apdu->u.initRequest->maximumRecordSize = 1024*1024;
+
+       Z_CharSetandLanguageNegotiation *charSetandLangRecord =
+           yaz_get_charneg_record(*oi);
+       
+       // Save proposal charsets and langs.
+       if (ODR_MASK_GET(apdu->u.initRequest->options,
+                        Z_Options_negotiationModel) 
+           && charSetandLangRecord)
+       {
+           
+           yaz_get_proposal_charneg(m_referenceId_mem,
+                                    charSetandLangRecord,
+                                    &m_initRequest_oi_negotiation_charsets,
+                                    &m_initRequest_oi_negotiation_num_charsets,
+                                    &m_initRequest_oi_negotiation_langs,
+                                    &m_initRequest_oi_negotiation_num_langs,
+                                    &m_initRequest_oi_negotiation_selected);
+           
+           for (int i = 0; i<m_initRequest_oi_negotiation_num_charsets; i++)
+           {
+               yaz_log(YLOG_LOG, "%scharacters set proposal: %s",
+                       m_session_str,(m_initRequest_oi_negotiation_charsets[i])?
+                       m_initRequest_oi_negotiation_charsets[i]:"none");
+           }
+           for (int i=0; i<m_initRequest_oi_negotiation_num_langs; i++)
+           {
+               yaz_log(YLOG_LOG, "%slanguages proposal: %s",
+                       m_session_str, (m_initRequest_oi_negotiation_langs[i])?
+                       m_initRequest_oi_negotiation_langs[i]:"none");
+           }
+           yaz_log(YLOG_LOG, "%sselected proposal: %d (boolean)",
+                   m_session_str, m_initRequest_oi_negotiation_selected);
+       }       
+       // save init options for the response..
+       m_initRequest_options = apdu->u.initRequest->options;
+       
+       apdu->u.initRequest->options = 
+           (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
+                                      sizeof(Odr_bitmask));
+       ODR_MASK_ZERO(apdu->u.initRequest->options);
+       int i;
+       for (i = 0; i<= 24; i++)
+           ODR_MASK_SET(apdu->u.initRequest->options, i);
+       // check negotiation option
+       if (!ODR_MASK_GET(m_initRequest_options,
+                         Z_Options_negotiationModel))
+       {
+           ODR_MASK_CLEAR(apdu->u.initRequest->options,
+                          Z_Options_negotiationModel);
+       }
+       ODR_MASK_CLEAR(apdu->u.initRequest->options,
+                      Z_Options_concurrentOperations);
+       // make new version
+       m_initRequest_version = apdu->u.initRequest->protocolVersion;
+       apdu->u.initRequest->protocolVersion = 
+           (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
+                                      sizeof(Odr_bitmask));
+       ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
+       
+       for (i = 0; i<= 8; i++)
+           ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
+    }
+    if (m_client->m_init_flag)
+    {
+       if (handle_init_response_for_invalid_session(apdu))
+           return;
+       if (m_client->m_initResponse)
+       {
+           Z_APDU *apdu2 = m_client->m_initResponse;
+           apdu2->u.initResponse->otherInfo = 0;
+           if (m_client->m_cookie && *m_client->m_cookie)
+               set_otherInformationString(apdu2, VAL_COOKIE, 1,
+                                          m_client->m_cookie);
+           apdu2->u.initResponse->referenceId =
+               apdu->u.initRequest->referenceId;
+           apdu2->u.initResponse->options = m_client->m_initResponse_options;
+           apdu2->u.initResponse->protocolVersion = 
+               m_client->m_initResponse_version;
+           
+           send_to_client(apdu2);
+           return;
+       }
+    }
+    m_client->m_init_flag = 1;
+    
+#if USE_AUTH_MSG
+    Auth_Msg *m = new Auth_Msg;
+    m->m_proxy = this;
+    z_APDU(odr_encode(), &apdu, 0, "encode");
+    char *apdu_buf = odr_getbuf(odr_encode(), &m->m_apdu_len, 0);
+    m->m_apdu_buf = (char*) nmem_malloc(m->m_nmem, m->m_apdu_len);
+    memcpy(m->m_apdu_buf, apdu_buf, m->m_apdu_len);
+    odr_reset(odr_encode());
+
+    inc_ref();
+    m_my_thread->put(m);
+#else
+    int ret = handle_authentication(apdu);
+    result_authentication(apdu, ret);
+#endif
 }
 
 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
@@ -2203,11 +2897,12 @@ void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
     else
        m_referenceId = 0;
 
-    if (!m_client && m_invalid_session)
+    if (!m_client && m_flag_invalid_session)
     {
-       m_apdu_invalid_session = apdu;
+       // Got request for a session that is invalid..
+       m_apdu_invalid_session = apdu; // save package
        m_mem_invalid_session = odr_extract_mem(odr_decode());
-       apdu = m_initRequest_apdu;
+       apdu = m_initRequest_apdu;     // but throw an init to the target
     }
     
     // Determine our client.
@@ -2216,86 +2911,30 @@ void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi));
     if (!m_client)
     {
-       delete this;
-       return;
+       if (m_http_version)
+       {
+           send_http_response(404);
+           return;
+       }
+       else
+       {
+           delete this;
+           return;
+       }
     }
+    
     m_client->m_server = this;
 
     if (apdu->which == Z_APDU_initRequest)
-    {
-       if (apdu->u.initRequest->implementationId)
-           yaz_log(YLOG_LOG, "%simplementationId: %s",
-                   m_session_str, apdu->u.initRequest->implementationId);
-       if (apdu->u.initRequest->implementationName)
-           yaz_log(YLOG_LOG, "%simplementationName: %s",
-                   m_session_str, apdu->u.initRequest->implementationName);
-       if (apdu->u.initRequest->implementationVersion)
-           yaz_log(YLOG_LOG, "%simplementationVersion: %s",
-                   m_session_str, apdu->u.initRequest->implementationVersion);
-       if (m_initRequest_apdu == 0)
-       {
-           if (m_initRequest_mem)
-               nmem_destroy(m_initRequest_mem);
-           m_initRequest_apdu = apdu;
-           m_initRequest_mem = odr_extract_mem(odr_decode());
-
-           m_initRequest_preferredMessageSize = *apdu->u.initRequest->
-               preferredMessageSize;
-           *apdu->u.initRequest->preferredMessageSize = 1024*1024;
-           m_initRequest_maximumRecordSize = *apdu->u.initRequest->
-               maximumRecordSize;
-           *apdu->u.initRequest->maximumRecordSize = 1024*1024;
-
-           // save init options for the response..
-           m_initRequest_options = apdu->u.initRequest->options;
-           
-           apdu->u.initRequest->options = 
-               (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
-                                          sizeof(Odr_bitmask));
-           ODR_MASK_ZERO(apdu->u.initRequest->options);
-           int i;
-           for (i = 0; i<= 24; i++)
-               ODR_MASK_SET(apdu->u.initRequest->options, i);
-           ODR_MASK_CLEAR(apdu->u.initRequest->options,
-                          Z_Options_negotiationModel);
-           ODR_MASK_CLEAR(apdu->u.initRequest->options,
-                          Z_Options_concurrentOperations);
-
-           // make new version
-           m_initRequest_version = apdu->u.initRequest->protocolVersion;
-           apdu->u.initRequest->protocolVersion = 
-               (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
-                                          sizeof(Odr_bitmask));
-           ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
+       handle_init(apdu);
+    else
+       handle_incoming_Z_PDU_2(apdu);
+}
 
-           for (i = 0; i<= 8; i++)
-               ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
-       }
-       if (m_client->m_init_flag)
-       {
-           if (handle_init_response_for_invalid_session(apdu))
-               return;
-           if (m_client->m_initResponse)
-           {
-               Z_APDU *apdu2 = m_client->m_initResponse;
-               apdu2->u.initResponse->otherInfo = 0;
-               if (m_client->m_cookie && *m_client->m_cookie)
-                   set_otherInformationString(apdu2, VAL_COOKIE, 1,
-                                              m_client->m_cookie);
-               apdu2->u.initResponse->referenceId =
-                   apdu->u.initRequest->referenceId;
-               apdu2->u.initResponse->options = m_client->m_initResponse_options;
-               apdu2->u.initResponse->protocolVersion = 
-                   m_client->m_initResponse_version;
-               
-               send_to_client(apdu2);
-               return;
-           }
-       }
-       m_client->m_init_flag = 1;
-    }
+void Yaz_Proxy::handle_incoming_Z_PDU_2(Z_APDU *apdu)
+{
     handle_max_record_retrieve(apdu);
-
+    
     if (apdu)
        apdu = handle_syntax_validation(apdu);
 
@@ -2303,20 +2942,29 @@ void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
        apdu = handle_query_transformation(apdu);
 
     if (apdu)
+       apdu = handle_target_charset_conversion(apdu);
+
+    if (apdu)
        apdu = handle_query_validation(apdu);
 
     if (apdu)
        apdu = result_set_optimize(apdu);
+
     if (!apdu)
     {
        m_client->timeout(m_target_idletime);  // mark it active even 
        // though we didn't use it
        return;
     }
+    // Add otherInformation entry in APDU if
+    // negotiation is in use.
+    if (apdu)
+       handle_charset_lang_negotiation(apdu);
 
     // delete other info construct completely if 0 elements
+    Z_OtherInformation **oi;
     get_otherInfoAPDU(apdu, &oi);
-    if (*oi && (*oi)->num_elements == 0)
+    if (oi && *oi && (*oi)->num_elements == 0)
         *oi = 0;
 
     if (apdu->which == Z_APDU_presentRequest &&
@@ -2346,7 +2994,7 @@ void Yaz_Proxy::releaseClient()
 {
     xfree(m_proxyTarget);
     m_proxyTarget = 0;
-    m_invalid_session = 0;
+    m_flag_invalid_session = 0;
     // only keep if keep_alive flag is set...
     if (m_client && 
        m_client->m_pdu_recv < m_keepalive_limit_pdu &&
@@ -2389,10 +3037,17 @@ void Yaz_Proxy::releaseClient()
        m_parent->pre_init();
 }
 
-void Yaz_Proxy::shutdown()
+bool Yaz_Proxy::dec_ref()
 {
-    releaseClient();
-    delete this;
+    --m_ref_count;
+    assert(m_ref_count >= 0);
+    bool last = (m_ref_count == 0);
+    if (m_ref_count == 0)
+    {
+       releaseClient();
+       delete this;
+    }
+    return last;
 }
 
 const char *Yaz_ProxyClient::get_session_str() 
@@ -2415,7 +3070,7 @@ void Yaz_Proxy::failNotify()
     inc_request_no();
     yaz_log (YLOG_LOG, "%sConnection closed by client",
             get_session_str());
-    shutdown();
+    dec_ref();
 }
 
 void Yaz_ProxyClient::failNotify()
@@ -2438,8 +3093,8 @@ void Yaz_ProxyClient::connectNotify()
        pre_init_client();
 }
 
-IYaz_PDU_Observer *Yaz_ProxyClient::sessionNotify(IYaz_PDU_Observable
-                                                 *the_PDU_Observable, int fd)
+IPDU_Observer *Yaz_ProxyClient::sessionNotify(IPDU_Observable
+                                             *the_PDU_Observable, int fd)
 {
     return new Yaz_ProxyClient(the_PDU_Observable, 0);
 }
@@ -2495,6 +3150,8 @@ void Yaz_Proxy::pre_init()
     int pre_init;
     const char *cql2rpn = 0;
     const char *authentication = 0;
+    const char *negotiation_charset = 0;
+    const char *negotiation_lang = 0;
 
     Yaz_ProxyConfig *cfg = check_reconfigure();
 
@@ -2513,7 +3170,10 @@ void Yaz_Proxy::pre_init()
                                          &keepalive_limit_pdu,
                                          &pre_init,
                                          &cql2rpn,
-                                         &authentication) ; i++)
+                                         &authentication,
+                                         &negotiation_charset,
+                                         &negotiation_lang,
+                                         0) ; i++)
     {
        if (pre_init)
        {
@@ -2581,25 +3241,26 @@ void Yaz_Proxy::timeoutNotify()
 {
     if (m_parent)
     {
-       if (m_bw_hold_PDU)
-       {
-           timeout(m_client_idletime);
-           Z_GDU *apdu = m_bw_hold_PDU;
-           m_bw_hold_PDU = 0;
-           
-           if (apdu->which == Z_GDU_Z3950)
-               handle_incoming_Z_PDU(apdu->u.z3950);
-           else if (apdu->which == Z_GDU_HTTP_Request)
-               handle_incoming_HTTP(apdu->u.HTTP_Request);
-       }
-       else if (m_stylesheet_nprl)
-           convert_xsl_delay();
-       else
+       GDU *gdu;
+       switch(m_timeout_mode)
        {
+       case timeout_normal:
            inc_request_no();
-
+           m_in_queue.clear();
            yaz_log (YLOG_LOG, "%sTimeout (client to proxy)", m_session_str);
-           shutdown();
+           dec_ref();
+           break;
+       case timeout_reduce:
+           timeout(m_client_idletime);
+           m_timeout_mode = timeout_normal;
+           gdu = m_timeout_gdu;
+           m_timeout_gdu = 0;
+           recv_GDU_normal(gdu);
+           break;
+       case timeout_xsl:
+           assert(m_stylesheet_nprl);
+           convert_xsl_delay();
+           recv_GDU_more();
        }
     }
     else
@@ -2612,7 +3273,7 @@ void Yaz_Proxy::timeoutNotify()
 void Yaz_Proxy::markInvalid()
 {
     m_client = 0;
-    m_invalid_session = 1;
+    m_flag_invalid_session = 1;
 }
 
 void Yaz_ProxyClient::timeoutNotify()
@@ -2635,9 +3296,9 @@ void Yaz_ProxyClient::timeoutNotify()
     shutdown();
 }
 
-Yaz_ProxyClient::Yaz_ProxyClient(IYaz_PDU_Observable *the_PDU_Observable,
+Yaz_ProxyClient::Yaz_ProxyClient(IPDU_Observable *the_PDU_Observable,
                                 Yaz_Proxy *parent) :
-    Yaz_Z_Assoc (the_PDU_Observable)
+    Z_Assoc (the_PDU_Observable)
 {
     m_cookie = 0;
     m_next = 0;
@@ -2693,9 +3354,9 @@ void Yaz_ProxyClient::recv_GDU(Z_GDU *apdu, int len)
 
 int Yaz_Proxy::handle_init_response_for_invalid_session(Z_APDU *apdu)
 {
-    if (!m_invalid_session)
+    if (!m_flag_invalid_session)
        return 0;
-    m_invalid_session = 0;
+    m_flag_invalid_session = 0;
     handle_incoming_Z_PDU(m_apdu_invalid_session);
     assert (m_mem_invalid_session);
     nmem_destroy(m_mem_invalid_session);
@@ -2789,14 +3450,16 @@ void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
     }
     if (m_cookie)
        set_otherInformationString (apdu, VAL_COOKIE, 1, m_cookie);
-    if (m_server)
-    {
-       m_server->send_to_client(apdu);
-    }
+
+    Yaz_Proxy *server = m_server; // save it. send_to_client may destroy us
+
+    if (server)
+       server->send_to_client(apdu);
     if (apdu->which == Z_APDU_close)
-    {
        shutdown();
-    }
+    else if (server)
+       server->recv_GDU_more();
+    
 }
 
 void Yaz_Proxy::low_socket_close()
@@ -2822,7 +3485,7 @@ void Yaz_Proxy::low_socket_open()
 
 int Yaz_Proxy::server(const char *addr)
 {
-    int r = Yaz_Z_Assoc::server(addr);
+    int r = Z_Assoc::server(addr);
     if (!r)
     {
        yaz_log(YLOG_LOG, "%sStarted proxy "