Bump version to 1.1.0
[yazproxy-moved-to-github.git] / src / yaz-proxy.cpp
index 6857d0c..e59613e 100644 (file)
@@ -1,5 +1,5 @@
-/* $Id: yaz-proxy.cpp,v 1.13 2004-12-03 14:28:18 adam Exp $
-   Copyright (c) 1998-2004, Index Data.
+/* $Id: yaz-proxy.cpp,v 1.27 2005-05-27 18:07:49 adam Exp $
+   Copyright (c) 1998-2005, Index Data.
 
 This file is part of the yaz-proxy.
 
@@ -20,27 +20,38 @@ Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  */
 
 #ifdef WIN32
-#else
-#include <unistd.h>
+#define HAVE_SYS_STAT_H 1
+#define HAVE_SYS_TYPES_H 1
 #endif
 
-#if HAVE_GETTIMEOFDAY
+#if HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#if HAVE_SYS_TIME_H
 #include <sys/time.h>
 #endif
+#if HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#if HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
 
 #include <assert.h>
+#include <stdlib.h>
 #include <time.h>
-#include <sys/types.h>
 #include <fcntl.h>
 
 #include <yaz/srw.h>
 #include <yaz/marcdisp.h>
 #include <yaz/yaz-iconv.h>
-#include <yaz/ylog.h>
+#include <yaz/log.h>
 #include <yaz/diagbib1.h>
 #include "proxyp.h"
 #include <yaz/pquery.h>
 #include <yaz/otherinfo.h>
+#include <yaz/charneg.h>
+
 
 static const char *apdu_name(Z_APDU *apdu)
 {
@@ -93,7 +104,6 @@ static const char *gdu_name(Z_GDU *gdu)
     }
     return "Unknown request/response";
 }
-
 Yaz_Proxy::Yaz_Proxy(IYaz_PDU_Observable *the_PDU_Observable,
                     Yaz_Proxy *parent) :
     Yaz_Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60)
@@ -107,7 +117,9 @@ Yaz_Proxy::Yaz_Proxy(IYaz_PDU_Observable *the_PDU_Observable,
     m_keepalive_limit_pdu = 1000;
     m_proxyTarget = 0;
     m_default_target = 0;
-    m_proxy_authentication = 0;
+    m_proxy_negotiation_charset = 0;
+    m_proxy_negotiation_lang = 0;
+    m_charset_converter = new Yaz_CharsetConverter;
     m_max_clients = 150;
     m_log_mask = 0;
     m_seed = time(0);
@@ -129,7 +141,7 @@ Yaz_Proxy::Yaz_Proxy(IYaz_PDU_Observable *the_PDU_Observable,
     m_referenceId = 0;
     m_referenceId_mem = nmem_create();
     m_config = 0;
-    m_marcxml_flag = 0;
+    m_marcxml_mode = none;
     m_stylesheet_xsp = 0;
     m_stylesheet_nprl = 0;
     m_s2z_stylesheet = 0;
@@ -144,6 +156,11 @@ Yaz_Proxy::Yaz_Proxy(IYaz_PDU_Observable *the_PDU_Observable,
     m_initRequest_maximumRecordSize = 0;
     m_initRequest_options = 0;
     m_initRequest_version = 0;
+    m_initRequest_oi_negotiation_charsets = 0;
+    m_initRequest_oi_negotiation_num_charsets = 0;
+    m_initRequest_oi_negotiation_langs = 0;
+    m_initRequest_oi_negotiation_num_langs = 0;
+    m_initRequest_oi_negotiation_selected = 0;
     m_apdu_invalid_session = 0;
     m_mem_invalid_session = 0;
     m_s2z_odr_init = 0;
@@ -180,7 +197,9 @@ Yaz_Proxy::~Yaz_Proxy()
 
     xfree(m_proxyTarget);
     xfree(m_default_target);
-    xfree(m_proxy_authentication);
+    xfree(m_proxy_negotiation_charset);
+    xfree(m_proxy_negotiation_lang);
+    delete m_charset_converter;
     xfree(m_optimize);
 
 #if HAVE_XSLT
@@ -204,6 +223,11 @@ Yaz_Proxy::~Yaz_Proxy()
     delete m_config;
 }
 
+void Yaz_Proxy::set_debug_mode(int mode)
+{
+    m_debug_mode = mode;
+}
+
 int Yaz_Proxy::set_config(const char *config)
 {
     delete m_config;
@@ -224,12 +248,18 @@ void Yaz_Proxy::set_default_target(const char *target)
        m_default_target = (char *) xstrdup (target);
 }
 
-void Yaz_Proxy::set_proxy_authentication (const char *auth)
+void Yaz_Proxy::set_proxy_negotiation (const char *charset, const char *lang)
 {
-    xfree (m_proxy_authentication);
-    m_proxy_authentication = 0;
-    if (auth)
-       m_proxy_authentication = (char *) xstrdup (auth);
+    yaz_log(YLOG_LOG, "%sSet the proxy negotiation: charset to '%s', "
+       "language to '%s'", m_session_str, charset?charset:"none",
+       lang?lang:"none");
+    xfree (m_proxy_negotiation_charset);
+    xfree (m_proxy_negotiation_lang);
+    m_proxy_negotiation_charset = m_proxy_negotiation_lang = 0;
+    if (charset)
+       m_proxy_negotiation_charset = (char *) xstrdup (charset);
+    if (lang)
+       m_proxy_negotiation_lang = (char *) xstrdup (lang);
 }
 
 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
@@ -278,11 +308,12 @@ IYaz_PDU_Observer *Yaz_Proxy::sessionNotify(IYaz_PDU_Observable
        new_proxy->set_APDU_yazlog(1);
     else
        new_proxy->set_APDU_yazlog(0);
-    new_proxy->set_proxy_authentication(m_proxy_authentication);
     sprintf(new_proxy->m_session_str, "%ld:%d ", (long) time(0), m_session_no);
     m_session_no++;
     yaz_log (YLOG_LOG, "%sNew session %s", new_proxy->m_session_str,
             the_PDU_Observable->getpeername());
+    new_proxy->set_proxy_negotiation(m_proxy_negotiation_charset,
+       m_proxy_negotiation_lang);
     return new_proxy;
 }
 
@@ -302,7 +333,6 @@ char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
        return oi->information.characterInfo;
     return 0;
 }
-
 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
 {
     int oid[OID_SIZE];
@@ -317,7 +347,6 @@ char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
        return oi->information.characterInfo;
     return 0;
 }
-
 const char *Yaz_Proxy::load_balance(const char **url)
 {
     int zurl_in_use[MAX_ZURL_PLEX];
@@ -361,12 +390,6 @@ const char *Yaz_Proxy::load_balance(const char **url)
            max_spare = zurl_in_spare[i];
        }
     }
-    // use the one with minimum connections if spare is > 3
-    if (spare_for_min > 3)
-       return ret_min;
-    // use one with most spares (if any)
-    if (max_spare > 0)
-       return ret_spare;
     return ret_min;
 }
 
@@ -383,18 +406,21 @@ Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
        Yaz_ProxyConfig *cfg = check_reconfigure();
        if (proxy_host)
        {
-#if 0
-/* only to be enabled for debugging... */
-           if (!strcmp(proxy_host, "stop"))
-               exit(0);
-#endif
+           if (parent && parent->m_debug_mode)
+           {
+                // only to be enabled for debugging...
+               if (!strcmp(proxy_host, "stop"))
+                   exit(0);
+           }
            xfree(m_default_target);
            m_default_target = xstrdup(proxy_host);
        }
        proxy_host = m_default_target;
        int client_idletime = -1;
        const char *cql2rpn_fname = 0;
-       const char *authentication = 0;
+       const char *negotiation_charset = 0;
+       const char *negotiation_lang = 0;
+       const char *query_charset = 0;
        url[0] = m_default_target;
        url[1] = 0;
        if (cfg)
@@ -408,7 +434,9 @@ Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
                                 &m_keepalive_limit_pdu,
                                 &pre_init,
                                 &cql2rpn_fname,
-                                &authentication);
+                                &negotiation_charset,
+                                &negotiation_lang,
+                                &query_charset);
        }
        if (client_idletime != -1)
        {
@@ -417,8 +445,12 @@ Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
        }
        if (cql2rpn_fname)
            m_cql2rpn.set_pqf_file(cql2rpn_fname);
-       if (authentication)
-           set_proxy_authentication(authentication);
+       if (negotiation_charset || negotiation_lang)
+       {
+           set_proxy_negotiation(negotiation_charset,
+               negotiation_lang);
+       }
+       m_charset_converter->set_target_query_charset(query_charset);
        if (!url[0])
        {
            yaz_log(YLOG_LOG, "%sNo default target", m_session_str);
@@ -476,11 +508,14 @@ Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
            }
        }
     }
-    else if (!c && apdu->which == Z_APDU_initRequest
-            && apdu->u.initRequest->idAuthentication == 0)
+    else if (!c &&
+           apdu->which == Z_APDU_initRequest &&
+           apdu->u.initRequest->idAuthentication == 0 &&
+           !ODR_MASK_GET(apdu->u.initRequest->options, Z_Options_negotiationModel))
     {
        // anonymous sessions without cookie.
        // if authentication is set it is NOT anonymous se we can't share them.
+       // If charset and lang negotiation is use it is NOT anonymous session too.
        for (c = parent->m_clientPool; c; c = c->m_next)
        {
            assert(c->m_prev);
@@ -519,23 +554,9 @@ Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
        }
         Z_InitRequest *initRequest = apdu->u.initRequest;
 
-        if (!initRequest->idAuthentication)
-        {
-            if (m_proxy_authentication)
-            {
-                initRequest->idAuthentication =
-                    (Z_IdAuthentication *)
-                    odr_malloc (odr_encode(),
-                                sizeof(*initRequest->idAuthentication));
-                initRequest->idAuthentication->which =
-                    Z_IdAuthentication_open;
-                initRequest->idAuthentication->u.open =
-                    odr_strdup (odr_encode(), m_proxy_authentication);
-            }
-        }
-       else
+        if (initRequest->idAuthentication)
        {
-           // the client use authentication. We set the keepalive PDU
+           // the client uses authentication. We set the keepalive PDU
            // to 0 so we don't cache it in releaseClient
            m_keepalive_limit_pdu = 0;
        }
@@ -637,7 +658,6 @@ Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
     yaz_log (YLOG_DEBUG, "get_client 3 %p %p", this, c);
     return c;
 }
-
 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
 {
     int i;
@@ -688,8 +708,8 @@ int Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p, Z_APDU *apdu)
 
 void Yaz_Proxy::convert_xsl_delay()
 {
-    Z_NamePlusRecord *npr = m_stylesheet_nprl->records[m_stylesheet_offset];
 #if HAVE_XSLT
+    Z_NamePlusRecord *npr = m_stylesheet_nprl->records[m_stylesheet_offset];
     if (npr->which == Z_NamePlusRecord_databaseRecord)
     {
        Z_External *r = npr->u.databaseRecord;
@@ -803,6 +823,7 @@ void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p)
                        continue;
                    }
 #endif
+/* HAVE_USEMARCON */
                    npr->u.databaseRecord =
                        z_ext_record(odr_encode(),
                                     m_frontend_type,
@@ -814,11 +835,78 @@ void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p)
     }
 }
 
+void Yaz_Proxy::convert_records_charset(Z_NamePlusRecordList *p,
+                                       const char *backend_charset)
+{
+    yaz_log(YLOG_LOG, "%sconvert_to_marc", m_session_str);
+    int sel =  m_charset_converter->get_client_charset_selected();
+    const char *client_record_charset =
+       m_charset_converter->get_client_query_charset();
+    if (sel && backend_charset && client_record_charset &&
+       strcmp(backend_charset, client_record_charset))
+    {
+       int i;
+       yaz_iconv_t cd = yaz_iconv_open(client_record_charset,
+                                       backend_charset);
+       yaz_marc_t mt = yaz_marc_create();
+       yaz_marc_xml(mt, YAZ_MARC_ISO2709);
+       yaz_marc_iconv(mt, cd);
+       for (i = 0; i < p->num_records; i++)
+       {
+           Z_NamePlusRecord *npr = p->records[i];
+           if (npr->which == Z_NamePlusRecord_databaseRecord)
+           {
+               Z_External *r = npr->u.databaseRecord;
+               oident *ent = oid_getentbyoid(r->direct_reference);
+               if (!ent || ent->value == VAL_NONE)
+                   continue;
+
+               if (ent->value == VAL_SUTRS)
+               {
+                   WRBUF w = wrbuf_alloc();
+
+                   wrbuf_iconv_write(w, cd,  (char*) r->u.octet_aligned->buf,
+                                     r->u.octet_aligned->len);
+                   npr->u.databaseRecord =
+                       z_ext_record(odr_encode(), ent->value, wrbuf_buf(w),
+                                    wrbuf_len(w));
+                   wrbuf_free(w, 1);
+               }
+               else if (ent->value == VAL_TEXT_XML)
+               {
+                   ;
+               }
+               else if (r->which == Z_External_octet)
+               {
+                   int rlen;
+                   char *result;
+                   if (yaz_marc_decode_buf(mt,
+                                           (char*) r->u.octet_aligned->buf,
+                                           r->u.octet_aligned->len,
+                                           &result, &rlen))
+                   {
+                       npr->u.databaseRecord =
+                           z_ext_record(odr_encode(), ent->value, result, rlen);
+                       yaz_log(YLOG_LOG, "%sRecoding MARC record",
+                               m_session_str);
+                   }
+               }
+           }
+       }
+       if (cd)
+           yaz_iconv_close(cd);
+       yaz_marc_destroy(mt);
+    }
+    else
+    {
+       yaz_log(YLOG_LOG, "%sSkipping marc convert", m_session_str);
+    }
+}
+
 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p,
                                   const char *backend_charset)
 {
     int i;
-
     if (!backend_charset)
        backend_charset = "MARC-8";
     yaz_iconv_t cd = yaz_iconv_open("UTF-8", backend_charset);
@@ -831,7 +919,18 @@ void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p,
        if (npr->which == Z_NamePlusRecord_databaseRecord)
        {
            Z_External *r = npr->u.databaseRecord;
-           if (r->which == Z_External_octet)
+           if (r->which == Z_External_OPAC)
+           {
+               WRBUF w = wrbuf_alloc();
+
+               yaz_display_OPAC(w, r->u.opac, 0);
+               npr->u.databaseRecord = z_ext_record(
+                   odr_encode(), VAL_TEXT_XML,
+                   wrbuf_buf(w), wrbuf_len(w)
+                   );
+               wrbuf_free(w, 1);
+           }
+           else if (r->which == Z_External_octet)
            {
                int rlen;
                char *result;
@@ -839,9 +938,8 @@ void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p,
                                        r->u.octet_aligned->len,
                                        &result, &rlen))
                {
-                   npr->u.databaseRecord = z_ext_record(odr_encode(),
-                                                        VAL_TEXT_XML,
-                                                        result, rlen);
+                   npr->u.databaseRecord =
+                       z_ext_record(odr_encode(), VAL_TEXT_XML, result, rlen);
                }
            }
        }
@@ -1160,9 +1258,12 @@ int Yaz_Proxy::send_to_client(Z_APDU *apdu)
 #endif
                    )
                    convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
-               if (m_marcxml_flag)
+               if (m_marcxml_mode == marcxml)
                    convert_to_marcxml(p->u.databaseOrSurDiagnostics,
                                       m_backend_charset);
+               else
+                   convert_records_charset(p->u.databaseOrSurDiagnostics,
+                                           m_backend_charset);
                if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
                    return 0;
                    
@@ -1205,15 +1306,22 @@ int Yaz_Proxy::send_to_client(Z_APDU *apdu)
 #endif
                )
                convert_to_frontend_type(p->u.databaseOrSurDiagnostics);
-           if (m_marcxml_flag)
+           if (m_marcxml_mode == marcxml)
                convert_to_marcxml(p->u.databaseOrSurDiagnostics,
                                   m_backend_charset);
+           else
+               convert_records_charset(p->u.databaseOrSurDiagnostics,
+                                       m_backend_charset);
            if (convert_xsl(p->u.databaseOrSurDiagnostics, apdu))
                return 0;
        }
     }
     else if (apdu->which == Z_APDU_initResponse)
     {
+       //Get and check negotiation record
+       //from init response.
+       handle_charset_lang_negotiation(apdu);
+       
        if (m_initRequest_options)
        {
            Z_Options *nopt = 
@@ -1255,6 +1363,7 @@ int Yaz_Proxy::send_to_client(Z_APDU *apdu)
                       m_initRequest_maximumRecordSize :
                       m_client->m_initResponse_maximumRecordSize);
     }
+    
     int r = send_PDU_convert(apdu);
     if (r)
        return r;
@@ -1560,6 +1669,7 @@ void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
            reduce = (reduce > nreduce) ? reduce : nreduce;
        }
     }
+    m_http_version = 0;
     if (reduce)  
     {
        yaz_log(YLOG_LOG, "%sdelay=%d bw=%d pdu=%d limit-bw=%d limit-pdu=%d",
@@ -1588,6 +1698,180 @@ void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
        }
     }
 }
+void Yaz_Proxy::handle_charset_lang_negotiation(Z_APDU *apdu)
+{
+    if (apdu->which == Z_APDU_initRequest)
+    {
+       yaz_log(YLOG_LOG, "%shandle_charset_lang_negotiation", 
+               m_session_str);
+       if (m_initRequest_options &&
+           !ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel) &&
+           (m_proxy_negotiation_charset || m_proxy_negotiation_lang))
+       {
+           // There is no negotiation proposal from
+           // client's side. OK. The proxy negotiation
+           // in use, only.
+           Z_InitRequest *initRequest = apdu->u.initRequest;
+           Z_OtherInformation **otherInfo;  
+           Z_OtherInformationUnit *oi;
+           get_otherInfoAPDU(apdu, &otherInfo);
+           oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
+           if (oi)
+           {
+               ODR_MASK_SET(initRequest->options,
+                   Z_Options_negotiationModel);
+               oi->which = Z_OtherInfo_externallyDefinedInfo;    
+               oi->information.externallyDefinedInfo =
+               yaz_set_proposal_charneg(odr_encode(),
+                   (const char**)&m_proxy_negotiation_charset,
+                   m_proxy_negotiation_charset ? 1:0,
+                   (const char**)&m_proxy_negotiation_lang,
+                   m_proxy_negotiation_lang ? 1:0,
+                   1);
+           }
+       }
+       else if (m_initRequest_options &&
+                ODR_MASK_GET(m_initRequest_options,
+                             Z_Options_negotiationModel) &&
+                m_charset_converter->get_target_query_charset())
+       {
+           yaz_log(YLOG_LOG, "%sManaged charset negotiation: charset=%s",
+                   m_session_str,
+                   m_charset_converter->get_target_query_charset());
+           Z_InitRequest *initRequest = apdu->u.initRequest;
+           Z_CharSetandLanguageNegotiation *negotiation =
+               yaz_get_charneg_record (initRequest->otherInfo);        
+           if (negotiation &&
+               negotiation->which == Z_CharSetandLanguageNegotiation_proposal)
+           {
+               NMEM nmem = nmem_create();
+               char **charsets = 0;
+               int num_charsets = 0;
+               char **langs = 0;
+               int num_langs = 0;
+               int selected = 0;
+               yaz_get_proposal_charneg (nmem, negotiation,
+                                         &charsets, &num_charsets,
+                                         &langs, &num_langs, &selected);
+               int i;
+               for (i = 0; i<num_charsets; i++)
+                   yaz_log(YLOG_LOG, "%scharset %s", m_session_str,
+                           charsets[i]);
+               for (i = 0; i<num_langs; i++)
+                   yaz_log(YLOG_LOG, "%slang %s", m_session_str,
+                           langs[i]);
+
+               const char *t_charset =
+                   m_charset_converter->get_target_query_charset();
+               // sweep through charsets and pick the first supported
+               // conversion
+               for (i = 0; i<num_charsets; i++)
+               {
+                   const char *c_charset = charsets[i];
+                   if (!odr_set_charset(odr_decode(), t_charset, c_charset))
+                       break;
+               }
+               if (i != num_charsets)
+               {
+                   // got one .. set up ODR for reverse direction
+                   const char *c_charset = charsets[i];
+                   odr_set_charset(odr_encode(), c_charset, t_charset);
+                   m_charset_converter->set_client_query_charset(c_charset);
+                   m_charset_converter->set_client_charset_selected(selected);
+               }
+               nmem_destroy(nmem);
+               ODR_MASK_CLEAR(m_initRequest_options, 
+                              Z_Options_negotiationModel);
+               yaz_del_charneg_record(&initRequest->otherInfo);
+           }
+           else
+           {
+               yaz_log(YLOG_WARN, "%sUnable to decode charset package",
+                       m_session_str);
+           }
+       }
+    }
+    else if (apdu->which == Z_APDU_initResponse)
+    {
+       Z_InitResponse *initResponse = apdu->u.initResponse;    
+       Z_OtherInformation **otherInfo;  
+       
+       if (ODR_MASK_GET(initResponse->options, Z_Options_negotiationModel))
+       {
+           char *charset = 0;
+           char *lang = 0;
+           int selected = 0;
+           
+           get_otherInfoAPDU(apdu, &otherInfo);
+           
+           if (!otherInfo && !(*otherInfo))
+               return;
+           
+           Z_CharSetandLanguageNegotiation *charneg =
+               yaz_get_charneg_record(*otherInfo);
+               
+           if (!charneg)
+               return;
+                               
+           yaz_get_response_charneg(m_referenceId_mem, charneg,
+               &charset, &lang, &selected);
+                               
+           yaz_log(YLOG_LOG, "%sAccepted charset - '%s' and lang - '%s'",
+               m_session_str, (charset)?charset:"none", (lang)?lang:"none");
+               
+           if (m_initRequest_options &&
+               ODR_MASK_GET(m_initRequest_options, Z_Options_negotiationModel))
+           {
+               yaz_log(YLOG_LOG, "%sClient's negotiation record in use",
+                   m_session_str);          
+           }
+           else if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
+           {
+               // negotiation-charset, negotiation-lang
+               // elements of config file in use.
+
+               yaz_log(YLOG_LOG, "%sProxy's negotiation record in use",
+                   m_session_str);
+
+               // clear negotiation option.
+               ODR_MASK_CLEAR(initResponse->options, Z_Options_negotiationModel);
+               
+               // Delete negotiation (charneg-3) entry.
+               yaz_del_charneg_record(otherInfo);
+           }
+       }
+       else
+       {
+           if (m_proxy_negotiation_charset || m_proxy_negotiation_lang)
+           {
+               yaz_log(YLOG_LOG, "%sTarget did not honor negotiation",
+                       m_session_str);
+           }
+           else if (m_charset_converter->get_client_query_charset())
+           {
+               Z_OtherInformation **otherInfo;  
+               Z_OtherInformationUnit *oi;
+               get_otherInfoAPDU(apdu, &otherInfo);
+               oi = update_otherInformation(otherInfo, 1, NULL, 0, 0);
+               if (oi)
+               {
+                   ODR_MASK_SET(initResponse->options,
+                                Z_Options_negotiationModel);
+                   ODR_MASK_SET(m_initRequest_options,
+                                Z_Options_negotiationModel);
+
+                   oi->which = Z_OtherInfo_externallyDefinedInfo;    
+                   oi->information.externallyDefinedInfo =
+                       yaz_set_response_charneg(
+                           odr_encode(),
+                           m_charset_converter->get_client_query_charset(),
+                           0 /* no lang */,
+                           m_charset_converter->get_client_charset_selected());
+               }
+           }
+       }
+    }
+}
 
 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
                                                     int error,
@@ -1657,6 +1941,24 @@ Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
     return apdu;
 }
 
+Z_APDU *Yaz_Proxy::handle_target_charset_conversion(Z_APDU *apdu)
+{
+    if (apdu->which == Z_APDU_searchRequest &&
+       apdu->u.searchRequest->query)
+    {
+       if (apdu->u.searchRequest->query->which == Z_Query_type_1
+           || apdu->u.searchRequest->query->which == Z_Query_type_101)
+       {
+           if (m_http_version)
+               m_charset_converter->set_client_query_charset("UTF-8");
+           Z_RPNQuery *rpnquery = apdu->u.searchRequest->query->u.type_1;
+           m_charset_converter->convert_type_1(rpnquery, odr_encode());
+       }
+    }
+    return apdu;
+}
+
+
 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
 {
     if (apdu->which == Z_APDU_searchRequest)
@@ -1686,9 +1988,47 @@ Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
     return apdu;
 }
 
+int Yaz_Proxy::handle_authentication(Z_APDU *apdu)
+{
+    if (apdu->which != Z_APDU_initRequest)
+       return 1;  // pass if no init request
+    Z_InitRequest *req = apdu->u.initRequest;
+
+    Yaz_ProxyConfig *cfg = check_reconfigure();
+    if (!cfg)
+       return 1;  // pass if no config
+
+    int ret;
+    if (req->idAuthentication == 0)
+    {
+       ret = cfg->client_authentication(m_default_target, 0, 0, 0);
+    }
+    else if (req->idAuthentication->which == Z_IdAuthentication_idPass)
+    {
+       ret = cfg->client_authentication(m_default_target,
+           req->idAuthentication->u.idPass->userId,
+           req->idAuthentication->u.idPass->groupId,
+           req->idAuthentication->u.idPass->password);
+    }
+    else if (req->idAuthentication->which == Z_IdAuthentication_open)
+    {
+       char user[64], pass[64];
+       *user = '\0';
+       *pass = '\0';
+       sscanf(req->idAuthentication->u.open, "%63[^/]/%63s", user, pass);
+       ret = cfg->client_authentication(m_default_target, user, 0, pass);
+    }
+    else
+       ret = cfg->client_authentication(m_default_target, 0, 0, 0);
+
+    cfg->target_authentication(m_default_target, odr_encode(), req);
+
+    return ret;
+}
+
 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
 {
-    m_marcxml_flag = 0;
+    m_marcxml_mode = none;
     if (apdu->which == Z_APDU_searchRequest)
     {
        Z_SearchRequest *sr = apdu->u.searchRequest;
@@ -1741,6 +2081,7 @@ Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
        {
            sr->smallSetElementSetNames = 0;
            sr->mediumSetElementSetNames = 0;
+           m_marcxml_mode = marcxml;
            if (m_backend_type)
            {
                
@@ -1752,7 +2093,6 @@ Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
                sr->preferredRecordSyntax =
                    yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
                                           VAL_USMARC);
-           m_marcxml_flag = 1;
        }
        else if (err)
        {
@@ -1817,6 +2157,7 @@ Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
        if (err == -1)
        {
            pr->recordComposition = 0;
+           m_marcxml_mode = marcxml;
            if (m_backend_type)
            {
                
@@ -1828,7 +2169,6 @@ Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
                pr->preferredRecordSyntax =
                    yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
                                           VAL_USMARC);
-           m_marcxml_flag = 1;
        }
        else if (err)
        {
@@ -1882,6 +2222,78 @@ void Yaz_Proxy::srw_get_client(const char *db, const char **backend_db)
        m_default_target = xstrdup(t);
     }
 }
+
+int Yaz_Proxy::file_access(Z_HTTP_Request *hreq)
+{
+    struct stat sbuf;
+    if (strcmp(hreq->method, "GET"))
+       return 0;
+    if (hreq->path[0] != '/')
+    {
+       yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
+       return 0;
+    }
+    const char *cp = hreq->path;
+    while (*cp)
+    {
+       if (*cp == '/' && strchr("/.", cp[1]))
+       {
+           yaz_log(YLOG_WARN, "Bad path: %s", hreq->path);
+           return 0;
+       }
+       cp++;
+    }
+    const char *fname = hreq->path+1;
+    if (stat(fname, &sbuf))
+    {
+       yaz_log(YLOG_WARN|YLOG_ERRNO, "%s: stat failed", fname);
+       return 0;
+    }
+    if ((sbuf.st_mode & S_IFMT) != S_IFREG)
+    {
+       yaz_log(YLOG_WARN, "%s: not a regular file", fname);
+       return 0;
+    }
+    if (sbuf.st_size > (off_t) 1000000)
+    {
+       yaz_log(YLOG_WARN, "%s: too large for transfer", fname);
+       return 0;
+    }
+    
+    ODR o = odr_encode();
+    Yaz_ProxyConfig *cfg = check_reconfigure();
+    const char *ctype = cfg->check_mime_type(fname);
+    Z_GDU *gdu = z_get_HTTP_Response(o, 200);
+    Z_HTTP_Response *hres = gdu->u.HTTP_Response;
+    if (m_http_version)
+       hres->version = odr_strdup(o, m_http_version);
+    z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
+    if (m_http_keepalive)
+        z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
+    else
+       timeout(0);
+
+    hres->content_len = sbuf.st_size;
+    hres->content_buf = (char*) odr_malloc(o, hres->content_len);
+    FILE *f = fopen(fname, "rb");
+    if (f)
+    {
+       fread(hres->content_buf, 1, hres->content_len, f);
+       fclose(f);
+    }
+    else
+    {
+       return 0;
+    }
+    if (m_log_mask & PROXY_LOG_REQ_CLIENT)
+    {
+       yaz_log (YLOG_LOG, "%sSending file %s to client", m_session_str,
+                fname);
+    }
+    int len;
+    send_GDU(gdu, &len);
+    return 1;
+}
        
 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
 {
@@ -1922,10 +2334,15 @@ void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
     char *charset = 0;
     Z_SRW_diagnostic *diagnostic = 0;
     int num_diagnostic = 0;
-    if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
-                      &charset) == 0
-       || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
-                         &charset, &diagnostic, &num_diagnostic) == 0)
+
+    if (file_access(hreq))
+    {
+       return;
+    }
+    else if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
+                           &charset) == 0
+            || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
+                              &charset, &diagnostic, &num_diagnostic) == 0)
     {
        m_s2z_odr_init = odr_createmem(ODR_ENCODE);
        m_s2z_odr_search = odr_createmem(ODR_ENCODE);
@@ -2181,10 +2598,7 @@ void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
            send_to_srw_client_error(4, 0);
         }
     }
-    int len = 0;
-    Z_GDU *p = z_get_HTTP_Response(odr_encode(), 400);
-    timeout(0);
-    send_GDU(p, &len);
+    send_http_response(400);
 }
 
 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
@@ -2216,9 +2630,18 @@ void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
     m_client = get_client(apdu, get_cookie(oi), get_proxy(oi));
     if (!m_client)
     {
-       delete this;
-       return;
+       if (m_http_version)
+       {
+           send_http_response(404);
+           return;
+       }
+       else
+       {
+           delete this;
+           return;
+       }
     }
+    
     m_client->m_server = this;
 
     if (apdu->which == Z_APDU_initRequest)
@@ -2236,6 +2659,7 @@ void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
        {
            if (m_initRequest_mem)
                nmem_destroy(m_initRequest_mem);
+
            m_initRequest_apdu = apdu;
            m_initRequest_mem = odr_extract_mem(odr_decode());
 
@@ -2245,7 +2669,37 @@ void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
            m_initRequest_maximumRecordSize = *apdu->u.initRequest->
                maximumRecordSize;
            *apdu->u.initRequest->maximumRecordSize = 1024*1024;
-
+                   
+           // Save proposal charsets and langs.
+           if (ODR_MASK_GET(apdu->u.initRequest->options,
+               Z_Options_negotiationModel))
+           {
+               Z_CharSetandLanguageNegotiation *charSetandLangRecord =
+                   yaz_get_charneg_record(*oi);
+    
+               yaz_get_proposal_charneg(m_referenceId_mem,
+                   charSetandLangRecord,
+                   &m_initRequest_oi_negotiation_charsets,
+                   &m_initRequest_oi_negotiation_num_charsets,
+                   &m_initRequest_oi_negotiation_langs,
+                   &m_initRequest_oi_negotiation_num_langs,
+                   &m_initRequest_oi_negotiation_selected);
+       
+               for (int i=0; i<m_initRequest_oi_negotiation_num_charsets; i++)
+               {
+                   yaz_log(YLOG_LOG, "%scharacters set proposal: %s",
+                       m_session_str,(m_initRequest_oi_negotiation_charsets[i])?
+                       m_initRequest_oi_negotiation_charsets[i]:"none");
+               }
+               for (int i=0; i<m_initRequest_oi_negotiation_num_langs; i++)
+               {
+                   yaz_log(YLOG_LOG, "%slanguages proposal: %s",
+                       m_session_str, (m_initRequest_oi_negotiation_langs[i])?
+                       m_initRequest_oi_negotiation_langs[i]:"none");
+               }
+               yaz_log(YLOG_LOG, "%sselected proposal: %d (boolean)",
+                   m_session_str, m_initRequest_oi_negotiation_selected);
+           }   
            // save init options for the response..
            m_initRequest_options = apdu->u.initRequest->options;
            
@@ -2256,11 +2710,15 @@ void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
            int i;
            for (i = 0; i<= 24; i++)
                ODR_MASK_SET(apdu->u.initRequest->options, i);
+           // check negotiation option
+           if (!ODR_MASK_GET(m_initRequest_options,
+               Z_Options_negotiationModel))
+           {
+               ODR_MASK_CLEAR(apdu->u.initRequest->options,
+                   Z_Options_negotiationModel);
+           }
            ODR_MASK_CLEAR(apdu->u.initRequest->options,
-                          Z_Options_negotiationModel);
-           ODR_MASK_CLEAR(apdu->u.initRequest->options,
-                          Z_Options_concurrentOperations);
-
+               Z_Options_concurrentOperations);
            // make new version
            m_initRequest_version = apdu->u.initRequest->protocolVersion;
            apdu->u.initRequest->protocolVersion = 
@@ -2294,6 +2752,17 @@ void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
        }
        m_client->m_init_flag = 1;
     }
+    
+    if (!handle_authentication(apdu))
+    {
+       Z_APDU *apdu_reject = zget_APDU(odr_encode(), Z_APDU_initResponse);
+       *apdu_reject->u.initResponse->result = 0;
+       send_to_client(apdu_reject);
+
+       shutdown();
+       return;
+    }
+
     handle_max_record_retrieve(apdu);
 
     if (apdu)
@@ -2303,20 +2772,28 @@ void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
        apdu = handle_query_transformation(apdu);
 
     if (apdu)
+       apdu = handle_target_charset_conversion(apdu);
+
+    if (apdu)
        apdu = handle_query_validation(apdu);
 
     if (apdu)
        apdu = result_set_optimize(apdu);
+
     if (!apdu)
     {
        m_client->timeout(m_target_idletime);  // mark it active even 
        // though we didn't use it
        return;
     }
+    // Add otherInformation entry in APDU if
+    // negotiatoin in use.
+    if (apdu)
+       handle_charset_lang_negotiation(apdu);
 
     // delete other info construct completely if 0 elements
     get_otherInfoAPDU(apdu, &oi);
-    if (*oi && (*oi)->num_elements == 0)
+    if (oi && *oi && (*oi)->num_elements == 0)
         *oi = 0;
 
     if (apdu->which == Z_APDU_presentRequest &&
@@ -2495,6 +2972,8 @@ void Yaz_Proxy::pre_init()
     int pre_init;
     const char *cql2rpn = 0;
     const char *authentication = 0;
+    const char *negotiation_charset = 0;
+    const char *negotiation_lang = 0;
 
     Yaz_ProxyConfig *cfg = check_reconfigure();
 
@@ -2513,7 +2992,10 @@ void Yaz_Proxy::pre_init()
                                          &keepalive_limit_pdu,
                                          &pre_init,
                                          &cql2rpn,
-                                         &authentication) ; i++)
+                                         &authentication,
+                                         &negotiation_charset,
+                                         &negotiation_lang,
+                                         0) ; i++)
     {
        if (pre_init)
        {