Using yaz_daemon rather than own code. Fixed bug #2060.
[yazproxy-moved-to-github.git] / src / yaz-proxy.cpp
index a6930ea..258c5b4 100644 (file)
@@ -1,5 +1,5 @@
-/* $Id: yaz-proxy.cpp,v 1.50 2006-04-06 12:04:20 adam Exp $
-   Copyright (c) 1998-2006, Index Data.
+/* $Id: yaz-proxy.cpp,v 1.78 2008-02-21 09:33:23 adam Exp $
+   Copyright (c) 1998-2007, Index Data.
 
 This file is part of the yazproxy.
 
@@ -51,6 +51,7 @@ Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
 #include <yaz/pquery.h>
 #include <yaz/otherinfo.h>
 #include <yaz/charneg.h>
+#include <yaz/oid_db.h>
 #include "msg-thread.h"
 
 using namespace yazpp_1;
@@ -59,9 +60,6 @@ using namespace yazpp_1;
 #define strncasecmp _strnicmp
 #endif
 
-#define USE_AUTH_MSG 1
-
-#if USE_AUTH_MSG
 class YAZ_EXPORT Auth_Msg : public IMsg_Thread {
 public:
     int m_ret;
@@ -106,7 +104,7 @@ IMsg_Thread *Auth_Msg::handle()
 
 void Auth_Msg::result()
 {
-    if (m_proxy->dec_ref(false))
+    if (m_proxy->dec_ref())
     {
         yaz_log(YLOG_LOG, "Auth_Msg::proxy deleted meanwhile");
     }
@@ -122,8 +120,6 @@ void Auth_Msg::result()
     delete this;
 }
 
-#endif
-
 void Yaz_Proxy::result_authentication(Z_APDU *apdu, int ret)
 {
     if (apdu == 0 || ret == 0)
@@ -131,7 +127,7 @@ void Yaz_Proxy::result_authentication(Z_APDU *apdu, int ret)
         Z_APDU *apdu_reject = zget_APDU(odr_encode(), Z_APDU_initResponse);
         *apdu_reject->u.initResponse->result = 0;
         send_to_client(apdu_reject);
-        dec_ref(false);
+        dec_ref();
     }
     else
     {
@@ -232,8 +228,10 @@ Yaz_Proxy::Yaz_Proxy(IPDU_Observable *the_PDU_Observable,
     m_bw_max = 0;
     m_pdu_max = 0;
     m_search_max = 0;
-    m_connect_max = 0;
+    m_max_connect = 0;
+    m_max_connect_period = 0;
     m_limit_connect = 0;
+    m_limit_connect_period = 0;
     m_timeout_mode = timeout_normal;
     m_timeout_gdu = 0;
     m_max_record_retrieve = 0;
@@ -253,7 +251,7 @@ Yaz_Proxy::Yaz_Proxy(IPDU_Observable *the_PDU_Observable,
     m_schema = 0;
     m_backend_type = 0;
     m_backend_charset = 0;
-    m_frontend_type = 0;
+    m_frontend_type[0] = -1;
     m_initRequest_apdu = 0;
     m_initRequest_mem = 0;
     m_initRequest_preferredMessageSize = 0;
@@ -293,7 +291,7 @@ Yaz_Proxy::Yaz_Proxy(IPDU_Observable *the_PDU_Observable,
     m_ref_count = 1;
     m_main_ptr_dec = false;
     m_peername = 0;
-    m_initial_reduce = 0;
+    m_num_msg_threads = 0;
 }
 
 void Yaz_Proxy::inc_ref()
@@ -317,7 +315,7 @@ Yaz_Proxy::~Yaz_Proxy()
     delete m_charset_converter;
     xfree(m_optimize);
 
-#if HAVE_XSLT
+#if YAZ_HAVE_XSLT
     if (m_stylesheet_xsp)
         xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 #endif
@@ -354,8 +352,13 @@ int Yaz_Proxy::set_config(const char *config)
     m_config_fname = xstrdup(config);
     int r = m_config->read_xml(config);
     if (!r)
+    {
+        int period = 60;
         m_config->get_generic_info(&m_log_mask, &m_max_clients,
-                                   &m_connect_max, &m_limit_connect);
+                                   &m_max_connect, &m_limit_connect, &period,
+                                   &m_num_msg_threads);
+        m_connect.set_period(period);
+    }
     return r;
 }
 
@@ -370,7 +373,7 @@ void Yaz_Proxy::set_default_target(const char *target)
 void Yaz_Proxy::set_proxy_negotiation (const char *charset, const char *lang,
                                        const char *default_charset)
 {
-    yaz_log(YLOG_LOG, "%sSet the proxy negotiation: charset to '%s', "
+    yaz_log(YLOG_DEBUG, "%sSet the proxy negotiation: charset to '%s', "
         "default charset to '%s', language to '%s'", m_session_str, 
         charset?charset:"none",
         default_charset?default_charset:"none",
@@ -396,7 +399,6 @@ Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
     if (m_reconfig_flag)
     {
         yaz_log(YLOG_LOG, "reconfigure");
-        yaz_log_reopen();
         if (m_config_fname && cfg)
         {
             yaz_log(YLOG_LOG, "reconfigure config %s", m_config_fname);
@@ -406,8 +408,11 @@ Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
             else
             {
                 m_log_mask = 0;
+                int period = 60;
                 cfg->get_generic_info(&m_log_mask, &m_max_clients,
-                                      &m_connect_max, &m_limit_connect);
+                                      &m_max_connect, &m_limit_connect,
+                                      &period, &m_num_msg_threads);
+                m_connect.set_period(period);
             }
         }
         else
@@ -424,36 +429,22 @@ IPDU_Observer *Yaz_Proxy::sessionNotify(IPDU_Observable
 
     char session_str[200];
     const char *peername = the_PDU_Observable->getpeername();
+    if (!peername)
+        peername = "nullpeer";
+
     if (m_log_mask & PROXY_LOG_IP_CLIENT)
-        sprintf(session_str, "%ld:%d %s 0 ",
-                (long) time(0), m_session_no, peername);
+        sprintf(session_str, "%ld:%d %.80s %d ",
+                (long) time(0), m_session_no, peername, 0);
     else
-        sprintf(session_str, "%ld:%d 0 ",
-                (long) time(0), m_session_no);        
+        sprintf(session_str, "%ld:%d %d ",
+                (long) time(0), m_session_no, 0);
     m_session_no++;
 
     yaz_log (YLOG_LOG, "%sNew session %s", session_str, peername);
 
-    m_connect.cleanup(false);
-    m_connect.add_connect(peername);
-
-    int connect_total = m_connect.get_total(peername);
-    int connect_max = m_connect_max;
-    if (connect_max && connect_total > connect_max)
-    {
-        yaz_log(YLOG_LOG, "%sconnect not accepted total=%d max=%d",
-                session_str, connect_total, connect_max);
-        return 0;
-    }
-    yaz_log(YLOG_LOG, "%sconnect accepted total=%d", session_str,
-            connect_total);
-    
     Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable,
                                          m_socket_observable, this);
 
-    if (m_limit_connect)
-        new_proxy->m_initial_reduce = connect_total / m_limit_connect;
-
     new_proxy->m_config = 0;
     new_proxy->m_config_fname = 0;
     new_proxy->timeout(m_client_idletime);
@@ -461,8 +452,17 @@ IPDU_Observer *Yaz_Proxy::sessionNotify(IPDU_Observable
     new_proxy->set_default_target(m_default_target);
     new_proxy->m_max_clients = m_max_clients;
     new_proxy->m_log_mask = m_log_mask;
+    new_proxy->m_session_no = m_session_no;
+    new_proxy->m_num_msg_threads = m_num_msg_threads;
+
+#if 0
+    // in case we want to watch a particular client..
+    if (!strcmp(peername, "tcp:163.121.19.82")) // NIS GROUP
+        new_proxy->m_log_mask = 255;
+#endif
+
     new_proxy->set_APDU_log(get_APDU_log());
-    if (m_log_mask & PROXY_LOG_APDU_CLIENT)
+    if (new_proxy->m_log_mask & PROXY_LOG_APDU_CLIENT)
         new_proxy->set_APDU_yazlog(1);
     else
         new_proxy->set_APDU_yazlog(0);
@@ -471,39 +471,32 @@ IPDU_Observer *Yaz_Proxy::sessionNotify(IPDU_Observable
     new_proxy->set_proxy_negotiation(m_proxy_negotiation_charset,
         m_proxy_negotiation_lang, m_proxy_negotiation_default_charset);
     // create thread object the first time we get an incoming connection
-    if (!m_my_thread)
-        m_my_thread = new Msg_Thread(m_socket_observable, 1);
+    if (!m_my_thread && m_num_msg_threads > 0)
+    {
+        yaz_log (YLOG_LOG, "%sStarting message thread management. number=%d",
+                 session_str, m_num_msg_threads);
+        m_my_thread = new Msg_Thread(m_socket_observable, m_num_msg_threads);
+    }
     new_proxy->m_my_thread = m_my_thread;
     return new_proxy;
 }
 
 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
 {
-    int oid[OID_SIZE];
-    Z_OtherInformationUnit *oi;
-    struct oident ent;
-    ent.proto = PROTO_Z3950;
-    ent.oclass = CLASS_USERINFO;
-    ent.value = (oid_value) VAL_COOKIE;
-    assert (oid_ent_to_oid (&ent, oid));
-
-    if (oid_ent_to_oid (&ent, oid) &&
-        (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
-        oi->which == Z_OtherInfo_characterInfo)
+    Z_OtherInformationUnit *oi =
+        update_otherInformation(otherInfo, 0, yaz_oid_userinfo_cookie, 1, 1);
+    
+    if (oi && oi->which == Z_OtherInfo_characterInfo)
         return oi->information.characterInfo;
     return 0;
 }
+
 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
 {
-    int oid[OID_SIZE];
-    Z_OtherInformationUnit *oi;
-    struct oident ent;
-    ent.proto = PROTO_Z3950;
-    ent.oclass = CLASS_USERINFO;
-    ent.value = (oid_value) VAL_PROXY;
-    if (oid_ent_to_oid (&ent, oid) &&
-        (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
-        oi->which == Z_OtherInfo_characterInfo)
+    Z_OtherInformationUnit *oi =
+        update_otherInformation(otherInfo, 0, yaz_oid_userinfo_proxy, 1, 1);
+    
+    if (oi && oi->which == Z_OtherInfo_characterInfo)
         return oi->information.characterInfo;
     return 0;
 }
@@ -606,8 +599,14 @@ Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
             m_client_idletime = client_idletime;
             timeout(m_client_idletime);
         }
+
+        // get those FILE descriptors available 
+        m_parent->low_socket_close();
         if (cql2rpn_fname)
             m_cql2rpn.set_pqf_file(cql2rpn_fname);
+        // reserve them again
+        m_parent->low_socket_open();
+        
         if (negotiation_charset || negotiation_lang || default_client_query_charset)
         {
             set_proxy_negotiation(negotiation_charset,
@@ -739,8 +738,7 @@ Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
                 yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Destroy %d",
                          m_session_str, parent->m_max_clients, c->m_seqno);
                 if (c->m_server && c->m_server != this)
-                    delete c->m_server;   // PROBLEM: m_ref_count!
-                c->m_server = 0;
+                    c->m_server->dec_ref();
             }
             else
             {
@@ -755,7 +753,7 @@ Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
                 if (c->m_server && c->m_server != this)
                 {
                     c->m_server->m_client = 0;
-                    delete c->m_server;   // PROBLEM: m_ref_count!
+                    c->m_server->dec_ref();
                 }
                 (parent->m_seqno)++;
                 c->m_target_idletime = m_target_idletime;
@@ -819,7 +817,6 @@ void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
     int i;
     for (i = 0; i<num; i++)
     {
-        oident *ent;
         Z_DefaultDiagFormat *r;
         Z_DiagRec *p = pp[i];
         if (p->which != Z_DiagRec_defaultFormat)
@@ -829,9 +826,6 @@ void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
         }
         else
             r = p->u.defaultFormat;
-        if (!(ent = oid_getentbyoid(r->diagnosticSetId)) ||
-            ent->oclass != CLASS_DIAGSET || ent->value != VAL_BIB1)
-            yaz_log(YLOG_LOG, "%sError unknown diagnostic set", m_session_str);
         switch (r->which)
         {
         case Z_DefaultDiagFormat_v2Addinfo:
@@ -868,7 +862,7 @@ int Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p, Z_APDU *apdu)
 
 void Yaz_Proxy::convert_xsl_delay()
 {
-#if HAVE_XSLT
+#if YAZ_HAVE_XSLT
     Z_NamePlusRecord *npr = m_stylesheet_nprl->records[m_stylesheet_offset];
     if (npr->which == Z_NamePlusRecord_databaseRecord)
     {
@@ -893,11 +887,10 @@ void Yaz_Proxy::convert_xsl_delay()
                 xmlChar *out_buf;
                 int out_len;
                 xmlDocDumpFormatMemory (res, &out_buf, &out_len, 1);
-
                 m_stylesheet_nprl->records[m_stylesheet_offset]->
                     u.databaseRecord =
-                    z_ext_record(odr_encode(), VAL_TEXT_XML,
-                                 (char*) out_buf, out_len);
+                    z_ext_record_oid(odr_encode(), yaz_oid_recsyn_xml,
+                                     (char*) out_buf, out_len);
                 xmlFree(out_buf);
                 xmlFreeDoc(res);
             }
@@ -911,7 +904,7 @@ void Yaz_Proxy::convert_xsl_delay()
     {
         m_timeout_mode = timeout_normal;
         m_stylesheet_nprl = 0;
-#if HAVE_XSLT
+#if YAZ_HAVE_XSLT
         if (m_stylesheet_xsp)
             xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
 #endif
@@ -925,7 +918,7 @@ void Yaz_Proxy::convert_xsl_delay()
 
 void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p)
 {
-    if (m_frontend_type != VAL_NONE)
+    if (m_frontend_type[0] != -1)
     {
         int i;
         for (i = 0; i < p->num_records; i++)
@@ -971,10 +964,10 @@ void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p)
                                 }
                             }
                             npr->u.databaseRecord =
-                                z_ext_record(odr_encode(),
-                                             m_frontend_type,
-                                             converted,
-                                             strlen(converted));
+                                z_ext_record_oid(odr_encode(),
+                                                 m_frontend_type,
+                                                 converted,
+                                                 strlen(converted));
                             free(converted);
                         }
                         else
@@ -986,10 +979,10 @@ void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p)
 #endif
 /* HAVE_USEMARCON */
                     npr->u.databaseRecord =
-                        z_ext_record(odr_encode(),
-                                     m_frontend_type,
-                                     (char*) r->u.octet_aligned->buf,
-                                     r->u.octet_aligned->len);
+                        z_ext_record_oid(odr_encode(),
+                                         m_frontend_type,
+                                         (char*) r->u.octet_aligned->buf,
+                                         r->u.octet_aligned->len);
                 }
             }
         }
@@ -1017,36 +1010,36 @@ void Yaz_Proxy::convert_records_charset(Z_NamePlusRecordList *p,
             if (npr->which == Z_NamePlusRecord_databaseRecord)
             {
                 Z_External *r = npr->u.databaseRecord;
-                oident *ent = oid_getentbyoid(r->direct_reference);
-                if (!ent || ent->value == VAL_NONE)
+                const Odr_oid *oid = r->direct_reference;
+                if (!oid)
                     continue;
 
-                if (ent->value == VAL_SUTRS)
+                if (!oid_oidcmp(oid, yaz_oid_recsyn_sutrs))
                 {
                     WRBUF w = wrbuf_alloc();
 
                     wrbuf_iconv_write(w, cd,  (char*) r->u.octet_aligned->buf,
                                       r->u.octet_aligned->len);
                     npr->u.databaseRecord =
-                        z_ext_record(odr_encode(), ent->value, wrbuf_buf(w),
-                                     wrbuf_len(w));
-                    wrbuf_free(w, 1);
+                        z_ext_record_oid(odr_encode(), oid, wrbuf_buf(w),
+                                         wrbuf_len(w));
+                    wrbuf_destroy(w);
                 }
-                else if (ent->value == VAL_TEXT_XML)
+                else if (!oid_oidcmp(oid, yaz_oid_recsyn_xml))
                 {
                     ;
                 }
                 else if (r->which == Z_External_octet)
                 {
-                    int rlen;
-                    char *result;
+                    size_t rlen;
+                    const char *result;
                     if (yaz_marc_decode_buf(mt,
                                             (char*) r->u.octet_aligned->buf,
                                             r->u.octet_aligned->len,
                                             &result, &rlen))
                     {
                         npr->u.databaseRecord =
-                            z_ext_record(odr_encode(), ent->value, result, rlen);
+                            z_ext_record_oid(odr_encode(), oid, result, rlen);
                         yaz_log(YLOG_LOG, "%sRecoding MARC record",
                                 m_session_str);
                     }
@@ -1080,22 +1073,22 @@ void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p,
                 WRBUF w = wrbuf_alloc();
 
                 yaz_opac_decode_wrbuf(mt, r->u.opac, w);
-                npr->u.databaseRecord = z_ext_record(
-                    odr_encode(), VAL_TEXT_XML,
-                    wrbuf_buf(w), wrbuf_len(w)
-                    );
-                wrbuf_free(w, 1);
+                npr->u.databaseRecord = z_ext_record_oid(
+                    odr_encode(), yaz_oid_recsyn_xml,
+                    wrbuf_buf(w), wrbuf_len(w));
+                wrbuf_destroy(w);
             }
             else if (r->which == Z_External_octet)
             {
-                int rlen;
-                char *result;
+                size_t rlen;
+                const char *result;
                 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
                                         r->u.octet_aligned->len,
                                         &result, &rlen))
                 {
                     npr->u.databaseRecord =
-                        z_ext_record(odr_encode(), VAL_TEXT_XML, result, rlen);
+                        z_ext_record_oid(odr_encode(), yaz_oid_recsyn_xml,
+                                         result, rlen);
                 }
             }
         }
@@ -1135,6 +1128,10 @@ int Yaz_Proxy::send_http_response(int code)
         z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
     else
         timeout(0);
+    if (code == 401)
+        z_HTTP_header_add(o, &hres->headers, "WWW-Authenticate", 
+                          "Basic realm=\"YAZ Proxy\"");
+
 
     if (m_log_mask & PROXY_LOG_REQ_CLIENT)
     {
@@ -1169,7 +1166,7 @@ int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu, int http_code /* = 200 */)
         z_HTTP_header_add(o, &hres->headers, "WWW-Authenticate", "Basic realm=\"YAZ Proxy\"");
 
     static Z_SOAP_Handler soap_handlers[2] = {
-#if HAVE_XSLT
+#if YAZ_HAVE_XSLT
         {"http://www.loc.gov/zing/srw/", 0,
          (Z_SOAP_fun) yaz_srw_codec},
 #endif
@@ -1255,8 +1252,9 @@ int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
                 continue;
             }
             Z_External *r = npr->u.databaseRecord;
-            oident *ent = oid_getentbyoid(r->direct_reference);
-            if (r->which == Z_External_octet && ent->value == VAL_TEXT_XML)
+
+            if (r->which == Z_External_octet 
+                && !oid_oidcmp(r->direct_reference, yaz_oid_recsyn_xml))
             {
                 srw_res->records[i].recordSchema = m_schema;
                 srw_res->records[i].recordPacking = m_s2z_packing;
@@ -1617,8 +1615,21 @@ Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
         {
             Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
             new_apdu->u.presentResponse->records =
-                create_nonSurrogateDiagnostics(odr_encode(), 30,
-                                               pr->resultSetId);
+                create_nonSurrogateDiagnostics(
+                    odr_encode(), 
+                    YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
+                    pr->resultSetId);
+            send_to_client(new_apdu);
+            return 0;
+        }
+        if (start < 1 || toget < 0)
+        {
+            Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
+            new_apdu->u.presentResponse->records =
+                create_nonSurrogateDiagnostics(
+                    odr_encode(), 
+                    YAZ_BIB1_PRESENT_REQUEST_OUT_OF_RANGE, 
+                    0);
             send_to_client(new_apdu);
             return 0;
         }
@@ -1628,7 +1639,10 @@ Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
             {
                 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
                 new_apdu->u.presentResponse->records =
-                    create_nonSurrogateDiagnostics(odr_encode(), 13, 0);
+                    create_nonSurrogateDiagnostics(
+                        odr_encode(), 
+                        YAZ_BIB1_PRESENT_REQUEST_OUT_OF_RANGE,
+                        0);
                 send_to_client(new_apdu);
                 return 0;
             }
@@ -1674,7 +1688,22 @@ Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
 
     this_query->set_Z_Query(sr->query);
 
-    char query_str[120];
+    // Check for non-negative piggyback params.
+    if (*sr->smallSetUpperBound < 0
+        || *sr->largeSetLowerBound < 0
+        || *sr->mediumSetPresentNumber < 0)
+    {
+        Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
+        // Not a present request.. But can't find better diagnostic
+        new_apdu->u.searchResponse->records =
+            create_nonSurrogateDiagnostics(
+                odr_encode(), 
+                YAZ_BIB1_PRESENT_REQUEST_OUT_OF_RANGE, 0);
+        send_to_client(new_apdu);
+        return 0;
+    }
+
+    char query_str[4096];
     this_query->print(query_str, sizeof(query_str)-1);
     yaz_log(YLOG_LOG, "%sSearch %s", m_session_str, query_str);
 
@@ -1694,7 +1723,7 @@ Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
 
             if (toget > m_client->m_last_resultCount)
                 toget = m_client->m_last_resultCount;
-
+            
             if (sr->mediumSetElementSetNames)
             {
                 comp = (Z_RecordComposition *)
@@ -1834,6 +1863,7 @@ Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
 
 void Yaz_Proxy::inc_request_no()
 {
+    m_request_no++;
     char *cp = m_session_str + strlen(m_session_str)-1;
     if (*cp == ' ')
         cp--;
@@ -1853,6 +1883,25 @@ void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
         yaz_log (YLOG_LOG, "%sReceiving %s from client %d bytes",
                  m_session_str, gdu_name(apdu), len);
 
+#if 0
+    // try to make a _bad_ attribute set ID .. Don't enable this in prod.
+    if (apdu->which == Z_GDU_Z3950 
+        && apdu->u.z3950->which == Z_APDU_searchRequest)
+    {
+        Z_SearchRequest *req = apdu->u.z3950->u.searchRequest;
+        if (req->query && req->query->which == Z_Query_type_1)
+        {
+            Z_RPNQuery *rpnquery = req->query->u.type_1;
+            if (rpnquery->attributeSetId)
+            {
+                rpnquery->attributeSetId[0] = -2;
+                rpnquery->attributeSetId[1] = -1;
+                yaz_log(YLOG_WARN, "%sBAD FIXUP TEST", m_session_str);
+            }
+        }
+    }
+#endif
+
 #if HAVE_GETTIMEOFDAY
     gettimeofday((struct timeval *) m_time_tv, 0);
 #endif
@@ -1860,15 +1909,90 @@ void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
     m_pdu_stat.add_bytes(1);
 
     GDU *gdu = new GDU(apdu);
-    m_in_queue.enqueue(gdu);
 
+    if (gdu->get() == 0)
+    {
+        delete gdu;
+        yaz_log(YLOG_LOG, "%sUnable to encode package", m_session_str);
+        m_in_queue.clear();
+        dec_ref();
+        return;
+    }
+    m_in_queue.enqueue(gdu);
     recv_GDU_more(false);
 }
 
+void Yaz_Proxy::HTTP_Forwarded(Z_GDU *z_gdu)
+{
+    if (z_gdu->which == Z_GDU_HTTP_Request)
+    {
+        Z_HTTP_Request *hreq = z_gdu->u.HTTP_Request;
+        const char *x_forwarded_for =
+            z_HTTP_header_lookup(hreq->headers, "X-Forwarded-For");
+        if (x_forwarded_for)
+        {
+            xfree(m_peername);
+            m_peername = (char*) xmalloc(strlen(x_forwarded_for)+5);
+            sprintf(m_peername, "tcp:%s", x_forwarded_for);
+            
+            yaz_log(YLOG_LOG, "%sHTTP Forwarded from %s", m_session_str,
+                    m_peername);
+            if (m_log_mask & PROXY_LOG_IP_CLIENT)
+                sprintf(m_session_str, "%ld:%d %.80s %d ",
+                        (long) time(0), m_session_no, m_peername, m_request_no);
+            else
+                sprintf(m_session_str, "%ld:%d %d ",
+                        (long) time(0), m_session_no, m_request_no);
+        }
+    }
+}
+
+void Yaz_Proxy::connect_stat(bool &block, int &reduce)
+{
+
+    m_parent->m_connect.cleanup(false);
+    m_parent->m_connect.add_connect(m_peername);
+
+    int connect_total = m_parent->m_connect.get_total(m_peername);
+    int max_connect = m_parent->m_max_connect;
+
+    if (max_connect && connect_total > max_connect)
+    {
+        yaz_log(YLOG_LOG, "%sconnect not accepted total=%d max=%d",
+                m_session_str, connect_total, max_connect);
+        block = true;
+    }
+    else 
+        block = false;
+    yaz_log(YLOG_LOG, "%sconnect accepted total=%d", m_session_str,
+            connect_total);
+    
+    int limit_connect = m_parent->m_limit_connect;
+    if (limit_connect)
+        reduce = connect_total / limit_connect;
+    else
+        reduce = 0;
+}
+
 void Yaz_Proxy::recv_GDU_reduce(GDU *gdu)
 {
-    int reduce = m_initial_reduce; // initial reduce from connect phase..
-    m_initial_reduce = 0;  // reset it..
+    HTTP_Forwarded(gdu->get());
+
+    int reduce = 0;
+    
+    if (m_request_no == 1)
+    {
+        bool block = false;
+        
+        connect_stat(block, reduce);
+
+        if (block)
+        {
+            m_timeout_mode = timeout_busy;
+            timeout(0);
+            return;
+        }
+    }
 
     int bw_total = m_bw_stat.get_total();
     int pdu_total = m_pdu_stat.get_total();
@@ -1920,7 +2044,10 @@ void Yaz_Proxy::recv_GDU_more(bool normal)
     while (m_timeout_mode == timeout_normal && (g = m_in_queue.dequeue()))
     {
         m_timeout_mode = timeout_busy;
+        inc_ref();
         recv_GDU_reduce(g);
+        if (dec_ref())
+            break;
     }
 }
 
@@ -2139,8 +2266,7 @@ Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
     *err = error;
     rec->which = Z_Records_NSD;
     rec->u.nonSurrogateDiagnostic = dr;
-    dr->diagnosticSetId =
-        yaz_oidval_to_z3950oid (odr, CLASS_DIAGSET, VAL_BIB1);
+    dr->diagnosticSetId = odr_oiddup(odr, yaz_oid_diagset_bib_1);
     dr->condition = err;
     dr->which = Z_DefaultDiagFormat_v2Addinfo;
     dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
@@ -2279,6 +2405,44 @@ int Yaz_Proxy::handle_authentication(Z_APDU *apdu)
     return ret;
 }
 
+int Yaz_Proxy::handle_global_authentication(Z_APDU *apdu)
+{
+    if (apdu->which != Z_APDU_initRequest)
+        return 1;  // pass if no init request
+    Z_InitRequest *req = apdu->u.initRequest;
+
+    Yaz_ProxyConfig *cfg = check_reconfigure();
+    if (!cfg)
+        return 1;  // pass if no config
+
+    int ret;
+    if (req->idAuthentication == 0)
+    {
+        ret = cfg->global_client_authentication(0, 0, 0,
+                                                m_peername);
+    }
+    else if (req->idAuthentication->which == Z_IdAuthentication_idPass)
+    {
+        ret = cfg->global_client_authentication(
+            req->idAuthentication->u.idPass->userId,
+            req->idAuthentication->u.idPass->groupId,
+            req->idAuthentication->u.idPass->password,
+            m_peername);
+    }
+    else if (req->idAuthentication->which == Z_IdAuthentication_open)
+    {
+        char user[64], pass[64];
+        *user = '\0';
+        *pass = '\0';
+        sscanf(req->idAuthentication->u.open, "%63[^/]/%63s", user, pass);
+        ret = cfg->global_client_authentication(user, 0, pass,
+                                                m_peername);
+    }
+    else
+        ret = cfg->global_client_authentication(0, 0, 0, m_peername);
+    return ret;
+}
+
 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
 {
     m_marcxml_mode = none;
@@ -2298,13 +2462,9 @@ Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
         }
 
         if (sr->preferredRecordSyntax)
-        {
-            struct oident *ent;
-            ent = oid_getentbyoid(sr->preferredRecordSyntax);
-            m_frontend_type = ent->value;
-        }
+            oid_oidcpy(m_frontend_type, sr->preferredRecordSyntax);
         else
-            m_frontend_type = VAL_NONE;
+            m_frontend_type[0] = -1;
 
         char *stylesheet_name = 0;
         if (cfg)
@@ -2319,7 +2479,7 @@ Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
         {
             m_parent->low_socket_close();
 
-#if HAVE_XSLT
+#if YAZ_HAVE_XSLT
             if (m_stylesheet_xsp)
                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
@@ -2335,17 +2495,11 @@ Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
             sr->smallSetElementSetNames = 0;
             sr->mediumSetElementSetNames = 0;
             m_marcxml_mode = marcxml;
-            if (m_backend_type)
-            {
-
-                sr->preferredRecordSyntax =
-                    yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
-                                        m_backend_type);
-            }
-            else
-                sr->preferredRecordSyntax =
-                    yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
-                                           VAL_USMARC);
+            sr->preferredRecordSyntax =
+                yaz_string_to_oid_odr(
+                    yaz_oid_std(), CLASS_RECSYN,
+                    m_backend_type ? m_backend_type : "usmarc", 
+                    odr_encode());
         }
         else if (err)
         {
@@ -2363,7 +2517,8 @@ Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
         else if (m_backend_type)
         {
             sr->preferredRecordSyntax =
-                yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
+                yaz_string_to_oid_odr(yaz_oid_std(), CLASS_RECSYN, 
+                                      m_backend_type, odr_encode());
         }
     }
     else if (apdu->which == Z_APDU_presentRequest)
@@ -2374,13 +2529,9 @@ Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
         Yaz_ProxyConfig *cfg = check_reconfigure();
 
         if (pr->preferredRecordSyntax)
-        {
-            struct oident *ent;
-            ent = oid_getentbyoid(pr->preferredRecordSyntax);
-            m_frontend_type = ent->value;
-        }
+            oid_oidcpy(m_frontend_type, pr->preferredRecordSyntax);
         else
-            m_frontend_type = VAL_NONE;
+            m_frontend_type[0] = -1;
 
         char *stylesheet_name = 0;
         if (cfg)
@@ -2396,7 +2547,7 @@ Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
         {
             m_parent->low_socket_close();
 
-#if HAVE_XSLT
+#if YAZ_HAVE_XSLT
             if (m_stylesheet_xsp)
                 xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp);
             m_stylesheet_xsp = xsltParseStylesheetFile((const xmlChar*)
@@ -2411,17 +2562,12 @@ Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
         {
             pr->recordComposition = 0;
             m_marcxml_mode = marcxml;
-            if (m_backend_type)
-            {
 
-                pr->preferredRecordSyntax =
-                    yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN,
-                                        m_backend_type);
-            }
-            else
-                pr->preferredRecordSyntax =
-                    yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN,
-                                           VAL_USMARC);
+            pr->preferredRecordSyntax =
+                yaz_string_to_oid_odr(
+                    yaz_oid_std(), CLASS_RECSYN,
+                    m_backend_type ? m_backend_type : "usmarc", 
+                    odr_encode());
         }
         else if (err)
         {
@@ -2440,7 +2586,9 @@ Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
         else if (m_backend_type)
         {
             pr->preferredRecordSyntax =
-                yaz_str_to_z3950oid(odr_encode(), CLASS_RECSYN, m_backend_type);
+                yaz_string_to_oid_odr(yaz_oid_std(),
+                                      CLASS_RECSYN, m_backend_type, 
+                                      odr_encode());
         }
     }
     return apdu;
@@ -2487,11 +2635,7 @@ int Yaz_Proxy::file_access(Z_HTTP_Request *hreq)
     while (*cp)
     {
         if (*cp == '/' && strchr("/.", cp[1]))
-        {
-            yaz_log(YLOG_LOG, "%sRejecting path %s", m_session_str,
-                    hreq->path);
             return 0;
-        }
         cp++;
     }
 
@@ -2601,6 +2745,8 @@ void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
     Z_SRW_diagnostic *diagnostic = 0;
     int num_diagnostic = 0;
 
+    yaz_log(YLOG_LOG, "%s%s %s", m_session_str, hreq->method, hreq->path);
+
     if (file_access(hreq))
     {
         return;
@@ -2618,24 +2764,51 @@ void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
         m_s2z_present_apdu = 0;
 
         m_s2z_stylesheet = 0;
-
+        
         Z_IdAuthentication *auth = NULL;
-        if (*authorization_str)
+        if (srw_pdu->username && srw_pdu->password)
         {
+            yaz_log(YLOG_LOG, "username/password: %s/%s\n",
+                    srw_pdu->username, srw_pdu->password);
             auth = (Z_IdAuthentication *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdAuthentication));
             auth->which = Z_IdAuthentication_idPass;
             auth->u.idPass = (Z_IdPass *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdPass));
             auth->u.idPass->groupId = NULL;
-            char *p = strchr(authorization_str, ':');
-            if (p)
+            auth->u.idPass->password = odr_strdup(m_s2z_odr_init, srw_pdu->password);
+            auth->u.idPass->userId = odr_strdup(m_s2z_odr_init, srw_pdu->username);
+        }
+        else
+        {
+            if (*authorization_str)
             {
-                *p = '\0';
-                p++;
-                auth->u.idPass->password = odr_strdup(m_s2z_odr_init, p);
+                yaz_log(YLOG_LOG, "authorization_str present: %s\n", authorization_str);
+                auth = (Z_IdAuthentication *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdAuthentication));
+                auth->which = Z_IdAuthentication_idPass;
+                auth->u.idPass = (Z_IdPass *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdPass));
+                auth->u.idPass->groupId = NULL;
+                char *p = strchr(authorization_str, ':');
+                if (p)
+                {
+                    *p = '\0';
+                    p++;
+                    auth->u.idPass->password = odr_strdup(m_s2z_odr_init, p);
+                }
+                auth->u.idPass->userId = odr_strdup(m_s2z_odr_init, authorization_str);
             }
-            auth->u.idPass->userId = odr_strdup(m_s2z_odr_init, authorization_str);
-        }
-
+            else
+            {
+                // Use _client_ IP as shown in the log entries...!
+                yaz_log(YLOG_LOG, "No authorization_str present: use client IP: %s\n", m_peername);
+                
+                auth = (Z_IdAuthentication *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdAuthentication));
+                   auth->which = Z_IdAuthentication_idPass;
+                   auth->u.idPass = (Z_IdPass *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdPass));
+                   auth->u.idPass->groupId  = NULL;
+                   auth->u.idPass->password = NULL;
+                   auth->u.idPass->userId   = odr_strdup(m_s2z_odr_init, m_peername);
+            }
+        }              
+        
         if (srw_pdu->which == Z_SRW_searchRetrieve_request)
         {
 
@@ -2766,8 +2939,8 @@ void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
                     *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
 
                     z_searchRequest->preferredRecordSyntax =
-                        yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
-                                               VAL_TEXT_XML);
+                        odr_oiddup(m_s2z_odr_search, yaz_oid_recsyn_xml);
+
                     if (srw_req->recordSchema)
                     {
                         z_searchRequest->smallSetElementSetNames =
@@ -2784,9 +2957,9 @@ void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
                         m_s2z_present_apdu->u.presentRequest;
                     *z_presentRequest->resultSetStartPoint = start;
                     *z_presentRequest->numberOfRecordsRequested = max;
+
                     z_presentRequest->preferredRecordSyntax =
-                        yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
-                                               VAL_TEXT_XML);
+                        odr_oiddup(m_s2z_odr_search, yaz_oid_recsyn_xml);
                     if (srw_req->recordSchema)
                     {
                         z_presentRequest->recordComposition =
@@ -2991,8 +3164,8 @@ void Yaz_Proxy::handle_init(Z_APDU *apdu)
             Z_APDU *apdu2 = m_client->m_initResponse;
             apdu2->u.initResponse->otherInfo = 0;
             if (m_client->m_cookie && *m_client->m_cookie)
-                set_otherInformationString(apdu2, VAL_COOKIE, 1,
-                                           m_client->m_cookie);
+                set_otherInformationString(apdu2, yaz_oid_userinfo_cookie, 
+                                           1, m_client->m_cookie);
             apdu2->u.initResponse->referenceId =
                 apdu->u.initRequest->referenceId;
             apdu2->u.initResponse->options = m_client->m_initResponse_options;
@@ -3001,28 +3174,32 @@ void Yaz_Proxy::handle_init(Z_APDU *apdu)
 
             handle_charset_lang_negotiation(apdu2);
 
+           if (m_timeout_mode == timeout_busy)
+               m_timeout_mode = timeout_normal;
             send_to_client(apdu2);
-            m_timeout_mode = timeout_normal;
             return;
         }
     }
     m_client->m_init_flag = 1;
 
-#if USE_AUTH_MSG
-    Auth_Msg *m = new Auth_Msg;
-    m->m_proxy = this;
-    z_APDU(odr_encode(), &apdu, 0, "encode");
-    char *apdu_buf = odr_getbuf(odr_encode(), &m->m_apdu_len, 0);
-    m->m_apdu_buf = (char*) nmem_malloc(m->m_nmem, m->m_apdu_len);
-    memcpy(m->m_apdu_buf, apdu_buf, m->m_apdu_len);
-    odr_reset(odr_encode());
-
-    inc_ref();
-    m_my_thread->put(m);
-#else
-    int ret = handle_authentication(apdu);
-    result_authentication(apdu, ret);
-#endif
+    if (m_num_msg_threads && m_my_thread)
+    {
+        Auth_Msg *m = new Auth_Msg;
+        m->m_proxy = this;
+        z_APDU(odr_encode(), &apdu, 0, "encode");
+        char *apdu_buf = odr_getbuf(odr_encode(), &m->m_apdu_len, 0);
+        m->m_apdu_buf = (char*) nmem_malloc(m->m_nmem, m->m_apdu_len);
+        memcpy(m->m_apdu_buf, apdu_buf, m->m_apdu_len);
+        odr_reset(odr_encode());
+        
+        inc_ref();
+        m_my_thread->put(m);
+    }
+    else
+    {
+        int ret = handle_authentication(apdu);
+        result_authentication(apdu, ret);
+    }
 }
 
 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
@@ -3052,6 +3229,23 @@ void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
     if (apdu->which == Z_APDU_searchRequest)
         m_search_stat.add_bytes(1);
 
+    // Handle global authentication
+    if (!handle_global_authentication(apdu))
+    {
+        if (m_http_version)
+        {   // HTTP. Send unauthorized
+            send_http_response(401);
+            return;
+        }
+        else
+        {
+            // Z39.50 just shutdown
+            timeout(0);
+            return;
+        }
+        return;
+    }
+
     // Determine our client.
     Z_OtherInformation **oi;
     get_otherInfoAPDU(apdu, &oi);
@@ -3066,7 +3260,7 @@ void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
         else
         {
             // Z39.50 just shutdown
-            delete this;
+            timeout(0);
             return;
         }
     }
@@ -3180,16 +3374,8 @@ void Yaz_Proxy::releaseClient()
         m_parent->pre_init();
 }
 
-bool Yaz_Proxy::dec_ref(bool main_ptr)
+bool Yaz_Proxy::dec_ref()
 {
-    assert(m_ref_count > 0);
-    if (main_ptr)
-    {
-        if (m_main_ptr_dec)
-            return false;
-        m_main_ptr_dec = true;
-    }
-
     m_http_keepalive = 0;
 
     --m_ref_count;
@@ -3216,8 +3402,8 @@ void Yaz_ProxyClient::shutdown()
 
     if (m_server)
     {
-        m_waiting = 1;      // ensure it's released from Proxy in releaseClient
-        m_server->dec_ref(true);
+        m_waiting = 1;   // ensure it's released from Yaz_Proxy::releaseClient
+        m_server->dec_ref();
     }
     else
         delete this;
@@ -3226,9 +3412,8 @@ void Yaz_ProxyClient::shutdown()
 void Yaz_Proxy::failNotify()
 {
     inc_request_no();
-    yaz_log (YLOG_LOG, "%sConnection closed by client",
-             get_session_str());
-    dec_ref(true);
+    yaz_log (YLOG_LOG, "%sConnection closed by client", get_session_str());
+    dec_ref();
 }
 
 void Yaz_Proxy::send_response_fail_client(const char *addr)
@@ -3429,7 +3614,7 @@ void Yaz_Proxy::timeoutNotify()
             inc_request_no();
             m_in_queue.clear();
             yaz_log (YLOG_LOG, "%sTimeout (client to proxy)", m_session_str);
-            dec_ref(true);
+            dec_ref();
             break;
         case timeout_reduce:
             timeout(m_client_idletime);
@@ -3467,9 +3652,12 @@ void Yaz_ProxyClient::timeoutNotify()
 
     if (m_server)
         m_server->send_response_fail_client(get_hostname());
+
+    Yaz_Proxy *proxy_root = m_root;
+
     shutdown();
 
-    m_root->pre_init();
+    proxy_root->pre_init();
 }
 
 Yaz_ProxyClient::Yaz_ProxyClient(IPDU_Observable *the_PDU_Observable,
@@ -3568,8 +3756,21 @@ void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
             *apdu->u.initResponse->maximumRecordSize;
 
         Z_InitResponse *ir = apdu->u.initResponse;
+       
+        // apply YAZ Proxy version
+        char *imv0 = ir->implementationVersion;
+        char *imv1 = (char*)
+            odr_malloc(m_init_odr, 20 + (imv0 ? strlen(imv0) : 0));
+        *imv1 = '\0';
+        if (imv0)
+            strcat(imv1, imv0);
+#ifdef VERSION
+        strcat(imv1, "/" VERSION);
+#endif
+        ir->implementationVersion = imv1;
+        
+        // apply YAZ Proxy implementation name
         char *im0 = ir->implementationName;
-
         char *im1 = (char*)
             odr_malloc(m_init_odr, 20 + (im0 ? strlen(im0) : 0));
         *im1 = '\0';
@@ -3628,7 +3829,7 @@ void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
         }
     }
     if (m_cookie)
-        set_otherInformationString (apdu, VAL_COOKIE, 1, m_cookie);
+        set_otherInformationString(apdu, yaz_oid_userinfo_cookie, 1, m_cookie);
 
     Yaz_Proxy *server = m_server; // save it. send_to_client may destroy us