X-Git-Url: http://git.indexdata.com/?p=yazproxy-moved-to-github.git;a=blobdiff_plain;f=src%2Fyaz-proxy.cpp;h=8dc5e604aa7fc23e9d22258ac926ed1e73d02c07;hp=6b04f3845fe807eb7dc0c8808662d026f54e2055;hb=9fcdfed653560d961df0477ca6f04c41c8576706;hpb=02098acac83c12e26589fea83a0afd8f4e5018ca diff --git a/src/yaz-proxy.cpp b/src/yaz-proxy.cpp index 6b04f38..8dc5e60 100644 --- a/src/yaz-proxy.cpp +++ b/src/yaz-proxy.cpp @@ -1,7 +1,5 @@ -/* $Id: yaz-proxy.cpp,v 1.75 2007-04-30 19:46:34 adam Exp $ - Copyright (c) 1998-2007, Index Data. - -This file is part of the yazproxy. +/* This file is part of YAZ proxy + Copyright (C) 1998-2011 Index Data YAZ proxy is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free @@ -14,10 +12,9 @@ FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License -along with YAZ proxy; see the file LICENSE. If not, write to the -Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA -02111-1307, USA. - */ +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +*/ #ifdef WIN32 #define HAVE_SYS_STAT_H 1 @@ -54,6 +51,12 @@ Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA #include #include "msg-thread.h" +#if YAZ_VERSIONL >= 0x03001D +#define YAZ_HAS_MK_SURROGATE 1 +#else +#define YAZ_HAS_MK_SURROGATE 0 +#endif + using namespace yazpp_1; #ifdef WIN32 @@ -135,7 +138,7 @@ void Yaz_Proxy::result_authentication(Z_APDU *apdu, int ret) { Yaz_ProxyConfig *cfg = check_reconfigure(); if (cfg) - cfg->target_authentication(m_default_target, odr_encode(), + cfg->target_authentication(m_default_target, odr_encode(), apdu->u.initRequest); } handle_incoming_Z_PDU_2(apdu); @@ -220,7 +223,8 @@ Yaz_Proxy::Yaz_Proxy(IPDU_Observable *the_PDU_Observable, m_seed = time(0); m_client_idletime = 600; m_target_idletime = 600; - m_optimize = xstrdup ("1"); + m_max_sockets = 1024; + m_optimize = xstrdup("1"); strcpy(m_session_str, "0 "); m_session_no = 0; m_bytes_sent = 0; @@ -267,9 +271,11 @@ Yaz_Proxy::Yaz_Proxy(IPDU_Observable *the_PDU_Observable, m_mem_invalid_session = 0; m_s2z_odr_init = 0; m_s2z_odr_search = 0; + m_s2z_odr_scan = 0; m_s2z_init_apdu = 0; m_s2z_search_apdu = 0; m_s2z_present_apdu = 0; + m_s2z_scan_apdu = 0; m_http_keepalive = 0; m_http_version = 0; m_soap_ns = 0; @@ -287,6 +293,7 @@ Yaz_Proxy::Yaz_Proxy(IPDU_Observable *the_PDU_Observable, m_usemarcon = new Yaz_usemarcon(); if (!m_parent) low_socket_open(); + m_backend_elementset = 0; m_my_thread = 0; m_ref_count = 1; m_main_ptr_dec = false; @@ -319,19 +326,22 @@ Yaz_Proxy::~Yaz_Proxy() if (m_stylesheet_xsp) xsltFreeStylesheet((xsltStylesheetPtr) m_stylesheet_xsp); #endif - xfree (m_time_tv); - - xfree (m_peername); - xfree (m_schema); - xfree (m_backend_type); - xfree (m_backend_charset); - xfree (m_usemarcon_ini_stage1); - xfree (m_usemarcon_ini_stage2); + xfree(m_time_tv); + + xfree(m_peername); + xfree(m_schema); + xfree(m_backend_type); + xfree(m_backend_charset); + xfree(m_usemarcon_ini_stage1); + xfree(m_usemarcon_ini_stage2); + xfree(m_backend_elementset); delete m_usemarcon; if (m_s2z_odr_init) odr_destroy(m_s2z_odr_init); if (m_s2z_odr_search) odr_destroy(m_s2z_odr_search); + if (m_s2z_odr_scan) + odr_destroy(m_s2z_odr_scan); if (!m_parent) low_socket_close(); if (!m_parent) @@ -364,30 +374,30 @@ int Yaz_Proxy::set_config(const char *config) void Yaz_Proxy::set_default_target(const char *target) { - xfree (m_default_target); + xfree(m_default_target); m_default_target = 0; if (target) - m_default_target = (char *) xstrdup (target); + m_default_target = (char *) xstrdup(target); } -void Yaz_Proxy::set_proxy_negotiation (const char *charset, const char *lang, +void Yaz_Proxy::set_proxy_negotiation(const char *charset, const char *lang, const char *default_charset) { yaz_log(YLOG_DEBUG, "%sSet the proxy negotiation: charset to '%s', " - "default charset to '%s', language to '%s'", m_session_str, + "default charset to '%s', language to '%s'", m_session_str, charset?charset:"none", default_charset?default_charset:"none", lang?lang:"none"); - xfree (m_proxy_negotiation_charset); - xfree (m_proxy_negotiation_lang); + xfree(m_proxy_negotiation_charset); + xfree(m_proxy_negotiation_lang); m_proxy_negotiation_charset = m_proxy_negotiation_lang = 0; if (charset) - m_proxy_negotiation_charset = (char *) xstrdup (charset); + m_proxy_negotiation_charset = (char *) xstrdup(charset); if (lang) - m_proxy_negotiation_lang = (char *) xstrdup (lang); + m_proxy_negotiation_lang = (char *) xstrdup(lang); if (default_charset) m_proxy_negotiation_default_charset = - (char *) xstrdup (default_charset); + (char *) xstrdup(default_charset); } Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure() @@ -399,7 +409,6 @@ Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure() if (m_reconfig_flag) { yaz_log(YLOG_LOG, "reconfigure"); - yaz_log_reopen(); if (m_config_fname && cfg) { yaz_log(YLOG_LOG, "reconfigure config %s", m_config_fname); @@ -441,7 +450,7 @@ IPDU_Observer *Yaz_Proxy::sessionNotify(IPDU_Observable (long) time(0), m_session_no, 0); m_session_no++; - yaz_log (YLOG_LOG, "%sNew session %s", session_str, peername); + yaz_log(YLOG_LOG, "%sNew session %s", session_str, peername); Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable, m_socket_observable, this); @@ -474,7 +483,7 @@ IPDU_Observer *Yaz_Proxy::sessionNotify(IPDU_Observable // create thread object the first time we get an incoming connection if (!m_my_thread && m_num_msg_threads > 0) { - yaz_log (YLOG_LOG, "%sStarting message thread management. number=%d", + yaz_log(YLOG_LOG, "%sStarting message thread management. number=%d", session_str, m_num_msg_threads); m_my_thread = new Msg_Thread(m_socket_observable, m_num_msg_threads); } @@ -486,8 +495,8 @@ char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo) { Z_OtherInformationUnit *oi = update_otherInformation(otherInfo, 0, yaz_oid_userinfo_cookie, 1, 1); - - if (oi->which == Z_OtherInfo_characterInfo) + + if (oi && oi->which == Z_OtherInfo_characterInfo) return oi->information.characterInfo; return 0; } @@ -496,8 +505,8 @@ char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo) { Z_OtherInformationUnit *oi = update_otherInformation(otherInfo, 0, yaz_oid_userinfo_proxy, 1, 1); - - if (oi->which == Z_OtherInfo_characterInfo) + + if (oi && oi->which == Z_OtherInfo_characterInfo) return oi->information.characterInfo; return 0; } @@ -524,10 +533,8 @@ const char *Yaz_Proxy::load_balance(const char **url) } } int min_use = 100000; - int spare_for_min = 0; int max_spare = 0; const char *ret_min = 0; - const char *ret_spare = 0; for (i = 0; url[i]; i++) { yaz_log(YLOG_DEBUG, "%szurl=%s use=%d spare=%d", @@ -536,21 +543,38 @@ const char *Yaz_Proxy::load_balance(const char **url) { ret_min = url[i]; min_use = zurl_in_use[i]; - spare_for_min = zurl_in_spare[i]; } if (max_spare < zurl_in_spare[i]) { - ret_spare = url[i]; max_spare = zurl_in_spare[i]; } } return ret_min; } +int Yaz_Proxy::get_number_of_connections() +{ + int no_connections = 0; + Yaz_ProxyClient *c; + + for (c = m_parent->m_clientPool; c; c = c->m_next) + { + assert(c->m_prev); + assert(*c->m_prev == c); + if (!strcmp(m_proxyTarget, c->get_hostname())) + { + no_connections++; + } + } + yaz_log(YLOG_LOG, "%sExisting %s connections: %d", m_session_str, m_proxyTarget, + no_connections); + return no_connections; +} + Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie, - const char *proxy_host) + const char *proxy_host, int *http_code) { - assert (m_parent); + assert(m_parent); Yaz_Proxy *parent = m_parent; Yaz_ProxyClient *c = m_client; @@ -585,6 +609,7 @@ Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie, &m_pdu_max, &m_max_record_retrieve, &m_search_max, &m_target_idletime, &client_idletime, + &m_max_sockets, &parent->m_max_clients, &m_keepalive_limit_bw, &m_keepalive_limit_pdu, @@ -601,13 +626,13 @@ Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie, timeout(m_client_idletime); } - // get those FILE descriptors available + // get those FILE descriptors available m_parent->low_socket_close(); if (cql2rpn_fname) m_cql2rpn.set_pqf_file(cql2rpn_fname); // reserve them again m_parent->low_socket_open(); - + if (negotiation_charset || negotiation_lang || default_client_query_charset) { set_proxy_negotiation(negotiation_charset, @@ -631,8 +656,8 @@ Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie, { // search in sessions with a cookie for (c = parent->m_clientPool; c; c = c->m_next) { - assert (c->m_prev); - assert (*c->m_prev == c); + assert(c->m_prev); + assert(*c->m_prev == c); if (c->m_cookie && !strcmp(cookie,c->m_cookie) && !strcmp(m_proxyTarget, c->get_hostname())) { @@ -642,7 +667,7 @@ Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie, // we have an initRequest we can safely do re-open if (c->m_waiting && apdu->which == Z_APDU_initRequest) { - yaz_log (YLOG_LOG, "%s REOPEN target=%s", m_session_str, + yaz_log(YLOG_LOG, "%s REOPEN target=%s", m_session_str, c->get_hostname()); c->close(); c->m_init_flag = 0; @@ -666,7 +691,7 @@ Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie, c->m_server->m_client = 0; c->m_server = this; (parent->m_seqno)++; - yaz_log (YLOG_DEBUG, "get_client 1 %p %p", this, c); + yaz_log(YLOG_DEBUG, "get_client 1 %p %p", this, c); return c; } } @@ -680,15 +705,15 @@ Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie, { assert(c->m_prev); assert(*c->m_prev == c); - if (c->m_server == 0 && c->m_cookie == 0 && c->m_waiting == 0 + if (c->m_server == 0 && c->m_cookie == 0 && c->m_waiting == 0 && c->compare_idAuthentication(apdu) && c->compare_charset(apdu) && !strcmp(m_proxyTarget, c->get_hostname())) { // found it in cache - yaz_log (YLOG_LOG, "%sREUSE %d %s", + yaz_log(YLOG_LOG, "%sREUSE %d %s", m_session_str, parent->m_seqno, c->get_hostname()); - + c->m_seqno = parent->m_seqno; assert(c->m_server == 0); c->m_server = this; @@ -710,7 +735,17 @@ Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie, { if (apdu->which != Z_APDU_initRequest) { - yaz_log (YLOG_LOG, "%sno init request as first PDU", m_session_str); + yaz_log(YLOG_LOG, "%sno init request as first PDU", m_session_str); + *http_code = 500; + return 0; + } + + int no_in_use = get_number_of_connections(); + if (no_in_use >= m_max_sockets) + { + yaz_log(YLOG_LOG, "%smax sockets reached %d", m_session_str, + m_max_sockets); + *http_code = 500; return 0; } // go through list of clients - and find the lowest/oldest one. @@ -718,10 +753,10 @@ Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie, int min_seq = -1; int no_of_clients = 0; if (parent->m_clientPool) - yaz_log (YLOG_DEBUG, "Existing sessions"); + yaz_log(YLOG_DEBUG, "Existing sessions"); for (c = parent->m_clientPool; c; c = c->m_next) { - yaz_log (YLOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno, + yaz_log(YLOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno, c->m_waiting, c->get_hostname(), c->m_cookie ? c->m_cookie : ""); no_of_clients++; @@ -736,17 +771,17 @@ Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie, c = c_min; if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname())) { - yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Destroy %d", + yaz_log(YLOG_LOG, "%sMAXCLIENTS %d Destroy %d", m_session_str, parent->m_max_clients, c->m_seqno); if (c->m_server && c->m_server != this) c->m_server->dec_ref(); } else { - yaz_log (YLOG_LOG, "%sMAXCLIENTS %d Reuse %d %d %s", + yaz_log(YLOG_LOG, "%sMAXCLIENTS %d Reuse %d %d %s", m_session_str, parent->m_max_clients, c->m_seqno, parent->m_seqno, c->get_hostname()); - xfree (c->m_cookie); + xfree(c->m_cookie); c->m_cookie = 0; if (cookie) c->m_cookie = xstrdup(cookie); @@ -770,7 +805,8 @@ Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie, } else { - yaz_log (YLOG_LOG, "%sNEW %d %s", + + yaz_log(YLOG_LOG, "%sNEW %d %s", m_session_str, parent->m_seqno, m_proxyTarget); c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent); c->m_next = parent->m_clientPool; @@ -780,7 +816,7 @@ Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie, c->m_prev = &parent->m_clientPool; } - xfree (c->m_cookie); + xfree(c->m_cookie); c->m_cookie = 0; if (cookie) c->m_cookie = xstrdup(cookie); @@ -809,7 +845,7 @@ Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie, c->set_idAuthentication(apdu); } - yaz_log (YLOG_DEBUG, "get_client 3 %p %p", this, c); + yaz_log(YLOG_DEBUG, "get_client 3 %p %p", this, c); return c; } @@ -830,13 +866,13 @@ void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num) switch (r->which) { case Z_DefaultDiagFormat_v2Addinfo: - yaz_log(YLOG_LOG, "%sError %d %s:%s", + yaz_log(YLOG_LOG, "%sError " ODR_INT_PRINTF " %s:%s", m_session_str, *r->condition, diagbib1_str(*r->condition), r->u.v2Addinfo); break; case Z_DefaultDiagFormat_v3Addinfo: - yaz_log(YLOG_LOG, "%sError %d %s:%s", + yaz_log(YLOG_LOG, "%sError " ODR_INT_PRINTF " %s:%s", m_session_str, *r->condition, diagbib1_str(*r->condition), r->u.v3Addinfo); @@ -887,7 +923,7 @@ void Yaz_Proxy::convert_xsl_delay() { xmlChar *out_buf; int out_len; - xmlDocDumpFormatMemory (res, &out_buf, &out_len, 1); + xmlDocDumpFormatMemory(res, &out_buf, &out_len, 1); m_stylesheet_nprl->records[m_stylesheet_offset]-> u.databaseRecord = z_ext_record_oid(odr_encode(), yaz_oid_recsyn_xml, @@ -930,53 +966,30 @@ void Yaz_Proxy::convert_to_frontend_type(Z_NamePlusRecordList *p) Z_External *r = npr->u.databaseRecord; if (r->which == Z_External_octet) { -#if HAVE_USEMARCON +#if !HAVE_USEMARCON if (m_usemarcon_ini_stage1 && *m_usemarcon_ini_stage1) + yaz_log(YLOG_LOG, "%sError: USEMARCON requested but not available", + m_session_str); +#endif +#if HAVE_USEMARCON + yaz_log(YLOG_DEBUG, "%sUSEMARCON stage1=%s stage2=%s", + m_session_str, + m_usemarcon_ini_stage1 ? m_usemarcon_ini_stage1 : "(none)", + m_usemarcon_ini_stage2 ? m_usemarcon_ini_stage2 : "(none)"); + char *converted; + int convlen; + if (m_usemarcon->convert(m_usemarcon_ini_stage1, m_usemarcon_ini_stage2, + (char*) r->u.octet_aligned->buf, r->u.octet_aligned->len, + &converted, &convlen)) { - if (!m_usemarcon->m_stage1) - { - m_usemarcon->m_stage1 = new CDetails(); - } - m_usemarcon->m_stage1->SetIniFileName(m_usemarcon_ini_stage1); - m_usemarcon->m_stage1->SetMarcRecord((char*) r->u.octet_aligned->buf, r->u.octet_aligned->len); - int res = m_usemarcon->m_stage1->Start(); - if (res == 0) - { - char *converted; - int convlen; - m_usemarcon->m_stage1->GetMarcRecord(converted, convlen); - if (m_usemarcon_ini_stage2 && *m_usemarcon_ini_stage2) - { - if (!m_usemarcon->m_stage2) - { - m_usemarcon->m_stage2 = new CDetails(); - } - m_usemarcon->m_stage2->SetIniFileName(m_usemarcon_ini_stage2); - m_usemarcon->m_stage2->SetMarcRecord(converted, convlen); - res = m_usemarcon->m_stage2->Start(); - if (res == 0) - { - free(converted); - m_usemarcon->m_stage2->GetMarcRecord(converted, convlen); - } - else - { - yaz_log(YLOG_LOG, "%sUSEMARCON stage 2 error %d", m_session_str, res); - } - } - npr->u.databaseRecord = - z_ext_record_oid(odr_encode(), - m_frontend_type, - converted, - strlen(converted)); - free(converted); - } - else - { - yaz_log(YLOG_LOG, "%sUSEMARCON stage 1 error %d", m_session_str, res); - } - continue; + npr->u.databaseRecord = + z_ext_record_oid(odr_encode(), + m_frontend_type, + converted, + strlen(converted)); + free(converted); } + else #endif /* HAVE_USEMARCON */ npr->u.databaseRecord = @@ -1011,7 +1024,7 @@ void Yaz_Proxy::convert_records_charset(Z_NamePlusRecordList *p, if (npr->which == Z_NamePlusRecord_databaseRecord) { Z_External *r = npr->u.databaseRecord; - const int *oid = r->direct_reference; + const Odr_oid *oid = r->direct_reference; if (!oid) continue; @@ -1130,7 +1143,7 @@ int Yaz_Proxy::send_http_response(int code) else timeout(0); if (code == 401) - z_HTTP_header_add(o, &hres->headers, "WWW-Authenticate", + z_HTTP_header_add(o, &hres->headers, "WWW-Authenticate", "Basic realm=\"YAZ Proxy\""); @@ -1168,7 +1181,7 @@ int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu, int http_code /* = 200 */) static Z_SOAP_Handler soap_handlers[2] = { #if YAZ_HAVE_XSLT - {"http://www.loc.gov/zing/srw/", 0, + { (char*) "http://www.loc.gov/zing/srw/", 0, (Z_SOAP_fun) yaz_srw_codec}, #endif {0, 0, 0} @@ -1187,8 +1200,8 @@ int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu, int http_code /* = 200 */) soap_handlers, 0, m_s2z_stylesheet); if (m_log_mask & PROXY_LOG_REQ_CLIENT) { - yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str, - gdu_name(gdu)); + yaz_log(YLOG_LOG, "%sSending %s to client", m_session_str, + gdu_name(gdu)); } int len; int r = send_GDU(gdu, &len); @@ -1226,6 +1239,55 @@ int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res, return 0; } +int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_scanResponse *srw_res, + Z_DiagRec *dr) +{ + if (dr->which == Z_DiagRec_defaultFormat) + { + int bib1_code = *dr->u.defaultFormat->condition; + if (bib1_code == 109) + return 404; + srw_res->num_diagnostics = 1; + srw_res->diagnostics = (Z_SRW_diagnostic *) + odr_malloc(o, sizeof(*srw_res->diagnostics)); + yaz_mk_std_diagnostic(o, srw_res->diagnostics, + yaz_diag_bib1_to_srw(*dr->u.defaultFormat->condition), + dr->u.defaultFormat->u.v2Addinfo); + } + return 0; +} + +#if YAZ_HAS_MK_SURROGATE +#else +static void yazproxy_mk_sru_surrogate(ODR o, Z_SRW_record *record, int pos, + int code, const char *details) +{ + const char *message = yaz_diag_srw_str(code); + int len = 200; + if (message) + len += strlen(message); + if (details) + len += strlen(details); + + record->recordData_buf = (char *) odr_malloc(o, len); + + sprintf(record->recordData_buf, "\n" + " info:srw/diagnostic/1/%d\n", code); + if (details) + sprintf(record->recordData_buf + strlen(record->recordData_buf), + "
%s
\n", details); + if (message) + sprintf(record->recordData_buf + strlen(record->recordData_buf), + " %s\n", message); + sprintf(record->recordData_buf + strlen(record->recordData_buf), + "\n"); + record->recordData_len = strlen(record->recordData_buf); + record->recordPosition = odr_intdup(o, pos); + record->recordSchema = odr_strdup(o, "info:srw/schema/1/diagnostics-v1.1"); +} +#endif + int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start) { ODR o = odr_encode(); @@ -1245,16 +1307,20 @@ int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start) Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i]; if (npr->which != Z_NamePlusRecord_databaseRecord) { - srw_res->records[i].recordSchema = "diagnostic"; - srw_res->records[i].recordPacking = m_s2z_packing; - srw_res->records[i].recordData_buf = "67"; - srw_res->records[i].recordData_len = 2; - srw_res->records[i].recordPosition = odr_intdup(o, i+start); +#if YAZ_HAS_MK_SURROGATE + yaz_mk_sru_surrogate( + o, srw_res->records + i, i+start, + YAZ_SRW_RECORD_NOT_AVAILABLE_IN_THIS_SCHEMA, 0); +#else + yazproxy_mk_sru_surrogate( + o, srw_res->records + i, i+start, + YAZ_SRW_RECORD_NOT_AVAILABLE_IN_THIS_SCHEMA, 0); +#endif continue; } Z_External *r = npr->u.databaseRecord; - if (r->which == Z_External_octet + if (r->which == Z_External_octet && !oid_oidcmp(r->direct_reference, yaz_oid_recsyn_xml)) { srw_res->records[i].recordSchema = m_schema; @@ -1266,11 +1332,15 @@ int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start) } else { - srw_res->records[i].recordSchema = "diagnostic"; - srw_res->records[i].recordPacking = m_s2z_packing; - srw_res->records[i].recordData_buf = "67"; - srw_res->records[i].recordData_len = 2; - srw_res->records[i].recordPosition = odr_intdup(o, i+start); +#if YAZ_HAS_MK_SURROGATE + yaz_mk_sru_surrogate( + o, srw_res->records + i, i+start, + YAZ_SRW_RECORD_NOT_AVAILABLE_IN_THIS_SCHEMA, 0); +#else + yazproxy_mk_sru_surrogate( + o, srw_res->records + i, i+start, + YAZ_SRW_RECORD_NOT_AVAILABLE_IN_THIS_SCHEMA, 0); +#endif } } } @@ -1286,6 +1356,58 @@ int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start) } +int Yaz_Proxy::send_to_srw_client_ok(Z_ListEntries *entries) +{ + ODR o = odr_encode(); + Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_scan_response); + Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response; + + if (entries && entries->num_entries > 0) + { + srw_res->num_terms = entries->num_entries; + int i; + srw_res->terms = (Z_SRW_scanTerm *) + odr_malloc(o, srw_res->num_terms * sizeof(Z_SRW_scanTerm)); + for (i = 0; i < srw_res->num_terms; i++) + { + if (entries->entries[i]->which == Z_Entry_termInfo) + { + switch(entries->entries[i]->u.termInfo->term->which) + { + case Z_Term_general: + srw_res->terms[i].value = odr_strdupn(o, + (char *) entries->entries[i]->u.termInfo->term->u.general->buf, + entries->entries[i]->u.termInfo->term->u.general->len); + break; + default: + srw_res->terms[i].value = NULL; + } + if (entries->entries[i]->u.termInfo->globalOccurrences != NULL) + srw_res->terms[i].numberOfRecords = odr_intdup(o, + *entries->entries[i]->u.termInfo->globalOccurrences); + else + srw_res->terms[i].numberOfRecords = NULL; + if (entries->entries[i]->u.termInfo->displayTerm != NULL) + srw_res->terms[i].displayTerm = odr_strdup(o, + entries->entries[i]->u.termInfo->displayTerm); + else + srw_res->terms[i].displayTerm = NULL; + srw_res->terms[i].whereInList = NULL; + } + } + } + if (entries && entries->num_nonsurrogateDiagnostics > 0) + { + int http_code; + http_code = z_to_srw_diag(odr_encode(), srw_res, + entries->nonsurrogateDiagnostics[0]); + if (http_code) + return send_http_response(http_code); + } + return send_srw_response(srw_pdu); + +} + int Yaz_Proxy::send_srw_search_response(Z_SRW_diagnostic *diagnostics, int num_diagnostics, int http_code /* = 200 */) { @@ -1298,15 +1420,28 @@ int Yaz_Proxy::send_srw_search_response(Z_SRW_diagnostic *diagnostics, return send_srw_response(srw_pdu, http_code); } +int Yaz_Proxy::send_srw_scan_response(Z_SRW_diagnostic *diagnostics, + int num_diagnostics, int http_code /* = 200 */) +{ + ODR o = odr_encode(); + Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_scan_response); + Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response; + + srw_res->num_diagnostics = num_diagnostics; + srw_res->diagnostics = diagnostics; + return send_srw_response(srw_pdu, http_code); +} + int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics, int num_diagnostics) { + int http_status = 404; Yaz_ProxyConfig *cfg = check_reconfigure(); if (cfg) { int len; char *b = cfg->get_explain_doc(odr_encode(), 0 /* target */, - m_s2z_database, &len); + m_s2z_database, &len, &http_status); if (b) { Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response); @@ -1315,14 +1450,15 @@ int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics, er->record.recordData_buf = b; er->record.recordData_len = len; er->record.recordPacking = m_s2z_packing; - er->record.recordSchema = "http://explain.z3950.org/dtd/2.0/"; + er->record.recordSchema = odr_strdup(odr_encode(), + "http://explain.z3950.org/dtd/2.0/"); er->diagnostics = diagnostics; er->num_diagnostics = num_diagnostics; return send_srw_response(res); } } - return send_http_response(404); + return send_http_response(http_status); } int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu) @@ -1336,13 +1472,17 @@ int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu) { send_to_srw_client_error(3, 0); } - else if (!m_s2z_search_apdu) + else if (m_s2z_search_apdu) { - send_srw_explain_response(0, 0); + handle_incoming_Z_PDU(m_s2z_search_apdu); + } + else if (m_s2z_scan_apdu) + { + handle_incoming_Z_PDU(m_s2z_scan_apdu); } else { - handle_incoming_Z_PDU(m_s2z_search_apdu); + send_srw_explain_response(0, 0); } } else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse) @@ -1383,12 +1523,17 @@ int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu) Z_PresentResponse *res = apdu->u.presentResponse; send_to_srw_client_ok(m_s2z_hit_count, res->records, start); } + else if (m_s2z_scan_apdu && apdu->which == Z_APDU_scanResponse) + { + Z_ScanResponse *res = apdu->u.scanResponse; + send_to_srw_client_ok(res->entries); + } } else { int len = 0; if (m_log_mask & PROXY_LOG_REQ_CLIENT) - yaz_log (YLOG_LOG, "%sSending %s to client", m_session_str, + yaz_log(YLOG_LOG, "%sSending %s to client", m_session_str, apdu_name(apdu)); int r = send_Z_PDU(apdu, &len); m_bytes_sent += len; @@ -1442,7 +1587,7 @@ int Yaz_Proxy::send_to_client(Z_APDU *apdu) } if (sr->resultCount) { - yaz_log(YLOG_LOG, "%s%d hits", m_session_str, + yaz_log(YLOG_LOG, "%s" ODR_INT_PRINTF " hits", m_session_str, *sr->resultCount); if (*sr->resultCount < 0) { @@ -1551,11 +1696,11 @@ int Yaz_Proxy::send_to_client(Z_APDU *apdu) void Yaz_ProxyClient::set_idAuthentication(Z_APDU *apdu) { Z_IdAuthentication *t = apdu->u.initRequest->idAuthentication; - + odr_reset(m_idAuthentication_odr); z_IdAuthentication(m_idAuthentication_odr, &t, 1, 0); m_idAuthentication_ber_buf = - odr_getbuf(m_idAuthentication_odr, + odr_getbuf(m_idAuthentication_odr, &m_idAuthentication_ber_size, 0); } @@ -1591,7 +1736,7 @@ int Yaz_ProxyClient::send_to_target(Z_APDU *apdu) const char *apdu_name_tmp = apdu_name(apdu); int r = send_Z_PDU(apdu, &len); if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER) - yaz_log (YLOG_LOG, "%sSending %s to %s %d bytes", + yaz_log(YLOG_LOG, "%sSending %s to %s %d bytes", get_session_str(), apdu_name_tmp, get_hostname(), len); m_bytes_sent += len; @@ -1617,7 +1762,7 @@ Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu) Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse); new_apdu->u.presentResponse->records = create_nonSurrogateDiagnostics( - odr_encode(), + odr_encode(), YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST, pr->resultSetId); send_to_client(new_apdu); @@ -1628,8 +1773,8 @@ Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu) Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse); new_apdu->u.presentResponse->records = create_nonSurrogateDiagnostics( - odr_encode(), - YAZ_BIB1_PRESENT_REQUEST_OUT_OF_RANGE, + odr_encode(), + YAZ_BIB1_PRESENT_REQUEST_OUT_OF_RANGE, 0); send_to_client(new_apdu); return 0; @@ -1641,7 +1786,7 @@ Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu) Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse); new_apdu->u.presentResponse->records = create_nonSurrogateDiagnostics( - odr_encode(), + odr_encode(), YAZ_BIB1_PRESENT_REQUEST_OUT_OF_RANGE, 0); send_to_client(new_apdu); @@ -1657,7 +1802,7 @@ Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu) pr->preferredRecordSyntax, pr->recordComposition)) { - yaz_log (YLOG_LOG, "%sReturned cached records for present request", + yaz_log(YLOG_LOG, "%sReturned cached records for present request", m_session_str); Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse); new_apdu->u.presentResponse->referenceId = pr->referenceId; @@ -1698,7 +1843,7 @@ Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu) // Not a present request.. But can't find better diagnostic new_apdu->u.searchResponse->records = create_nonSurrogateDiagnostics( - odr_encode(), + odr_encode(), YAZ_BIB1_PRESENT_REQUEST_OUT_OF_RANGE, 0); send_to_client(new_apdu); return 0; @@ -1724,7 +1869,7 @@ Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu) if (toget > m_client->m_last_resultCount) toget = m_client->m_last_resultCount; - + if (sr->mediumSetElementSetNames) { comp = (Z_RecordComposition *) @@ -1736,7 +1881,7 @@ Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu) if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget, sr->preferredRecordSyntax, comp)) { - yaz_log (YLOG_LOG, "%sReturned cached records for medium set", + yaz_log(YLOG_LOG, "%sReturned cached records for medium set", m_session_str); Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse); new_apdu->u.searchResponse->referenceId = sr->referenceId; @@ -1761,7 +1906,7 @@ Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu) { // medium Set // send present request (medium size) - yaz_log (YLOG_LOG, "%sOptimizing search for medium set", + yaz_log(YLOG_LOG, "%sOptimizing search for medium set", m_session_str); Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest); @@ -1779,7 +1924,7 @@ Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu) m_client->m_last_resultCount <= 0) { // large set. Return pseudo-search response immediately - yaz_log (YLOG_LOG, "%sOptimizing search for large set", + yaz_log(YLOG_LOG, "%sOptimizing search for large set", m_session_str); Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse); new_apdu->u.searchResponse->referenceId = sr->referenceId; @@ -1807,7 +1952,7 @@ Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu) if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget, sr->preferredRecordSyntax, comp)) { - yaz_log (YLOG_LOG, "%sReturned cached records for small set", + yaz_log(YLOG_LOG, "%sReturned cached records for small set", m_session_str); Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse); new_apdu->u.searchResponse->referenceId = sr->referenceId; @@ -1830,7 +1975,7 @@ Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu) } else { - yaz_log (YLOG_LOG, "%sOptimizing search for small set", + yaz_log(YLOG_LOG, "%sOptimizing search for small set", m_session_str); Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest); Z_PresentRequest *pr = new_apdu->u.presentRequest; @@ -1852,8 +1997,8 @@ Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu) m_client->m_cache.clear(); m_client->m_resultSetStartPoint = 0; - xfree (m_client->m_last_resultSetId); - m_client->m_last_resultSetId = xstrdup (sr->resultSetName); + xfree(m_client->m_last_resultSetId); + m_client->m_last_resultSetId = xstrdup(sr->resultSetName); m_client->m_last_databases.set(sr->num_databaseNames, (const char **) sr->databaseNames); @@ -1881,12 +2026,12 @@ void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len) m_bytes_recv += len; if (m_log_mask & PROXY_LOG_REQ_CLIENT) - yaz_log (YLOG_LOG, "%sReceiving %s from client %d bytes", + yaz_log(YLOG_LOG, "%sReceiving %s from client %d bytes", m_session_str, gdu_name(apdu), len); #if 0 // try to make a _bad_ attribute set ID .. Don't enable this in prod. - if (apdu->which == Z_GDU_Z3950 + if (apdu->which == Z_GDU_Z3950 && apdu->u.z3950->which == Z_APDU_searchRequest) { Z_SearchRequest *req = apdu->u.z3950->u.searchRequest; @@ -1935,7 +2080,7 @@ void Yaz_Proxy::HTTP_Forwarded(Z_GDU *z_gdu) xfree(m_peername); m_peername = (char*) xmalloc(strlen(x_forwarded_for)+5); sprintf(m_peername, "tcp:%s", x_forwarded_for); - + yaz_log(YLOG_LOG, "%sHTTP Forwarded from %s", m_session_str, m_peername); if (m_log_mask & PROXY_LOG_IP_CLIENT) @@ -1963,11 +2108,11 @@ void Yaz_Proxy::connect_stat(bool &block, int &reduce) m_session_str, connect_total, max_connect); block = true; } - else + else block = false; yaz_log(YLOG_LOG, "%sconnect accepted total=%d", m_session_str, connect_total); - + int limit_connect = m_parent->m_limit_connect; if (limit_connect) reduce = connect_total / limit_connect; @@ -1980,11 +2125,11 @@ void Yaz_Proxy::recv_GDU_reduce(GDU *gdu) HTTP_Forwarded(gdu->get()); int reduce = 0; - + if (m_request_no == 1) { bool block = false; - + connect_stat(block, reduce); if (block) @@ -2180,10 +2325,10 @@ void Yaz_Proxy::handle_charset_lang_negotiation(Z_APDU *apdu) Z_InitResponse *initResponse = apdu->u.initResponse; Z_OtherInformation **otherInfo; get_otherInfoAPDU(apdu, &otherInfo); - + Z_CharSetandLanguageNegotiation *charneg = 0; - if (otherInfo && *otherInfo && + if (otherInfo && *otherInfo && ODR_MASK_GET(initResponse->options, Z_Options_negotiationModel) && (charneg = yaz_get_charneg_record(*otherInfo))) { @@ -2238,8 +2383,8 @@ void Yaz_Proxy::handle_charset_lang_negotiation(Z_APDU *apdu) if (m_initRequest_options) ODR_MASK_SET(m_initRequest_options, Z_Options_negotiationModel); - - oi->which = Z_OtherInfo_externallyDefinedInfo; + + oi->which = Z_OtherInfo_externallyDefinedInfo; oi->information.externallyDefinedInfo = yaz_set_response_charneg( odr_encode(), @@ -2258,7 +2403,7 @@ Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr, { Z_Records *rec = (Z_Records *) odr_malloc (odr, sizeof(*rec)); - int *err = (int *) + Odr_int *err = (Odr_int *) odr_malloc (odr, sizeof(*err)); Z_DiagRec *drec = (Z_DiagRec *) odr_malloc (odr, sizeof(*drec)); @@ -2274,6 +2419,34 @@ Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr, return rec; } +Z_ListEntries *Yaz_Proxy::create_nonSurrogateDiagnostics2(ODR odr, + int error, + const char *addinfo) +{ + Z_ListEntries *rec = (Z_ListEntries *) + odr_malloc (odr, sizeof(*rec)); + Odr_int *err = (Odr_int *) + odr_malloc (odr, sizeof(*err)); + Z_DiagRec *drec = (Z_DiagRec *) + odr_malloc (odr, sizeof(*drec)); + Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *) + odr_malloc (odr, sizeof(*dr)); + *err = error; + drec->which = Z_DiagRec_defaultFormat; + drec->u.defaultFormat = dr; + rec->num_entries = 0; + rec->entries = NULL; + rec->num_nonsurrogateDiagnostics = 1; + rec->nonsurrogateDiagnostics = + (Z_DiagRec **)odr_malloc(odr, sizeof(Z_DiagRec *)); + rec->nonsurrogateDiagnostics[0] = drec; + dr->diagnosticSetId = odr_oiddup(odr, yaz_oid_diagset_bib_1); + dr->condition = err; + dr->which = Z_DefaultDiagFormat_v2Addinfo; + dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : ""); + return rec; +} + Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu) { if (apdu->which == Z_APDU_searchRequest && @@ -2316,6 +2489,44 @@ Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu) } return apdu; } + else if (apdu->which == Z_APDU_scanRequest) + { + Z_RPNQuery *rpnquery = 0; + Z_ScanRequest *sr = apdu->u.scanRequest; + char *addinfo = 0; + + yaz_log(YLOG_LOG, "%sCQL: %s", m_session_str, + sr->termListAndStartPoint->term->u.characterString); + + int r = m_cql2rpn.query_transform(sr->termListAndStartPoint->term->u.characterString, + &rpnquery, odr_encode(), + &addinfo); + if (r == -3) + yaz_log(YLOG_LOG, "%sNo CQL to RPN table", m_session_str); + else if (r) + { + yaz_log(YLOG_LOG, "%sCQL Conversion error %d", m_session_str, r); + Z_APDU *new_apdu = create_Z_PDU(Z_APDU_scanResponse); + + new_apdu->u.scanResponse->referenceId = sr->referenceId; + new_apdu->u.scanResponse->entries = + create_nonSurrogateDiagnostics2(odr_encode(), + yaz_diag_srw_to_bib1(r), + addinfo); + *new_apdu->u.scanResponse->scanStatus = Z_Scan_failure; + + send_to_client(new_apdu); + + return 0; + } + else + { + sr->attributeSet = rpnquery->attributeSetId; + if (rpnquery->RPNStructure->which == Z_RPNStructure_simple) + sr->termListAndStartPoint = rpnquery->RPNStructure->u.simple->u.attributesPlusTerm; + } + return apdu; + } return apdu; } @@ -2333,6 +2544,15 @@ Z_APDU *Yaz_Proxy::handle_target_charset_conversion(Z_APDU *apdu) m_charset_converter->convert_type_1(rpnquery, odr_encode()); } } + else if (apdu->which == Z_APDU_scanRequest && + apdu->u.scanRequest->termListAndStartPoint) + { + if (apdu->u.scanRequest->termListAndStartPoint->term) + if (m_http_version) + m_charset_converter->set_client_query_charset("UTF-8"); + Z_Term *term = apdu->u.scanRequest->termListAndStartPoint->term; + m_charset_converter->convert_term(term, odr_encode()); + } return apdu; } @@ -2363,6 +2583,34 @@ Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu) return 0; } } + else if (apdu->which == Z_APDU_scanRequest) + { + Z_ScanRequest *sr = apdu->u.scanRequest; + int err = 0; + char *addinfo = 0; + +#if 0 + Yaz_ProxyConfig *cfg = check_reconfigure(); +// Something like this needs to be implemented later: + if (cfg) + err = cfg->check_type_1_attributes(odr_encode(), m_default_target, + sr->termListAndStartPoint->attributes, &addinfo); +#endif + if (err) + { + Z_APDU *new_apdu = create_Z_PDU(Z_APDU_scanResponse); + + new_apdu->u.scanResponse->referenceId = sr->referenceId; + new_apdu->u.scanResponse->entries = + create_nonSurrogateDiagnostics2(odr_encode(), err, addinfo); + *new_apdu->u.scanResponse->scanStatus = Z_Scan_failure; + + send_to_client(new_apdu); + + return 0; + } + } + return apdu; } @@ -2475,7 +2723,8 @@ Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu) &addinfo, &stylesheet_name, &m_schema, &m_backend_type, &m_backend_charset, &m_usemarcon_ini_stage1, - &m_usemarcon_ini_stage2); + &m_usemarcon_ini_stage2, + &m_backend_elementset); if (stylesheet_name) { m_parent->low_socket_close(); @@ -2499,7 +2748,7 @@ Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu) sr->preferredRecordSyntax = yaz_string_to_oid_odr( yaz_oid_std(), CLASS_RECSYN, - m_backend_type ? m_backend_type : "usmarc", + m_backend_type ? m_backend_type : "usmarc", odr_encode()); } else if (err) @@ -2518,9 +2767,18 @@ Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu) else if (m_backend_type) { sr->preferredRecordSyntax = - yaz_string_to_oid_odr(yaz_oid_std(), CLASS_RECSYN, + yaz_string_to_oid_odr(yaz_oid_std(), CLASS_RECSYN, m_backend_type, odr_encode()); } + if (m_backend_elementset) + { + Z_ElementSetNames *esn = + mk_esn_from_schema( + odr_encode(), + *m_backend_elementset ? m_backend_elementset : 0); + sr->smallSetElementSetNames = esn; + sr->mediumSetElementSetNames = esn; + } } else if (apdu->which == Z_APDU_presentRequest) { @@ -2542,8 +2800,8 @@ Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu) &addinfo, &stylesheet_name, &m_schema, &m_backend_type, &m_backend_charset, &m_usemarcon_ini_stage1, - &m_usemarcon_ini_stage2 - ); + &m_usemarcon_ini_stage2, + &m_backend_elementset); if (stylesheet_name) { m_parent->low_socket_close(); @@ -2567,7 +2825,7 @@ Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu) pr->preferredRecordSyntax = yaz_string_to_oid_odr( yaz_oid_std(), CLASS_RECSYN, - m_backend_type ? m_backend_type : "usmarc", + m_backend_type ? m_backend_type : "usmarc", odr_encode()); } else if (err) @@ -2588,9 +2846,21 @@ Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu) { pr->preferredRecordSyntax = yaz_string_to_oid_odr(yaz_oid_std(), - CLASS_RECSYN, m_backend_type, + CLASS_RECSYN, m_backend_type, odr_encode()); } + if (m_backend_elementset) + { + Z_ElementSetNames *esn = + mk_esn_from_schema( + odr_encode(), + *m_backend_elementset ? m_backend_elementset : 0); + Z_RecordComposition *comp = (Z_RecordComposition *) + odr_malloc(odr_encode(), sizeof(Z_RecordComposition)); + comp->which = Z_RecordComp_simple; + comp->u.simple = esn; + pr->recordComposition = comp; + } } return apdu; } @@ -2691,7 +2961,7 @@ int Yaz_Proxy::file_access(Z_HTTP_Request *hreq) } if (m_log_mask & PROXY_LOG_REQ_CLIENT) { - yaz_log (YLOG_LOG, "%sSending file %s to client", m_session_str, + yaz_log(YLOG_LOG, "%sSending file %s to client", m_session_str, fname); } int len; @@ -2712,6 +2982,11 @@ void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq) odr_destroy(m_s2z_odr_search); m_s2z_odr_search = 0; } + if (m_s2z_odr_scan) + { + odr_destroy(m_s2z_odr_scan); + m_s2z_odr_scan = 0; + } m_http_keepalive = 0; m_http_version = 0; @@ -2759,15 +3034,24 @@ void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq) { m_s2z_odr_init = odr_createmem(ODR_ENCODE); m_s2z_odr_search = odr_createmem(ODR_ENCODE); + m_s2z_odr_scan = odr_createmem(ODR_ENCODE); m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns); m_s2z_init_apdu = 0; m_s2z_search_apdu = 0; m_s2z_present_apdu = 0; + m_s2z_scan_apdu = 0; m_s2z_stylesheet = 0; - + Z_IdAuthentication *auth = NULL; - if (srw_pdu->username && srw_pdu->password) + if (srw_pdu->username && !srw_pdu->password) + { + yaz_log(YLOG_LOG, "username: %s\n", srw_pdu->username); + auth = (Z_IdAuthentication *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdAuthentication)); + auth->which = Z_IdAuthentication_open; + auth->u.open = odr_strdup(m_s2z_odr_init, srw_pdu->username); + } + else if (srw_pdu->username && srw_pdu->password) { yaz_log(YLOG_LOG, "username/password: %s/%s\n", srw_pdu->username, srw_pdu->password); @@ -2796,20 +3080,8 @@ void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq) } auth->u.idPass->userId = odr_strdup(m_s2z_odr_init, authorization_str); } - else - { - // Use _client_ IP as shown in the log entries...! - yaz_log(YLOG_LOG, "No authorization_str present: use client IP: %s\n", m_peername); - - auth = (Z_IdAuthentication *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdAuthentication)); - auth->which = Z_IdAuthentication_idPass; - auth->u.idPass = (Z_IdPass *) odr_malloc(m_s2z_odr_init, sizeof(Z_IdPass)); - auth->u.idPass->groupId = NULL; - auth->u.idPass->password = NULL; - auth->u.idPass->userId = odr_strdup(m_s2z_odr_init, m_peername); - } - } - + } + if (srw_pdu->which == Z_SRW_searchRetrieve_request) { @@ -2875,7 +3147,13 @@ void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq) odr_malloc(m_s2z_odr_search, sizeof(Z_Query)); z_searchRequest->query = query; - if (srw_req->query_type == Z_SRW_query_type_cql) + if ( +#ifdef Z_SRW_query_type_cql + srw_req->query_type == Z_SRW_query_type_cql +#else + !srw_req->queryType || !strcmp(srw_req->queryType, "cql") +#endif + ) { Z_External *ext = (Z_External *) odr_malloc(m_s2z_odr_search, sizeof(*ext)); @@ -2884,12 +3162,22 @@ void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq) ext->indirect_reference = 0; ext->descriptor = 0; ext->which = Z_External_CQL; +#ifdef Z_SRW_query_type_cql ext->u.cql = srw_req->query.cql; +#else + ext->u.cql = srw_req->query; +#endif query->which = Z_Query_type_104; query->u.type_104 = ext; } - else if (srw_req->query_type == Z_SRW_query_type_pqf) + else if ( +#ifdef Z_SRW_query_type_pqf + srw_req->query_type == Z_SRW_query_type_pqf +#else + srw_req->queryType && !strcmp(srw_req->queryType, "pqf") +#endif + ) { Z_RPNQuery *RPNquery; YAZ_PQF_Parser pqf_parser; @@ -2897,7 +3185,12 @@ void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq) pqf_parser = yaz_pqf_create (); RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search, - srw_req->query.pqf); +#ifdef Z_SRW_query_type_pqf + srw_req->query.pqf +#else + srw_req->query +#endif + ); if (!RPNquery) { const char *pqf_msg; @@ -2980,6 +3273,7 @@ void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq) m_s2z_init_apdu = zget_APDU(m_s2z_odr_init, Z_APDU_initRequest); + ODR_MASK_SET(m_s2z_init_apdu->u.initRequest->options, Z_Options_scan); m_s2z_init_apdu->u.initRequest->idAuthentication = auth; // prevent m_initRequest_apdu memory from being grabbed @@ -3025,8 +3319,9 @@ void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq) m_s2z_init_apdu = zget_APDU(m_s2z_odr_init, Z_APDU_initRequest); + ODR_MASK_SET(m_s2z_init_apdu->u.initRequest->options, Z_Options_scan); m_s2z_init_apdu->u.initRequest->idAuthentication = auth; - + // prevent m_initRequest_apdu memory from being grabbed // in Yaz_Proxy::handle_incoming_Z_PDU m_initRequest_apdu = m_s2z_init_apdu; @@ -3038,21 +3333,81 @@ void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq) } else if (srw_pdu->which == Z_SRW_scan_request) { + Z_SRW_scanRequest *srw_req = srw_pdu->u.scan_request; + + const char *backend_db = srw_req->database; + srw_get_client(srw_req->database, &backend_db); + m_s2z_database = odr_strdup(m_s2z_odr_init, - srw_pdu->u.scan_request->database); - - yaz_add_srw_diagnostic(odr_decode(), - &diagnostic, &num_diagnostic, - 4, "scan"); - Z_SRW_PDU *srw_pdu = - yaz_srw_get(odr_encode(), - Z_SRW_scan_response); - Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response; - - srw_res->diagnostics = diagnostic; - srw_res->num_diagnostics = num_diagnostic; - send_srw_response(srw_pdu); - return; + srw_req->database); + // save stylesheet + if (srw_req->stylesheet) + m_s2z_stylesheet = + odr_strdup(m_s2z_odr_init, srw_req->stylesheet); + + // prepare scan PDU + m_s2z_scan_apdu = zget_APDU(m_s2z_odr_scan, + Z_APDU_scanRequest); + Z_ScanRequest *z_scanRequest = + m_s2z_scan_apdu->u.scanRequest; + + z_scanRequest->num_databaseNames = 1; + z_scanRequest->databaseNames = (char**) + odr_malloc(m_s2z_odr_scan, sizeof(char *)); + z_scanRequest->databaseNames[0] = odr_strdup(m_s2z_odr_scan, + backend_db); + + // query transformation + if ( +#ifdef Z_SRW_query_type_cql + srw_req->query_type == Z_SRW_query_type_cql +#else + !srw_req->queryType || !strcmp(srw_req->queryType, "cql") +#endif + ) + { + z_scanRequest->termListAndStartPoint = + (Z_AttributesPlusTerm *)odr_malloc(m_s2z_odr_scan, sizeof(Z_AttributesPlusTerm)); + z_scanRequest->termListAndStartPoint->attributes = NULL; + z_scanRequest->termListAndStartPoint->term = + (Z_Term *)odr_malloc(m_s2z_odr_scan, sizeof(Z_Term)); + z_scanRequest->termListAndStartPoint->term->which = + Z_Term_characterString; + z_scanRequest->termListAndStartPoint->term->u.characterString = + odr_strdup(m_s2z_odr_scan, +#ifdef Z_SRW_query_type_cql + srw_req->scanClause.cql +#else + srw_req->scanClause +#endif + ); + } + + if (srw_req->responsePosition) + z_scanRequest->preferredPositionInResponse = + odr_intdup(m_s2z_odr_scan, *srw_req->responsePosition); + if (srw_req->maximumTerms) + *z_scanRequest->numberOfTermsRequested = *srw_req->maximumTerms; + + if (!m_client) + { + m_s2z_init_apdu = zget_APDU(m_s2z_odr_init, + Z_APDU_initRequest); + + ODR_MASK_SET(m_s2z_init_apdu->u.initRequest->options, Z_Options_scan); + m_s2z_init_apdu->u.initRequest->idAuthentication = auth; + + // prevent m_initRequest_apdu memory from being grabbed + // in Yaz_Proxy::handle_incoming_Z_PDU + m_initRequest_apdu = m_s2z_init_apdu; + handle_incoming_Z_PDU(m_s2z_init_apdu); + return; + } + else + { + handle_incoming_Z_PDU(m_s2z_scan_apdu); + return; + } } else { @@ -3089,10 +3444,10 @@ void Yaz_Proxy::handle_init(Z_APDU *apdu) m_initRequest_preferredMessageSize = *apdu->u.initRequest-> preferredMessageSize; - *apdu->u.initRequest->preferredMessageSize = 1024*1024; + *apdu->u.initRequest->preferredMessageSize = 64*1024*1024; m_initRequest_maximumRecordSize = *apdu->u.initRequest-> maximumRecordSize; - *apdu->u.initRequest->maximumRecordSize = 1024*1024; + *apdu->u.initRequest->maximumRecordSize = 64*1024*1024; Z_CharSetandLanguageNegotiation *charSetandLangRecord = yaz_get_charneg_record(*oi); @@ -3165,7 +3520,7 @@ void Yaz_Proxy::handle_init(Z_APDU *apdu) Z_APDU *apdu2 = m_client->m_initResponse; apdu2->u.initResponse->otherInfo = 0; if (m_client->m_cookie && *m_client->m_cookie) - set_otherInformationString(apdu2, yaz_oid_userinfo_cookie, + set_otherInformationString(apdu2, yaz_oid_userinfo_cookie, 1, m_client->m_cookie); apdu2->u.initResponse->referenceId = apdu->u.initRequest->referenceId; @@ -3192,7 +3547,7 @@ void Yaz_Proxy::handle_init(Z_APDU *apdu) m->m_apdu_buf = (char*) nmem_malloc(m->m_nmem, m->m_apdu_len); memcpy(m->m_apdu_buf, apdu_buf, m->m_apdu_len); odr_reset(odr_encode()); - + inc_ref(); m_my_thread->put(m); } @@ -3211,10 +3566,15 @@ void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu) { m_referenceId = (Z_ReferenceId *) nmem_malloc(m_referenceId_mem, sizeof(*m_referenceId)); - m_referenceId->len = m_referenceId->size = (*refid)->len; + m_referenceId->len = (*refid)->len; +#if YAZ_VERSIONL < 0x50000 + m_referenceId->size = m_referenceId->len; m_referenceId->buf = (unsigned char *) - nmem_malloc(m_referenceId_mem, (*refid)->len); - memcpy(m_referenceId->buf, (*refid)->buf, (*refid)->len); + nmem_strdupn(m_referenceId_mem, (*refid)->buf, (*refid)->len); +#else + m_referenceId->buf = + nmem_strdupn(m_referenceId_mem, (*refid)->buf, (*refid)->len); +#endif } else m_referenceId = 0; @@ -3250,12 +3610,13 @@ void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu) // Determine our client. Z_OtherInformation **oi; get_otherInfoAPDU(apdu, &oi); - m_client = get_client(apdu, get_cookie(oi), get_proxy(oi)); + int http_code = 404; + m_client = get_client(apdu, get_cookie(oi), get_proxy(oi), &http_code); if (!m_client) { if (m_http_version) { // HTTP. Send not found - send_http_response(404); + send_http_response(http_code); return; } else @@ -3353,7 +3714,7 @@ void Yaz_Proxy::releaseClient() } else if (m_client) { - yaz_log (YLOG_LOG, "%sShutdown (client to proxy) close %s", + yaz_log(YLOG_LOG, "%sShutdown (client to proxy) close %s", m_session_str, m_client->get_hostname()); assert (m_client->m_waiting != 2); @@ -3362,13 +3723,13 @@ void Yaz_Proxy::releaseClient() } else if (!m_parent) { - yaz_log (YLOG_LOG, "%sshutdown (client to proxy) bad state", + yaz_log(YLOG_LOG, "%sshutdown (client to proxy) bad state", m_session_str); assert (m_parent); } else { - yaz_log (YLOG_LOG, "%sShutdown (client to proxy)", + yaz_log(YLOG_LOG, "%sShutdown (client to proxy)", m_session_str); } if (m_parent) @@ -3398,7 +3759,7 @@ const char *Yaz_ProxyClient::get_session_str() void Yaz_ProxyClient::shutdown() { - yaz_log (YLOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(), + yaz_log(YLOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(), get_hostname()); if (m_server) @@ -3413,7 +3774,7 @@ void Yaz_ProxyClient::shutdown() void Yaz_Proxy::failNotify() { inc_request_no(); - yaz_log (YLOG_LOG, "%sConnection closed by client", get_session_str()); + yaz_log(YLOG_LOG, "%sConnection closed by client", get_session_str()); dec_ref(); } @@ -3423,21 +3784,24 @@ void Yaz_Proxy::send_response_fail_client(const char *addr) { Z_SRW_diagnostic *diagnostic = 0; int num_diagnostic = 0; - + yaz_add_srw_diagnostic(odr_encode(), &diagnostic, &num_diagnostic, YAZ_SRW_SYSTEM_TEMPORARILY_UNAVAILABLE, addr); if (m_s2z_search_apdu) send_srw_search_response(diagnostic, num_diagnostic); + else if (m_s2z_scan_apdu) + send_srw_scan_response(diagnostic, num_diagnostic); else send_srw_explain_response(diagnostic, num_diagnostic); - } + } } + void Yaz_ProxyClient::failNotify() { if (m_server) m_server->inc_request_no(); - yaz_log (YLOG_LOG, "%sConnection closed by target %s", + yaz_log(YLOG_LOG, "%sConnection closed by target %s", get_session_str(), get_hostname()); if (m_server) @@ -3449,7 +3813,7 @@ void Yaz_ProxyClient::connectNotify() { const char *s = get_session_str(); const char *h = get_hostname(); - yaz_log (YLOG_LOG, "%sConnection accepted by %s timeout=%d", s, h, + yaz_log(YLOG_LOG, "%sConnection accepted by %s timeout=%d", s, h, m_target_idletime); timeout(m_target_idletime); if (!m_server) @@ -3472,8 +3836,8 @@ Yaz_ProxyClient::~Yaz_ProxyClient() odr_destroy(m_init_odr); odr_destroy(m_idAuthentication_odr); delete m_last_query; - xfree (m_last_resultSetId); - xfree (m_cookie); + xfree(m_last_resultSetId); + xfree(m_cookie); } void Yaz_ProxyClient::pre_init_client() @@ -3509,6 +3873,7 @@ void Yaz_Proxy::pre_init() const char *zurl_in_use[MAX_ZURL_PLEX]; int limit_bw, limit_pdu, limit_req, limit_search; int target_idletime, client_idletime; + int max_sockets = m_max_sockets; int max_clients; int keepalive_limit_bw, keepalive_limit_pdu; int pre_init; @@ -3530,6 +3895,7 @@ void Yaz_Proxy::pre_init() &limit_bw, &limit_pdu, &limit_req, &limit_search, &target_idletime, &client_idletime, + &max_sockets, &max_clients, &keepalive_limit_bw, &keepalive_limit_pdu, @@ -3573,7 +3939,8 @@ void Yaz_Proxy::pre_init() "sparew=%d preinit=%d",m_session_str, name, zurl_in_use[j], in_use, other, spare, spare_waiting, pre_init); - if (spare + spare_waiting < pre_init) + if (spare + spare_waiting < pre_init + && in_use + spare + spare_waiting + other < max_sockets) { c = new Yaz_ProxyClient(m_PDU_Observable->clone(), this); c->m_next = m_clientPool; @@ -3614,7 +3981,7 @@ void Yaz_Proxy::timeoutNotify() case timeout_busy: inc_request_no(); m_in_queue.clear(); - yaz_log (YLOG_LOG, "%sTimeout (client to proxy)", m_session_str); + yaz_log(YLOG_LOG, "%sTimeout (client to proxy)", m_session_str); dec_ref(); break; case timeout_reduce: @@ -3648,7 +4015,7 @@ void Yaz_ProxyClient::timeoutNotify() if (m_server) m_server->inc_request_no(); - yaz_log (YLOG_LOG, "%sTimeout (proxy to target) %s", get_session_str(), + yaz_log(YLOG_LOG, "%sTimeout (proxy to target) %s", get_session_str(), get_hostname()); if (m_server) @@ -3697,8 +4064,8 @@ const char *Yaz_Proxy::option(const char *name, const char *value) { if (!strcmp (name, "optimize")) { if (value) { - xfree (m_optimize); - m_optimize = xstrdup (value); + xfree(m_optimize); + m_optimize = xstrdup(value); } return m_optimize; } @@ -3739,7 +4106,7 @@ void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len) m_pdu_recv++; m_waiting = 0; if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER) - yaz_log (YLOG_LOG, "%sReceiving %s from %s %d bytes", get_session_str(), + yaz_log(YLOG_LOG, "%sReceiving %s from %s %d bytes", get_session_str(), apdu_name(apdu), get_hostname(), len); if (apdu->which == Z_APDU_initResponse) { @@ -3757,7 +4124,7 @@ void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len) *apdu->u.initResponse->maximumRecordSize; Z_InitResponse *ir = apdu->u.initResponse; - + // apply YAZ Proxy version char *imv0 = ir->implementationVersion; char *imv1 = (char*) @@ -3769,7 +4136,7 @@ void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len) strcat(imv1, "/" VERSION); #endif ir->implementationVersion = imv1; - + // apply YAZ Proxy implementation name char *im0 = ir->implementationName; char *im1 = (char*) @@ -3897,19 +4264,19 @@ void Yaz_Proxy::base64_decode(const char *base64, char *buf, int buf_len) char ch = (char) (ch_ptr - base64_chars); switch (index) { - case 1: - buf[buf_pos] = ch << 2; - break; - case 2: - buf[buf_pos++] += (ch & 0x30) >> 4; - buf[buf_pos] = (ch & 0x0f) << 4; - break; - case 3: - buf[buf_pos++] += (ch & 0x3c) >> 2; - buf[buf_pos] = (ch & 0x03) << 6; - break; - case 4: - buf[buf_pos++] += ch; + case 1: + buf[buf_pos] = ch << 2; + break; + case 2: + buf[buf_pos++] += (ch & 0x30) >> 4; + buf[buf_pos] = (ch & 0x0f) << 4; + break; + case 3: + buf[buf_pos++] += (ch & 0x3c) >> 2; + buf[buf_pos] = (ch & 0x03) << 6; + break; + case 4: + buf[buf_pos++] += ch; } if (index < 4) index++; @@ -3922,6 +4289,7 @@ void Yaz_Proxy::base64_decode(const char *base64, char *buf, int buf_len) /* * Local variables: * c-basic-offset: 4 + * c-file-style: "Stroustrup" * indent-tabs-mode: nil * End: * vim: shiftwidth=4 tabstop=8 expandtab