From: Adam Dickmeiss Date: Mon, 29 Jun 2009 14:05:48 +0000 (+0200) Subject: Implement limit filter (bug #2697, bug #2698) X-Git-Tag: v1.0.18~12 X-Git-Url: http://git.indexdata.com/?a=commitdiff_plain;h=ddc156360c0c46ef669accde05fea1106cc6b244;p=metaproxy-moved-to-github.git Implement limit filter (bug #2697, bug #2698) --- diff --git a/doc/Makefile.am b/doc/Makefile.am index 0f51cfd..f9bed9e 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -13,6 +13,7 @@ XMLMAN = metaproxy.xml \ cql_rpn.xml \ frontend_net.xml \ http_file.xml \ + limit.xml \ load_balance.xml \ log.xml \ multi.xml \ @@ -32,6 +33,7 @@ MANFILES = metaproxy.1 \ cql_rpn.3mp \ frontend_net.3mp \ http_file.3mp \ + limit.3mp \ load_balance.3mp \ log.3mp \ multi.3mp query_rewrite.3mp \ diff --git a/doc/limit.xml b/doc/limit.xml new file mode 100644 index 0000000..67431f2 --- /dev/null +++ b/doc/limit.xml @@ -0,0 +1,73 @@ + + + %idcommon; +]> + + + limit + 3mp + Metaproxy Module + + + + limit + Metaproxy Module for imposing resource limits + + + DESCRIPTION + + This filter offers a way to limit access for a single session to + a resource (target) in order to obtain a fair resource sharing. + + + The limit section specifies bandwidth/pdu requests limits for an active + session. The filter records bandwidth/pdu requests during the last 60 + seconds (1 minute). The limit may include the elements bandwidth, pdu, + retrieve and search. The bandwidth measures the number of bytes + transferred within the last minute. The pdu is the number of requests + in the last minute. The retrieve holds the maximum records to which may + be retrieved in one Present Request. + The search is the maximum number of searches within the last minute. + + + + EXAMPLES + + Configuration: + + + +]]> + + + + + SEE ALSO + + + metaproxy + 1 + + + + + ©right; + + + diff --git a/etc/config-shared1.xml b/etc/config-shared1.xml index 141aaa8..0a3ca95 100644 --- a/etc/config-shared1.xml +++ b/etc/config-shared1.xml @@ -15,6 +15,9 @@ F + + + diff --git a/src/Makefile.am b/src/Makefile.am index 8448fa3..c369ae8 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -46,7 +46,8 @@ libmetaproxy_la_SOURCES = \ sru_util.cpp sru_util.hpp \ thread_pool_observer.cpp thread_pool_observer.hpp \ util.cpp util.hpp \ - xmlutil.cpp xmlutil.hpp + xmlutil.cpp xmlutil.hpp \ + filter_limit.cpp filter_limit.hpp libmetaproxy_la_LIBADD = $(YAZPPLALIB) $(BOOST_LIB) $(BOOST_THREAD_LIB) diff --git a/src/factory_static.cpp b/src/factory_static.cpp index 73ecf51..b8599ea 100644 --- a/src/factory_static.cpp +++ b/src/factory_static.cpp @@ -44,6 +44,7 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA #include "filter_virt_db.hpp" #include "filter_z3950_client.hpp" #include "filter_zeerex_explain.hpp" +#include "filter_limit.hpp" namespace mp = metaproxy_1; @@ -67,6 +68,7 @@ mp::FactoryStatic::FactoryStatic() &metaproxy_1_filter_virt_db, &metaproxy_1_filter_z3950_client, &metaproxy_1_filter_zeerex_explain, + &metaproxy_1_filter_limit, 0 }; int i; diff --git a/src/filter_limit.cpp b/src/filter_limit.cpp new file mode 100644 index 0000000..3158ee0 --- /dev/null +++ b/src/filter_limit.cpp @@ -0,0 +1,265 @@ +/* This file is part of Metaproxy. + Copyright (C) 2005-2009 Index Data + +Metaproxy is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. + +Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include "config.hpp" +#include "filter_limit.hpp" + +#include +#include +#include "package.hpp" +#include "util.hpp" + +namespace mp = metaproxy_1; +namespace yf = mp::filter; + +namespace metaproxy_1 { + namespace filter { + class Limit::Ses { + public: + Yaz_bw bw_stat; + Yaz_bw pdu_stat; + Yaz_bw search_stat; + Ses() : bw_stat(60), pdu_stat(60), search_stat(60) {}; + }; + class Limit::Impl { + public: + Impl(); + ~Impl(); + void process(metaproxy_1::Package & package); + void configure(const xmlNode * ptr); + private: + + boost::mutex m_session_mutex; + std::map m_sessions; + + int m_bw_max; + int m_pdu_max; + int m_search_max; + int m_max_record_retrieve; + }; + } +} + +// define Pimpl wrapper forwarding to Impl + +yf::Limit::Limit() : m_p(new Impl) +{ +} + +yf::Limit::~Limit() +{ // must have a destructor because of boost::scoped_ptr +} + +void yf::Limit::configure(const xmlNode *xmlnode, bool test_only) +{ + m_p->configure(xmlnode); +} + +void yf::Limit::process(mp::Package &package) const +{ + m_p->process(package); +} + + +// define Implementation stuff + +yf::Limit::Impl::Impl() : m_bw_max(0), m_pdu_max(0), m_search_max(0), + m_max_record_retrieve(0) +{ +} + +yf::Limit::Impl::~Impl() +{ +} + +void yf::Limit::Impl::configure(const xmlNode *ptr) +{ + for (ptr = ptr->children; ptr; ptr = ptr->next) + { + if (ptr->type != XML_ELEMENT_NODE) + continue; + if (!strcmp((const char *) ptr->name, "limit")) + { + const struct _xmlAttr *attr; + for (attr = ptr->properties; attr; attr = attr->next) + { + if (!strcmp((const char *) attr->name, "bandwidth")) + m_bw_max = mp::xml::get_int(attr->children, 0); + else if (!strcmp((const char *) attr->name, "pdu")) + m_pdu_max = mp::xml::get_int(attr->children, 0); + else if (!strcmp((const char *) attr->name, "search")) + m_search_max = mp::xml::get_int(attr->children, 0); + else if (!strcmp((const char *) attr->name, "retrieve")) + m_max_record_retrieve = + mp::xml::get_int(attr->children, 0); + else + throw mp::filter::FilterException( + "Bad attribute " + std::string((const char *) + attr->name)); + } + } + else + { + throw mp::filter::FilterException("Bad element " + + std::string((const char *) + ptr->name)); + } + } +} + +void yf::Limit::Impl::process(mp::Package &package) +{ + package.move(); + int reduce = 0; + + { + boost::mutex::scoped_lock scoped_lock(m_session_mutex); + + yf::Limit::Ses *ses = 0; + + std::map::iterator it = + m_sessions.find(package.session()); + if (it != m_sessions.end()) + ses = it->second; + else + { + ses = new yf::Limit::Ses; + m_sessions[package.session()] = ses; + } + + int sz = package.request().get_size() + package.response().get_size(); + + ses->bw_stat.add_bytes(sz); + ses->pdu_stat.add_bytes(1); + + Z_GDU *gdu = package.request().get(); + if (gdu && gdu->which == Z_GDU_Z3950) + { + // we're getting a Z39.50 package + Z_APDU *apdu = gdu->u.z3950; + if (apdu->which == Z_APDU_searchRequest) + ses->search_stat.add_bytes(1); + if (m_max_record_retrieve) + { + if (apdu->which == Z_APDU_presentRequest) + { + Z_PresentRequest *pr = apdu->u.presentRequest; + if (pr->numberOfRecordsRequested && + *pr->numberOfRecordsRequested > m_max_record_retrieve) + *pr->numberOfRecordsRequested = m_max_record_retrieve; + } + } + } + + yaz_log(YLOG_LOG, "sz = %d . total = %d", sz, + ses->bw_stat.get_total()); + + int bw_total = ses->bw_stat.get_total(); + int pdu_total = ses->pdu_stat.get_total(); + int search_total = ses->search_stat.get_total(); + + if (m_search_max) + reduce += search_total / m_search_max; + if (m_bw_max) + reduce += (bw_total/m_bw_max); + if (m_pdu_max) + { + if (pdu_total > m_pdu_max) + { + int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max; + reduce = (reduce > nreduce) ? reduce : nreduce; + } + } + if (package.session().is_closed()) + m_sessions.erase(package.session()); + } + if (reduce) + { + yaz_log(YLOG_LOG, "sleeping %d seconds", reduce); + sleep(reduce); + } +} + + +static mp::filter::Base* filter_creator() +{ + return new mp::filter::Limit; +} + +extern "C" { + struct metaproxy_1_filter_struct metaproxy_1_filter_limit = { + 0, + "limit", + filter_creator + }; +} + +// bandwidth class (taken from YAZ Proxy) + +Yaz_bw::Yaz_bw(int sz) +{ + m_sec = 0; + m_size = sz; + m_bucket = new int[m_size]; + m_ptr = 0; +} + +Yaz_bw::~Yaz_bw() +{ + delete [] m_bucket; +} + +int Yaz_bw::get_total() +{ + add_bytes(0); + int bw = 0; + int i; + for (i = 0; i= m_sec) + { + int d = now - m_sec; + if (d > m_size) + d = m_size; + while (--d >= 0) + { + if (++m_ptr == m_size) + m_ptr = 0; + m_bucket[m_ptr] = 0; + } + m_bucket[m_ptr] += b; + } + m_sec = now; +} + +/* + * Local variables: + * c-basic-offset: 4 + * c-file-style: "Stroustrup" + * indent-tabs-mode: nil + * End: + * vim: shiftwidth=4 tabstop=8 expandtab + */ + diff --git a/src/filter_limit.hpp b/src/filter_limit.hpp new file mode 100644 index 0000000..3b5fd57 --- /dev/null +++ b/src/filter_limit.hpp @@ -0,0 +1,67 @@ +/* This file is part of Metaproxy. + Copyright (C) 2005-2009 Index Data + +Metaproxy is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. + +Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#ifndef FILTER_LIMIT_HPP +#define FILTER_LIMIT_HPP + +#include + +#include "filter.hpp" + +namespace metaproxy_1 { + namespace filter { + class Limit : public Base { + class Impl; + class Ses; + boost::scoped_ptr m_p; + public: + Limit(); + ~Limit(); + void process(metaproxy_1::Package & package) const; + void configure(const xmlNode * ptr, bool test_only); + }; + } +} + +extern "C" { + extern struct metaproxy_1_filter_struct metaproxy_1_filter_limit; +} + +class Yaz_bw { + public: + Yaz_bw(int sz); + ~Yaz_bw(); + void add_bytes(int m); + int get_total(); + private: + long m_sec; // time of most recent bucket + int *m_bucket; + int m_ptr; + int m_size; +}; +#endif + +/* + * Local variables: + * c-basic-offset: 4 + * c-file-style: "Stroustrup" + * indent-tabs-mode: nil + * End: + * vim: shiftwidth=4 tabstop=8 expandtab + */ + diff --git a/win/makefile b/win/makefile index e3e4359..4c6755e 100644 --- a/win/makefile +++ b/win/makefile @@ -240,6 +240,7 @@ PROJECT_DLL_OBJS = \ $(OBJDIR)\filter_virt_db.obj \ $(OBJDIR)\filter_z3950_client.obj \ $(OBJDIR)\filter_zeerex_explain.obj \ + $(OBJDIR)\filter_limit.obj \ $(OBJDIR)\gduutil.obj \ $(OBJDIR)\origin.obj \ $(OBJDIR)\package.obj \ diff --git a/xml/schema/metaproxy.rnc b/xml/schema/metaproxy.rnc index 6c5686c..ebb819f 100644 --- a/xml/schema/metaproxy.rnc +++ b/xml/schema/metaproxy.rnc @@ -62,6 +62,7 @@ filter = | filter_sru_z3950 | filter_virt_db | filter_z3950_client + | filter_limit # | filter_zeerex_explain } @@ -198,6 +199,15 @@ filter_z3950_client = element mp:default_target { xsd:string }?, element mp:force_target { xsd:string }? +filter_limit = + attribute type { "limit" }, + element mp:limit { + attribute bandwidth { xsd:integer }?, + attribute pdu { xsd:integer }?, + attribute search { xsd:integer }?, + attribute retrieve { xsd:integer }? + }? + #filter_zeerex_explain = # attribute type { "zeerex_explain" }, # attribute id { xsd:NCName }?, diff --git a/xml/schema/metaproxy.rng b/xml/schema/metaproxy.rng index cccda2d..245993c 100644 --- a/xml/schema/metaproxy.rng +++ b/xml/schema/metaproxy.rng @@ -99,6 +99,7 @@ + @@ -519,6 +520,35 @@ + + + limit + + + + + + + + + + + + + + + + + + + + + + + + + +