1 /* This file is part of Metaproxy.
2 Copyright (C) 2005-2013 Index Data
4 Metaproxy is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
9 Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
20 #include <metaproxy/filter.hpp>
21 #include <metaproxy/package.hpp>
22 #include <metaproxy/util.hpp>
23 #include "filter_http_rewrite.hpp"
24 #include "html_parser.hpp"
30 #include <boost/regex.hpp>
31 #include <boost/lexical_cast.hpp>
32 #include <boost/algorithm/string.hpp>
36 namespace mp = metaproxy_1;
37 namespace yf = mp::filter;
39 namespace metaproxy_1 {
41 class HttpRewrite::Replace {
47 std::map<int, std::string> group_index;
49 const std::map<std::string, std::string> & vars) const;
50 void parse_groups(std::string pattern);
53 class HttpRewrite::Rule {
55 std::list<Replace> replace_list;
56 const std::string test_patterns(
57 std::map<std::string, std::string> & vars,
58 const std::string & txt, bool anchor);
60 class HttpRewrite::Within {
69 class HttpRewrite::Content {
72 boost::regex content_re;
73 std::list<Within> within_list;
74 void configure(const xmlNode *ptr,
75 std::map<std::string, RulePtr > &rules);
76 void quoted_literal(std::string &content,
77 std::map<std::string, std::string> &vars) const;
79 class HttpRewrite::Phase {
83 std::list<Content> content_list;
84 void rewrite_reqline(mp::odr & o, Z_HTTP_Request *hreq,
85 std::map<std::string, std::string> & vars) const;
86 void rewrite_headers(mp::odr & o, Z_HTTP_Header *headers,
87 std::map<std::string, std::string> & vars) const;
88 void rewrite_body(mp::odr & o,
89 const char *content_type,
90 char **content_buf, int *content_len,
91 std::map<std::string, std::string> & vars) const;
93 class HttpRewrite::Event : public HTMLParserEvent {
94 void openTagStart(const char *tag, int tag_len);
95 void anyTagEnd(const char *tag, int tag_len, int close_it);
96 void attribute(const char *tag, int tag_len,
97 const char *attr, int attr_len,
98 const char *value, int val_len,
100 void closeTag(const char *tag, int tag_len);
101 void text(const char *value, int len);
102 const Content *m_content;
104 std::stack<std::list<Within>::const_iterator> s_within;
105 std::map<std::string, std::string> &m_vars;
107 Event(const Content *p, std::map<std::string, std::string> &vars);
109 const char *result();
114 yf::HttpRewrite::HttpRewrite() :
115 req_phase(new Phase), res_phase(new Phase)
119 yf::HttpRewrite::~HttpRewrite()
123 void yf::HttpRewrite::process(mp::Package & package) const
125 yaz_log(YLOG_LOG, "HttpRewrite begins....");
126 Z_GDU *gdu = package.request().get();
127 //map of request/response vars
128 std::map<std::string, std::string> vars;
129 //we have an http req
130 if (gdu && gdu->which == Z_GDU_HTTP_Request)
132 Z_HTTP_Request *hreq = gdu->u.HTTP_Request;
134 req_phase->rewrite_reqline(o, hreq, vars);
135 yaz_log(YLOG_LOG, ">> Request headers");
136 req_phase->rewrite_headers(o, hreq->headers, vars);
137 req_phase->rewrite_body(o,
138 z_HTTP_header_lookup(hreq->headers,
140 &hreq->content_buf, &hreq->content_len,
142 package.request() = gdu;
145 gdu = package.response().get();
146 if (gdu && gdu->which == Z_GDU_HTTP_Response)
148 Z_HTTP_Response *hres = gdu->u.HTTP_Response;
149 yaz_log(YLOG_LOG, "Response code %d", hres->code);
151 yaz_log(YLOG_LOG, "<< Respose headers");
152 res_phase->rewrite_headers(o, hres->headers, vars);
153 res_phase->rewrite_body(o,
154 z_HTTP_header_lookup(hres->headers,
156 &hres->content_buf, &hres->content_len,
158 package.response() = gdu;
162 void yf::HttpRewrite::Phase::rewrite_reqline (mp::odr & o,
163 Z_HTTP_Request *hreq,
164 std::map<std::string, std::string> & vars) const
166 //rewrite the request line
168 if (strstr(hreq->path, "http://") == hreq->path)
170 yaz_log(YLOG_LOG, "Path in the method line is absolute, "
171 "possibly a proxy request");
176 //TODO what about proto
177 const char *host = z_HTTP_header_lookup(hreq->headers, "Host");
186 std::list<Content>::const_iterator cit = content_list.begin();
187 for (; cit != content_list.end(); cit++)
188 if (cit->type == "headers")
191 if (cit == content_list.end())
194 std::list<Within>::const_iterator it = cit->within_list.begin();
195 for (; it != cit->within_list.end(); it++)
198 RulePtr rule = it->rule;
199 yaz_log(YLOG_LOG, "Proxy request URL is %s", path.c_str());
200 std::string npath = rule->test_patterns(vars, path, true);
203 yaz_log(YLOG_LOG, "Rewritten request URL is %s", npath.c_str());
204 hreq->path = odr_strdup(o, npath.c_str());
209 void yf::HttpRewrite::Phase::rewrite_headers(mp::odr & o,
210 Z_HTTP_Header *headers,
211 std::map<std::string, std::string> & vars) const
213 std::list<Content>::const_iterator cit = content_list.begin();
214 for (; cit != content_list.end(); cit++)
215 if (cit->type == "headers")
218 if (cit == content_list.end())
221 for (Z_HTTP_Header *header = headers; header; header = header->next)
223 std::list<Within>::const_iterator it = cit->within_list.begin();
224 for (; it != cit->within_list.end(); it++)
226 if (it->header.length() > 0 &&
227 yaz_strcasecmp(it->header.c_str(), header->name) == 0)
229 std::string sheader(header->name);
231 sheader += header->value;
233 RulePtr rule = it->rule;
234 std::string out = rule->test_patterns(vars, sheader, true);
237 size_t pos = out.find(": ");
238 if (pos == std::string::npos)
240 yaz_log(YLOG_LOG, "Header malformed during rewrite, ignoring");
243 header->name = odr_strdup(o, out.substr(0, pos).c_str());
244 header->value = odr_strdup(o,
246 std::string::npos).c_str());
253 void yf::HttpRewrite::Phase::rewrite_body(
255 const char *content_type,
258 std::map<std::string, std::string> & vars) const
260 std::list<Content>::const_iterator cit = content_list.begin();
261 for (; cit != content_list.end(); cit++)
263 yaz_log(YLOG_LOG, "rewrite_body: content_type=%s type=%s",
264 content_type, cit->type.c_str());
265 if (cit->type != "headers"
266 && regex_match(content_type, cit->content_re))
269 if (cit == content_list.end())
275 for (i = 0; i < *content_len; i++)
276 if ((*content_buf)[i] == 0)
277 return; // binary content. skip
279 if (cit->type == "html")
282 Event ev(&*cit, vars);
284 parser.set_verbose(m_verbose);
286 std::string buf(*content_buf, *content_len);
288 parser.parse(ev, buf.c_str());
289 const char *res = ev.result();
290 *content_buf = odr_strdup(o, res);
291 *content_len = strlen(res);
293 if (cit->type == "quoted-literal")
295 std::string content(*content_buf, *content_len);
296 cit->quoted_literal(content, vars);
297 *content_buf = odr_strdup(o, content.c_str());
298 *content_len = strlen(*content_buf);
303 yf::HttpRewrite::Event::Event(const Content *p,
304 std::map<std::string, std::string> & vars
305 ) : m_content(p), m_vars(vars)
310 yf::HttpRewrite::Event::~Event()
315 const char *yf::HttpRewrite::Event::result()
317 return wrbuf_cstr(m_w);
320 void yf::HttpRewrite::Event::openTagStart(const char *tag, int tag_len)
322 wrbuf_putc(m_w, '<');
323 wrbuf_write(m_w, tag, tag_len);
325 std::string t(tag, tag_len);
326 std::list<Within>::const_iterator it = m_content->within_list.begin();
327 for (; it != m_content->within_list.end(); it++)
329 if (it->tag.length() > 0 && yaz_strcasecmp(it->tag.c_str(),
332 std::vector<std::string> attr;
333 boost::split(attr, it->attr, boost::is_any_of(","));
335 for (i = 0; i < attr.size(); i++)
337 if (attr[i].compare("#text") == 0)
347 void yf::HttpRewrite::Event::anyTagEnd(const char *tag, int tag_len,
352 if (!s_within.empty())
354 std::list<Within>::const_iterator it = s_within.top();
355 std::string t(tag, tag_len);
356 if (yaz_strcasecmp(it->tag.c_str(), t.c_str()) == 0)
361 wrbuf_putc(m_w, '/');
362 wrbuf_putc(m_w, '>');
365 void yf::HttpRewrite::Event::attribute(const char *tag, int tag_len,
366 const char *attr, int attr_len,
367 const char *value, int val_len,
370 std::list<Within>::const_iterator it = m_content->within_list.begin();
373 for (; it != m_content->within_list.end(); it++)
375 std::string t(tag, tag_len);
376 if (it->tag.length() == 0 ||
377 yaz_strcasecmp(it->tag.c_str(), t.c_str()) == 0)
379 std::string a(attr, attr_len);
380 std::vector<std::string> attr;
381 boost::split(attr, it->attr, boost::is_any_of(","));
383 for (i = 0; i < attr.size(); i++)
385 if (attr[i].compare("#text") &&
386 yaz_strcasecmp(attr[i].c_str(), a.c_str()) == 0)
394 wrbuf_putc(m_w, ' ');
395 wrbuf_write(m_w, attr, attr_len);
398 wrbuf_puts(m_w, "=");
399 wrbuf_puts(m_w, sep);
404 std::string input(value, val_len);
405 output = it->rule->test_patterns(m_vars, input, true);
408 wrbuf_write(m_w, value, val_len);
410 wrbuf_puts(m_w, output.c_str());
411 wrbuf_puts(m_w, sep);
415 void yf::HttpRewrite::Event::closeTag(const char *tag, int tag_len)
417 if (!s_within.empty())
419 std::list<Within>::const_iterator it = s_within.top();
420 std::string t(tag, tag_len);
421 if (yaz_strcasecmp(it->tag.c_str(), t.c_str()) == 0)
424 wrbuf_puts(m_w, "</");
425 wrbuf_write(m_w, tag, tag_len);
428 void yf::HttpRewrite::Event::text(const char *value, int len)
430 std::list<Within>::const_iterator it = m_content->within_list.end();
431 if (!s_within.empty())
434 if (it != m_content->within_list.end())
436 std::string input(value, len);
437 output = it->rule->test_patterns(m_vars, input, false);
440 wrbuf_write(m_w, value, len);
442 wrbuf_puts(m_w, output.c_str());
445 const std::string yf::HttpRewrite::Rule::test_patterns(
446 std::map<std::string, std::string> & vars,
447 const std::string & txt, bool anchor)
451 std::string::const_iterator start, end;
456 std::list<Replace>::iterator bit = replace_list.end();
458 std::string::const_iterator best_pos = txt.end();
459 std::list<Replace>::iterator it = replace_list.begin();
460 for (; it != replace_list.end(); it++)
462 if (it->start_anchor && !first)
464 if (regex_search(start, end, it->what, it->re))
466 if (it->what[0].first < best_pos)
468 best_pos = it->what[0].first;
473 if (bit == replace_list.end())
478 for (i = 1; i < bit->what.size(); ++i)
480 //check if the group is named
481 std::map<int, std::string>::const_iterator git
482 = bit->group_index.find(i);
483 if (git != bit->group_index.end())
485 vars[git->second] = bit->what[i];
489 //prepare replacement string
490 std::string rvalue = bit->sub_vars(vars);
491 yaz_log(YLOG_LOG, "! Rewritten '%s' to '%s'",
492 bit->what.str(0).c_str(), rvalue.c_str());
493 out.append(start, bit->what[0].first);
495 start = bit->what[0].second; //move search forward
497 if (start != txt.begin())
498 out.append(start, end);
502 void yf::HttpRewrite::Replace::parse_groups(std::string pattern)
506 const std::string &str = pattern;
508 start_anchor = str[0] == '^';
509 yaz_log(YLOG_LOG, "Parsing groups from '%s'", str.c_str());
510 for (size_t i = 0; i < str.size(); ++i)
513 if (!esc && str[i] == '\\')
518 if (!esc && str[i] == '(') //group starts
521 if (i+1 < str.size() && str[i+1] == '?') //group with attrs
524 if (i+1 < str.size() && str[i+1] == ':') //non-capturing
526 if (gnum > 0) gnum--;
532 if (i+1 < str.size() && str[i+1] == 'P') //optional, python
534 if (i+1 < str.size() && str[i+1] == '<') //named
539 while (++i < str.size())
541 if (str[i] == '>') { term = true; break; }
542 if (!isalnum(str[i]))
543 throw mp::filter::FilterException
544 ("Only alphanumeric chars allowed, found "
548 + boost::lexical_cast<std::string>(i));
552 throw mp::filter::FilterException
553 ("Unterminated group name '" + gname
554 + " in '" + str +"'");
555 group_index[gnum] = gname;
556 yaz_log(YLOG_LOG, "Found named group '%s' at $%d",
557 gname.c_str(), gnum);
566 std::string yf::HttpRewrite::Replace::sub_vars(
567 const std::map<std::string, std::string> & vars) const
571 const std::string & in = recipe;
572 for (size_t i = 0; i < in.size(); ++i)
574 if (!esc && in[i] == '\\')
579 if (!esc && in[i] == '$') //var
581 if (i+1 < in.size() && in[i+1] == '{') //ref prefix
586 while (++i < in.size())
588 if (in[i] == '}') { term = true; break; }
591 if (!term) throw mp::filter::FilterException
592 ("Unterminated var ref in '"+in+"' at "
593 + boost::lexical_cast<std::string>(i));
594 std::map<std::string, std::string>::const_iterator it
596 if (it != vars.end())
603 throw mp::filter::FilterException
604 ("Malformed or trimmed var ref in '"
605 +in+"' at "+boost::lexical_cast<std::string>(i));
616 yf::HttpRewrite::Phase::Phase() : m_verbose(0)
620 void yf::HttpRewrite::Content::quoted_literal(
621 std::string &content,
622 std::map<std::string, std::string> &vars) const
625 const char *cp = content.c_str();
626 const char *cp0 = cp;
629 if (*cp == '"' || *cp == '\'')
633 res.append(cp0, cp - cp0);
637 if (cp[-1] != '\\' && *cp == m)
645 std::list<Within>::const_iterator it = within_list.begin();
646 std::string s(cp0, cp - cp0);
647 if (it != within_list.end())
649 RulePtr rule = it->rule;
651 r = rule->test_patterns(vars, s, true);
658 else if (*cp == '/' && cp[1] == '/')
660 while (cp[1] && cp[1] != '\n')
665 res.append(cp0, cp - cp0);
669 void yf::HttpRewrite::Content::configure(
670 const xmlNode *ptr, std::map<std::string, RulePtr > &rules)
672 for (; ptr; ptr = ptr->next)
674 if (ptr->type != XML_ELEMENT_NODE)
676 if (!strcmp((const char *) ptr->name, "within"))
678 static const char *names[6] =
679 { "header", "attr", "tag", "rule", "reqline", 0 };
680 std::string values[5];
681 mp::xml::parse_attr(ptr, names, values);
683 w.header = values[0];
686 std::map<std::string,RulePtr>::const_iterator it =
687 rules.find(values[3]);
688 if (it == rules.end())
689 throw mp::filter::FilterException
690 ("Reference to non-existing rule '" + values[3] +
691 "' in http_rewrite filter");
693 w.reqline = values[4] == "1";
694 within_list.push_back(w);
699 void yf::HttpRewrite::configure_phase(const xmlNode *ptr, Phase &phase)
701 static const char *names[2] = { "verbose", 0 };
702 std::string values[1];
704 mp::xml::parse_attr(ptr, names, values);
706 phase.m_verbose = atoi(values[0].c_str());
708 std::map<std::string, RulePtr > rules;
709 for (ptr = ptr->children; ptr; ptr = ptr->next)
711 if (ptr->type != XML_ELEMENT_NODE)
713 else if (!strcmp((const char *) ptr->name, "rule"))
715 static const char *names[2] = { "name", 0 };
716 std::string values[1];
717 values[0] = "default";
718 mp::xml::parse_attr(ptr, names, values);
720 RulePtr rule(new Rule);
721 for (xmlNode *p = ptr->children; p; p = p->next)
723 if (p->type != XML_ELEMENT_NODE)
725 if (!strcmp((const char *) p->name, "rewrite"))
729 const struct _xmlAttr *attr;
730 for (attr = p->properties; attr; attr = attr->next)
732 if (!strcmp((const char *) attr->name, "from"))
733 from = mp::xml::get_text(attr->children);
734 else if (!strcmp((const char *) attr->name, "to"))
735 replace.recipe = mp::xml::get_text(attr->children);
737 throw mp::filter::FilterException
739 + std::string((const char *) attr->name)
740 + " in rewrite section of http_rewrite");
742 yaz_log(YLOG_LOG, "Found rewrite rule from '%s' to '%s'",
743 from.c_str(), replace.recipe.c_str());
746 replace.parse_groups(from);
747 rule->replace_list.push_back(replace);
751 throw mp::filter::FilterException
753 + std::string((const char *) p->name)
754 + " in http_rewrite filter");
756 rules[values[0]] = rule;
758 else if (!strcmp((const char *) ptr->name, "content"))
760 static const char *names[3] =
761 { "type", "mime", 0 };
762 std::string values[2];
763 mp::xml::parse_attr(ptr, names, values);
764 if (values[0].empty())
766 throw mp::filter::FilterException
767 ("Missing attribute, type for for element "
768 + std::string((const char *) ptr->name)
769 + " in http_rewrite filter");
774 // if (!values[1].empty())
775 c.content_re = values[1];
776 c.configure(ptr->children, rules);
777 phase.content_list.push_back(c);
781 throw mp::filter::FilterException
783 + std::string((const char *) ptr->name)
784 + " in http_rewrite filter");
789 void yf::HttpRewrite::configure(const xmlNode * ptr, bool test_only,
792 for (ptr = ptr->children; ptr; ptr = ptr->next)
794 if (ptr->type != XML_ELEMENT_NODE)
796 else if (!strcmp((const char *) ptr->name, "request"))
798 configure_phase(ptr, *req_phase);
800 else if (!strcmp((const char *) ptr->name, "response"))
802 configure_phase(ptr, *res_phase);
806 throw mp::filter::FilterException
808 + std::string((const char *) ptr->name)
809 + " in http_rewrite1 filter");
814 static mp::filter::Base* filter_creator()
816 return new mp::filter::HttpRewrite;
820 struct metaproxy_1_filter_struct metaproxy_1_filter_http_rewrite = {
831 * c-file-style: "Stroustrup"
832 * indent-tabs-mode: nil
834 * vim: shiftwidth=4 tabstop=8 expandtab