X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=src%2Ffilter_http_rewrite.cpp;h=1badb44cee656f8a0744288b30307b207cb3a38e;hb=586d78659d671683f33ec55f4a7d32b28e345ccd;hp=80a876a86620e3a88f6ad9d4dffdd2fd75c0f103;hpb=fafb98623457b4b29bbe3a32af19e901d92106dc;p=metaproxy-moved-to-github.git

diff --git a/src/filter_http_rewrite.cpp b/src/filter_http_rewrite.cpp
index 80a876a..1badb44 100644
--- a/src/filter_http_rewrite.cpp
+++ b/src/filter_http_rewrite.cpp
@@ -1,5 +1,5 @@
 /* This file is part of Metaproxy.
-   Copyright (C) 2005-2013 Index Data
+   Copyright (C) Index Data
 
 Metaproxy is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free
@@ -42,7 +42,6 @@ namespace metaproxy_1 {
         public:
             bool start_anchor;
             boost::regex re;
-            boost::smatch what;
             std::string recipe;
             std::map<int, std::string> group_index;
             std::string sub_vars(
@@ -55,18 +54,20 @@ namespace metaproxy_1 {
             std::list<Replace> replace_list;
             bool test_patterns(
                 std::map<std::string, std::string> &vars,
-                std::string &txt, bool anchor);
+                std::string &txt, bool anchor,
+                std::list<boost::regex> &skip_list);
         };
         class HttpRewrite::Within {
         public:
-            std::string header;
-            std::string attr;
-            std::string tag;
+            boost::regex header;
+            boost::regex attr;
+            boost::regex tag;
             std::string type;
             bool reqline;
             RulePtr rule;
             bool exec(std::map<std::string, std::string> &vars,
-                      std::string &txt, bool anchor) const;
+                      std::string &txt, bool anchor,
+                      std::list<boost::regex> &skip_list) const;
         };
 
         class HttpRewrite::Content {
@@ -77,15 +78,19 @@ namespace metaproxy_1 {
             void configure(const xmlNode *ptr,
                            std::map<std::string, RulePtr > &rules);
             void quoted_literal(std::string &content,
-                                std::map<std::string, std::string> &vars) const;
+                                std::map<std::string, std::string> &vars,
+                                std::list<boost::regex> & skip_list) const;
             void parse(int verbose, std::string &content,
-                       std::map<std::string, std::string> & vars) const;
+                       std::map<std::string, std::string> & vars,
+                       std::list<boost::regex> & skip_list ) const;
         };
         class HttpRewrite::Phase {
         public:
             Phase();
             int m_verbose;
             std::list<Content> content_list;
+            void read_skip_headers(Z_HTTP_Request *hreq,
+                                   std::list<boost::regex> &skip_list);
             void rewrite_reqline(mp::odr & o, Z_HTTP_Request *hreq,
                 std::map<std::string, std::string> & vars) const;
             void rewrite_headers(mp::odr & o, Z_HTTP_Header *headers,
@@ -93,7 +98,8 @@ namespace metaproxy_1 {
             void rewrite_body(mp::odr & o,
                               const char *content_type,
                               char **content_buf, int *content_len,
-                              std::map<std::string, std::string> & vars) const;
+                              std::map<std::string, std::string> & vars,
+                              std::list<boost::regex> & skip_list ) const;
         };
         class HttpRewrite::Event : public HTMLParserEvent {
             void openTagStart(const char *tag, int tag_len);
@@ -108,8 +114,11 @@ namespace metaproxy_1 {
             WRBUF m_w;
             std::stack<std::list<Within>::const_iterator> s_within;
             std::map<std::string, std::string> &m_vars;
+            std::list<boost::regex> & m_skips;
         public:
-            Event(const Content *p, std::map<std::string, std::string> &vars);
+            Event(const Content *p,
+                  std::map<std::string, std::string> &vars,
+                  std::list<boost::regex> & skip_list );
             ~Event();
             const char *result();
         };
@@ -132,18 +141,22 @@ void yf::HttpRewrite::process(mp::Package & package) const
     //map of request/response vars
     std::map<std::string, std::string> vars;
     //we have an http req
+
+    std::list<boost::regex> skip_list;
+    
     if (gdu && gdu->which == Z_GDU_HTTP_Request)
     {
         Z_HTTP_Request *hreq = gdu->u.HTTP_Request;
         mp::odr o;
         req_phase->rewrite_reqline(o, hreq, vars);
+        res_phase->read_skip_headers(hreq, skip_list);  
         yaz_log(YLOG_LOG, ">> Request headers");
         req_phase->rewrite_headers(o, hreq->headers, vars);
         req_phase->rewrite_body(o,
                                 z_HTTP_header_lookup(hreq->headers,
                                                      "Content-Type"),
                                 &hreq->content_buf, &hreq->content_len,
-                                vars);
+                                vars, skip_list);
         package.request() = gdu;
     }
     package.move();
@@ -159,11 +172,49 @@ void yf::HttpRewrite::process(mp::Package & package) const
                                 z_HTTP_header_lookup(hres->headers,
                                                      "Content-Type"),
                                 &hres->content_buf, &hres->content_len,
-                                vars);
+                                vars, skip_list);
         package.response() = gdu;
     }
 }
 
+// Read (and remove) the X-Metaproxy-SkipLink headers
+void yf::HttpRewrite::Phase::read_skip_headers(Z_HTTP_Request *hreq,
+                                 std::list<boost::regex> &skip_list )
+{
+    std::string url(hreq->path);
+    if ( url.substr(0,7) != "http://" )
+    { // path was relative, as it usually is
+        const char *host =  z_HTTP_header_lookup(hreq->headers, "Host");
+        if (host)
+          url = "http://" + std::string(host) + hreq->path ;
+    }
+
+    while ( const char *hv = z_HTTP_header_remove( &(hreq->headers),
+        "X-Metaproxy-SkipLink") )
+    {
+        yaz_log(YLOG_LOG,"Found SkipLink '%s'", hv );
+        const char *p = strchr(hv,' ');
+        if (!p)
+            continue; // should not happen
+        std::string page(hv,p);
+        std::string link(p+1);
+        boost::regex pagere(page);
+        if ( boost::regex_search(url, pagere) )
+        {
+            yaz_log(YLOG_LOG,"SkipLink '%s' matches URL %s",
+                    page.c_str(), url.c_str() );
+            boost::regex linkre(link);
+            skip_list.push_back(linkre);
+        }
+        else
+        {
+            yaz_log(YLOG_LOG,"SkipLink ignored, '%s' does not match '%s'",
+                    url.c_str(), page.c_str() );
+        }
+    }
+}
+
+
 void yf::HttpRewrite::Phase::rewrite_reqline (mp::odr & o,
         Z_HTTP_Request *hreq,
         std::map<std::string, std::string> & vars) const
@@ -201,7 +252,8 @@ void yf::HttpRewrite::Phase::rewrite_reqline (mp::odr & o,
         if (it->reqline)
         {
             yaz_log(YLOG_LOG, "Proxy request URL is %s", path.c_str());
-            if (it->exec(vars, path, true))
+            std::list<boost::regex> dummy_skip_list; // no skips here!
+            if (it->exec(vars, path, true, dummy_skip_list))
             {
                 yaz_log(YLOG_LOG, "Rewritten request URL is %s", path.c_str());
                 hreq->path = odr_strdup(o, path.c_str());
@@ -211,7 +263,7 @@ void yf::HttpRewrite::Phase::rewrite_reqline (mp::odr & o,
 
 void yf::HttpRewrite::Phase::rewrite_headers(mp::odr & o,
         Z_HTTP_Header *headers,
-        std::map<std::string, std::string> & vars) const
+        std::map<std::string, std::string> & vars ) const
 {
     std::list<Content>::const_iterator cit = content_list.begin();
     for (; cit != content_list.end(); cit++)
@@ -226,25 +278,15 @@ void yf::HttpRewrite::Phase::rewrite_headers(mp::odr & o,
         std::list<Within>::const_iterator it = cit->within_list.begin();
         for (; it != cit->within_list.end(); it++)
         {
-            if (it->header.length() > 0 &&
-                yaz_strcasecmp(it->header.c_str(), header->name) == 0)
+            if (!it->header.empty() &&
+                regex_match(header->name, it->header))
             {
-                std::string sheader(header->name);
-                sheader += ": ";
-                sheader += header->value;
-
-                if (it->exec(vars, sheader, true))
+                // Match and replace only the header value
+                std::string hval(header->value);
+                std::list<boost::regex> dummy_skip_list; // no skips here!
+                if (it->exec(vars, hval, true, dummy_skip_list))
                 {
-                    size_t pos = sheader.find(": ");
-                    if (pos == std::string::npos)
-                    {
-                        yaz_log(YLOG_LOG, "Header malformed during rewrite, ignoring");
-                        continue;
-                    }
-                    header->name = odr_strdup(
-                        o, sheader.substr(0, pos).c_str());
-                    header->value = odr_strdup(
-                        o, sheader.substr(pos + 2, std::string::npos).c_str());
+                    header->value = odr_strdup(o, hval.c_str());
                 }
             }
         }
@@ -256,10 +298,15 @@ void yf::HttpRewrite::Phase::rewrite_body(
     const char *content_type,
     char **content_buf,
     int *content_len,
-    std::map<std::string, std::string> & vars) const
+    std::map<std::string, std::string> & vars,
+    std::list<boost::regex> & skip_list ) const
 {
     if (*content_len == 0)
         return;
+    if (!content_type) {
+        yaz_log(YLOG_LOG, "rewrite_body: null content_type, can not rewrite");
+        return;
+    }
     std::list<Content>::const_iterator cit = content_list.begin();
     for (; cit != content_list.end(); cit++)
     {
@@ -269,23 +316,29 @@ void yf::HttpRewrite::Phase::rewrite_body(
             && regex_match(content_type, cit->content_re))
             break;
     }
-    if (cit == content_list.end())
+    if (cit == content_list.end()) {
+        yaz_log(YLOG_LOG,"rewrite_body: No content rule matched %s, not rewriting",
+                content_type );  
         return;
+    }
 
     int i;
     for (i = 0; i < *content_len; i++)
-        if ((*content_buf)[i] == 0)
+        if ((*content_buf)[i] == 0) {
+            yaz_log(YLOG_LOG,"rewrite_body: Looks like binary stuff, not rewriting");
             return;  // binary content. skip
+        }
 
     std::string content(*content_buf, *content_len);
-    cit->parse(m_verbose, content, vars);
+    cit->parse(m_verbose, content, vars, skip_list);
     *content_buf = odr_strdup(o, content.c_str());
     *content_len = strlen(*content_buf);
 }
 
 yf::HttpRewrite::Event::Event(const Content *p,
-                              std::map<std::string, std::string> & vars
-    ) : m_content(p), m_vars(vars)
+                              std::map<std::string, std::string> & vars,
+                              std::list<boost::regex> & skip_list 
+    ) : m_content(p), m_vars(vars), m_skips(skip_list)
 {
     m_w = wrbuf_alloc();
 }
@@ -309,19 +362,12 @@ void yf::HttpRewrite::Event::openTagStart(const char *tag, int tag_len)
     std::list<Within>::const_iterator it = m_content->within_list.begin();
     for (; it != m_content->within_list.end(); it++)
     {
-        if (it->tag.length() > 0 && yaz_strcasecmp(it->tag.c_str(),
-                                                   t.c_str()) == 0)
+        if (!it->tag.empty() && regex_match(t, it->tag))
         {
-            std::vector<std::string> attr;
-            boost::split(attr, it->attr, boost::is_any_of(","));
-            size_t i;
-            for (i = 0; i < attr.size(); i++)
+            if (!it->attr.empty() && regex_match("#text", it->attr))
             {
-                if (attr[i].compare("#text") == 0)
-                {
-                    s_within.push(it);
-                    return;
-                }
+                s_within.push(it);
+                return;
             }
         }
     }
@@ -336,7 +382,7 @@ void yf::HttpRewrite::Event::anyTagEnd(const char *tag, int tag_len,
         {
             std::list<Within>::const_iterator it = s_within.top();
             std::string t(tag, tag_len);
-            if (yaz_strcasecmp(it->tag.c_str(), t.c_str()) == 0)
+            if (regex_match(t, it->tag))
                 s_within.pop();
         }
     }
@@ -356,19 +402,11 @@ void yf::HttpRewrite::Event::attribute(const char *tag, int tag_len,
     for (; it != m_content->within_list.end(); it++)
     {
         std::string t(tag, tag_len);
-        if (it->tag.length() == 0 ||
-            yaz_strcasecmp(it->tag.c_str(), t.c_str()) == 0)
+        if (it->tag.empty() || regex_match(t, it->tag))
         {
             std::string a(attr, attr_len);
-            std::vector<std::string> attr;
-            boost::split(attr, it->attr, boost::is_any_of(","));
-            size_t i;
-            for (i = 0; i < attr.size(); i++)
-            {
-                if (attr[i].compare("#text") &&
-                    yaz_strcasecmp(attr[i].c_str(), a.c_str()) == 0)
-                    subst = true;
-            }
+            if (!it->attr.empty() && regex_match(a, it->attr))
+                subst = true;
         }
         if (subst)
             break;
@@ -385,7 +423,7 @@ void yf::HttpRewrite::Event::attribute(const char *tag, int tag_len,
         if (subst)
         {
             std::string s(value, val_len);
-            it->exec(m_vars, s, true);
+            it->exec(m_vars, s, true, m_skips);
             wrbuf_puts(m_w, s.c_str());
         }
         else
@@ -400,7 +438,7 @@ void yf::HttpRewrite::Event::closeTag(const char *tag, int tag_len)
     {
         std::list<Within>::const_iterator it = s_within.top();
         std::string t(tag, tag_len);
-        if (yaz_strcasecmp(it->tag.c_str(), t.c_str()) == 0)
+        if (regex_match(t, it->tag))
             s_within.pop();
     }
     wrbuf_puts(m_w, "</");
@@ -415,7 +453,7 @@ void yf::HttpRewrite::Event::text(const char *value, int len)
     if (it != m_content->within_list.end())
     {
         std::string s(value, len);
-        it->exec(m_vars, s, false);
+        it->exec(m_vars, s, false, m_skips);
         wrbuf_puts(m_w, s.c_str());
     }
     else
@@ -425,7 +463,9 @@ void yf::HttpRewrite::Event::text(const char *value, int len)
 static bool embed_quoted_literal(
     std::string &content,
     std::map<std::string, std::string> &vars,
-    mp::filter::HttpRewrite::RulePtr ruleptr)
+    mp::filter::HttpRewrite::RulePtr ruleptr,
+    bool html_context,
+    std::list<boost::regex> &skip_list)
 {
     bool replace = false;
     std::string res;
@@ -433,7 +473,28 @@ static bool embed_quoted_literal(
     const char *cp0 = cp;
     while (*cp)
     {
-        if (*cp == '"' || *cp == '\'')
+        if (html_context && !strncmp(cp, "&quot;", 6))
+        {
+            cp += 6;
+            res.append(cp0, cp - cp0);
+            cp0 = cp;
+            while (*cp)
+            {
+                if (!strncmp(cp, "&quot;", 6))
+                    break;
+                if (*cp == '\n')
+                    break;
+                cp++;
+            }
+            if (!*cp)
+                break;
+            std::string s(cp0, cp - cp0);
+            if (ruleptr->test_patterns(vars, s, true, skip_list))
+                replace = true;
+            cp0 = cp;
+            res.append(s);
+        }
+        else if (*cp == '"' || *cp == '\'')
         {
             int m = *cp;
             cp++;
@@ -450,7 +511,7 @@ static bool embed_quoted_literal(
             if (!*cp)
                 break;
             std::string s(cp0, cp - cp0);
-            if (ruleptr->test_patterns(vars, s, true))
+            if (ruleptr->test_patterns(vars, s, true, skip_list))
                 replace = true;
             cp0 = cp;
             res.append(s);
@@ -469,21 +530,23 @@ static bool embed_quoted_literal(
 
 bool yf::HttpRewrite::Within::exec(
     std::map<std::string, std::string> & vars,
-    std::string & txt, bool anchor) const
+    std::string & txt, bool anchor,
+    std::list<boost::regex> & skip_list) const
 {
     if (type == "quoted-literal")
     {
-        return embed_quoted_literal(txt, vars, rule);
+        return embed_quoted_literal(txt, vars, rule, true, skip_list);
     }
     else
     {
-        return rule->test_patterns(vars, txt, anchor);
+        return rule->test_patterns(vars, txt, anchor, skip_list);
     }
 }
 
 bool yf::HttpRewrite::Rule::test_patterns(
     std::map<std::string, std::string> & vars,
-    std::string & txt, bool anchor)
+    std::string & txt, bool anchor,
+    std::list<boost::regex> & skip_list )
 {
     bool replaces = false;
     bool first = anchor;
@@ -494,46 +557,68 @@ bool yf::HttpRewrite::Rule::test_patterns(
     while (1)
     {
         std::list<Replace>::iterator bit = replace_list.end();
+        boost::smatch bwhat;
+        bool match_one = false;
         {
-            std::string::const_iterator best_pos = txt.end();
             std::list<Replace>::iterator it = replace_list.begin();
             for (; it != replace_list.end(); it++)
             {
                 if (it->start_anchor && !first)
                     continue;
-                if (regex_search(start, end, it->what, it->re))
+                boost::smatch what;
+                if (regex_search(start, end, what, it->re))
                 {
-                    if (it->what[0].first < best_pos)
+                    if (!match_one || what[0].first < bwhat[0].first)
                     {
-                        best_pos = it->what[0].first;
+                        bwhat = what;
                         bit = it;
                     }
+                    match_one = true;
                 }
             }
-            if (bit == replace_list.end())
+            if (!match_one)
                 break;
         }
         first = false;
         replaces = true;
         size_t i;
-        for (i = 1; i < bit->what.size(); ++i)
+        for (i = 1; i < bwhat.size(); ++i)
         {
             //check if the group is named
             std::map<int, std::string>::const_iterator git
                 = bit->group_index.find(i);
             if (git != bit->group_index.end())
             {   //it is
-                vars[git->second] = bit->what[i];
+                vars[git->second] = bwhat[i];
             }
 
         }
+        // Compare against skip_list
+        bool skipthis = false;
+        std::list<boost::regex>::iterator si = skip_list.begin();
+        for ( ; si != skip_list.end(); si++) {
+            if ( boost::regex_search(bwhat.str(0), *si) )
+            {
+                skipthis = true;
+                break;
+            }
+        }
         //prepare replacement string
         std::string rvalue = bit->sub_vars(vars);
-        yaz_log(YLOG_LOG, "! Rewritten '%s' to '%s'",
-                bit->what.str(0).c_str(), rvalue.c_str());
-        out.append(start, bit->what[0].first);
-        out.append(rvalue);
-        start = bit->what[0].second; //move search forward
+        out.append(start, bwhat[0].first);
+        if ( skipthis )
+        {
+            yaz_log(YLOG_LOG,"! Not rewriting '%s', skiplist match",
+                    bwhat.str(0).c_str() );
+            out.append(bwhat.str(0).c_str());
+        }
+        else
+        {
+            yaz_log(YLOG_LOG, "! Rewritten '%s' to '%s'",
+                    bwhat.str(0).c_str(), rvalue.c_str());
+            out.append(rvalue);
+        }
+        start = bwhat[0].second; //move search forward
     }
     out.append(start, end);
     txt = out;
@@ -661,12 +746,13 @@ yf::HttpRewrite::Phase::Phase() : m_verbose(0)
 void yf::HttpRewrite::Content::parse(
     int verbose,
     std::string &content,
-    std::map<std::string, std::string> &vars) const
+    std::map<std::string, std::string> &vars,
+    std::list<boost::regex> & skip_list ) const
 {
     if (type == "html")
     {
         HTMLParser parser;
-        Event ev(this, vars);
+        Event ev(this, vars, skip_list);
 
         parser.set_verbose(verbose);
 
@@ -675,17 +761,18 @@ void yf::HttpRewrite::Content::parse(
     }
     if (type == "quoted-literal")
     {
-        quoted_literal(content, vars);
+        quoted_literal(content, vars, skip_list);
     }
 }
 
 void yf::HttpRewrite::Content::quoted_literal(
     std::string &content,
-    std::map<std::string, std::string> &vars) const
+    std::map<std::string, std::string> &vars,
+    std::list<boost::regex> & skip_list ) const
 {
     std::list<Within>::const_iterator it = within_list.begin();
     if (it != within_list.end())
-        embed_quoted_literal(content, vars, it->rule);
+        embed_quoted_literal(content, vars, it->rule, false, skip_list);
 }
 
 void yf::HttpRewrite::Content::configure(
@@ -702,16 +789,52 @@ void yf::HttpRewrite::Content::configure(
             std::string values[6];
             mp::xml::parse_attr(ptr, names, values);
             Within w;
-            w.header = values[0];
-            w.attr = values[1];
-            w.tag = values[2];
-            std::map<std::string,RulePtr>::const_iterator it =
-                rules.find(values[3]);
-            if (it == rules.end())
+            if (values[0].length() > 0)
+                w.header.assign(values[0], boost::regex_constants::icase);
+            if (values[1].length() > 0)
+                w.attr.assign(values[1], boost::regex_constants::icase);
+            if (values[2].length() > 0)
+                w.tag.assign(values[2], boost::regex_constants::icase);
+
+            std::vector<std::string> rulenames;
+            boost::split(rulenames, values[3], boost::is_any_of(","));
+            if (rulenames.size() == 0)
+            {
                 throw mp::filter::FilterException
-                    ("Reference to non-existing rule '" + values[3] +
+                    ("Empty rule in '" + values[3] +
                      "' in http_rewrite filter");
-            w.rule = it->second;
+            }
+            else if (rulenames.size() == 1)
+            {
+                std::map<std::string,RulePtr>::const_iterator it =
+                    rules.find(rulenames[0]);
+                if (it == rules.end())
+                    throw mp::filter::FilterException
+                        ("Reference to non-existing rule '" + rulenames[0] +
+                         "' in http_rewrite filter");
+                w.rule = it->second;
+
+            }
+            else
+            {
+                RulePtr rule(new Rule);
+                size_t i;
+                for (i = 0; i < rulenames.size(); i++)
+                {
+                    std::map<std::string,RulePtr>::const_iterator it =
+                        rules.find(rulenames[i]);
+                    if (it == rules.end())
+                        throw mp::filter::FilterException
+                            ("Reference to non-existing rule '" + rulenames[i] +
+                             "' in http_rewrite filter");
+                    RulePtr subRule = it->second;
+                    std::list<Replace>::iterator rit =
+                        subRule->replace_list.begin();
+                    for (; rit != subRule->replace_list.end(); rit++)
+                        rule->replace_list.push_back(*rit);
+                }
+                w.rule = rule;
+            }
             w.reqline = values[4] == "1";
             w.type = values[5];
             if (w.type.empty() || w.type == "quoted-literal")
@@ -800,8 +923,8 @@ void yf::HttpRewrite::configure_phase(const xmlNode *ptr, Phase &phase)
             Content c;
 
             c.type = values[0];
-            // if (!values[1].empty())
-                c.content_re = values[1];
+            if (!values[1].empty())
+                c.content_re.assign(values[1], boost::regex::icase);
             c.configure(ptr->children, rules);
             phase.content_list.push_back(c);
         }