src/filter_http_rewrite.cpp

   1 /* This file is part of Metaproxy.
   2    Copyright (C) 2005-2013 Index Data
   3
   4 Metaproxy is free software; you can redistribute it and/or modify it under
   5 the terms of the GNU General Public License as published by the Free
   6 Software Foundation; either version 2, or (at your option) any later
   7 version.
   8
   9 Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY
  10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 for more details.
  13
  14 You should have received a copy of the GNU General Public License
  15 along with this program; if not, write to the Free Software
  16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  17 */
  18
  19 #include "config.hpp"
  20 #include <metaproxy/filter.hpp>
  21 #include <metaproxy/package.hpp>
  22 #include <metaproxy/util.hpp>
  23 #include "filter_http_rewrite.hpp"
  24 #include "html_parser.hpp"
  25
  26 #include <yaz/zgdu.h>
  27 #include <yaz/log.h>
  28
  29 #include <boost/regex.hpp>
  30 #include <boost/lexical_cast.hpp>
  31 #include <boost/algorithm/string.hpp>
  32
  33 #include <map>
  34
  35 namespace mp = metaproxy_1;
  36 namespace yf = mp::filter;
  37
  38 namespace metaproxy_1 {
  39     namespace filter {
  40         class HttpRewrite::Replace {
  41         public:
  42             std::string regex;
  43             std::string recipe;
  44             std::map<int, std::string> group_index;
  45             const std::string search_replace(
  46                 std::map<std::string, std::string> & vars,
  47                 const std::string & txt) const;
  48             std::string sub_vars (
  49                 const std::map<std::string, std::string> & vars) const;
  50             void parse_groups();
  51         };
  52
  53         class HttpRewrite::Rule {
  54         public:
  55             std::list<Replace> replace_list;
  56             const std::string test_patterns(
  57                 std::map<std::string, std::string> & vars,
  58                 const std::string & txt) const;
  59         };
  60         class HttpRewrite::Within {
  61         public:
  62             std::string header;
  63             std::string attr;
  64             std::string tag;
  65             RulePtr rule;
  66         };
  67
  68         class HttpRewrite::Phase {
  69         public:
  70             std::list<Within> within_list;
  71             void rewrite_reqline(mp::odr & o, Z_HTTP_Request *hreq,
  72                 std::map<std::string, std::string> & vars) const;
  73             void rewrite_headers(mp::odr & o, Z_HTTP_Header *headers,
  74                 std::map<std::string, std::string> & vars) const;
  75             void rewrite_body(mp::odr & o,
  76                 char **content_buf, int *content_len,
  77                 std::map<std::string, std::string> & vars) const;
  78         };
  79         class HttpRewrite::Event : public HTMLParserEvent {
  80             void openTagStart(const char *name);
  81             void anyTagEnd(const char *name);
  82             void attribute(const char *tagName,
  83                            const char *name,
  84                            const char *value,
  85                            int val_len);
  86             void closeTag(const char *name);
  87             void text(const char *value, int len);
  88             const Phase *m_phase;
  89             WRBUF m_w;
  90             std::list<Within>::const_iterator enabled_within;
  91         public:
  92             Event(const Phase *p);
  93             ~Event();
  94             const char *result();
  95         };
  96     }
  97 }
  98
  99 yf::HttpRewrite::HttpRewrite() :
 100     req_phase(new Phase), res_phase(new Phase)
 101 {
 102 }
 103
 104 yf::HttpRewrite::~HttpRewrite()
 105 {
 106 }
 107
 108 void yf::HttpRewrite::process(mp::Package & package) const
 109 {
 110     yaz_log(YLOG_LOG, "HttpRewrite begins....");
 111     Z_GDU *gdu = package.request().get();
 112     //map of request/response vars
 113     std::map<std::string, std::string> vars;
 114     //we have an http req
 115     if (gdu && gdu->which == Z_GDU_HTTP_Request)
 116     {
 117         Z_HTTP_Request *hreq = gdu->u.HTTP_Request;
 118         mp::odr o;
 119         req_phase->rewrite_reqline(o, hreq, vars);
 120         yaz_log(YLOG_LOG, ">> Request headers");
 121         req_phase->rewrite_headers(o, hreq->headers, vars);
 122         req_phase->rewrite_body(o,
 123                 &hreq->content_buf, &hreq->content_len, vars);
 124         package.request() = gdu;
 125     }
 126     package.move();
 127     gdu = package.response().get();
 128     if (gdu && gdu->which == Z_GDU_HTTP_Response)
 129     {
 130         Z_HTTP_Response *hres = gdu->u.HTTP_Response;
 131         yaz_log(YLOG_LOG, "Response code %d", hres->code);
 132         mp::odr o;
 133         yaz_log(YLOG_LOG, "<< Respose headers");
 134         res_phase->rewrite_headers(o, hres->headers, vars);
 135         res_phase->rewrite_body(o, &hres->content_buf,
 136                 &hres->content_len, vars);
 137         package.response() = gdu;
 138     }
 139 }
 140
 141 void yf::HttpRewrite::Phase::rewrite_reqline (mp::odr & o,
 142         Z_HTTP_Request *hreq,
 143         std::map<std::string, std::string> & vars) const
 144 {
 145     //rewrite the request line
 146     std::string path;
 147     if (strstr(hreq->path, "http://") == hreq->path)
 148     {
 149         yaz_log(YLOG_LOG, "Path in the method line is absolute, "
 150             "possibly a proxy request");
 151         path += hreq->path;
 152     }
 153     else
 154     {
 155         //TODO what about proto
 156         path += "http://";
 157         path += z_HTTP_header_lookup(hreq->headers, "Host");
 158         path += hreq->path;
 159     }
 160
 161     std::list<Within>::const_iterator it = within_list.begin();
 162     if (it != within_list.end())
 163     {
 164         RulePtr rule = it->rule;
 165
 166         yaz_log(YLOG_LOG, "Proxy request URL is %s", path.c_str());
 167         std::string npath = rule->test_patterns(vars, path);
 168         if (!npath.empty())
 169         {
 170             yaz_log(YLOG_LOG, "Rewritten request URL is %s", npath.c_str());
 171             hreq->path = odr_strdup(o, npath.c_str());
 172         }
 173     }
 174 }
 175
 176 void yf::HttpRewrite::Phase::rewrite_headers(mp::odr & o,
 177         Z_HTTP_Header *headers,
 178         std::map<std::string, std::string> & vars) const
 179 {
 180     for (Z_HTTP_Header *header = headers;
 181             header != 0;
 182             header = header->next)
 183     {
 184         std::string sheader(header->name);
 185         sheader += ": ";
 186         sheader += header->value;
 187         yaz_log(YLOG_LOG, "%s: %s", header->name, header->value);
 188
 189         std::list<Within>::const_iterator it = within_list.begin();
 190         if (it == within_list.end())
 191             continue;
 192         RulePtr rule = it->rule;
 193
 194         std::string out = rule->test_patterns(vars, sheader);
 195         if (!out.empty())
 196         {
 197             size_t pos = out.find(": ");
 198             if (pos == std::string::npos)
 199             {
 200                 yaz_log(YLOG_LOG, "Header malformed during rewrite, ignoring");
 201                 continue;
 202             }
 203             header->name = odr_strdup(o, out.substr(0, pos).c_str());
 204             header->value = odr_strdup(o, out.substr(pos+2,
 205                                                      std::string::npos).c_str());
 206         }
 207     }
 208 }
 209
 210 void yf::HttpRewrite::Phase::rewrite_body(mp::odr & o,
 211         char **content_buf,
 212         int *content_len,
 213         std::map<std::string, std::string> & vars) const
 214 {
 215     if (*content_buf)
 216     {
 217         HTMLParser parser;
 218         Event ev(this);
 219         std::string buf(*content_buf, *content_len);
 220
 221         parser.parse(ev, buf.c_str());
 222         std::cout << "RES\n" << ev.result() << std::endl;
 223         std::cout << "-----" << std::endl;
 224
 225
 226         std::list<Within>::const_iterator it = within_list.begin();
 227         if (it != within_list.end())
 228         {
 229             RulePtr rule = it->rule;
 230
 231             std::string body(*content_buf);
 232             std::string nbody = rule->test_patterns(vars, body);
 233             if (!nbody.empty())
 234             {
 235                 *content_buf = odr_strdup(o, nbody.c_str());
 236                 *content_len = nbody.size();
 237             }
 238         }
 239     }
 240 }
 241
 242 yf::HttpRewrite::Event::Event(const Phase *p) : m_phase(p)
 243 {
 244     m_w = wrbuf_alloc();
 245     enabled_within = m_phase->within_list.end();
 246 }
 247
 248 yf::HttpRewrite::Event::~Event()
 249 {
 250     wrbuf_destroy(m_w);
 251 }
 252
 253 const char *yf::HttpRewrite::Event::result()
 254 {
 255     return wrbuf_cstr(m_w);
 256 }
 257
 258 void yf::HttpRewrite::Event::openTagStart(const char *name)
 259 {
 260     // check if there is <within tag="x" .. />
 261     if (enabled_within == m_phase->within_list.end())
 262     {
 263         std::list<Within>::const_iterator it =
 264             m_phase->within_list.begin();
 265         for (; it != m_phase->within_list.end(); it++)
 266         {
 267             if (it->tag.length() > 0 && it->tag.compare(name) == 0)
 268             {
 269                 enabled_within = it;
 270             }
 271         }
 272     }
 273     wrbuf_putc(m_w, '<');
 274     wrbuf_puts(m_w, name);
 275 }
 276
 277 void yf::HttpRewrite::Event::anyTagEnd(const char *name)
 278 {
 279     std::list<Within>::const_iterator it = enabled_within;
 280     if (it != m_phase->within_list.end())
 281     {
 282         if (it->tag.compare(name) == 0)
 283         {
 284             enabled_within = m_phase->within_list.end();
 285         }
 286     }
 287     wrbuf_putc(m_w, '>');
 288 }
 289
 290 void yf::HttpRewrite::Event::attribute(const char *tagName,
 291                                          const char *name,
 292                                          const char *value,
 293                                          int val_len)
 294 {
 295     std::list<Within>::const_iterator it = enabled_within;
 296     bool subst = false;
 297
 298     if (it == m_phase->within_list.end())
 299     {
 300         // no active within tag.. see if a attr rule without tag applies
 301         it = m_phase->within_list.begin();
 302         for (; it != m_phase->within_list.end(); it++)
 303         {
 304             if (it->attr.length() > 0 && it->tag.length() == 0)
 305                 break;
 306         }
 307     }
 308     if (it != m_phase->within_list.end())
 309     {
 310         std::vector<std::string> attr;
 311         boost::split(attr, it->attr, boost::is_any_of(","));
 312         size_t i;
 313         for (i = 0; i < attr.size(); i++)
 314         {
 315             if (attr[i].compare("#text") && attr[i].compare(tagName) == 0)
 316             {
 317                 subst = true;
 318             }
 319         }
 320     }
 321
 322     wrbuf_putc(m_w, ' ');
 323     wrbuf_puts(m_w, name);
 324     wrbuf_puts(m_w, "\"");
 325     wrbuf_write(m_w, value, val_len);
 326     if (subst)
 327         wrbuf_puts(m_w, " SUBST");
 328     wrbuf_puts(m_w, "\"");
 329 }
 330
 331 void yf::HttpRewrite::Event::closeTag(const char *name)
 332 {
 333     std::list<Within>::const_iterator it = enabled_within;
 334     if (it != m_phase->within_list.end())
 335     {
 336         if (it->tag.compare(name) == 0)
 337         {
 338             enabled_within = m_phase->within_list.end();
 339         }
 340     }
 341     wrbuf_puts(m_w, "</");
 342     wrbuf_puts(m_w, name);
 343 }
 344
 345 void yf::HttpRewrite::Event::text(const char *value, int len)
 346 {
 347     std::list<Within>::const_iterator it = enabled_within;
 348     bool subst = false;
 349
 350     if (it != m_phase->within_list.end())
 351     {
 352         subst = true;
 353         if (it->attr.length() > 0)
 354         {
 355             subst = false;
 356             std::vector<std::string> attr;
 357             boost::split(attr, it->attr, boost::is_any_of(","));
 358             size_t i;
 359             for (i = 0; i < attr.size(); i++)
 360             {
 361                 if (attr[i].compare("#text") == 0)
 362                 {
 363                     subst = true;
 364                 }
 365             }
 366         }
 367     }
 368     wrbuf_write(m_w, value, len);
 369     if (subst)
 370         wrbuf_puts(m_w, "<!-- SUBST -->");
 371 }
 372
 373
 374 /**
 375  * Tests pattern from the vector in order and executes recipe on
 376  the first match.
 377  */
 378 const std::string yf::HttpRewrite::Rule::test_patterns(
 379         std::map<std::string, std::string> & vars,
 380         const std::string & txt) const
 381 {
 382     std::list<Replace>::const_iterator it = replace_list.begin();
 383
 384     for (; it != replace_list.end(); it++)
 385     {
 386         std::string out = it->search_replace(vars, txt);
 387         if (!out.empty()) return out;
 388     }
 389     return "";
 390 }
 391
 392 const std::string yf::HttpRewrite::Replace::search_replace(
 393         std::map<std::string, std::string> & vars,
 394         const std::string & txt) const
 395 {
 396     //exec regex against value
 397     boost::regex re(regex);
 398     boost::smatch what;
 399     std::string::const_iterator start, end;
 400     start = txt.begin();
 401     end = txt.end();
 402     std::string out;
 403     while (regex_search(start, end, what, re)) //find next full match
 404     {
 405         size_t i;
 406         for (i = 1; i < what.size(); ++i)
 407         {
 408             //check if the group is named
 409             std::map<int, std::string>::const_iterator it
 410                 = group_index.find(i);
 411             if (it != group_index.end())
 412             {   //it is
 413                 if (!what[i].str().empty())
 414                     vars[it->second] = what[i];
 415             }
 416
 417         }
 418         //prepare replacement string
 419         std::string rvalue = sub_vars(vars);
 420         yaz_log(YLOG_LOG, "! Rewritten '%s' to '%s'",
 421                 what.str(0).c_str(), rvalue.c_str());
 422         out.append(start, what[0].first);
 423         out.append(rvalue);
 424         start = what[0].second; //move search forward
 425     }
 426     //if we had a match cat the last part
 427     if (start != txt.begin())
 428         out.append(start, end);
 429     return out;
 430 }
 431
 432 void yf::HttpRewrite::Replace::parse_groups()
 433 {
 434     int gnum = 0;
 435     bool esc = false;
 436     const std::string & str = regex;
 437     std::string res;
 438     yaz_log(YLOG_LOG, "Parsing groups from '%s'", str.c_str());
 439     for (size_t i = 0; i < str.size(); ++i)
 440     {
 441         res += str[i];
 442         if (!esc && str[i] == '\\')
 443         {
 444             esc = true;
 445             continue;
 446         }
 447         if (!esc && str[i] == '(') //group starts
 448         {
 449             gnum++;
 450             if (i+1 < str.size() && str[i+1] == '?') //group with attrs
 451             {
 452                 i++;
 453                 if (i+1 < str.size() && str[i+1] == ':') //non-capturing
 454                 {
 455                     if (gnum > 0) gnum--;
 456                     res += str[i];
 457                     i++;
 458                     res += str[i];
 459                     continue;
 460                 }
 461                 if (i+1 < str.size() && str[i+1] == 'P') //optional, python
 462                     i++;
 463                 if (i+1 < str.size() && str[i+1] == '<') //named
 464                 {
 465                     i++;
 466                     std::string gname;
 467                     bool term = false;
 468                     while (++i < str.size())
 469                     {
 470                         if (str[i] == '>') { term = true; break; }
 471                         if (!isalnum(str[i]))
 472                             throw mp::filter::FilterException
 473                                 ("Only alphanumeric chars allowed, found "
 474                                  " in '"
 475                                  + str
 476                                  + "' at "
 477                                  + boost::lexical_cast<std::string>(i));
 478                         gname += str[i];
 479                     }
 480                     if (!term)
 481                         throw mp::filter::FilterException
 482                             ("Unterminated group name '" + gname
 483                              + " in '" + str +"'");
 484                     group_index[gnum] = gname;
 485                     yaz_log(YLOG_LOG, "Found named group '%s' at $%d",
 486                             gname.c_str(), gnum);
 487                 }
 488             }
 489         }
 490         esc = false;
 491     }
 492     regex = res;
 493 }
 494
 495 std::string yf::HttpRewrite::Replace::sub_vars (
 496         const std::map<std::string, std::string> & vars) const
 497 {
 498     std::string out;
 499     bool esc = false;
 500     const std::string & in = recipe;
 501     for (size_t i = 0; i < in.size(); ++i)
 502     {
 503         if (!esc && in[i] == '\\')
 504         {
 505             esc = true;
 506             continue;
 507         }
 508         if (!esc && in[i] == '$') //var
 509         {
 510             if (i+1 < in.size() && in[i+1] == '{') //ref prefix
 511             {
 512                 ++i;
 513                 std::string name;
 514                 bool term = false;
 515                 while (++i < in.size())
 516                 {
 517                     if (in[i] == '}') { term = true; break; }
 518                     name += in[i];
 519                 }
 520                 if (!term) throw mp::filter::FilterException
 521                     ("Unterminated var ref in '"+in+"' at "
 522                      + boost::lexical_cast<std::string>(i));
 523                 std::map<std::string, std::string>::const_iterator it
 524                     = vars.find(name);
 525                 if (it != vars.end())
 526                 {
 527                     out += it->second;
 528                 }
 529             }
 530             else
 531             {
 532                 throw mp::filter::FilterException
 533                     ("Malformed or trimmed var ref in '"
 534                      +in+"' at "+boost::lexical_cast<std::string>(i));
 535             }
 536             continue;
 537         }
 538         //passthru
 539         out += in[i];
 540         esc = false;
 541     }
 542     return out;
 543 }
 544
 545
 546 void yf::HttpRewrite::configure_phase(const xmlNode *ptr, Phase &phase)
 547 {
 548     std::map<std::string, RulePtr > rules;
 549     for (ptr = ptr->children; ptr; ptr = ptr->next)
 550     {
 551         if (ptr->type != XML_ELEMENT_NODE)
 552             continue;
 553         else if (!strcmp((const char *) ptr->name, "rule"))
 554         {
 555             static const char *names[2] = { "name", 0 };
 556             std::string values[1];
 557             values[0] = "default";
 558             mp::xml::parse_attr(ptr, names, values);
 559
 560             RulePtr rule(new Rule);
 561             for (xmlNode *p = ptr->children; p; p = p->next)
 562             {
 563                 if (p->type != XML_ELEMENT_NODE)
 564                     continue;
 565                 if (!strcmp((const char *) p->name, "rewrite"))
 566                 {
 567                     Replace replace;
 568                     const struct _xmlAttr *attr;
 569                     for (attr = p->properties; attr; attr = attr->next)
 570                     {
 571                         if (!strcmp((const char *) attr->name,  "from"))
 572                             replace.regex = mp::xml::get_text(attr->children);
 573                         else if (!strcmp((const char *) attr->name,  "to"))
 574                             replace.recipe = mp::xml::get_text(attr->children);
 575                         else
 576                             throw mp::filter::FilterException
 577                                 ("Bad attribute "
 578                                  + std::string((const char *) attr->name)
 579                                  + " in rewrite section of http_rewrite");
 580                     }
 581                     yaz_log(YLOG_LOG, "Found rewrite rule from '%s' to '%s'",
 582                             replace.regex.c_str(), replace.recipe.c_str());
 583                     replace.parse_groups();
 584                     if (!replace.regex.empty())
 585                         rule->replace_list.push_back(replace);
 586                 }
 587                 else
 588                     throw mp::filter::FilterException
 589                         ("Bad element "
 590                          + std::string((const char *) p->name)
 591                          + " in http_rewrite filter");
 592             }
 593             if (!rule->replace_list.empty())
 594                 rules[values[0]] = rule;
 595         }
 596         else if (!strcmp((const char *) ptr->name, "within"))
 597         {
 598             static const char *names[5] =
 599                 { "header", "attr", "tag", "rule", 0 };
 600             std::string values[4];
 601             mp::xml::parse_attr(ptr, names, values);
 602             Within w;
 603             w.header = values[0];
 604             w.attr = values[1];
 605             w.tag = values[2];
 606             std::map<std::string,RulePtr>::const_iterator it =
 607                 rules.find(values[3]);
 608             if (it == rules.end())
 609                 throw mp::filter::FilterException
 610                     ("Reference to non-existing rule '" + values[3] +
 611                      "' in http_rewrite filter");
 612             w.rule = it->second;
 613             phase.within_list.push_back(w);
 614         }
 615         else
 616         {
 617             throw mp::filter::FilterException
 618                 ("Bad element "
 619                  + std::string((const char *) ptr->name)
 620                  + " in http_rewrite filter");
 621         }
 622     }
 623 }
 624
 625 void yf::HttpRewrite::configure(const xmlNode * ptr, bool test_only,
 626         const char *path)
 627 {
 628     for (ptr = ptr->children; ptr; ptr = ptr->next)
 629     {
 630         if (ptr->type != XML_ELEMENT_NODE)
 631             continue;
 632         else if (!strcmp((const char *) ptr->name, "request"))
 633         {
 634             configure_phase(ptr, *req_phase);
 635         }
 636         else if (!strcmp((const char *) ptr->name, "response"))
 637         {
 638             configure_phase(ptr, *res_phase);
 639         }
 640         else
 641         {
 642             throw mp::filter::FilterException
 643                 ("Bad element "
 644                  + std::string((const char *) ptr->name)
 645                  + " in http_rewrite1 filter");
 646         }
 647     }
 648 }
 649
 650 static mp::filter::Base* filter_creator()
 651 {
 652     return new mp::filter::HttpRewrite;
 653 }
 654
 655 extern "C" {
 656     struct metaproxy_1_filter_struct metaproxy_1_filter_http_rewrite = {
 657         0,
 658         "http_rewrite",
 659         filter_creator
 660     };
 661 }
 662
 663
 664 /*
 665  * Local variables:
 666  * c-basic-offset: 4
 667  * c-file-style: "Stroustrup"
 668  * indent-tabs-mode: nil
 669  * End:
 670  * vim: shiftwidth=4 tabstop=8 expandtab
 671  */
 672