src/url.c

   1 /* This file is part of the YAZ toolkit.
   2  * Copyright (C) Index Data
   3  * See the file LICENSE for details.
   4  */
   5 /**
   6  * \file url.c
   7  * \brief URL fetch utility
   8  */
   9 #if HAVE_CONFIG_H
  10 #include <config.h>
  11 #endif
  12
  13 #include <yaz/url.h>
  14 #include <yaz/comstack.h>
  15 #include <yaz/log.h>
  16 #include <yaz/wrbuf.h>
  17 #include <yaz/cookie.h>
  18
  19 struct yaz_url {
  20     ODR odr_in;
  21     ODR odr_out;
  22     char *proxy;
  23     int proxy_mode;
  24     int max_redirects;
  25     WRBUF w_error;
  26     int verbose;
  27     yaz_cookies_t cookies;
  28 };
  29
  30 yaz_url_t yaz_url_create(void)
  31 {
  32     yaz_url_t p = xmalloc(sizeof(*p));
  33     p->odr_in = odr_createmem(ODR_DECODE);
  34     p->odr_out = odr_createmem(ODR_ENCODE);
  35     p->proxy = 0;
  36     p->proxy_mode = 0;
  37     p->max_redirects = 10;
  38     p->w_error = wrbuf_alloc();
  39     p->verbose = 0;
  40     p->cookies = yaz_cookies_create();
  41     return p;
  42 }
  43
  44 void yaz_url_destroy(yaz_url_t p)
  45 {
  46     if (p)
  47     {
  48         odr_destroy(p->odr_in);
  49         odr_destroy(p->odr_out);
  50         xfree(p->proxy);
  51         wrbuf_destroy(p->w_error);
  52         yaz_cookies_destroy(p->cookies);
  53         xfree(p);
  54     }
  55 }
  56
  57 void yaz_url_set_proxy(yaz_url_t p, const char *proxy)
  58 {
  59     xfree(p->proxy);
  60     p->proxy = 0;
  61     if (proxy && *proxy)
  62         p->proxy = xstrdup(proxy);
  63 }
  64
  65 void yaz_url_set_max_redirects(yaz_url_t p, int num)
  66 {
  67     p->max_redirects = num;
  68 }
  69
  70 void yaz_url_set_verbose(yaz_url_t p, int num)
  71 {
  72     p->verbose = num;
  73 }
  74
  75 static void extract_user_pass(NMEM nmem,
  76                               const char *uri,
  77                               char **uri_lean, char **http_user,
  78                               char **http_pass)
  79 {
  80     const char *cp1 = strchr(uri, '/');
  81     *uri_lean = 0;
  82     *http_user = 0;
  83     *http_pass = 0;
  84     if (cp1 && cp1 > uri)
  85     {
  86         cp1--;
  87
  88         if (!strncmp(cp1, "://", 3))
  89         {
  90             const char *cp3 = 0;
  91             const char *cp2 = cp1 + 3;
  92             while (*cp2 && *cp2 != '/' && *cp2 != '@')
  93             {
  94                 if (*cp2 == ':')
  95                     cp3 = cp2;
  96                 cp2++;
  97             }
  98             if (*cp2 == '@' && cp3)
  99             {
 100                 *uri_lean = nmem_malloc(nmem, strlen(uri) + 1);
 101                 memcpy(*uri_lean, uri, cp1 + 3 - uri);
 102                 strcpy(*uri_lean + (cp1 + 3 - uri), cp2 + 1);
 103
 104                 *http_user = nmem_strdupn(nmem, cp1 + 3, cp3 - (cp1 + 3));
 105                 *http_pass = nmem_strdupn(nmem, cp3 + 1, cp2 - (cp3 + 1));
 106             }
 107         }
 108     }
 109     if (*uri_lean == 0)
 110         *uri_lean = nmem_strdup(nmem, uri);
 111 }
 112
 113 const char *yaz_url_get_error(yaz_url_t p)
 114 {
 115     return wrbuf_cstr(p->w_error);
 116 }
 117
 118 static void log_warn(yaz_url_t p)
 119 {
 120     yaz_log(YLOG_WARN, "yaz_url: %s", wrbuf_cstr(p->w_error));
 121 }
 122
 123 Z_HTTP_Response *yaz_url_exec(yaz_url_t p, const char *uri,
 124                               const char *method,
 125                               Z_HTTP_Header *user_headers,
 126                               const char *buf, size_t len)
 127 {
 128     Z_HTTP_Response *res = 0;
 129     int number_of_redirects = 0;
 130
 131     yaz_cookies_reset(p->cookies);
 132     wrbuf_rewind(p->w_error);
 133     while (1)
 134     {
 135         void *add;
 136         COMSTACK conn = 0;
 137         int code;
 138         const char *location = 0;
 139         char *http_user = 0;
 140         char *http_pass = 0;
 141         char *uri_lean = 0;
 142         Z_GDU *gdu;
 143
 144         extract_user_pass(p->odr_out->mem, uri, &uri_lean,
 145                           &http_user, &http_pass);
 146
 147         gdu = z_get_HTTP_Request_uri(p->odr_out, uri_lean, 0, p->proxy_mode);
 148         gdu->u.HTTP_Request->method = odr_strdup(p->odr_out, method);
 149
 150         yaz_cookies_request(p->cookies, p->odr_out, gdu->u.HTTP_Request);
 151         for ( ; user_headers; user_headers = user_headers->next)
 152         {
 153             /* prefer new Host over user-supplied Host */
 154             if (!strcmp(user_headers->name, "Host"))
 155                 ;
 156             /* prefer user-supplied User-Agent over YAZ' own */
 157             else if (!strcmp(user_headers->name, "User-Agent"))
 158                 z_HTTP_header_set(p->odr_out, &gdu->u.HTTP_Request->headers,
 159                                   user_headers->name, user_headers->value);
 160             else
 161                 z_HTTP_header_add(p->odr_out, &gdu->u.HTTP_Request->headers,
 162                                   user_headers->name, user_headers->value);
 163         }
 164         if (http_user && http_pass)
 165             z_HTTP_header_add_basic_auth(p->odr_out,
 166                                          &gdu->u.HTTP_Request->headers,
 167                                          http_user, http_pass);
 168
 169         res = 0;
 170         if (buf && len)
 171         {
 172             gdu->u.HTTP_Request->content_buf = (char *) buf;
 173             gdu->u.HTTP_Request->content_len = len;
 174         }
 175         if (!z_GDU(p->odr_out, &gdu, 0, 0))
 176         {
 177             wrbuf_printf(p->w_error, "Can not encode HTTP request for URL %s",
 178                          uri);
 179             log_warn(p);
 180             return 0;
 181         }
 182         conn = cs_create_host2(uri_lean, 1, &add, p->proxy, &p->proxy_mode);
 183         if (!conn)
 184         {
 185             wrbuf_printf(p->w_error, "Can not resolve URL %s", uri);
 186             log_warn(p);
 187         }
 188         else if (cs_connect(conn, add) < 0)
 189         {
 190             wrbuf_printf(p->w_error, "Can not connect to URL %s", uri);
 191             log_warn(p);
 192         }
 193         else
 194         {
 195             int len;
 196             char *buf = odr_getbuf(p->odr_out, &len, 0);
 197
 198             if (p->verbose)
 199                 fwrite(buf, 1, len, stdout);
 200
 201             if (cs_put(conn, buf, len) < 0)
 202             {
 203                 wrbuf_printf(p->w_error, "cs_put fail for URL %s", uri);
 204                 log_warn(p);
 205             }
 206             else
 207             {
 208                 char *netbuffer = 0;
 209                 int netlen = 0;
 210                 int cs_res = cs_get(conn, &netbuffer, &netlen);
 211                 if (cs_res <= 0)
 212                 {
 213                     wrbuf_printf(p->w_error, "cs_get failed for URL %s", uri);
 214                     log_warn(p);
 215                 }
 216                 else
 217                 {
 218                     Z_GDU *gdu;
 219                     if (p->verbose)
 220                         fwrite(netbuffer, 1, cs_res, stdout);
 221                     odr_setbuf(p->odr_in, netbuffer, cs_res, 0);
 222                     if (!z_GDU(p->odr_in, &gdu, 0, 0)
 223                         || gdu->which != Z_GDU_HTTP_Response)
 224                     {
 225                         wrbuf_printf(p->w_error, "HTTP decoding fail for "
 226                                      "URL %s", uri);
 227                         log_warn(p);
 228                     }
 229                     else
 230                     {
 231                         res = gdu->u.HTTP_Response;
 232                     }
 233                 }
 234                 xfree(netbuffer);
 235             }
 236         }
 237         if (conn)
 238             cs_close(conn);
 239         if (!res)
 240             break;
 241         code = res->code;
 242         location = z_HTTP_header_lookup(res->headers, "Location");
 243         if (++number_of_redirects <= p->max_redirects &&
 244             location && (code == 301 || code == 302 || code == 307))
 245         {
 246             int host_change = 0;
 247             const char *nlocation = yaz_check_location(p->odr_in, uri,
 248                                                        location, &host_change);
 249
 250             odr_reset(p->odr_out);
 251             uri = odr_strdup(p->odr_out, nlocation);
 252         }
 253         else
 254             break;
 255         yaz_cookies_response(p->cookies, res);
 256         odr_reset(p->odr_in);
 257     }
 258     return res;
 259 }
 260
 261 /*
 262  * Local variables:
 263  * c-basic-offset: 4
 264  * c-file-style: "Stroustrup"
 265  * indent-tabs-mode: nil
 266  * End:
 267  * vim: shiftwidth=4 tabstop=8 expandtab
 268  */
 269