src/url.c

   1 /* This file is part of the YAZ toolkit.
   2  * Copyright (C) Index Data
   3  * See the file LICENSE for details.
   4  */
   5 /**
   6  * \file url.c
   7  * \brief URL fetch utility
   8  */
   9 #if HAVE_CONFIG_H
  10 #include <config.h>
  11 #endif
  12
  13 #include <yaz/url.h>
  14 #include <yaz/comstack.h>
  15 #include <yaz/log.h>
  16 #include <yaz/wrbuf.h>
  17 #include <yaz/cookie.h>
  18
  19 struct yaz_url {
  20     ODR odr_in;
  21     ODR odr_out;
  22     char *proxy;
  23     int max_redirects;
  24     WRBUF w_error;
  25     int verbose;
  26     yaz_cookies_t cookies;
  27 };
  28
  29 yaz_url_t yaz_url_create(void)
  30 {
  31     yaz_url_t p = xmalloc(sizeof(*p));
  32     p->odr_in = odr_createmem(ODR_DECODE);
  33     p->odr_out = odr_createmem(ODR_ENCODE);
  34     p->proxy = 0;
  35     p->max_redirects = 10;
  36     p->w_error = wrbuf_alloc();
  37     p->verbose = 0;
  38     p->cookies = yaz_cookies_create();
  39     return p;
  40 }
  41
  42 void yaz_url_destroy(yaz_url_t p)
  43 {
  44     if (p)
  45     {
  46         odr_destroy(p->odr_in);
  47         odr_destroy(p->odr_out);
  48         xfree(p->proxy);
  49         wrbuf_destroy(p->w_error);
  50         yaz_cookies_destroy(p->cookies);
  51         xfree(p);
  52     }
  53 }
  54
  55 void yaz_url_set_proxy(yaz_url_t p, const char *proxy)
  56 {
  57     xfree(p->proxy);
  58     p->proxy = 0;
  59     if (proxy && *proxy)
  60         p->proxy = xstrdup(proxy);
  61 }
  62
  63 void yaz_url_set_max_redirects(yaz_url_t p, int num)
  64 {
  65     p->max_redirects = num;
  66 }
  67
  68 void yaz_url_set_verbose(yaz_url_t p, int num)
  69 {
  70     p->verbose = num;
  71 }
  72
  73 static void extract_user_pass(NMEM nmem,
  74                               const char *uri,
  75                               char **uri_lean, char **http_user,
  76                               char **http_pass)
  77 {
  78     const char *cp1 = strchr(uri, '/');
  79     *uri_lean = 0;
  80     *http_user = 0;
  81     *http_pass = 0;
  82     if (cp1 && cp1 > uri)
  83     {
  84         cp1--;
  85
  86         if (!strncmp(cp1, "://", 3))
  87         {
  88             const char *cp3 = 0;
  89             const char *cp2 = cp1 + 3;
  90             while (*cp2 && *cp2 != '/' && *cp2 != '@')
  91             {
  92                 if (*cp2 == ':')
  93                     cp3 = cp2;
  94                 cp2++;
  95             }
  96             if (*cp2 == '@' && cp3)
  97             {
  98                 *uri_lean = nmem_malloc(nmem, strlen(uri) + 1);
  99                 memcpy(*uri_lean, uri, cp1 + 3 - uri);
 100                 strcpy(*uri_lean + (cp1 + 3 - uri), cp2 + 1);
 101
 102                 *http_user = nmem_strdupn(nmem, cp1 + 3, cp3 - (cp1 + 3));
 103                 *http_pass = nmem_strdupn(nmem, cp3 + 1, cp2 - (cp3 + 1));
 104             }
 105         }
 106     }
 107     if (*uri_lean == 0)
 108         *uri_lean = nmem_strdup(nmem, uri);
 109 }
 110
 111 const char *yaz_url_get_error(yaz_url_t p)
 112 {
 113     return wrbuf_cstr(p->w_error);
 114 }
 115
 116 static void log_warn(yaz_url_t p)
 117 {
 118     yaz_log(YLOG_WARN, "yaz_url: %s", wrbuf_cstr(p->w_error));
 119 }
 120
 121 Z_HTTP_Response *yaz_url_exec(yaz_url_t p, const char *uri,
 122                               const char *method,
 123                               Z_HTTP_Header *user_headers,
 124                               const char *buf, size_t len)
 125 {
 126     Z_HTTP_Response *res = 0;
 127     int number_of_redirects = 0;
 128
 129     yaz_cookies_reset(p->cookies);
 130     wrbuf_rewind(p->w_error);
 131     while (1)
 132     {
 133         void *add;
 134         COMSTACK conn = 0;
 135         int code;
 136         const char *location = 0;
 137         char *http_user = 0;
 138         char *http_pass = 0;
 139         char *uri_lean = 0;
 140         Z_GDU *gdu;
 141
 142         extract_user_pass(p->odr_out->mem, uri, &uri_lean,
 143                           &http_user, &http_pass);
 144
 145         gdu = z_get_HTTP_Request_uri(p->odr_out, uri_lean, 0, p->proxy ? 1 : 0);
 146         gdu->u.HTTP_Request->method = odr_strdup(p->odr_out, method);
 147
 148         yaz_cookies_request(p->cookies, p->odr_out, gdu->u.HTTP_Request);
 149         for ( ; user_headers; user_headers = user_headers->next)
 150         {
 151             /* prefer new Host over user-supplied Host */
 152             if (!strcmp(user_headers->name, "Host"))
 153                 ;
 154             /* prefer user-supplied User-Agent over YAZ' own */
 155             else if (!strcmp(user_headers->name, "User-Agent"))
 156                 z_HTTP_header_set(p->odr_out, &gdu->u.HTTP_Request->headers,
 157                                   user_headers->name, user_headers->value);
 158             else
 159                 z_HTTP_header_add(p->odr_out, &gdu->u.HTTP_Request->headers,
 160                                   user_headers->name, user_headers->value);
 161         }
 162         if (http_user && http_pass)
 163             z_HTTP_header_add_basic_auth(p->odr_out,
 164                                          &gdu->u.HTTP_Request->headers,
 165                                          http_user, http_pass);
 166
 167         res = 0;
 168         if (buf && len)
 169         {
 170             gdu->u.HTTP_Request->content_buf = (char *) buf;
 171             gdu->u.HTTP_Request->content_len = len;
 172         }
 173         if (!z_GDU(p->odr_out, &gdu, 0, 0))
 174         {
 175             wrbuf_printf(p->w_error, "Can not encode HTTP request for URL %s",
 176                          uri);
 177             log_warn(p);
 178             return 0;
 179         }
 180         conn = cs_create_host_proxy(uri_lean, 1, &add, p->proxy);
 181         if (!conn)
 182         {
 183             wrbuf_printf(p->w_error, "Can not resolve URL %s", uri);
 184             log_warn(p);
 185         }
 186         else if (cs_connect(conn, add) < 0)
 187         {
 188             wrbuf_printf(p->w_error, "Can not connect to URL %s", uri);
 189             log_warn(p);
 190         }
 191         else
 192         {
 193             int len;
 194             char *buf = odr_getbuf(p->odr_out, &len, 0);
 195
 196             if (p->verbose)
 197                 fwrite(buf, 1, len, stdout);
 198
 199             if (cs_put(conn, buf, len) < 0)
 200             {
 201                 wrbuf_printf(p->w_error, "cs_put fail for URL %s", uri);
 202                 log_warn(p);
 203             }
 204             else
 205             {
 206                 char *netbuffer = 0;
 207                 int netlen = 0;
 208                 int cs_res = cs_get(conn, &netbuffer, &netlen);
 209                 if (cs_res <= 0)
 210                 {
 211                     wrbuf_printf(p->w_error, "cs_get failed for URL %s", uri);
 212                     log_warn(p);
 213                 }
 214                 else
 215                 {
 216                     Z_GDU *gdu;
 217                     if (p->verbose)
 218                         fwrite(netbuffer, 1, cs_res, stdout);
 219                     odr_setbuf(p->odr_in, netbuffer, cs_res, 0);
 220                     if (!z_GDU(p->odr_in, &gdu, 0, 0)
 221                         || gdu->which != Z_GDU_HTTP_Response)
 222                     {
 223                         wrbuf_printf(p->w_error, "HTTP decoding fail for "
 224                                      "URL %s", uri);
 225                         log_warn(p);
 226                     }
 227                     else
 228                     {
 229                         res = gdu->u.HTTP_Response;
 230                     }
 231                 }
 232                 xfree(netbuffer);
 233             }
 234         }
 235         if (conn)
 236             cs_close(conn);
 237         if (!res)
 238             break;
 239         code = res->code;
 240         location = z_HTTP_header_lookup(res->headers, "Location");
 241         if (++number_of_redirects <= p->max_redirects &&
 242             location && (code == 301 || code == 302 || code == 307))
 243         {
 244             int host_change = 0;
 245             const char *nlocation = yaz_check_location(p->odr_in, uri,
 246                                                        location, &host_change);
 247
 248             odr_reset(p->odr_out);
 249             uri = odr_strdup(p->odr_out, nlocation);
 250         }
 251         else
 252             break;
 253         yaz_cookies_response(p->cookies, res);
 254         odr_reset(p->odr_in);
 255     }
 256     return res;
 257 }
 258
 259 /*
 260  * Local variables:
 261  * c-basic-offset: 4
 262  * c-file-style: "Stroustrup"
 263  * indent-tabs-mode: nil
 264  * End:
 265  * vim: shiftwidth=4 tabstop=8 expandtab
 266  */
 267