src/http.c

   1 /* This file is part of Pazpar2.
   2    Copyright (C) 2006-2011 Index Data
   3
   4 Pazpar2 is free software; you can redistribute it and/or modify it under
   5 the terms of the GNU General Public License as published by the Free
   6 Software Foundation; either version 2, or (at your option) any later
   7 version.
   8
   9 Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY
  10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 for more details.
  13
  14 You should have received a copy of the GNU General Public License
  15 along with this program; if not, write to the Free Software
  16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  17
  18 */
  19
  20 #if HAVE_CONFIG_H
  21 #include <config.h>
  22 #endif
  23
  24 #if HAVE_SYS_TIME_H
  25 #include <sys/time.h>
  26 #endif
  27
  28 #include <stdio.h>
  29 #ifdef WIN32
  30 #include <winsock.h>
  31 typedef int socklen_t;
  32 #endif
  33
  34 #if HAVE_SYS_SOCKET_H
  35 #include <sys/socket.h>
  36 #endif
  37
  38 #include <sys/types.h>
  39
  40 #include <yaz/snprintf.h>
  41 #if HAVE_UNISTD_H
  42 #include <unistd.h>
  43 #endif
  44
  45 #include <stdlib.h>
  46 #include <string.h>
  47 #include <ctype.h>
  48 #include <fcntl.h>
  49 #if HAVE_NETDB_H
  50 #include <netdb.h>
  51 #endif
  52
  53 #include <errno.h>
  54 #include <assert.h>
  55 #include <string.h>
  56
  57 #if HAVE_NETINET_IN_H
  58 #include <netinet/in.h>
  59 #endif
  60
  61 #if HAVE_ARPA_INET_H
  62 #include <arpa/inet.h>
  63 #endif
  64
  65 #include <yaz/yaz-util.h>
  66 #include <yaz/comstack.h>
  67 #include <yaz/nmem.h>
  68 #include <yaz/mutex.h>
  69
  70 #include "ppmutex.h"
  71 #include "session.h"
  72 #include "http.h"
  73
  74 #define MAX_HTTP_HEADER 4096
  75
  76 #ifdef WIN32
  77 #define strncasecmp _strnicmp
  78 #define strcasecmp _stricmp
  79 #endif
  80
  81 struct http_buf
  82 {
  83 #define HTTP_BUF_SIZE 4096
  84     char buf[4096];
  85     int offset;
  86     int len;
  87     struct http_buf *next;
  88 };
  89
  90
  91 static void proxy_io(IOCHAN i, int event);
  92 static struct http_channel *http_channel_create(http_server_t http_server,
  93                                                 const char *addr,
  94                                                 struct conf_server *server);
  95 static void http_channel_destroy(IOCHAN i);
  96 static http_server_t http_server_create(void);
  97 static void http_server_incref(http_server_t hs);
  98
  99 struct http_server
 100 {
 101     YAZ_MUTEX mutex;
 102     int listener_socket;
 103     int ref_count;
 104     http_sessions_t http_sessions;
 105     struct sockaddr_in *proxy_addr;
 106     FILE *record_file;
 107 };
 108
 109 struct http_channel_observer_s {
 110     void *data;
 111     void *data2;
 112     http_channel_destroy_t destroy;
 113     struct http_channel_observer_s *next;
 114     struct http_channel *chan;
 115 };
 116
 117
 118 const char *http_lookup_header(struct http_header *header,
 119                                const char *name)
 120 {
 121     for (; header; header = header->next)
 122         if (!strcasecmp(name, header->name))
 123             return header->value;
 124     return 0;
 125 }
 126
 127 static struct http_buf *http_buf_create(http_server_t hs)
 128 {
 129     struct http_buf *r = xmalloc(sizeof(*r));
 130     r->offset = 0;
 131     r->len = 0;
 132     r->next = 0;
 133     return r;
 134 }
 135
 136 static void http_buf_destroy(http_server_t hs, struct http_buf *b)
 137 {
 138     xfree(b);
 139 }
 140
 141 static void http_buf_destroy_queue(http_server_t hs, struct http_buf *b)
 142 {
 143     struct http_buf *p;
 144     while (b)
 145     {
 146         p = b->next;
 147         http_buf_destroy(hs, b);
 148         b = p;
 149     }
 150 }
 151
 152 static struct http_buf *http_buf_bybuf(http_server_t hs, char *b, int len)
 153 {
 154     struct http_buf *res = 0;
 155     struct http_buf **p = &res;
 156
 157     while (len)
 158     {
 159         int tocopy = len;
 160         if (tocopy > HTTP_BUF_SIZE)
 161             tocopy = HTTP_BUF_SIZE;
 162         *p = http_buf_create(hs);
 163         memcpy((*p)->buf, b, tocopy);
 164         (*p)->len = tocopy;
 165         len -= tocopy;
 166         b += tocopy;
 167         p = &(*p)->next;
 168     }
 169     return res;
 170 }
 171
 172 // Add a (chain of) buffers to the end of an existing queue.
 173 static void http_buf_enqueue(struct http_buf **queue, struct http_buf *b)
 174 {
 175     while (*queue)
 176         queue = &(*queue)->next;
 177     *queue = b;
 178 }
 179
 180 static struct http_buf *http_buf_bywrbuf(http_server_t hs, WRBUF wrbuf)
 181 {
 182     // Heavens to Betsy (buf)!
 183     return http_buf_bybuf(hs, wrbuf_buf(wrbuf), wrbuf_len(wrbuf));
 184 }
 185
 186 // Non-destructively collapse chain of buffers into a string (max *len)
 187 // Return
 188 static void http_buf_peek(struct http_buf *b, char *buf, int len)
 189 {
 190     int rd = 0;
 191     while (b && rd < len)
 192     {
 193         int toread = len - rd;
 194         if (toread > b->len)
 195             toread = b->len;
 196         memcpy(buf + rd, b->buf + b->offset, toread);
 197         rd += toread;
 198         b = b->next;
 199     }
 200     buf[rd] = '\0';
 201 }
 202
 203 static int http_buf_size(struct http_buf *b)
 204 {
 205     int sz = 0;
 206     for (; b; b = b->next)
 207         sz += b->len;
 208     return sz;
 209 }
 210
 211 // Ddestructively munch up to len  from head of queue.
 212 static int http_buf_read(http_server_t hs,
 213                          struct http_buf **b, char *buf, int len)
 214 {
 215     int rd = 0;
 216     while ((*b) && rd < len)
 217     {
 218         int toread = len - rd;
 219         if (toread > (*b)->len)
 220             toread = (*b)->len;
 221         memcpy(buf + rd, (*b)->buf + (*b)->offset, toread);
 222         rd += toread;
 223         if (toread < (*b)->len)
 224         {
 225             (*b)->len -= toread;
 226             (*b)->offset += toread;
 227             break;
 228         }
 229         else
 230         {
 231             struct http_buf *n = (*b)->next;
 232             http_buf_destroy(hs, *b);
 233             *b = n;
 234         }
 235     }
 236     buf[rd] = '\0';
 237     return rd;
 238 }
 239
 240 // Buffers may overlap.
 241 static void urldecode(char *i, char *o)
 242 {
 243     while (*i)
 244     {
 245         if (*i == '+')
 246         {
 247             *(o++) = ' ';
 248             i++;
 249         }
 250         else if (*i == '%' && i[1] && i[2])
 251         {
 252             int v;
 253             i++;
 254             sscanf(i, "%2x", &v);
 255             *o++ = v;
 256             i += 2;
 257         }
 258         else
 259             *(o++) = *(i++);
 260     }
 261     *o = '\0';
 262 }
 263
 264 // Warning: Buffers may not overlap
 265 void urlencode(const char *i, char *o)
 266 {
 267     while (*i)
 268     {
 269         if (strchr(" /:", *i))
 270         {
 271             sprintf(o, "%%%.2X", (int) *i);
 272             o += 3;
 273         }
 274         else
 275             *(o++) = *i;
 276         i++;
 277     }
 278     *o = '\0';
 279 }
 280
 281 void http_addheader(struct http_response *r, const char *name, const char *value)
 282 {
 283     struct http_channel *c = r->channel;
 284     struct http_header *h = nmem_malloc(c->nmem, sizeof *h);
 285     h->name = nmem_strdup(c->nmem, name);
 286     h->value = nmem_strdup(c->nmem, value);
 287     h->next = r->headers;
 288     r->headers = h;
 289 }
 290
 291 const char *http_argbyname(struct http_request *r, const char *name)
 292 {
 293     struct http_argument *p;
 294     if (!name)
 295         return 0;
 296     for (p = r->arguments; p; p = p->next)
 297         if (!strcmp(p->name, name))
 298             return p->value;
 299     return 0;
 300 }
 301
 302 const char *http_headerbyname(struct http_header *h, const char *name)
 303 {
 304     for (; h; h = h->next)
 305         if (!strcmp(h->name, name))
 306             return h->value;
 307     return 0;
 308 }
 309
 310 struct http_response *http_create_response(struct http_channel *c)
 311 {
 312     struct http_response *r = nmem_malloc(c->nmem, sizeof(*r));
 313     strcpy(r->code, "200");
 314     r->msg = "OK";
 315     r->channel = c;
 316     r->headers = 0;
 317     r->payload = 0;
 318     r->content_type = "text/xml";
 319     return r;
 320 }
 321
 322
 323 static const char *next_crlf(const char *cp, size_t *skipped)
 324 {
 325     const char *next_cp = strchr(cp, '\n');
 326     if (next_cp)
 327     {
 328         if (next_cp > cp && next_cp[-1] == '\r')
 329             *skipped = next_cp - cp - 1;
 330         else
 331             *skipped = next_cp - cp;
 332         next_cp++;
 333     }
 334     return next_cp;
 335 }
 336
 337 // Check if buf contains a package (minus payload)
 338 static int package_check(const char *buf, int sz)
 339 {
 340     int content_len = 0;
 341     int len = 0;
 342
 343     while (*buf)
 344     {
 345         size_t skipped = 0;
 346         const char *b = next_crlf(buf, &skipped);
 347
 348         if (!b)
 349         {
 350             // we did not find CRLF.. See if buffer is too large..
 351             if (sz >= MAX_HTTP_HEADER-1)
 352                 return MAX_HTTP_HEADER-1; // yes. Return that (will fail later)
 353             break;
 354         }
 355         len += (b - buf);
 356         if (skipped == 0)
 357         {
 358             // CRLF CRLF , i.e. end of header
 359             if (len + content_len <= sz)
 360                 return len + content_len;
 361             break;
 362         }
 363         buf = b;
 364         // following first skip of \r\n so that we don't consider Method
 365         if (!strncasecmp(buf, "Content-Length:", 15))
 366         {
 367             const char *cp = buf+15;
 368             while (*cp == ' ')
 369                 cp++;
 370             content_len = 0;
 371             while (*cp && isdigit(*(const unsigned char *)cp))
 372                 content_len = content_len*10 + (*cp++ - '0');
 373             if (content_len < 0) /* prevent negative offsets */
 374                 content_len = 0;
 375         }
 376     }
 377     return 0;     // incomplete request
 378 }
 379
 380 // Check if we have a request. Return 0 or length
 381 static int request_check(struct http_buf *queue)
 382 {
 383     char tmp[MAX_HTTP_HEADER];
 384
 385     // only peek at the header..
 386     http_buf_peek(queue, tmp, MAX_HTTP_HEADER-1);
 387     // still we only return non-zero if the complete request is received..
 388     return package_check(tmp, http_buf_size(queue));
 389 }
 390
 391 struct http_response *http_parse_response_buf(struct http_channel *c, const char *buf, int len)
 392 {
 393     char tmp[MAX_HTTP_HEADER];
 394     struct http_response *r = http_create_response(c);
 395     char *p, *p2;
 396     struct http_header **hp = &r->headers;
 397
 398     if (len >= MAX_HTTP_HEADER)
 399         return 0;
 400     memcpy(tmp, buf, len);
 401     for (p = tmp; *p && *p != ' '; p++) // Skip HTTP version
 402         ;
 403     p++;
 404     // Response code
 405     for (p2 = p; *p2 && *p2 != ' ' && p2 - p < 3; p2++)
 406         r->code[p2 - p] = *p2;
 407     if (!(p = strstr(tmp, "\r\n")))
 408         return 0;
 409     p += 2;
 410     while (*p)
 411     {
 412         if (!(p2 = strstr(p, "\r\n")))
 413             return 0;
 414         if (p == p2) // End of headers
 415             break;
 416         else
 417         {
 418             struct http_header *h = *hp = nmem_malloc(c->nmem, sizeof(*h));
 419             char *value = strchr(p, ':');
 420             if (!value)
 421                 return 0;
 422             *(value++) = '\0';
 423             h->name = nmem_strdup(c->nmem, p);
 424             while (isspace(*(const unsigned char *) value))
 425                 value++;
 426             if (value >= p2)  // Empty header;
 427             {
 428                 h->value = "";
 429                 p = p2 + 2;
 430                 continue;
 431             }
 432             *p2 = '\0';
 433             h->value = nmem_strdup(c->nmem, value);
 434             h->next = 0;
 435             hp = &h->next;
 436             p = p2 + 2;
 437         }
 438     }
 439     return r;
 440 }
 441
 442 static int http_parse_arguments(struct http_request *r, NMEM nmem,
 443                                 const char *args)
 444 {
 445     const char *p2 = args;
 446
 447     while (*p2)
 448     {
 449         struct http_argument *a;
 450         const char *equal = strchr(p2, '=');
 451         const char *eoa = strchr(p2, '&');
 452         if (!equal)
 453         {
 454             yaz_log(YLOG_WARN, "Expected '=' in argument");
 455             return -1;
 456         }
 457         if (!eoa)
 458             eoa = equal + strlen(equal); // last argument
 459         else if (equal > eoa)
 460         {
 461             yaz_log(YLOG_WARN, "Missing '&' in argument");
 462             return -1;
 463         }
 464         a = nmem_malloc(nmem, sizeof(struct http_argument));
 465         a->name = nmem_strdupn(nmem, p2, equal - p2);
 466         a->value = nmem_strdupn(nmem, equal+1, eoa - equal - 1);
 467         urldecode(a->name, a->name);
 468         urldecode(a->value, a->value);
 469         a->next = r->arguments;
 470         r->arguments = a;
 471         p2 = eoa;
 472         while (*p2 == '&')
 473             p2++;
 474     }
 475     return 0;
 476 }
 477
 478 struct http_request *http_parse_request(struct http_channel *c,
 479                                         struct http_buf **queue,
 480                                         int len)
 481 {
 482     struct http_request *r = nmem_malloc(c->nmem, sizeof(*r));
 483     char *p, *p2;
 484     char *start = nmem_malloc(c->nmem, len+1);
 485     char *buf = start;
 486
 487     if (http_buf_read(c->http_server, queue, buf, len) < len)
 488     {
 489         yaz_log(YLOG_WARN, "http_buf_read < len (%d)", len);
 490         return 0;
 491     }
 492     r->search = "";
 493     r->channel = c;
 494     r->arguments = 0;
 495     r->headers = 0;
 496     r->content_buf = 0;
 497     r->content_len = 0;
 498     // Parse first line
 499     for (p = buf, p2 = r->method; *p && *p != ' ' && p - buf < 19; p++)
 500         *(p2++) = *p;
 501     if (*p != ' ')
 502     {
 503         yaz_log(YLOG_WARN, "Unexpected HTTP method in request");
 504         return 0;
 505     }
 506     *p2 = '\0';
 507
 508     if (!(buf = strchr(buf, ' ')))
 509     {
 510         yaz_log(YLOG_WARN, "Missing Request-URI in HTTP request");
 511         return 0;
 512     }
 513     buf++;
 514     if (!(p = strchr(buf, ' ')))
 515     {
 516         yaz_log(YLOG_WARN, "HTTP Request-URI not terminated (too long?)");
 517         return 0;
 518     }
 519     *(p++) = '\0';
 520     if ((p2 = strchr(buf, '?'))) // Do we have arguments?
 521         *(p2++) = '\0';
 522     r->path = nmem_strdup(c->nmem, buf);
 523     if (p2)
 524     {
 525         r->search = nmem_strdup(c->nmem, p2);
 526         // Parse Arguments
 527         http_parse_arguments(r, c->nmem, p2);
 528     }
 529     buf = p;
 530
 531     if (strncmp(buf, "HTTP/", 5))
 532         strcpy(r->http_version, "1.0");
 533     else
 534     {
 535         size_t skipped;
 536         buf += 5; // strlen("HTTP/")
 537
 538         p = (char*) next_crlf(buf, &skipped);
 539         if (!p || skipped < 3 || skipped > 5)
 540             return 0;
 541
 542         memcpy(r->http_version, buf, skipped);
 543         r->http_version[skipped] = '\0';
 544         buf = p;
 545     }
 546     strcpy(c->version, r->http_version);
 547
 548     r->headers = 0;
 549     while (*buf)
 550     {
 551         size_t skipped;
 552
 553         p = (char *) next_crlf(buf, &skipped);
 554         if (!p)
 555         {
 556             return 0;
 557         }
 558         else if (skipped == 0)
 559         {
 560             buf = p;
 561             break;
 562         }
 563         else
 564         {
 565             char *cp;
 566             char *n_v = nmem_malloc(c->nmem, skipped+1);
 567             struct http_header *h = nmem_malloc(c->nmem, sizeof(*h));
 568
 569             memcpy(n_v, buf, skipped);
 570             n_v[skipped] = '\0';
 571
 572             if (!(cp = strchr(n_v, ':')))
 573                 return 0;
 574             h->name = nmem_strdupn(c->nmem, n_v, cp - n_v);
 575             cp++;
 576             while (isspace(*cp))
 577                 cp++;
 578             h->value = nmem_strdup(c->nmem, cp);
 579             h->next = r->headers;
 580             r->headers = h;
 581             buf = p;
 582         }
 583     }
 584
 585     // determine if we do keep alive
 586     if (!strcmp(c->version, "1.0"))
 587     {
 588         const char *v = http_lookup_header(r->headers, "Connection");
 589         if (v && !strcmp(v, "Keep-Alive"))
 590             c->keep_alive = 1;
 591         else
 592             c->keep_alive = 0;
 593     }
 594     else
 595     {
 596         const char *v = http_lookup_header(r->headers, "Connection");
 597         if (v && !strcmp(v, "close"))
 598             c->keep_alive = 0;
 599         else
 600             c->keep_alive = 1;
 601     }
 602     if (buf < start + len)
 603     {
 604         const char *content_type = http_lookup_header(r->headers,
 605                                                       "Content-Type");
 606         r->content_len = start + len - buf;
 607         r->content_buf = buf;
 608
 609         if (!yaz_strcmp_del("application/x-www-form-urlencoded",
 610                             content_type, "; "))
 611         {
 612             http_parse_arguments(r, c->nmem, r->content_buf);
 613         }
 614     }
 615     return r;
 616 }
 617
 618 static struct http_buf *http_serialize_response(struct http_channel *c,
 619         struct http_response *r)
 620 {
 621     struct http_header *h;
 622
 623     wrbuf_rewind(c->wrbuf);
 624     wrbuf_printf(c->wrbuf, "HTTP/%s %s %s\r\n", c->version, r->code, r->msg);
 625     for (h = r->headers; h; h = h->next)
 626         wrbuf_printf(c->wrbuf, "%s: %s\r\n", h->name, h->value);
 627     if (r->payload)
 628     {
 629         wrbuf_printf(c->wrbuf, "Content-Length: %d\r\n", r->payload ?
 630                 (int) strlen(r->payload) : 0);
 631         wrbuf_printf(c->wrbuf, "Content-Type: %s\r\n", r->content_type);
 632         if (!strcmp(r->content_type, "text/xml"))
 633         {
 634             xmlDoc *doc = xmlParseMemory(r->payload, strlen(r->payload));
 635             if (doc)
 636             {
 637                 xmlFreeDoc(doc);
 638             }
 639             else
 640             {
 641                 yaz_log(YLOG_WARN, "Sending non-wellformed "
 642                         "response (bug #1162");
 643                 yaz_log(YLOG_WARN, "payload: %s", r->payload);
 644             }
 645         }
 646     }
 647     wrbuf_puts(c->wrbuf, "\r\n");
 648
 649     if (r->payload)
 650         wrbuf_puts(c->wrbuf, r->payload);
 651
 652     return http_buf_bywrbuf(c->http_server, c->wrbuf);
 653 }
 654
 655 // Serialize a HTTP request
 656 static struct http_buf *http_serialize_request(struct http_request *r)
 657 {
 658     struct http_channel *c = r->channel;
 659     struct http_header *h;
 660
 661     wrbuf_rewind(c->wrbuf);
 662     wrbuf_printf(c->wrbuf, "%s %s%s%s", r->method, r->path,
 663                  *r->search ? "?" : "", r->search);
 664
 665     wrbuf_printf(c->wrbuf, " HTTP/%s\r\n", r->http_version);
 666
 667     for (h = r->headers; h; h = h->next)
 668         wrbuf_printf(c->wrbuf, "%s: %s\r\n", h->name, h->value);
 669
 670     wrbuf_puts(c->wrbuf, "\r\n");
 671
 672     if (r->content_buf)
 673         wrbuf_write(c->wrbuf, r->content_buf, r->content_len);
 674
 675 #if 0
 676     yaz_log(YLOG_LOG, "WRITING TO PROXY:\n%s\n----",
 677             wrbuf_cstr(c->wrbuf));
 678 #endif
 679     return http_buf_bywrbuf(c->http_server, c->wrbuf);
 680 }
 681
 682
 683 static int http_weshouldproxy(struct http_request *rq)
 684 {
 685     struct http_channel *c = rq->channel;
 686     if (c->server->http_server->proxy_addr && !strstr(rq->path, "search.pz2"))
 687         return 1;
 688     return 0;
 689 }
 690
 691
 692 struct http_header * http_header_append(struct http_channel *ch,
 693                                         struct http_header * hp,
 694                                         const char *name,
 695                                         const char *value)
 696 {
 697     struct http_header *hpnew = 0;
 698
 699     if (!hp | !ch)
 700         return 0;
 701
 702     while (hp && hp->next)
 703         hp = hp->next;
 704
 705     if(name && strlen(name)&& value && strlen(value)){
 706         hpnew = nmem_malloc(ch->nmem, sizeof *hpnew);
 707         hpnew->name = nmem_strdup(ch->nmem, name);
 708         hpnew->value = nmem_strdup(ch->nmem, value);
 709
 710         hpnew->next = 0;
 711         hp->next = hpnew;
 712         hp = hp->next;
 713
 714         return hpnew;
 715     }
 716
 717     return hp;
 718 }
 719
 720
 721 static int is_inprogress(void)
 722 {
 723 #ifdef WIN32
 724     if (WSAGetLastError() == WSAEWOULDBLOCK)
 725         return 1;
 726 #else
 727     if (errno == EINPROGRESS)
 728         return 1;
 729 #endif
 730     return 0;
 731 }
 732
 733 static void enable_nonblock(int sock)
 734 {
 735     int flags;
 736 #ifdef WIN32
 737     flags = (flags & CS_FLAGS_BLOCKING) ? 0 : 1;
 738     if (ioctlsocket(sock, FIONBIO, &flags) < 0)
 739         yaz_log(YLOG_FATAL|YLOG_ERRNO, "ioctlsocket");
 740 #else
 741     if ((flags = fcntl(sock, F_GETFL, 0)) < 0)
 742         yaz_log(YLOG_FATAL|YLOG_ERRNO, "fcntl");
 743     if (fcntl(sock, F_SETFL, flags | O_NONBLOCK) < 0)
 744         yaz_log(YLOG_FATAL|YLOG_ERRNO, "fcntl2");
 745 #endif
 746 }
 747
 748 static int http_proxy(struct http_request *rq)
 749 {
 750     struct http_channel *c = rq->channel;
 751     struct http_proxy *p = c->proxy;
 752     struct http_header *hp;
 753     struct http_buf *requestbuf;
 754     char server_port[16] = "";
 755     struct conf_server *ser = c->server;
 756
 757     if (!p) // This is a new connection. Create a proxy channel
 758     {
 759         int sock;
 760         struct protoent *pe;
 761         int one = 1;
 762
 763         if (!(pe = getprotobyname("tcp"))) {
 764             abort();
 765         }
 766         if ((sock = socket(PF_INET, SOCK_STREAM, pe->p_proto)) < 0)
 767         {
 768             yaz_log(YLOG_WARN|YLOG_ERRNO, "socket");
 769             return -1;
 770         }
 771         if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (char*)
 772                         &one, sizeof(one)) < 0)
 773             abort();
 774         enable_nonblock(sock);
 775         if (connect(sock, (struct sockaddr *)
 776                     c->server->http_server->proxy_addr,
 777                     sizeof(*c->server->http_server->proxy_addr)) < 0)
 778         {
 779             if (!is_inprogress())
 780             {
 781                 yaz_log(YLOG_WARN|YLOG_ERRNO, "Proxy connect");
 782                 return -1;
 783             }
 784         }
 785         p = xmalloc(sizeof(struct http_proxy));
 786         p->oqueue = 0;
 787         p->channel = c;
 788         p->first_response = 1;
 789         c->proxy = p;
 790         // We will add EVENT_OUTPUT below
 791         p->iochan = iochan_create(sock, proxy_io, EVENT_INPUT, "http_proxy");
 792         iochan_setdata(p->iochan, p);
 793
 794         iochan_add(ser->iochan_man, p->iochan);
 795     }
 796
 797     // Do _not_ modify Host: header, just checking it's existence
 798
 799     if (!http_lookup_header(rq->headers, "Host"))
 800     {
 801         yaz_log(YLOG_WARN, "Failed to find Host header in proxy");
 802         return -1;
 803     }
 804
 805     // Add new header about paraz2 version, host, remote client address, etc.
 806     {
 807         char server_via[128];
 808
 809         hp = rq->headers;
 810         hp = http_header_append(c, hp,
 811                                 "X-Pazpar2-Version", PACKAGE_VERSION);
 812         hp = http_header_append(c, hp,
 813                                 "X-Pazpar2-Server-Host", ser->host);
 814         sprintf(server_port, "%d",  ser->port);
 815         hp = http_header_append(c, hp,
 816                                 "X-Pazpar2-Server-Port", server_port);
 817         yaz_snprintf(server_via, sizeof(server_via),
 818                      "1.1 %s:%s (%s/%s)",
 819                      ser->host ? ser->host : "@",
 820                      server_port, PACKAGE_NAME, PACKAGE_VERSION);
 821         hp = http_header_append(c, hp, "Via" , server_via);
 822         hp = http_header_append(c, hp, "X-Forwarded-For", c->addr);
 823     }
 824
 825     requestbuf = http_serialize_request(rq);
 826
 827     http_buf_enqueue(&p->oqueue, requestbuf);
 828     iochan_setflag(p->iochan, EVENT_OUTPUT);
 829     return 0;
 830 }
 831
 832 void http_send_response(struct http_channel *ch)
 833 {
 834     struct http_response *rs = ch->response;
 835     struct http_buf *hb;
 836
 837     assert(rs);
 838     hb = http_serialize_response(ch, rs);
 839     if (!hb)
 840     {
 841         yaz_log(YLOG_WARN, "Failed to serialize HTTP response");
 842         http_channel_destroy(ch->iochan);
 843     }
 844     else
 845     {
 846         http_buf_enqueue(&ch->oqueue, hb);
 847         iochan_setflag(ch->iochan, EVENT_OUTPUT);
 848         ch->state = Http_Idle;
 849     }
 850 }
 851
 852 static void http_error(struct http_channel *hc, int no, const char *msg)
 853 {
 854     struct http_response *rs = http_create_response(hc);
 855
 856     hc->response = rs;
 857     hc->keep_alive = 0;  // not keeping this HTTP session alive
 858
 859     sprintf(rs->code, "%d", no);
 860
 861     rs->msg = nmem_strdup(hc->nmem, msg);
 862     rs->payload = nmem_malloc(hc->nmem, 100);
 863     yaz_snprintf(rs->payload, 99, "<error>HTTP Error %d: %s</error>\n",
 864                  no, msg);
 865     http_send_response(hc);
 866 }
 867
 868 static void http_io(IOCHAN i, int event)
 869 {
 870     struct http_channel *hc = iochan_getdata(i);
 871     while (event)
 872     {
 873         if (event == EVENT_INPUT)
 874         {
 875             int res, reqlen;
 876             struct http_buf *htbuf;
 877
 878             htbuf = http_buf_create(hc->http_server);
 879             res = recv(iochan_getfd(i), htbuf->buf, HTTP_BUF_SIZE -1, 0);
 880             if (res == -1 && errno == EAGAIN)
 881             {
 882                 http_buf_destroy(hc->http_server, htbuf);
 883                 return;
 884             }
 885             if (res <= 0)
 886             {
 887 #if HAVE_SYS_TIME_H
 888                 if (hc->http_server->record_file)
 889                 {
 890                     struct timeval tv;
 891                     gettimeofday(&tv, 0);
 892                     fprintf(hc->http_server->record_file, "r %lld %lld %lld 0\n",
 893                             (long long) tv.tv_sec, (long long) tv.tv_usec,
 894                             (long long) iochan_getfd(i));
 895                 }
 896 #endif
 897                 http_buf_destroy(hc->http_server, htbuf);
 898                 fflush(hc->http_server->record_file);
 899                 http_channel_destroy(i);
 900                 return;
 901             }
 902             htbuf->buf[res] = '\0';
 903             htbuf->len = res;
 904             http_buf_enqueue(&hc->iqueue, htbuf);
 905
 906             while (1)
 907             {
 908                 if (hc->state == Http_Busy)
 909                     return;
 910                 reqlen = request_check(hc->iqueue);
 911                 if (reqlen <= 2)
 912                     return;
 913                 // we have a complete HTTP request
 914                 nmem_reset(hc->nmem);
 915 #if HAVE_SYS_TIME_H
 916                 if (hc->http_server->record_file)
 917                 {
 918                     struct timeval tv;
 919                     int sz = 0;
 920                     struct http_buf *hb;
 921                     for (hb = hc->iqueue; hb; hb = hb->next)
 922                         sz += hb->len;
 923                     gettimeofday(&tv, 0);
 924                     fprintf(hc->http_server->record_file, "r %lld %lld %lld %d\n",
 925                             (long long) tv.tv_sec, (long long) tv.tv_usec,
 926                             (long long) iochan_getfd(i), sz);
 927                     for (hb = hc->iqueue; hb; hb = hb->next)
 928                         fwrite(hb->buf, 1, hb->len, hc->http_server->record_file);
 929                     fflush(hc->http_server->record_file);
 930                 }
 931  #endif
 932                 if (!(hc->request = http_parse_request(hc, &hc->iqueue, reqlen)))
 933                 {
 934                     yaz_log(YLOG_WARN, "Failed to parse request");
 935                     http_error(hc, 400, "Bad Request");
 936                     return;
 937                 }
 938                 hc->response = 0;
 939                 yaz_log(YLOG_LOG, "Request: %s %s%s%s", hc->request->method,
 940                         hc->request->path,
 941                         *hc->request->search ? "?" : "",
 942                         hc->request->search);
 943                 if (hc->request->content_buf)
 944                     yaz_log(YLOG_LOG, "%s", hc->request->content_buf);
 945                 if (http_weshouldproxy(hc->request))
 946                     http_proxy(hc->request);
 947                 else
 948                 {
 949                     // Execute our business logic!
 950                     hc->state = Http_Busy;
 951                     http_command(hc);
 952                 }
 953             }
 954         }
 955         else if (event == EVENT_OUTPUT)
 956         {
 957             event = 0;
 958             if (hc->oqueue)
 959             {
 960                 struct http_buf *wb = hc->oqueue;
 961                 int res;
 962                 res = send(iochan_getfd(hc->iochan),
 963                            wb->buf + wb->offset, wb->len, 0);
 964                 if (res <= 0)
 965                 {
 966                     yaz_log(YLOG_WARN|YLOG_ERRNO, "write");
 967                     http_channel_destroy(i);
 968                     return;
 969                 }
 970                 if (res == wb->len)
 971                 {
 972 #if HAVE_SYS_TIME_H
 973                     if (hc->http_server->record_file)
 974                     {
 975                         struct timeval tv;
 976                         int sz = wb->offset + wb->len;
 977                         gettimeofday(&tv, 0);
 978                         fprintf(hc->http_server->record_file, "w %lld %lld %lld %d\n",
 979                                 (long long) tv.tv_sec, (long long) tv.tv_usec,
 980                                 (long long) iochan_getfd(i), sz);
 981                         fwrite(wb->buf, 1, wb->offset + wb->len,
 982                                hc->http_server->record_file);
 983                         fputc('\n', hc->http_server->record_file);
 984                         fflush(hc->http_server->record_file);
 985                     }
 986  #endif
 987                     hc->oqueue = hc->oqueue->next;
 988                     http_buf_destroy(hc->http_server, wb);
 989                 }
 990                 else
 991                 {
 992                     wb->len -= res;
 993                     wb->offset += res;
 994                 }
 995                 if (!hc->oqueue)
 996                 {
 997                     if (!hc->keep_alive)
 998                     {
 999                         http_channel_destroy(i);
1000                         return;
1001                     }
1002                     else
1003                     {
1004                         iochan_clearflag(i, EVENT_OUTPUT);
1005                         if (hc->iqueue)
1006                             event = EVENT_INPUT;
1007                     }
1008                 }
1009             }
1010             if (!hc->oqueue && hc->proxy && !hc->proxy->iochan)
1011                 http_channel_destroy(i); // Server closed; we're done
1012         }
1013         else
1014         {
1015             yaz_log(YLOG_WARN, "Unexpected event on connection");
1016             http_channel_destroy(i);
1017             event = 0;
1018         }
1019     }
1020 }
1021
1022 // Handles I/O on a client connection to a backend web server (proxy mode)
1023 static void proxy_io(IOCHAN pi, int event)
1024 {
1025     struct http_proxy *pc = iochan_getdata(pi);
1026     struct http_channel *hc = pc->channel;
1027
1028     switch (event)
1029     {
1030         int res;
1031         struct http_buf *htbuf;
1032
1033         case EVENT_INPUT:
1034             htbuf = http_buf_create(hc->http_server);
1035             res = recv(iochan_getfd(pi), htbuf->buf, HTTP_BUF_SIZE -1, 0);
1036             if (res == 0 || (res < 0 && !is_inprogress()))
1037             {
1038                 if (hc->oqueue)
1039                 {
1040                     yaz_log(YLOG_WARN, "Proxy read came up short");
1041                     // Close channel and alert client HTTP channel that we're gone
1042                     http_buf_destroy(hc->http_server, htbuf);
1043 #ifdef WIN32
1044                     closesocket(iochan_getfd(pi));
1045 #else
1046                     close(iochan_getfd(pi));
1047 #endif
1048                     iochan_destroy(pi);
1049                     pc->iochan = 0;
1050                 }
1051                 else
1052                 {
1053                     http_channel_destroy(hc->iochan);
1054                     return;
1055                 }
1056             }
1057             else
1058             {
1059                 htbuf->buf[res] = '\0';
1060                 htbuf->offset = 0;
1061                 htbuf->len = res;
1062                 // Write any remaining payload
1063                 if (htbuf->len - htbuf->offset > 0)
1064                     http_buf_enqueue(&hc->oqueue, htbuf);
1065             }
1066             iochan_setflag(hc->iochan, EVENT_OUTPUT);
1067             break;
1068         case EVENT_OUTPUT:
1069             if (!(htbuf = pc->oqueue))
1070             {
1071                 iochan_clearflag(pi, EVENT_OUTPUT);
1072                 return;
1073             }
1074             res = send(iochan_getfd(pi), htbuf->buf + htbuf->offset, htbuf->len, 0);
1075             if (res <= 0)
1076             {
1077                 yaz_log(YLOG_WARN|YLOG_ERRNO, "write");
1078                 http_channel_destroy(hc->iochan);
1079                 return;
1080             }
1081             if (res == htbuf->len)
1082             {
1083                 struct http_buf *np = htbuf->next;
1084                 http_buf_destroy(hc->http_server, htbuf);
1085                 pc->oqueue = np;
1086             }
1087             else
1088             {
1089                 htbuf->len -= res;
1090                 htbuf->offset += res;
1091             }
1092
1093             if (!pc->oqueue) {
1094                 iochan_setflags(pi, EVENT_INPUT); // Turns off output flag
1095             }
1096             break;
1097         default:
1098             yaz_log(YLOG_WARN, "Unexpected event on connection");
1099             http_channel_destroy(hc->iochan);
1100     }
1101 }
1102
1103 static void http_fire_observers(struct http_channel *c);
1104 static void http_destroy_observers(struct http_channel *c);
1105
1106 // Cleanup channel
1107 static void http_channel_destroy(IOCHAN i)
1108 {
1109     struct http_channel *s = iochan_getdata(i);
1110     http_server_t http_server;
1111
1112     if (s->proxy)
1113     {
1114         if (s->proxy->iochan)
1115         {
1116 #ifdef WIN32
1117             closesocket(iochan_getfd(s->proxy->iochan));
1118 #else
1119             close(iochan_getfd(s->proxy->iochan));
1120 #endif
1121             iochan_destroy(s->proxy->iochan);
1122         }
1123         http_buf_destroy_queue(s->http_server, s->proxy->oqueue);
1124         xfree(s->proxy);
1125     }
1126     http_buf_destroy_queue(s->http_server, s->iqueue);
1127     http_buf_destroy_queue(s->http_server, s->oqueue);
1128     http_fire_observers(s);
1129     http_destroy_observers(s);
1130
1131     http_server = s->http_server; /* save it for destroy (decref) */
1132
1133     http_server_destroy(http_server);
1134
1135 #ifdef WIN32
1136     closesocket(iochan_getfd(i));
1137 #else
1138     close(iochan_getfd(i));
1139 #endif
1140     iochan_destroy(i);
1141     nmem_destroy(s->nmem);
1142     wrbuf_destroy(s->wrbuf);
1143     xfree(s);
1144 }
1145
1146 static struct http_channel *http_channel_create(http_server_t hs,
1147                                                 const char *addr,
1148                                                 struct conf_server *server)
1149 {
1150     struct http_channel *r;
1151
1152     r = xmalloc(sizeof(struct http_channel));
1153     r->nmem = nmem_create();
1154     r->wrbuf = wrbuf_alloc();
1155
1156     http_server_incref(hs);
1157     r->http_server = hs;
1158     r->http_sessions = hs->http_sessions;
1159     assert(r->http_sessions);
1160     r->server = server;
1161     r->proxy = 0;
1162     r->iochan = 0;
1163     r->iqueue = r->oqueue = 0;
1164     r->state = Http_Idle;
1165     r->keep_alive = 0;
1166     r->request = 0;
1167     r->response = 0;
1168     if (!addr)
1169     {
1170         yaz_log(YLOG_WARN, "Invalid HTTP forward address");
1171         exit(1);
1172     }
1173     strcpy(r->addr, addr);
1174     r->observers = 0;
1175     return r;
1176 }
1177
1178
1179 /* Accept a new command connection */
1180 static void http_accept(IOCHAN i, int event)
1181 {
1182     struct sockaddr_in addr;
1183     int fd = iochan_getfd(i);
1184     socklen_t len;
1185     int s;
1186     IOCHAN c;
1187     struct http_channel *ch;
1188     struct conf_server *server = iochan_getdata(i);
1189
1190     len = sizeof addr;
1191     if ((s = accept(fd, (struct sockaddr *) &addr, &len)) < 0)
1192     {
1193         yaz_log(YLOG_WARN|YLOG_ERRNO, "accept");
1194         return;
1195     }
1196     enable_nonblock(s);
1197
1198     yaz_log(YLOG_DEBUG, "New command connection");
1199     c = iochan_create(s, http_io, EVENT_INPUT | EVENT_EXCEPT, "http_session_socket");
1200
1201     ch = http_channel_create(server->http_server, inet_ntoa(addr.sin_addr),
1202                              server);
1203     ch->iochan = c;
1204     iochan_setdata(c, ch);
1205     iochan_add(server->iochan_man, c);
1206 }
1207
1208 /* Create a http-channel listener, syntax [host:]port */
1209 int http_init(const char *addr, struct conf_server *server,
1210               const char *record_fname)
1211 {
1212     IOCHAN c;
1213     int l;
1214     struct protoent *p;
1215     struct sockaddr_in myaddr;
1216     int one = 1;
1217     const char *pp;
1218     short port;
1219     FILE *record_file = 0;
1220
1221     yaz_log(YLOG_LOG, "HTTP listener %s", addr);
1222
1223
1224     if (record_fname)
1225     {
1226         record_file = fopen(record_fname, "wb");
1227         if (!record_file)
1228         {
1229             yaz_log(YLOG_FATAL|YLOG_ERRNO, "fopen %s", record_fname);
1230             return 1;
1231         }
1232     }
1233
1234     memset(&myaddr, 0, sizeof myaddr);
1235     myaddr.sin_family = AF_INET;
1236     pp = strchr(addr, ':');
1237     if (pp)
1238     {
1239         WRBUF w = wrbuf_alloc();
1240         struct hostent *he;
1241
1242         wrbuf_write(w, addr, pp - addr);
1243         wrbuf_puts(w, "");
1244
1245         he = gethostbyname(wrbuf_cstr(w));
1246         wrbuf_destroy(w);
1247         if (!he)
1248         {
1249             yaz_log(YLOG_FATAL, "Unable to resolve '%s'", addr);
1250             return 1;
1251         }
1252         memcpy(&myaddr.sin_addr.s_addr, he->h_addr_list[0], he->h_length);
1253         port = atoi(pp + 1);
1254     }
1255     else
1256     {
1257         port = atoi(addr);
1258         myaddr.sin_addr.s_addr = INADDR_ANY;
1259     }
1260
1261     myaddr.sin_port = htons(port);
1262
1263     if (!(p = getprotobyname("tcp"))) {
1264         return 1;
1265     }
1266     if ((l = socket(PF_INET, SOCK_STREAM, p->p_proto)) < 0)
1267         yaz_log(YLOG_FATAL|YLOG_ERRNO, "socket");
1268     if (setsockopt(l, SOL_SOCKET, SO_REUSEADDR, (char*)
1269                     &one, sizeof(one)) < 0)
1270         return 1;
1271
1272     if (bind(l, (struct sockaddr *) &myaddr, sizeof myaddr) < 0)
1273     {
1274         yaz_log(YLOG_FATAL|YLOG_ERRNO, "bind");
1275         return 1;
1276     }
1277     if (listen(l, SOMAXCONN) < 0)
1278     {
1279         yaz_log(YLOG_FATAL|YLOG_ERRNO, "listen");
1280         return 1;
1281     }
1282
1283     server->http_server = http_server_create();
1284
1285     server->http_server->record_file = record_file;
1286     server->http_server->listener_socket = l;
1287
1288     c = iochan_create(l, http_accept, EVENT_INPUT | EVENT_EXCEPT, "http_server");
1289     iochan_setdata(c, server);
1290
1291     iochan_add(server->iochan_man, c);
1292     return 0;
1293 }
1294
1295 void http_close_server(struct conf_server *server)
1296 {
1297     /* break the event_loop (select) by closing down the HTTP listener sock */
1298     if (server->http_server->listener_socket)
1299     {
1300 #ifdef WIN32
1301         closesocket(server->http_server->listener_socket);
1302 #else
1303         close(server->http_server->listener_socket);
1304 #endif
1305     }
1306 }
1307
1308 void http_set_proxyaddr(const char *host, struct conf_server *server)
1309 {
1310     const char *p;
1311     short port;
1312     struct hostent *he;
1313     WRBUF w = wrbuf_alloc();
1314
1315     yaz_log(YLOG_LOG, "HTTP backend  %s", host);
1316
1317     p = strchr(host, ':');
1318     if (p)
1319     {
1320         port = atoi(p + 1);
1321         wrbuf_write(w, host, p - host);
1322         wrbuf_puts(w, "");
1323     }
1324     else
1325     {
1326         port = 80;
1327         wrbuf_puts(w, host);
1328     }
1329     if (!(he = gethostbyname(wrbuf_cstr(w))))
1330     {
1331         fprintf(stderr, "Failed to lookup '%s'\n", wrbuf_cstr(w));
1332         exit(1);
1333     }
1334     wrbuf_destroy(w);
1335
1336     server->http_server->proxy_addr = xmalloc(sizeof(struct sockaddr_in));
1337     server->http_server->proxy_addr->sin_family = he->h_addrtype;
1338     memcpy(&server->http_server->proxy_addr->sin_addr.s_addr,
1339            he->h_addr_list[0], he->h_length);
1340     server->http_server->proxy_addr->sin_port = htons(port);
1341 }
1342
1343 static void http_fire_observers(struct http_channel *c)
1344 {
1345     http_channel_observer_t p = c->observers;
1346     while (p)
1347     {
1348         p->destroy(p->data, c, p->data2);
1349         p = p->next;
1350     }
1351 }
1352
1353 static void http_destroy_observers(struct http_channel *c)
1354 {
1355     while (c->observers)
1356     {
1357         http_channel_observer_t obs = c->observers;
1358         c->observers = obs->next;
1359         xfree(obs);
1360     }
1361 }
1362
1363 http_channel_observer_t http_add_observer(struct http_channel *c, void *data,
1364                                           http_channel_destroy_t des)
1365 {
1366     http_channel_observer_t obs = xmalloc(sizeof(*obs));
1367     obs->chan = c;
1368     obs->data = data;
1369     obs->data2 = 0;
1370     obs->destroy= des;
1371     obs->next = c->observers;
1372     c->observers = obs;
1373     return obs;
1374 }
1375
1376 void http_remove_observer(http_channel_observer_t obs)
1377 {
1378     struct http_channel *c = obs->chan;
1379     http_channel_observer_t found, *p = &c->observers;
1380     while (*p != obs)
1381         p = &(*p)->next;
1382     found = *p;
1383     assert(found);
1384     *p = (*p)->next;
1385     xfree(found);
1386 }
1387
1388 struct http_channel *http_channel_observer_chan(http_channel_observer_t obs)
1389 {
1390     return obs->chan;
1391 }
1392
1393 void http_observer_set_data2(http_channel_observer_t obs, void *data2)
1394 {
1395     obs->data2 = data2;
1396 }
1397
1398 http_server_t http_server_create(void)
1399 {
1400     http_server_t hs = xmalloc(sizeof(*hs));
1401     hs->mutex = 0;
1402     hs->proxy_addr = 0;
1403     hs->ref_count = 1;
1404     hs->http_sessions = 0;
1405
1406     hs->record_file = 0;
1407     return hs;
1408 }
1409
1410 void http_server_destroy(http_server_t hs)
1411 {
1412     if (hs)
1413     {
1414         int r;
1415
1416         yaz_mutex_enter(hs->mutex); /* OK: hs->mutex may be NULL */
1417         r = --(hs->ref_count);
1418         yaz_mutex_leave(hs->mutex);
1419
1420         if (r == 0)
1421         {
1422             http_sessions_destroy(hs->http_sessions);
1423             xfree(hs->proxy_addr);
1424             yaz_mutex_destroy(&hs->mutex);
1425             if (hs->record_file)
1426                 fclose(hs->record_file);
1427             xfree(hs);
1428         }
1429     }
1430 }
1431
1432 void http_server_incref(http_server_t hs)
1433 {
1434     assert(hs);
1435     yaz_mutex_enter(hs->mutex);
1436     (hs->ref_count)++;
1437     yaz_mutex_leave(hs->mutex);
1438 }
1439
1440 void http_mutex_init(struct conf_server *server)
1441 {
1442     assert(server);
1443
1444     assert(server->http_server->mutex == 0);
1445     pazpar2_mutex_create(&server->http_server->mutex, "http_server");
1446     server->http_server->http_sessions = http_sessions_create();
1447 }
1448
1449 /*
1450  * Local variables:
1451  * c-basic-offset: 4
1452  * c-file-style: "Stroustrup"
1453  * indent-tabs-mode: nil
1454  * End:
1455  * vim: shiftwidth=4 tabstop=8 expandtab
1456  */
1457