Merge branch 'master' of ssh://git.indexdata.com:222/home/git/pub/pazpar2
[pazpar2-moved-to-github.git] / src / http.c
1 /* This file is part of Pazpar2.
2    Copyright (C) 2006-2011 Index Data
3
4 Pazpar2 is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
7 version.
8
9 Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12 for more details.
13
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
17
18 */
19
20 #if HAVE_CONFIG_H
21 #include <config.h>
22 #endif
23
24 #include <stdio.h>
25 #ifdef WIN32
26 #include <winsock.h>
27 typedef int socklen_t;
28 #endif
29
30 #if HAVE_SYS_SOCKET_H
31 #include <sys/socket.h>
32 #endif
33
34 #include <sys/types.h>
35
36 #include <yaz/snprintf.h>
37 #if HAVE_UNISTD_H
38 #include <unistd.h>
39 #endif
40
41 #include <stdlib.h>
42 #include <string.h>
43 #include <ctype.h>
44 #include <fcntl.h>
45 #if HAVE_NETDB_H
46 #include <netdb.h>
47 #endif
48
49 #include <errno.h>
50 #include <assert.h>
51 #include <string.h>
52
53 #if HAVE_NETINET_IN_H
54 #include <netinet/in.h>
55 #endif
56
57 #if HAVE_ARPA_INET_H
58 #include <arpa/inet.h>
59 #endif
60
61 #include <yaz/yaz-util.h>
62 #include <yaz/comstack.h>
63 #include <yaz/nmem.h>
64 #include <yaz/mutex.h>
65
66 #include "ppmutex.h"
67 #include "session.h"
68 #include "http.h"
69
70 #define MAX_HTTP_HEADER 4096
71
72 #ifdef WIN32
73 #define strncasecmp _strnicmp
74 #define strcasecmp _stricmp
75 #endif
76
77 struct http_buf
78 {
79 #define HTTP_BUF_SIZE 4096
80     char buf[4096];
81     int offset;
82     int len;
83     struct http_buf *next;
84 };
85
86
87 static void proxy_io(IOCHAN i, int event);
88 static struct http_channel *http_channel_create(http_server_t http_server,
89                                                 const char *addr,
90                                                 struct conf_server *server);
91 static void http_channel_destroy(IOCHAN i);
92 static http_server_t http_server_create(void);
93 static void http_server_incref(http_server_t hs);
94
95 struct http_server
96 {
97     struct http_buf *http_buf_freelist;
98     int http_buf_freelist_count;
99     int http_buf_freelist_max;
100
101     struct http_channel *http_channel_freelist;
102     int http_channel_freelist_count;
103     int http_channel_freelist_max;
104     YAZ_MUTEX mutex;
105     int listener_socket;
106     int ref_count;
107     http_sessions_t http_sessions;
108     struct sockaddr_in *proxy_addr;
109 };
110
111 struct http_channel_observer_s {
112     void *data;
113     void *data2;
114     http_channel_destroy_t destroy;
115     struct http_channel_observer_s *next;
116     struct http_channel *chan;
117 };
118
119
120 const char *http_lookup_header(struct http_header *header,
121                                const char *name)
122 {
123     for (; header; header = header->next)
124         if (!strcasecmp(name, header->name))
125             return header->value;
126     return 0;
127 }
128
129 static struct http_buf *http_buf_create(http_server_t hs)
130 {
131     struct http_buf *r = 0;
132
133     yaz_mutex_enter(hs->mutex);
134     if (hs->http_buf_freelist)
135     {
136         r = hs->http_buf_freelist;
137         hs->http_buf_freelist = hs->http_buf_freelist->next;
138         hs->http_buf_freelist_count--;
139     }
140     yaz_mutex_leave(hs->mutex);
141     if (!r)
142         r = xmalloc(sizeof(struct http_buf));
143     r->offset = 0;
144     r->len = 0;
145     r->next = 0;
146     return r;
147 }
148
149 static void http_buf_destroy(http_server_t hs, struct http_buf *b)
150 {
151     yaz_mutex_enter(hs->mutex);
152     if (hs->http_buf_freelist_max > 0 && hs->http_buf_freelist_count >= hs->http_buf_freelist_max) {
153         xfree(b);
154         while ((b = hs->http_buf_freelist)) {
155             xfree(b);
156             hs->http_buf_freelist = hs->http_buf_freelist->next;
157         }
158         hs->http_buf_freelist_count = 0;
159     }
160     else {
161         b->next = hs->http_buf_freelist;
162         hs->http_buf_freelist = b;
163         hs->http_buf_freelist_count++;
164 #if 0 
165         yaz_log(YLOG_DEBUG, "Free %d http buffers on server.", hs->http_buf_freelist_count);
166 #endif
167     }
168     yaz_mutex_leave(hs->mutex);
169 }
170
171 static void http_buf_destroy_queue(http_server_t hs, struct http_buf *b)
172 {
173     struct http_buf *p;
174     while (b)
175     {
176         p = b->next;
177         http_buf_destroy(hs, b);
178         b = p;
179     }
180 }
181
182 static struct http_buf *http_buf_bybuf(http_server_t hs, char *b, int len)
183 {
184     struct http_buf *res = 0;
185     struct http_buf **p = &res;
186
187     while (len)
188     {
189         int tocopy = len;
190         if (tocopy > HTTP_BUF_SIZE)
191             tocopy = HTTP_BUF_SIZE;
192         *p = http_buf_create(hs);
193         memcpy((*p)->buf, b, tocopy);
194         (*p)->len = tocopy;
195         len -= tocopy;
196         b += tocopy;
197         p = &(*p)->next;
198     }
199     return res;
200 }
201
202 // Add a (chain of) buffers to the end of an existing queue.
203 static void http_buf_enqueue(struct http_buf **queue, struct http_buf *b)
204 {
205     while (*queue)
206         queue = &(*queue)->next;
207     *queue = b;
208 }
209
210 static struct http_buf *http_buf_bywrbuf(http_server_t hs, WRBUF wrbuf)
211 {
212     // Heavens to Betsy (buf)!
213     return http_buf_bybuf(hs, wrbuf_buf(wrbuf), wrbuf_len(wrbuf));
214 }
215
216 // Non-destructively collapse chain of buffers into a string (max *len)
217 // Return
218 static void http_buf_peek(struct http_buf *b, char *buf, int len)
219 {
220     int rd = 0;
221     while (b && rd < len)
222     {
223         int toread = len - rd;
224         if (toread > b->len)
225             toread = b->len;
226         memcpy(buf + rd, b->buf + b->offset, toread);
227         rd += toread;
228         b = b->next;
229     }
230     buf[rd] = '\0';
231 }
232
233 static int http_buf_size(struct http_buf *b)
234 {
235     int sz = 0;
236     for (; b; b = b->next)
237         sz += b->len;
238     return sz;
239 }
240
241 // Ddestructively munch up to len  from head of queue.
242 static int http_buf_read(http_server_t hs,
243                          struct http_buf **b, char *buf, int len)
244 {
245     int rd = 0;
246     while ((*b) && rd < len)
247     {
248         int toread = len - rd;
249         if (toread > (*b)->len)
250             toread = (*b)->len;
251         memcpy(buf + rd, (*b)->buf + (*b)->offset, toread);
252         rd += toread;
253         if (toread < (*b)->len)
254         {
255             (*b)->len -= toread;
256             (*b)->offset += toread;
257             break;
258         }
259         else
260         {
261             struct http_buf *n = (*b)->next;
262             http_buf_destroy(hs, *b);
263             *b = n;
264         }
265     }
266     buf[rd] = '\0';
267     return rd;
268 }
269
270 // Buffers may overlap.
271 static void urldecode(char *i, char *o)
272 {
273     while (*i)
274     {
275         if (*i == '+')
276         {
277             *(o++) = ' ';
278             i++;
279         }
280         else if (*i == '%' && i[1] && i[2])
281         {
282             int v;
283             i++;
284             sscanf(i, "%2x", &v);
285             *o++ = v;
286             i += 2;
287         }
288         else
289             *(o++) = *(i++);
290     }
291     *o = '\0';
292 }
293
294 // Warning: Buffers may not overlap
295 void urlencode(const char *i, char *o)
296 {
297     while (*i)
298     {
299         if (strchr(" /:", *i))
300         {
301             sprintf(o, "%%%.2X", (int) *i);
302             o += 3;
303         }
304         else
305             *(o++) = *i;
306         i++;
307     }
308     *o = '\0';
309 }
310
311 void http_addheader(struct http_response *r, const char *name, const char *value)
312 {
313     struct http_channel *c = r->channel;
314     struct http_header *h = nmem_malloc(c->nmem, sizeof *h);
315     h->name = nmem_strdup(c->nmem, name);
316     h->value = nmem_strdup(c->nmem, value);
317     h->next = r->headers;
318     r->headers = h;
319 }
320
321 const char *http_argbyname(struct http_request *r, const char *name)
322 {
323     struct http_argument *p;
324     if (!name)
325         return 0;
326     for (p = r->arguments; p; p = p->next)
327         if (!strcmp(p->name, name))
328             return p->value;
329     return 0;
330 }
331
332 const char *http_headerbyname(struct http_header *h, const char *name)
333 {
334     for (; h; h = h->next)
335         if (!strcmp(h->name, name))
336             return h->value;
337     return 0;
338 }
339
340 struct http_response *http_create_response(struct http_channel *c)
341 {
342     struct http_response *r = nmem_malloc(c->nmem, sizeof(*r));
343     strcpy(r->code, "200");
344     r->msg = "OK";
345     r->channel = c;
346     r->headers = 0;
347     r->payload = 0;
348     r->content_type = "text/xml";
349     return r;
350 }
351
352
353 static const char *next_crlf(const char *cp, size_t *skipped)
354 {
355     const char *next_cp = strchr(cp, '\n');
356     if (next_cp)
357     {
358         if (next_cp > cp && next_cp[-1] == '\r')
359             *skipped = next_cp - cp - 1;
360         else
361             *skipped = next_cp - cp;
362         next_cp++;
363     }
364     return next_cp;
365 }
366
367 // Check if buf contains a package (minus payload)
368 static int package_check(const char *buf, int sz)
369 {
370     int content_len = 0;
371     int len = 0;
372
373     while (*buf)
374     {
375         size_t skipped = 0;
376         const char *b = next_crlf(buf, &skipped);
377
378         if (!b)
379         {
380             // we did not find CRLF.. See if buffer is too large..
381             if (sz >= MAX_HTTP_HEADER-1)
382                 return MAX_HTTP_HEADER-1; // yes. Return that (will fail later)
383             break;
384         }
385         len += (b - buf);
386         if (skipped == 0)
387         {
388             // CRLF CRLF , i.e. end of header
389             if (len + content_len <= sz)
390                 return len + content_len;
391             break;
392         }
393         buf = b;
394         // following first skip of \r\n so that we don't consider Method
395         if (!strncasecmp(buf, "Content-Length:", 15))
396         {
397             const char *cp = buf+15;
398             while (*cp == ' ')
399                 cp++;
400             content_len = 0;
401             while (*cp && isdigit(*(const unsigned char *)cp))
402                 content_len = content_len*10 + (*cp++ - '0');
403             if (content_len < 0) /* prevent negative offsets */
404                 content_len = 0;
405         }
406     }
407     return 0;     // incomplete request
408 }
409
410 // Check if we have a request. Return 0 or length
411 static int request_check(struct http_buf *queue)
412 {
413     char tmp[MAX_HTTP_HEADER];
414
415     // only peek at the header..
416     http_buf_peek(queue, tmp, MAX_HTTP_HEADER-1);
417     // still we only return non-zero if the complete request is received..
418     return package_check(tmp, http_buf_size(queue));
419 }
420
421 struct http_response *http_parse_response_buf(struct http_channel *c, const char *buf, int len)
422 {
423     char tmp[MAX_HTTP_HEADER];
424     struct http_response *r = http_create_response(c);
425     char *p, *p2;
426     struct http_header **hp = &r->headers;
427
428     if (len >= MAX_HTTP_HEADER)
429         return 0;
430     memcpy(tmp, buf, len);
431     for (p = tmp; *p && *p != ' '; p++) // Skip HTTP version
432         ;
433     p++;
434     // Response code
435     for (p2 = p; *p2 && *p2 != ' ' && p2 - p < 3; p2++)
436         r->code[p2 - p] = *p2;
437     if (!(p = strstr(tmp, "\r\n")))
438         return 0;
439     p += 2;
440     while (*p)
441     {
442         if (!(p2 = strstr(p, "\r\n")))
443             return 0;
444         if (p == p2) // End of headers
445             break;
446         else
447         {
448             struct http_header *h = *hp = nmem_malloc(c->nmem, sizeof(*h));
449             char *value = strchr(p, ':');
450             if (!value)
451                 return 0;
452             *(value++) = '\0';
453             h->name = nmem_strdup(c->nmem, p);
454             while (isspace(*(const unsigned char *) value))
455                 value++;
456             if (value >= p2)  // Empty header;
457             {
458                 h->value = "";
459                 p = p2 + 2;
460                 continue;
461             }
462             *p2 = '\0';
463             h->value = nmem_strdup(c->nmem, value);
464             h->next = 0;
465             hp = &h->next;
466             p = p2 + 2;
467         }
468     }
469     return r;
470 }
471
472 static int http_parse_arguments(struct http_request *r, NMEM nmem,
473                                 const char *args)
474 {
475     const char *p2 = args;
476
477     while (*p2)
478     {
479         struct http_argument *a;
480         const char *equal = strchr(p2, '=');
481         const char *eoa = strchr(p2, '&');
482         if (!equal)
483         {
484             yaz_log(YLOG_WARN, "Expected '=' in argument");
485             return -1;
486         }
487         if (!eoa)
488             eoa = equal + strlen(equal); // last argument
489         else if (equal > eoa)
490         {
491             yaz_log(YLOG_WARN, "Missing '&' in argument");
492             return -1;
493         }
494         a = nmem_malloc(nmem, sizeof(struct http_argument));
495         a->name = nmem_strdupn(nmem, p2, equal - p2);
496         a->value = nmem_strdupn(nmem, equal+1, eoa - equal - 1);
497         urldecode(a->name, a->name);
498         urldecode(a->value, a->value);
499         a->next = r->arguments;
500         r->arguments = a;
501         p2 = eoa;
502         while (*p2 == '&')
503             p2++;
504     }
505     return 0;
506 }
507
508 struct http_request *http_parse_request(struct http_channel *c,
509                                         struct http_buf **queue,
510                                         int len)
511 {
512     struct http_request *r = nmem_malloc(c->nmem, sizeof(*r));
513     char *p, *p2;
514     char *start = nmem_malloc(c->nmem, len+1);
515     char *buf = start;
516
517     if (http_buf_read(c->http_server, queue, buf, len) < len)
518     {
519         yaz_log(YLOG_WARN, "http_buf_read < len (%d)", len);
520         return 0;
521     }
522     r->search = "";
523     r->channel = c;
524     r->arguments = 0;
525     r->headers = 0;
526     r->content_buf = 0;
527     r->content_len = 0;
528     // Parse first line
529     for (p = buf, p2 = r->method; *p && *p != ' ' && p - buf < 19; p++)
530         *(p2++) = *p;
531     if (*p != ' ')
532     {
533         yaz_log(YLOG_WARN, "Unexpected HTTP method in request");
534         return 0;
535     }
536     *p2 = '\0';
537
538     if (!(buf = strchr(buf, ' ')))
539     {
540         yaz_log(YLOG_WARN, "Missing Request-URI in HTTP request");
541         return 0;
542     }
543     buf++;
544     if (!(p = strchr(buf, ' ')))
545     {
546         yaz_log(YLOG_WARN, "HTTP Request-URI not terminated (too long?)");
547         return 0;
548     }
549     *(p++) = '\0';
550     if ((p2 = strchr(buf, '?'))) // Do we have arguments?
551         *(p2++) = '\0';
552     r->path = nmem_strdup(c->nmem, buf);
553     if (p2)
554     {
555         r->search = nmem_strdup(c->nmem, p2);
556         // Parse Arguments
557         http_parse_arguments(r, c->nmem, p2);
558     }
559     buf = p;
560
561     if (strncmp(buf, "HTTP/", 5))
562         strcpy(r->http_version, "1.0");
563     else
564     {
565         size_t skipped;
566         buf += 5; // strlen("HTTP/")
567
568         p = (char*) next_crlf(buf, &skipped);
569         if (!p || skipped < 3 || skipped > 5)
570             return 0;
571
572         memcpy(r->http_version, buf, skipped);
573         r->http_version[skipped] = '\0';
574         buf = p;
575     }
576     strcpy(c->version, r->http_version);
577
578     r->headers = 0;
579     while (*buf)
580     {
581         size_t skipped;
582
583         p = (char *) next_crlf(buf, &skipped);
584         if (!p)
585         {
586             return 0;
587         }
588         else if (skipped == 0)
589         {
590             buf = p;
591             break;
592         }
593         else
594         {
595             char *cp;
596             char *n_v = nmem_malloc(c->nmem, skipped+1);
597             struct http_header *h = nmem_malloc(c->nmem, sizeof(*h));
598
599             memcpy(n_v, buf, skipped);
600             n_v[skipped] = '\0';
601
602             if (!(cp = strchr(n_v, ':')))
603                 return 0;
604             h->name = nmem_strdupn(c->nmem, n_v, cp - n_v);
605             cp++;
606             while (isspace(*cp))
607                 cp++;
608             h->value = nmem_strdup(c->nmem, cp);
609             h->next = r->headers;
610             r->headers = h;
611             buf = p;
612         }
613     }
614
615     // determine if we do keep alive
616     if (!strcmp(c->version, "1.0"))
617     {
618         const char *v = http_lookup_header(r->headers, "Connection");
619         if (v && !strcmp(v, "Keep-Alive"))
620             c->keep_alive = 1;
621         else
622             c->keep_alive = 0;
623     }
624     else
625     {
626         const char *v = http_lookup_header(r->headers, "Connection");
627         if (v && !strcmp(v, "close"))
628             c->keep_alive = 0;
629         else
630             c->keep_alive = 1;
631     }
632     if (buf < start + len)
633     {
634         const char *content_type = http_lookup_header(r->headers,
635                                                       "Content-Type");
636         r->content_len = start + len - buf;
637         r->content_buf = buf;
638
639         if (!yaz_strcmp_del("application/x-www-form-urlencoded",
640                             content_type, "; "))
641         {
642             http_parse_arguments(r, c->nmem, r->content_buf);
643         }
644     }
645     return r;
646 }
647
648 static struct http_buf *http_serialize_response(struct http_channel *c,
649         struct http_response *r)
650 {
651     struct http_header *h;
652
653     wrbuf_rewind(c->wrbuf);
654     wrbuf_printf(c->wrbuf, "HTTP/%s %s %s\r\n", c->version, r->code, r->msg);
655     for (h = r->headers; h; h = h->next)
656         wrbuf_printf(c->wrbuf, "%s: %s\r\n", h->name, h->value);
657     if (r->payload)
658     {
659         wrbuf_printf(c->wrbuf, "Content-Length: %d\r\n", r->payload ?
660                 (int) strlen(r->payload) : 0);
661         wrbuf_printf(c->wrbuf, "Content-Type: %s\r\n", r->content_type);
662         if (!strcmp(r->content_type, "text/xml"))
663         {
664             xmlDoc *doc = xmlParseMemory(r->payload, strlen(r->payload));
665             if (doc)
666             {
667                 xmlFreeDoc(doc);
668             }
669             else
670             {
671                 yaz_log(YLOG_WARN, "Sending non-wellformed "
672                         "response (bug #1162");
673                 yaz_log(YLOG_WARN, "payload: %s", r->payload);
674             }
675         }
676     }
677     wrbuf_puts(c->wrbuf, "\r\n");
678
679     if (r->payload)
680         wrbuf_puts(c->wrbuf, r->payload);
681
682     return http_buf_bywrbuf(c->http_server, c->wrbuf);
683 }
684
685 // Serialize a HTTP request
686 static struct http_buf *http_serialize_request(struct http_request *r)
687 {
688     struct http_channel *c = r->channel;
689     struct http_header *h;
690
691     wrbuf_rewind(c->wrbuf);
692     wrbuf_printf(c->wrbuf, "%s %s%s%s", r->method, r->path,
693                  *r->search ? "?" : "", r->search);
694
695     wrbuf_printf(c->wrbuf, " HTTP/%s\r\n", r->http_version);
696
697     for (h = r->headers; h; h = h->next)
698         wrbuf_printf(c->wrbuf, "%s: %s\r\n", h->name, h->value);
699
700     wrbuf_puts(c->wrbuf, "\r\n");
701
702     if (r->content_buf)
703         wrbuf_write(c->wrbuf, r->content_buf, r->content_len);
704
705 #if 0
706     yaz_log(YLOG_LOG, "WRITING TO PROXY:\n%s\n----",
707             wrbuf_cstr(c->wrbuf));
708 #endif
709     return http_buf_bywrbuf(c->http_server, c->wrbuf);
710 }
711
712
713 static int http_weshouldproxy(struct http_request *rq)
714 {
715     struct http_channel *c = rq->channel;
716     if (c->server->http_server->proxy_addr && !strstr(rq->path, "search.pz2"))
717         return 1;
718     return 0;
719 }
720
721
722 struct http_header * http_header_append(struct http_channel *ch, 
723                                         struct http_header * hp, 
724                                         const char *name, 
725                                         const char *value)
726 {
727     struct http_header *hpnew = 0; 
728
729     if (!hp | !ch)
730         return 0;
731
732     while (hp && hp->next)
733         hp = hp->next;
734
735     if(name && strlen(name)&& value && strlen(value)){
736         hpnew = nmem_malloc(ch->nmem, sizeof *hpnew);
737         hpnew->name = nmem_strdup(ch->nmem, name);
738         hpnew->value = nmem_strdup(ch->nmem, value);
739         
740         hpnew->next = 0;
741         hp->next = hpnew;
742         hp = hp->next;
743         
744         return hpnew;
745     }
746
747     return hp;
748 }
749
750    
751 static int is_inprogress(void)
752 {
753 #ifdef WIN32
754     if (WSAGetLastError() == WSAEWOULDBLOCK)
755         return 1;
756 #else
757     if (errno == EINPROGRESS)
758         return 1;
759 #endif
760     return 0;
761
762
763 static void enable_nonblock(int sock)
764 {
765     int flags;
766 #ifdef WIN32
767     flags = (flags & CS_FLAGS_BLOCKING) ? 0 : 1;
768     if (ioctlsocket(sock, FIONBIO, &flags) < 0)
769         yaz_log(YLOG_FATAL|YLOG_ERRNO, "ioctlsocket");
770 #else
771     if ((flags = fcntl(sock, F_GETFL, 0)) < 0) 
772         yaz_log(YLOG_FATAL|YLOG_ERRNO, "fcntl");
773     if (fcntl(sock, F_SETFL, flags | O_NONBLOCK) < 0)
774         yaz_log(YLOG_FATAL|YLOG_ERRNO, "fcntl2");
775 #endif
776 }
777
778 static int http_proxy(struct http_request *rq)
779 {
780     struct http_channel *c = rq->channel;
781     struct http_proxy *p = c->proxy;
782     struct http_header *hp;
783     struct http_buf *requestbuf;
784     char server_port[16] = "";
785     struct conf_server *ser = c->server;
786
787     if (!p) // This is a new connection. Create a proxy channel
788     {
789         int sock;
790         struct protoent *pe;
791         int one = 1;
792
793         if (!(pe = getprotobyname("tcp"))) {
794             abort();
795         }
796         if ((sock = socket(PF_INET, SOCK_STREAM, pe->p_proto)) < 0)
797         {
798             yaz_log(YLOG_WARN|YLOG_ERRNO, "socket");
799             return -1;
800         }
801         if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (char*)
802                         &one, sizeof(one)) < 0)
803             abort();
804         enable_nonblock(sock);
805         if (connect(sock, (struct sockaddr *)
806                     c->server->http_server->proxy_addr, 
807                     sizeof(*c->server->http_server->proxy_addr)) < 0)
808         {
809             if (!is_inprogress()) 
810             {
811                 yaz_log(YLOG_WARN|YLOG_ERRNO, "Proxy connect");
812                 return -1;
813             }
814         }
815         p = xmalloc(sizeof(struct http_proxy));
816         p->oqueue = 0;
817         p->channel = c;
818         p->first_response = 1;
819         c->proxy = p;
820         // We will add EVENT_OUTPUT below
821         p->iochan = iochan_create(sock, proxy_io, EVENT_INPUT, "http_proxy");
822         iochan_setdata(p->iochan, p);
823
824         iochan_add(ser->iochan_man, p->iochan);
825     }
826
827     // Do _not_ modify Host: header, just checking it's existence
828
829     if (!http_lookup_header(rq->headers, "Host"))
830     {
831         yaz_log(YLOG_WARN, "Failed to find Host header in proxy");
832         return -1;
833     }
834     
835     // Add new header about paraz2 version, host, remote client address, etc.
836     {
837         char server_via[128];
838
839         hp = rq->headers;
840         hp = http_header_append(c, hp, 
841                                 "X-Pazpar2-Version", PACKAGE_VERSION);
842         hp = http_header_append(c, hp, 
843                                 "X-Pazpar2-Server-Host", ser->host);
844         sprintf(server_port, "%d",  ser->port);
845         hp = http_header_append(c, hp, 
846                                 "X-Pazpar2-Server-Port", server_port);
847         yaz_snprintf(server_via, sizeof(server_via), 
848                      "1.1 %s:%s (%s/%s)",  
849                      ser->host ? ser->host : "@",
850                      server_port, PACKAGE_NAME, PACKAGE_VERSION);
851         hp = http_header_append(c, hp, "Via" , server_via);
852         hp = http_header_append(c, hp, "X-Forwarded-For", c->addr);
853     }
854     
855     requestbuf = http_serialize_request(rq);
856
857     http_buf_enqueue(&p->oqueue, requestbuf);
858     iochan_setflag(p->iochan, EVENT_OUTPUT);
859     return 0;
860 }
861
862 void http_send_response(struct http_channel *ch)
863 {
864     struct http_response *rs = ch->response;
865     struct http_buf *hb;
866
867     assert(rs);
868     hb = http_serialize_response(ch, rs);
869     if (!hb)
870     {
871         yaz_log(YLOG_WARN, "Failed to serialize HTTP response");
872         http_channel_destroy(ch->iochan);
873     }
874     else
875     {
876         http_buf_enqueue(&ch->oqueue, hb);
877         iochan_setflag(ch->iochan, EVENT_OUTPUT);
878         ch->state = Http_Idle;
879     }
880 }
881
882 static void http_error(struct http_channel *hc, int no, const char *msg)
883 {
884     struct http_response *rs = http_create_response(hc);
885
886     hc->response = rs;
887     hc->keep_alive = 0;  // not keeping this HTTP session alive
888
889     sprintf(rs->code, "%d", no);
890
891     rs->msg = nmem_strdup(hc->nmem, msg);
892     rs->payload = nmem_malloc(hc->nmem, 100);
893     yaz_snprintf(rs->payload, 99, "<error>HTTP Error %d: %s</error>\n",
894                  no, msg);
895     http_send_response(hc);
896 }
897
898 static void http_io(IOCHAN i, int event)
899 {
900     struct http_channel *hc = iochan_getdata(i);
901     while (event)
902     {
903         if (event == EVENT_INPUT)
904         {
905             int res, reqlen;
906             struct http_buf *htbuf;
907             
908             htbuf = http_buf_create(hc->http_server);
909             res = recv(iochan_getfd(i), htbuf->buf, HTTP_BUF_SIZE -1, 0);
910             if (res == -1 && errno == EAGAIN)
911             {
912                 http_buf_destroy(hc->http_server, htbuf);
913                 return;
914             }
915             if (res <= 0)
916             {
917                 http_buf_destroy(hc->http_server, htbuf);
918                 http_channel_destroy(i);
919                 return;
920             }
921             htbuf->buf[res] = '\0';
922             htbuf->len = res;
923             http_buf_enqueue(&hc->iqueue, htbuf);
924
925             while (1)
926             {
927                 if (hc->state == Http_Busy)
928                     return;
929                 reqlen = request_check(hc->iqueue);
930                 if (reqlen <= 2)
931                     return;
932                 // we have a complete HTTP request
933                 nmem_reset(hc->nmem);
934                 if (!(hc->request = http_parse_request(hc, &hc->iqueue, reqlen)))
935                 {
936                     yaz_log(YLOG_WARN, "Failed to parse request");
937                     http_error(hc, 400, "Bad Request");
938                     return;
939                 }
940                 hc->response = 0;
941                 yaz_log(YLOG_LOG, "Request: %s %s%s%s", hc->request->method,
942                         hc->request->path,
943                         *hc->request->search ? "?" : "",
944                         hc->request->search);
945                 if (hc->request->content_buf)
946                     yaz_log(YLOG_LOG, "%s", hc->request->content_buf);
947                 if (http_weshouldproxy(hc->request))
948                     http_proxy(hc->request);
949                 else
950                 {
951                     // Execute our business logic!
952                     hc->state = Http_Busy;
953                     http_command(hc);
954                 }
955             }
956         }
957         else if (event == EVENT_OUTPUT)
958         {
959             event = 0;
960             if (hc->oqueue)
961             {
962                 struct http_buf *wb = hc->oqueue;
963                 int res;
964                 res = send(iochan_getfd(hc->iochan),
965                            wb->buf + wb->offset, wb->len, 0);
966                 if (res <= 0)
967                 {
968                     yaz_log(YLOG_WARN|YLOG_ERRNO, "write");
969                     http_channel_destroy(i);
970                     return;
971                 }
972                 if (res == wb->len)
973                 {
974                     hc->oqueue = hc->oqueue->next;
975                     http_buf_destroy(hc->http_server, wb);
976                 }
977                 else
978                 {
979                     wb->len -= res;
980                     wb->offset += res;
981                 }
982                 if (!hc->oqueue)
983                 {
984                     if (!hc->keep_alive)
985                     {
986                         http_channel_destroy(i);
987                         return;
988                     }
989                     else
990                     {
991                         iochan_clearflag(i, EVENT_OUTPUT);
992                         if (hc->iqueue)
993                             event = EVENT_INPUT;
994                     }
995                 }
996             }
997             if (!hc->oqueue && hc->proxy && !hc->proxy->iochan) 
998                 http_channel_destroy(i); // Server closed; we're done
999         }
1000         else
1001         {
1002             yaz_log(YLOG_WARN, "Unexpected event on connection");
1003             http_channel_destroy(i);
1004             event = 0;
1005         }
1006     }
1007 }
1008
1009 // Handles I/O on a client connection to a backend web server (proxy mode)
1010 static void proxy_io(IOCHAN pi, int event)
1011 {
1012     struct http_proxy *pc = iochan_getdata(pi);
1013     struct http_channel *hc = pc->channel;
1014
1015     switch (event)
1016     {
1017         int res;
1018         struct http_buf *htbuf;
1019
1020         case EVENT_INPUT:
1021             htbuf = http_buf_create(hc->http_server);
1022             res = recv(iochan_getfd(pi), htbuf->buf, HTTP_BUF_SIZE -1, 0);
1023             if (res == 0 || (res < 0 && !is_inprogress()))
1024             {
1025                 if (hc->oqueue)
1026                 {
1027                     yaz_log(YLOG_WARN, "Proxy read came up short");
1028                     // Close channel and alert client HTTP channel that we're gone
1029                     http_buf_destroy(hc->http_server, htbuf);
1030 #ifdef WIN32
1031                     closesocket(iochan_getfd(pi));
1032 #else
1033                     close(iochan_getfd(pi));
1034 #endif
1035                     iochan_destroy(pi);
1036                     pc->iochan = 0;
1037                 }
1038                 else
1039                 {
1040                     http_channel_destroy(hc->iochan);
1041                     return;
1042                 }
1043             }
1044             else
1045             {
1046                 htbuf->buf[res] = '\0';
1047                 htbuf->offset = 0;
1048                 htbuf->len = res;
1049                 // Write any remaining payload
1050                 if (htbuf->len - htbuf->offset > 0)
1051                     http_buf_enqueue(&hc->oqueue, htbuf);
1052             }
1053             iochan_setflag(hc->iochan, EVENT_OUTPUT);
1054             break;
1055         case EVENT_OUTPUT:
1056             if (!(htbuf = pc->oqueue))
1057             {
1058                 iochan_clearflag(pi, EVENT_OUTPUT);
1059                 return;
1060             }
1061             res = send(iochan_getfd(pi), htbuf->buf + htbuf->offset, htbuf->len, 0);
1062             if (res <= 0)
1063             {
1064                 yaz_log(YLOG_WARN|YLOG_ERRNO, "write");
1065                 http_channel_destroy(hc->iochan);
1066                 return;
1067             }
1068             if (res == htbuf->len)
1069             { 
1070                 struct http_buf *np = htbuf->next;
1071                 http_buf_destroy(hc->http_server, htbuf);
1072                 pc->oqueue = np;
1073             }
1074             else
1075             {
1076                 htbuf->len -= res;
1077                 htbuf->offset += res;
1078             }
1079
1080             if (!pc->oqueue) {
1081                 iochan_setflags(pi, EVENT_INPUT); // Turns off output flag
1082             }
1083             break;
1084         default:
1085             yaz_log(YLOG_WARN, "Unexpected event on connection");
1086             http_channel_destroy(hc->iochan);
1087     }
1088 }
1089
1090 static void http_fire_observers(struct http_channel *c);
1091 static void http_destroy_observers(struct http_channel *c);
1092
1093 // Cleanup channel
1094 static void http_channel_destroy(IOCHAN i)
1095 {
1096     struct http_channel *s = iochan_getdata(i);
1097     http_server_t http_server;
1098
1099     if (s->proxy)
1100     {
1101         if (s->proxy->iochan)
1102         {
1103 #ifdef WIN32
1104             closesocket(iochan_getfd(s->proxy->iochan));
1105 #else
1106             close(iochan_getfd(s->proxy->iochan));
1107 #endif
1108             iochan_destroy(s->proxy->iochan);
1109         }
1110         http_buf_destroy_queue(s->http_server, s->proxy->oqueue);
1111         xfree(s->proxy);
1112     }
1113     http_buf_destroy_queue(s->http_server, s->iqueue);
1114     http_buf_destroy_queue(s->http_server, s->oqueue);
1115     http_fire_observers(s);
1116     http_destroy_observers(s);
1117
1118     http_server = s->http_server; /* save it for destroy (decref) */
1119
1120     yaz_mutex_enter(s->http_server->mutex);
1121     if (s->http_server->http_channel_freelist_max > 0 && s->http_server->http_channel_freelist_count >= s->http_server->http_channel_freelist_max) {
1122         while ((s->next = s->http_server->http_channel_freelist)) {
1123             nmem_destroy(s->next->nmem);
1124             wrbuf_destroy(s->next->wrbuf);
1125             xfree(s->next);
1126             s->http_server->http_channel_freelist = s->http_server->http_channel_freelist->next;
1127         }
1128         s->http_server->http_channel_freelist_count = 0;
1129     }
1130     else {
1131         s->next = s->http_server->http_channel_freelist;
1132         s->http_server->http_channel_freelist = s;
1133         s->http_server->http_channel_freelist_count++;
1134         yaz_log(YLOG_DEBUG, "Free %d channels on server.", s->http_server->http_channel_freelist_count);
1135     }
1136     yaz_mutex_leave(s->http_server->mutex);
1137
1138     http_server_destroy(http_server);
1139
1140 #ifdef WIN32
1141     closesocket(iochan_getfd(i));
1142 #else
1143     close(iochan_getfd(i));
1144 #endif
1145     iochan_destroy(i);
1146 }
1147
1148 static struct http_channel *http_channel_create(http_server_t hs,
1149                                                 const char *addr,
1150                                                 struct conf_server *server)
1151 {
1152     struct http_channel *r;
1153
1154     yaz_mutex_enter(hs->mutex);
1155     r = hs->http_channel_freelist;
1156     if (r) {
1157         hs->http_channel_freelist = r->next;
1158         hs->http_channel_freelist_count--;
1159     }
1160     yaz_mutex_leave(hs->mutex);
1161
1162     if (r)
1163     {
1164         nmem_reset(r->nmem);
1165         wrbuf_rewind(r->wrbuf);
1166     }
1167     else
1168     {
1169         r = xmalloc(sizeof(struct http_channel));
1170         r->nmem = nmem_create();
1171         r->wrbuf = wrbuf_alloc();
1172     }
1173     http_server_incref(hs);
1174     r->http_server = hs;
1175     r->http_sessions = hs->http_sessions;
1176     assert(r->http_sessions);
1177     r->server = server;
1178     r->proxy = 0;
1179     r->iochan = 0;
1180     r->iqueue = r->oqueue = 0;
1181     r->state = Http_Idle;
1182     r->keep_alive = 0;
1183     r->request = 0;
1184     r->response = 0;
1185     if (!addr)
1186     {
1187         yaz_log(YLOG_WARN, "Invalid HTTP forward address");
1188         exit(1);
1189     }
1190     strcpy(r->addr, addr);
1191     r->observers = 0;
1192     return r;
1193 }
1194
1195
1196 /* Accept a new command connection */
1197 static void http_accept(IOCHAN i, int event)
1198 {
1199     struct sockaddr_in addr;
1200     int fd = iochan_getfd(i);
1201     socklen_t len;
1202     int s;
1203     IOCHAN c;
1204     struct http_channel *ch;
1205     struct conf_server *server = iochan_getdata(i);
1206
1207     len = sizeof addr;
1208     if ((s = accept(fd, (struct sockaddr *) &addr, &len)) < 0)
1209     {
1210         yaz_log(YLOG_WARN|YLOG_ERRNO, "accept");
1211         return;
1212     }
1213     enable_nonblock(s);
1214
1215     yaz_log(YLOG_DEBUG, "New command connection");
1216     c = iochan_create(s, http_io, EVENT_INPUT | EVENT_EXCEPT, "http_session_socket");
1217     
1218     ch = http_channel_create(server->http_server, inet_ntoa(addr.sin_addr),
1219                              server);
1220     ch->iochan = c;
1221     iochan_setdata(c, ch);
1222     iochan_add(server->iochan_man, c);
1223 }
1224
1225 /* Create a http-channel listener, syntax [host:]port */
1226 int http_init(const char *addr, struct conf_server *server)
1227 {
1228     IOCHAN c;
1229     int l;
1230     struct protoent *p;
1231     struct sockaddr_in myaddr;
1232     int one = 1;
1233     const char *pp;
1234     short port;
1235
1236     yaz_log(YLOG_LOG, "HTTP listener %s", addr);
1237
1238     memset(&myaddr, 0, sizeof myaddr);
1239     myaddr.sin_family = AF_INET;
1240     pp = strchr(addr, ':');
1241     if (pp)
1242     {
1243         WRBUF w = wrbuf_alloc();
1244         struct hostent *he;
1245
1246         wrbuf_write(w, addr, pp - addr);
1247         wrbuf_puts(w, "");
1248
1249         he = gethostbyname(wrbuf_cstr(w));
1250         wrbuf_destroy(w);
1251         if (!he)
1252         {
1253             yaz_log(YLOG_FATAL, "Unable to resolve '%s'", addr);
1254             return 1;
1255         }
1256         memcpy(&myaddr.sin_addr.s_addr, he->h_addr_list[0], he->h_length);
1257         port = atoi(pp + 1);
1258     }
1259     else
1260     {
1261         port = atoi(addr);
1262         myaddr.sin_addr.s_addr = INADDR_ANY;
1263     }
1264
1265     myaddr.sin_port = htons(port);
1266
1267     if (!(p = getprotobyname("tcp"))) {
1268         return 1;
1269     }
1270     if ((l = socket(PF_INET, SOCK_STREAM, p->p_proto)) < 0)
1271         yaz_log(YLOG_FATAL|YLOG_ERRNO, "socket");
1272     if (setsockopt(l, SOL_SOCKET, SO_REUSEADDR, (char*)
1273                     &one, sizeof(one)) < 0)
1274         return 1;
1275
1276     if (bind(l, (struct sockaddr *) &myaddr, sizeof myaddr) < 0) 
1277     {
1278         yaz_log(YLOG_FATAL|YLOG_ERRNO, "bind");
1279         return 1;
1280     }
1281     if (listen(l, SOMAXCONN) < 0) 
1282     {
1283         yaz_log(YLOG_FATAL|YLOG_ERRNO, "listen");
1284         return 1;
1285     }
1286
1287     server->http_server = http_server_create();
1288
1289     server->http_server->listener_socket = l;
1290
1291     c = iochan_create(l, http_accept, EVENT_INPUT | EVENT_EXCEPT, "http_server");
1292     iochan_setdata(c, server);
1293
1294     iochan_add(server->iochan_man, c);
1295     return 0;
1296 }
1297
1298 void http_close_server(struct conf_server *server)
1299 {
1300     /* break the event_loop (select) by closing down the HTTP listener sock */
1301     if (server->http_server->listener_socket)
1302     {
1303 #ifdef WIN32
1304         closesocket(server->http_server->listener_socket);
1305 #else
1306         close(server->http_server->listener_socket);
1307 #endif
1308     }
1309 }
1310
1311 void http_set_proxyaddr(const char *host, struct conf_server *server)
1312 {
1313     const char *p;
1314     short port;
1315     struct hostent *he;
1316     WRBUF w = wrbuf_alloc();
1317
1318     yaz_log(YLOG_LOG, "HTTP backend  %s", host);
1319
1320     p = strchr(host, ':');
1321     if (p)
1322     {
1323         port = atoi(p + 1);
1324         wrbuf_write(w, host, p - host);
1325         wrbuf_puts(w, "");
1326     }
1327     else
1328     {
1329         port = 80;
1330         wrbuf_puts(w, host);
1331     }
1332     if (!(he = gethostbyname(wrbuf_cstr(w))))
1333     {
1334         fprintf(stderr, "Failed to lookup '%s'\n", wrbuf_cstr(w));
1335         exit(1);
1336     }
1337     wrbuf_destroy(w);
1338
1339     server->http_server->proxy_addr = xmalloc(sizeof(struct sockaddr_in));
1340     server->http_server->proxy_addr->sin_family = he->h_addrtype;
1341     memcpy(&server->http_server->proxy_addr->sin_addr.s_addr,
1342            he->h_addr_list[0], he->h_length);
1343     server->http_server->proxy_addr->sin_port = htons(port);
1344 }
1345
1346 static void http_fire_observers(struct http_channel *c)
1347 {
1348     http_channel_observer_t p = c->observers;
1349     while (p)
1350     {
1351         p->destroy(p->data, c, p->data2);
1352         p = p->next;
1353     }
1354 }
1355
1356 static void http_destroy_observers(struct http_channel *c)
1357 {
1358     while (c->observers)
1359     {
1360         http_channel_observer_t obs = c->observers;
1361         c->observers = obs->next;
1362         xfree(obs);
1363     }
1364 }
1365
1366 http_channel_observer_t http_add_observer(struct http_channel *c, void *data,
1367                                           http_channel_destroy_t des)
1368 {
1369     http_channel_observer_t obs = xmalloc(sizeof(*obs));
1370     obs->chan = c;
1371     obs->data = data;
1372     obs->data2 = 0;
1373     obs->destroy= des;
1374     obs->next = c->observers;
1375     c->observers = obs;
1376     return obs;
1377 }
1378
1379 void http_remove_observer(http_channel_observer_t obs)
1380 {
1381     struct http_channel *c = obs->chan;
1382     http_channel_observer_t found, *p = &c->observers;
1383     while (*p != obs)
1384         p = &(*p)->next;
1385     found = *p;
1386     assert(found);
1387     *p = (*p)->next;
1388     xfree(found);
1389 }
1390
1391 struct http_channel *http_channel_observer_chan(http_channel_observer_t obs)
1392 {
1393     return obs->chan;
1394 }
1395
1396 void http_observer_set_data2(http_channel_observer_t obs, void *data2)
1397 {
1398     obs->data2 = data2;
1399 }
1400
1401 http_server_t http_server_create(void)
1402 {
1403     http_server_t hs = xmalloc(sizeof(*hs));
1404     hs->mutex = 0;
1405     hs->proxy_addr = 0;
1406     hs->ref_count = 1;
1407     hs->http_sessions = 0;
1408
1409     hs->http_channel_freelist = 0;
1410     hs->http_channel_freelist_count = 0;
1411     /* Disable max check */
1412     hs->http_channel_freelist_max   = 0;
1413
1414     hs->http_buf_freelist = 0;
1415     hs->http_buf_freelist_count = 0;
1416     /* Disable max check */
1417     hs->http_buf_freelist_max = 0;
1418     return hs;
1419 }
1420
1421 void http_server_destroy(http_server_t hs)
1422 {
1423     if (hs)
1424     {
1425         int r;
1426
1427         yaz_mutex_enter(hs->mutex); /* OK: hs->mutex may be NULL */
1428         r = --(hs->ref_count);
1429         yaz_mutex_leave(hs->mutex);
1430
1431         if (r == 0)
1432         {
1433             struct http_buf *b = hs->http_buf_freelist;
1434             struct http_channel *c = hs->http_channel_freelist;
1435             while (b)
1436             {
1437                 struct http_buf *b_next = b->next;
1438                 xfree(b);
1439                 b = b_next;
1440             }
1441             while (c)
1442             {
1443                 struct http_channel *c_next = c->next;
1444                 nmem_destroy(c->nmem);
1445                 wrbuf_destroy(c->wrbuf);
1446                 xfree(c);
1447                 c = c_next;
1448             }
1449             http_sessions_destroy(hs->http_sessions);
1450             xfree(hs->proxy_addr);
1451             yaz_mutex_destroy(&hs->mutex);
1452             xfree(hs);
1453         }
1454     }
1455 }
1456
1457 void http_server_incref(http_server_t hs)
1458 {
1459     assert(hs);
1460     yaz_mutex_enter(hs->mutex);
1461     (hs->ref_count)++;
1462     yaz_mutex_leave(hs->mutex);
1463 }
1464
1465 void http_mutex_init(struct conf_server *server)
1466 {
1467     assert(server);
1468
1469     assert(server->http_server->mutex == 0);
1470     pazpar2_mutex_create(&server->http_server->mutex, "http_server");
1471     server->http_server->http_sessions = http_sessions_create();
1472 }
1473
1474 /*
1475  * Local variables:
1476  * c-basic-offset: 4
1477  * c-file-style: "Stroustrup"
1478  * indent-tabs-mode: nil
1479  * End:
1480  * vim: shiftwidth=4 tabstop=8 expandtab
1481  */
1482