e3d05dc89867bd69a35ad09fd3890ee4cd378d27
[pazpar2-moved-to-github.git] / src / http.c
1 /* This file is part of Pazpar2.
2    Copyright (C) Index Data
3
4 Pazpar2 is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
7 version.
8
9 Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12 for more details.
13
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
17
18 */
19
20 #if HAVE_CONFIG_H
21 #include <config.h>
22 #endif
23
24 #if HAVE_SYS_TIME_H
25 #include <sys/time.h>
26 #endif
27
28 #include <stdio.h>
29 #ifdef WIN32
30 #include <winsock2.h>
31 #include <ws2tcpip.h>
32 typedef int socklen_t;
33 #endif
34
35 #if HAVE_SYS_SOCKET_H
36 #include <sys/socket.h>
37 #endif
38
39 #include <sys/types.h>
40
41 #include <yaz/snprintf.h>
42 #if HAVE_UNISTD_H
43 #include <unistd.h>
44 #endif
45
46 #include <stdlib.h>
47 #include <string.h>
48 #include <ctype.h>
49 #include <fcntl.h>
50 #if HAVE_NETDB_H
51 #include <netdb.h>
52 #endif
53
54 #include <errno.h>
55 #include <assert.h>
56 #include <string.h>
57
58 #include <yaz/yaz-util.h>
59 #include <yaz/comstack.h>
60 #include <yaz/nmem.h>
61 #include <yaz/mutex.h>
62
63 #include "ppmutex.h"
64 #include "session.h"
65 #include "http.h"
66 #include "parameters.h"
67
68 #define MAX_HTTP_HEADER 4096
69
70 #ifdef WIN32
71 #define strncasecmp _strnicmp
72 #define strcasecmp _stricmp
73 #endif
74
75 struct http_buf
76 {
77 #define HTTP_BUF_SIZE 4096
78     char buf[4096];
79     int offset;
80     int len;
81     struct http_buf *next;
82 };
83
84
85 static void proxy_io(IOCHAN i, int event);
86 static struct http_channel *http_channel_create(http_server_t http_server,
87                                                 const char *addr,
88                                                 struct conf_server *server);
89 static void http_channel_destroy(IOCHAN i);
90 static http_server_t http_server_create(void);
91 static void http_server_incref(http_server_t hs);
92
93 #ifdef WIN32
94 #define CLOSESOCKET(x) closesocket(x)
95 #else
96 #define CLOSESOCKET(x) close(x)
97 #endif
98
99 struct http_server
100 {
101     YAZ_MUTEX mutex;
102     int listener_socket;
103     int ref_count;
104     http_sessions_t http_sessions;
105     struct sockaddr_in *proxy_addr;
106     FILE *record_file;
107 };
108
109 struct http_channel_observer_s {
110     void *data;
111     void *data2;
112     http_channel_destroy_t destroy;
113     struct http_channel_observer_s *next;
114     struct http_channel *chan;
115 };
116
117
118 const char *http_lookup_header(struct http_header *header,
119                                const char *name)
120 {
121     for (; header; header = header->next)
122         if (!strcasecmp(name, header->name))
123             return header->value;
124     return 0;
125 }
126
127 static struct http_buf *http_buf_create(http_server_t hs)
128 {
129     struct http_buf *r = xmalloc(sizeof(*r));
130     r->offset = 0;
131     r->len = 0;
132     r->next = 0;
133     return r;
134 }
135
136 static void http_buf_destroy(http_server_t hs, struct http_buf *b)
137 {
138     xfree(b);
139 }
140
141 static void http_buf_destroy_queue(http_server_t hs, struct http_buf *b)
142 {
143     struct http_buf *p;
144     while (b)
145     {
146         p = b->next;
147         http_buf_destroy(hs, b);
148         b = p;
149     }
150 }
151
152 static struct http_buf *http_buf_bybuf(http_server_t hs, char *b, int len)
153 {
154     struct http_buf *res = 0;
155     struct http_buf **p = &res;
156
157     while (len)
158     {
159         int tocopy = len;
160         if (tocopy > HTTP_BUF_SIZE)
161             tocopy = HTTP_BUF_SIZE;
162         *p = http_buf_create(hs);
163         memcpy((*p)->buf, b, tocopy);
164         (*p)->len = tocopy;
165         len -= tocopy;
166         b += tocopy;
167         p = &(*p)->next;
168     }
169     return res;
170 }
171
172 // Add a (chain of) buffers to the end of an existing queue.
173 static void http_buf_enqueue(struct http_buf **queue, struct http_buf *b)
174 {
175     while (*queue)
176         queue = &(*queue)->next;
177     *queue = b;
178 }
179
180 static struct http_buf *http_buf_bywrbuf(http_server_t hs, WRBUF wrbuf)
181 {
182     // Heavens to Betsy (buf)!
183     return http_buf_bybuf(hs, wrbuf_buf(wrbuf), wrbuf_len(wrbuf));
184 }
185
186 // Non-destructively collapse chain of buffers into a string (max *len)
187 // Return
188 static void http_buf_peek(struct http_buf *b, char *buf, int len)
189 {
190     int rd = 0;
191     while (b && rd < len)
192     {
193         int toread = len - rd;
194         if (toread > b->len)
195             toread = b->len;
196         memcpy(buf + rd, b->buf + b->offset, toread);
197         rd += toread;
198         b = b->next;
199     }
200     buf[rd] = '\0';
201 }
202
203 static int http_buf_size(struct http_buf *b)
204 {
205     int sz = 0;
206     for (; b; b = b->next)
207         sz += b->len;
208     return sz;
209 }
210
211 // Ddestructively munch up to len  from head of queue.
212 static int http_buf_read(http_server_t hs,
213                          struct http_buf **b, char *buf, int len)
214 {
215     int rd = 0;
216     while ((*b) && rd < len)
217     {
218         int toread = len - rd;
219         if (toread > (*b)->len)
220             toread = (*b)->len;
221         memcpy(buf + rd, (*b)->buf + (*b)->offset, toread);
222         rd += toread;
223         if (toread < (*b)->len)
224         {
225             (*b)->len -= toread;
226             (*b)->offset += toread;
227             break;
228         }
229         else
230         {
231             struct http_buf *n = (*b)->next;
232             http_buf_destroy(hs, *b);
233             *b = n;
234         }
235     }
236     buf[rd] = '\0';
237     return rd;
238 }
239
240 // Buffers may overlap.
241 static void urldecode(char *i, char *o)
242 {
243     while (*i)
244     {
245         if (*i == '+')
246         {
247             *(o++) = ' ';
248             i++;
249         }
250         else if (*i == '%' && i[1] && i[2])
251         {
252             int v;
253             i++;
254             sscanf(i, "%2x", &v);
255             *o++ = v;
256             i += 2;
257         }
258         else
259             *(o++) = *(i++);
260     }
261     *o = '\0';
262 }
263
264 // Warning: Buffers may not overlap
265 void urlencode(const char *i, char *o)
266 {
267     while (*i)
268     {
269         if (strchr(" /:", *i))
270         {
271             sprintf(o, "%%%.2X", (int) *i);
272             o += 3;
273         }
274         else
275             *(o++) = *i;
276         i++;
277     }
278     *o = '\0';
279 }
280
281 void http_addheader(struct http_response *r, const char *name, const char *value)
282 {
283     struct http_channel *c = r->channel;
284     struct http_header *h = nmem_malloc(c->nmem, sizeof *h);
285     h->name = nmem_strdup(c->nmem, name);
286     h->value = nmem_strdup(c->nmem, value);
287     h->next = r->headers;
288     r->headers = h;
289 }
290
291 const char *http_argbyname(struct http_request *r, const char *name)
292 {
293     struct http_argument *p;
294     if (!name)
295         return 0;
296     for (p = r->arguments; p; p = p->next)
297         if (!strcmp(p->name, name))
298             return p->value;
299     return 0;
300 }
301
302 const char *http_headerbyname(struct http_header *h, const char *name)
303 {
304     for (; h; h = h->next)
305         if (!strcmp(h->name, name))
306             return h->value;
307     return 0;
308 }
309
310 struct http_response *http_create_response(struct http_channel *c)
311 {
312     struct http_response *r = nmem_malloc(c->nmem, sizeof(*r));
313     strcpy(r->code, "200");
314     r->msg = "OK";
315     r->channel = c;
316     r->headers = 0;
317     r->payload = 0;
318     r->content_type = "text/xml";
319     return r;
320 }
321
322
323 static const char *next_crlf(const char *cp, size_t *skipped)
324 {
325     const char *next_cp = strchr(cp, '\n');
326     if (next_cp)
327     {
328         if (next_cp > cp && next_cp[-1] == '\r')
329             *skipped = next_cp - cp - 1;
330         else
331             *skipped = next_cp - cp;
332         next_cp++;
333     }
334     return next_cp;
335 }
336
337 // Check if buf contains a package (minus payload)
338 static int package_check(const char *buf, int sz)
339 {
340     int content_len = 0;
341     int len = 0;
342
343     while (*buf)
344     {
345         size_t skipped = 0;
346         const char *b = next_crlf(buf, &skipped);
347
348         if (!b)
349         {
350             // we did not find CRLF.. See if buffer is too large..
351             if (sz >= MAX_HTTP_HEADER-1)
352                 return MAX_HTTP_HEADER-1; // yes. Return that (will fail later)
353             break;
354         }
355         len += (b - buf);
356         if (skipped == 0)
357         {
358             // CRLF CRLF , i.e. end of header
359             if (len + content_len <= sz)
360                 return len + content_len;
361             break;
362         }
363         buf = b;
364         // following first skip of \r\n so that we don't consider Method
365         if (!strncasecmp(buf, "Content-Length:", 15))
366         {
367             const char *cp = buf+15;
368             while (*cp == ' ')
369                 cp++;
370             content_len = 0;
371             while (*cp && isdigit(*(const unsigned char *)cp))
372                 content_len = content_len*10 + (*cp++ - '0');
373             if (content_len < 0) /* prevent negative offsets */
374                 content_len = 0;
375         }
376     }
377     return 0;     // incomplete request
378 }
379
380 // Check if we have a request. Return 0 or length
381 static int request_check(struct http_buf *queue)
382 {
383     char tmp[MAX_HTTP_HEADER];
384
385     // only peek at the header..
386     http_buf_peek(queue, tmp, MAX_HTTP_HEADER-1);
387     // still we only return non-zero if the complete request is received..
388     return package_check(tmp, http_buf_size(queue));
389 }
390
391 struct http_response *http_parse_response_buf(struct http_channel *c, const char *buf, int len)
392 {
393     char tmp[MAX_HTTP_HEADER];
394     struct http_response *r = http_create_response(c);
395     char *p, *p2;
396     struct http_header **hp = &r->headers;
397
398     if (len >= MAX_HTTP_HEADER)
399         return 0;
400     memcpy(tmp, buf, len);
401     for (p = tmp; *p && *p != ' '; p++) // Skip HTTP version
402         ;
403     p++;
404     // Response code
405     for (p2 = p; *p2 && *p2 != ' ' && p2 - p < 3; p2++)
406         r->code[p2 - p] = *p2;
407     if (!(p = strstr(tmp, "\r\n")))
408         return 0;
409     p += 2;
410     while (*p)
411     {
412         if (!(p2 = strstr(p, "\r\n")))
413             return 0;
414         if (p == p2) // End of headers
415             break;
416         else
417         {
418             struct http_header *h = *hp = nmem_malloc(c->nmem, sizeof(*h));
419             char *value = strchr(p, ':');
420             if (!value)
421                 return 0;
422             *(value++) = '\0';
423             h->name = nmem_strdup(c->nmem, p);
424             while (isspace(*(const unsigned char *) value))
425                 value++;
426             if (value >= p2)  // Empty header;
427             {
428                 h->value = "";
429                 p = p2 + 2;
430                 continue;
431             }
432             *p2 = '\0';
433             h->value = nmem_strdup(c->nmem, value);
434             h->next = 0;
435             hp = &h->next;
436             p = p2 + 2;
437         }
438     }
439     return r;
440 }
441
442 static int http_parse_arguments(struct http_request *r, NMEM nmem,
443                                 const char *args)
444 {
445     const char *p2 = args;
446
447     while (*p2)
448     {
449         struct http_argument *a;
450         const char *equal = strchr(p2, '=');
451         const char *eoa = strchr(p2, '&');
452         if (!equal)
453         {
454             yaz_log(YLOG_WARN, "Expected '=' in argument");
455             return -1;
456         }
457         if (!eoa)
458             eoa = equal + strlen(equal); // last argument
459         else if (equal > eoa)
460         {
461             yaz_log(YLOG_WARN, "Missing '&' in argument");
462             return -1;
463         }
464         a = nmem_malloc(nmem, sizeof(struct http_argument));
465         a->name = nmem_strdupn(nmem, p2, equal - p2);
466         a->value = nmem_strdupn(nmem, equal+1, eoa - equal - 1);
467         urldecode(a->name, a->name);
468         urldecode(a->value, a->value);
469         a->next = r->arguments;
470         r->arguments = a;
471         p2 = eoa;
472         while (*p2 == '&')
473             p2++;
474     }
475     return 0;
476 }
477
478 struct http_request *http_parse_request(struct http_channel *c,
479                                         struct http_buf **queue,
480                                         int len)
481 {
482     struct http_request *r = nmem_malloc(c->nmem, sizeof(*r));
483     char *p, *p2;
484     char *start = nmem_malloc(c->nmem, len+1);
485     char *buf = start;
486
487     if (http_buf_read(c->http_server, queue, buf, len) < len)
488     {
489         yaz_log(YLOG_WARN, "http_buf_read < len (%d)", len);
490         return 0;
491     }
492     r->search = "";
493     r->channel = c;
494     r->arguments = 0;
495     r->headers = 0;
496     r->content_buf = 0;
497     r->content_len = 0;
498     // Parse first line
499     for (p = buf, p2 = r->method; *p && *p != ' ' && p - buf < 19; p++)
500         *(p2++) = *p;
501     if (*p != ' ')
502     {
503         yaz_log(YLOG_WARN, "Unexpected HTTP method in request");
504         return 0;
505     }
506     *p2 = '\0';
507
508     if (!(buf = strchr(buf, ' ')))
509     {
510         yaz_log(YLOG_WARN, "Missing Request-URI in HTTP request");
511         return 0;
512     }
513     buf++;
514     if (!(p = strchr(buf, ' ')))
515     {
516         yaz_log(YLOG_WARN, "HTTP Request-URI not terminated (too long?)");
517         return 0;
518     }
519     *(p++) = '\0';
520     if ((p2 = strchr(buf, '?'))) // Do we have arguments?
521         *(p2++) = '\0';
522     r->path = nmem_strdup(c->nmem, buf);
523     if (p2)
524     {
525         r->search = nmem_strdup(c->nmem, p2);
526         // Parse Arguments
527         http_parse_arguments(r, c->nmem, p2);
528     }
529     buf = p;
530
531     if (strncmp(buf, "HTTP/", 5))
532         strcpy(r->http_version, "1.0");
533     else
534     {
535         size_t skipped;
536         buf += 5; // strlen("HTTP/")
537
538         p = (char*) next_crlf(buf, &skipped);
539         if (!p || skipped < 3 || skipped > 5)
540             return 0;
541
542         memcpy(r->http_version, buf, skipped);
543         r->http_version[skipped] = '\0';
544         buf = p;
545     }
546     strcpy(c->version, r->http_version);
547
548     r->headers = 0;
549     while (*buf)
550     {
551         size_t skipped;
552
553         p = (char *) next_crlf(buf, &skipped);
554         if (!p)
555         {
556             return 0;
557         }
558         else if (skipped == 0)
559         {
560             buf = p;
561             break;
562         }
563         else
564         {
565             char *cp;
566             char *n_v = nmem_malloc(c->nmem, skipped+1);
567             struct http_header *h = nmem_malloc(c->nmem, sizeof(*h));
568
569             memcpy(n_v, buf, skipped);
570             n_v[skipped] = '\0';
571
572             if (!(cp = strchr(n_v, ':')))
573                 return 0;
574             h->name = nmem_strdupn(c->nmem, n_v, cp - n_v);
575             cp++;
576             while (isspace(*cp))
577                 cp++;
578             h->value = nmem_strdup(c->nmem, cp);
579             h->next = r->headers;
580             r->headers = h;
581             buf = p;
582         }
583     }
584
585     // determine if we do keep alive
586     if (!strcmp(c->version, "1.0"))
587     {
588         const char *v = http_lookup_header(r->headers, "Connection");
589         if (v && !strcmp(v, "Keep-Alive"))
590             c->keep_alive = 1;
591         else
592             c->keep_alive = 0;
593     }
594     else
595     {
596         const char *v = http_lookup_header(r->headers, "Connection");
597         if (v && !strcmp(v, "close"))
598             c->keep_alive = 0;
599         else
600             c->keep_alive = 1;
601     }
602     if (buf < start + len)
603     {
604         const char *content_type = http_lookup_header(r->headers,
605                                                       "Content-Type");
606         r->content_len = start + len - buf;
607         r->content_buf = buf;
608
609         if (!yaz_strcmp_del("application/x-www-form-urlencoded",
610                             content_type, "; "))
611         {
612             http_parse_arguments(r, c->nmem, r->content_buf);
613         }
614     }
615     return r;
616 }
617
618 static struct http_buf *http_serialize_response(struct http_channel *c,
619         struct http_response *r)
620 {
621     struct http_header *h;
622
623     wrbuf_rewind(c->wrbuf);
624
625     wrbuf_printf(c->wrbuf, "HTTP/%s %s %s\r\n", c->version, r->code, r->msg);
626     for (h = r->headers; h; h = h->next)
627         wrbuf_printf(c->wrbuf, "%s: %s\r\n", h->name, h->value);
628     if (r->payload)
629     {
630         wrbuf_printf(c->wrbuf, "Content-Length: %d\r\n", r->payload ?
631                 (int) strlen(r->payload) : 0);
632         wrbuf_printf(c->wrbuf, "Content-Type: %s\r\n", r->content_type);
633         if (!strcmp(r->content_type, "text/xml"))
634         {
635             xmlDoc *doc = xmlParseMemory(r->payload, strlen(r->payload));
636             if (doc)
637             {
638                 xmlFreeDoc(doc);
639             }
640             else
641             {
642                 yaz_log(YLOG_WARN, "Sending non-wellformed "
643                         "response (bug #1162");
644                 yaz_log(YLOG_WARN, "payload: %s", r->payload);
645             }
646         }
647     }
648     wrbuf_puts(c->wrbuf, "\r\n");
649
650     if (r->payload)
651         wrbuf_puts(c->wrbuf, r->payload);
652
653     if (global_parameters.dump_records > 1)
654     {
655         FILE *lf = yaz_log_file();
656         yaz_log(YLOG_LOG, "Response:");
657         fwrite(wrbuf_buf(c->wrbuf), 1, wrbuf_len(c->wrbuf), lf);
658     }
659     return http_buf_bywrbuf(c->http_server, c->wrbuf);
660 }
661
662 // Serialize a HTTP request
663 static struct http_buf *http_serialize_request(struct http_request *r)
664 {
665     struct http_channel *c = r->channel;
666     struct http_header *h;
667
668     wrbuf_rewind(c->wrbuf);
669     wrbuf_printf(c->wrbuf, "%s %s%s%s", r->method, r->path,
670                  *r->search ? "?" : "", r->search);
671
672     wrbuf_printf(c->wrbuf, " HTTP/%s\r\n", r->http_version);
673
674     for (h = r->headers; h; h = h->next)
675         wrbuf_printf(c->wrbuf, "%s: %s\r\n", h->name, h->value);
676
677     wrbuf_puts(c->wrbuf, "\r\n");
678
679     if (r->content_buf)
680         wrbuf_write(c->wrbuf, r->content_buf, r->content_len);
681
682 #if 0
683     yaz_log(YLOG_LOG, "WRITING TO PROXY:\n%s\n----",
684             wrbuf_cstr(c->wrbuf));
685 #endif
686     return http_buf_bywrbuf(c->http_server, c->wrbuf);
687 }
688
689
690 static int http_weshouldproxy(struct http_request *rq)
691 {
692     struct http_channel *c = rq->channel;
693     if (c->server->http_server->proxy_addr && !strstr(rq->path, "search.pz2"))
694         return 1;
695     return 0;
696 }
697
698
699 struct http_header * http_header_append(struct http_channel *ch,
700                                         struct http_header * hp,
701                                         const char *name,
702                                         const char *value)
703 {
704     struct http_header *hpnew = 0;
705
706     if (!hp | !ch)
707         return 0;
708
709     while (hp && hp->next)
710         hp = hp->next;
711
712     if(name && strlen(name)&& value && strlen(value)){
713         hpnew = nmem_malloc(ch->nmem, sizeof *hpnew);
714         hpnew->name = nmem_strdup(ch->nmem, name);
715         hpnew->value = nmem_strdup(ch->nmem, value);
716
717         hpnew->next = 0;
718         hp->next = hpnew;
719         hp = hp->next;
720
721         return hpnew;
722     }
723
724     return hp;
725 }
726
727
728 static int is_inprogress(void)
729 {
730 #ifdef WIN32
731     if (WSAGetLastError() == WSAEWOULDBLOCK)
732         return 1;
733 #else
734     if (errno == EINPROGRESS)
735         return 1;
736 #endif
737     return 0;
738 }
739
740 static void enable_nonblock(int sock)
741 {
742     int flags;
743 #ifdef WIN32
744     flags = (flags & CS_FLAGS_BLOCKING) ? 0 : 1;
745     if (ioctlsocket(sock, FIONBIO, &flags) < 0)
746         yaz_log(YLOG_FATAL|YLOG_ERRNO, "ioctlsocket");
747 #else
748     if ((flags = fcntl(sock, F_GETFL, 0)) < 0)
749         yaz_log(YLOG_FATAL|YLOG_ERRNO, "fcntl");
750     if (fcntl(sock, F_SETFL, flags | O_NONBLOCK) < 0)
751         yaz_log(YLOG_FATAL|YLOG_ERRNO, "fcntl2");
752 #endif
753 }
754
755 static int http_proxy(struct http_request *rq)
756 {
757     struct http_channel *c = rq->channel;
758     struct http_proxy *p = c->proxy;
759     struct http_header *hp;
760     struct http_buf *requestbuf;
761     struct conf_server *ser = c->server;
762
763     if (!p) // This is a new connection. Create a proxy channel
764     {
765         int sock;
766         struct protoent *pe;
767         int one = 1;
768
769         if (!(pe = getprotobyname("tcp"))) {
770             abort();
771         }
772         if ((sock = socket(PF_INET, SOCK_STREAM, pe->p_proto)) < 0)
773         {
774             yaz_log(YLOG_WARN|YLOG_ERRNO, "socket");
775             return -1;
776         }
777         if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (char*)
778                         &one, sizeof(one)) < 0)
779             abort();
780         enable_nonblock(sock);
781         if (connect(sock, (struct sockaddr *)
782                     c->server->http_server->proxy_addr,
783                     sizeof(*c->server->http_server->proxy_addr)) < 0)
784         {
785             if (!is_inprogress())
786             {
787                 yaz_log(YLOG_WARN|YLOG_ERRNO, "Proxy connect");
788                 return -1;
789             }
790         }
791         p = xmalloc(sizeof(struct http_proxy));
792         p->oqueue = 0;
793         p->channel = c;
794         p->first_response = 1;
795         c->proxy = p;
796         // We will add EVENT_OUTPUT below
797         p->iochan = iochan_create(sock, proxy_io, EVENT_INPUT, "http_proxy");
798         iochan_setdata(p->iochan, p);
799
800         iochan_add(ser->iochan_man, p->iochan);
801     }
802
803     // Do _not_ modify Host: header, just checking it's existence
804
805     if (!http_lookup_header(rq->headers, "Host"))
806     {
807         yaz_log(YLOG_WARN, "Failed to find Host header in proxy");
808         return -1;
809     }
810
811     // Add new header about paraz2 version, host, remote client address, etc.
812     {
813         char server_via[128];
814
815         hp = rq->headers;
816         hp = http_header_append(c, hp,
817                                 "X-Pazpar2-Version", PACKAGE_VERSION);
818         hp = http_header_append(c, hp,
819                                 "X-Pazpar2-Server-Host", ser->host);
820         hp = http_header_append(c, hp,
821                                 "X-Pazpar2-Server-Port", ser->port);
822         yaz_snprintf(server_via, sizeof(server_via),
823                      "1.1 %s:%s (%s/%s)",
824                      ser->host, ser->port,
825                      PACKAGE_NAME, PACKAGE_VERSION);
826         hp = http_header_append(c, hp, "Via" , server_via);
827         hp = http_header_append(c, hp, "X-Forwarded-For", c->addr);
828     }
829
830     requestbuf = http_serialize_request(rq);
831
832     http_buf_enqueue(&p->oqueue, requestbuf);
833     iochan_setflag(p->iochan, EVENT_OUTPUT);
834     return 0;
835 }
836
837 void http_send_response(struct http_channel *ch)
838 {
839     struct http_response *rs = ch->response;
840     struct http_buf *hb;
841
842     yaz_timing_stop(ch->yt);
843     if (ch->request)
844     {
845         yaz_log(YLOG_LOG, "Response: %6.5f %d %s%s%s ",
846                 yaz_timing_get_real(ch->yt),
847                 iochan_getfd(ch->iochan),
848                 ch->request->path,
849                 *ch->request->search ? "?" : "",
850                 ch->request->search);
851     }
852     assert(rs);
853     hb = http_serialize_response(ch, rs);
854     if (!hb)
855     {
856         yaz_log(YLOG_WARN, "Failed to serialize HTTP response");
857         http_channel_destroy(ch->iochan);
858     }
859     else
860     {
861         http_buf_enqueue(&ch->oqueue, hb);
862         iochan_setflag(ch->iochan, EVENT_OUTPUT);
863         ch->state = Http_Idle;
864     }
865 }
866
867 static void http_error(struct http_channel *hc, int no, const char *msg)
868 {
869     struct http_response *rs = http_create_response(hc);
870
871     hc->response = rs;
872     hc->keep_alive = 0;  // not keeping this HTTP session alive
873
874     sprintf(rs->code, "%d", no);
875
876     rs->msg = nmem_strdup(hc->nmem, msg);
877     rs->payload = nmem_malloc(hc->nmem, 100);
878     yaz_snprintf(rs->payload, 99, "<error>HTTP Error %d: %s</error>\n",
879                  no, msg);
880     http_send_response(hc);
881 }
882
883 static void http_io(IOCHAN i, int event)
884 {
885     struct http_channel *hc = iochan_getdata(i);
886     while (event)
887     {
888         if (event == EVENT_INPUT)
889         {
890             int res, reqlen;
891             struct http_buf *htbuf;
892
893             htbuf = http_buf_create(hc->http_server);
894             res = recv(iochan_getfd(i), htbuf->buf, HTTP_BUF_SIZE -1, 0);
895             if (res == -1 && errno == EAGAIN)
896             {
897                 http_buf_destroy(hc->http_server, htbuf);
898                 return;
899             }
900             if (res <= 0)
901             {
902 #if HAVE_SYS_TIME_H
903                 if (hc->http_server->record_file)
904                 {
905                     struct timeval tv;
906                     gettimeofday(&tv, 0);
907                     fprintf(hc->http_server->record_file, "r %lld %lld %lld 0\n",
908                             (long long) tv.tv_sec, (long long) tv.tv_usec,
909                             (long long) iochan_getfd(i));
910                 }
911 #endif
912                 http_buf_destroy(hc->http_server, htbuf);
913                 fflush(hc->http_server->record_file);
914                 http_channel_destroy(i);
915                 return;
916             }
917             htbuf->buf[res] = '\0';
918             htbuf->len = res;
919             http_buf_enqueue(&hc->iqueue, htbuf);
920
921             while (1)
922             {
923                 if (hc->state == Http_Busy)
924                     return;
925                 reqlen = request_check(hc->iqueue);
926                 if (reqlen <= 2)
927                     return;
928                 // we have a complete HTTP request
929                 nmem_reset(hc->nmem);
930 #if HAVE_SYS_TIME_H
931                 if (hc->http_server->record_file)
932                 {
933                     struct timeval tv;
934                     int sz = 0;
935                     struct http_buf *hb;
936                     for (hb = hc->iqueue; hb; hb = hb->next)
937                         sz += hb->len;
938                     gettimeofday(&tv, 0);
939                     fprintf(hc->http_server->record_file, "r %lld %lld %lld %d\n",
940                             (long long) tv.tv_sec, (long long) tv.tv_usec,
941                             (long long) iochan_getfd(i), sz);
942                     for (hb = hc->iqueue; hb; hb = hb->next)
943                         fwrite(hb->buf, 1, hb->len, hc->http_server->record_file);
944                     fflush(hc->http_server->record_file);
945                 }
946  #endif
947                 yaz_timing_start(hc->yt);
948                 if (!(hc->request = http_parse_request(hc, &hc->iqueue, reqlen)))
949                 {
950                     yaz_log(YLOG_WARN, "Failed to parse request");
951                     http_error(hc, 400, "Bad Request");
952                     return;
953                 }
954                 hc->response = 0;
955                 yaz_log(YLOG_LOG, "Request: - %d %s %s%s%s",
956                         iochan_getfd(i),
957                         hc->request->method,
958                         hc->request->path,
959                         *hc->request->search ? "?" : "",
960                         hc->request->search);
961                 if (hc->request->content_buf)
962                     yaz_log(YLOG_LOG, "%s", hc->request->content_buf);
963                 if (http_weshouldproxy(hc->request))
964                     http_proxy(hc->request);
965                 else
966                 {
967                     // Execute our business logic!
968                     hc->state = Http_Busy;
969                     http_command(hc);
970                 }
971             }
972         }
973         else if (event == EVENT_OUTPUT)
974         {
975             event = 0;
976             if (hc->oqueue)
977             {
978                 struct http_buf *wb = hc->oqueue;
979                 int res;
980                 res = send(iochan_getfd(hc->iochan),
981                            wb->buf + wb->offset, wb->len, 0);
982                 if (res <= 0)
983                 {
984                     yaz_log(YLOG_WARN|YLOG_ERRNO, "write");
985                     http_channel_destroy(i);
986                     return;
987                 }
988                 if (res == wb->len)
989                 {
990 #if HAVE_SYS_TIME_H
991                     if (hc->http_server->record_file)
992                     {
993                         struct timeval tv;
994                         int sz = wb->offset + wb->len;
995                         gettimeofday(&tv, 0);
996                         fprintf(hc->http_server->record_file, "w %lld %lld %lld %d\n",
997                                 (long long) tv.tv_sec, (long long) tv.tv_usec,
998                                 (long long) iochan_getfd(i), sz);
999                         fwrite(wb->buf, 1, wb->offset + wb->len,
1000                                hc->http_server->record_file);
1001                         fputc('\n', hc->http_server->record_file);
1002                         fflush(hc->http_server->record_file);
1003                     }
1004  #endif
1005                     hc->oqueue = hc->oqueue->next;
1006                     http_buf_destroy(hc->http_server, wb);
1007                 }
1008                 else
1009                 {
1010                     wb->len -= res;
1011                     wb->offset += res;
1012                 }
1013                 if (!hc->oqueue)
1014                 {
1015                     if (!hc->keep_alive)
1016                     {
1017                         http_channel_destroy(i);
1018                         return;
1019                     }
1020                     else
1021                     {
1022                         iochan_clearflag(i, EVENT_OUTPUT);
1023                         if (hc->iqueue)
1024                             event = EVENT_INPUT;
1025                     }
1026                 }
1027             }
1028             if (!hc->oqueue && hc->proxy && !hc->proxy->iochan)
1029                 http_channel_destroy(i); // Server closed; we're done
1030         }
1031         else
1032         {
1033             yaz_log(YLOG_WARN, "Unexpected event on connection");
1034             http_channel_destroy(i);
1035             event = 0;
1036         }
1037     }
1038 }
1039
1040 // Handles I/O on a client connection to a backend web server (proxy mode)
1041 static void proxy_io(IOCHAN pi, int event)
1042 {
1043     struct http_proxy *pc = iochan_getdata(pi);
1044     struct http_channel *hc = pc->channel;
1045
1046     switch (event)
1047     {
1048         int res;
1049         struct http_buf *htbuf;
1050
1051         case EVENT_INPUT:
1052             htbuf = http_buf_create(hc->http_server);
1053             res = recv(iochan_getfd(pi), htbuf->buf, HTTP_BUF_SIZE -1, 0);
1054             if (res == 0 || (res < 0 && !is_inprogress()))
1055             {
1056                 if (hc->oqueue)
1057                 {
1058                     yaz_log(YLOG_WARN, "Proxy read came up short");
1059                     // Close channel and alert client HTTP channel that we're gone
1060                     http_buf_destroy(hc->http_server, htbuf);
1061                     CLOSESOCKET(iochan_getfd(pi));
1062                     iochan_destroy(pi);
1063                     pc->iochan = 0;
1064                 }
1065                 else
1066                 {
1067                     http_channel_destroy(hc->iochan);
1068                     return;
1069                 }
1070             }
1071             else
1072             {
1073                 htbuf->buf[res] = '\0';
1074                 htbuf->offset = 0;
1075                 htbuf->len = res;
1076                 // Write any remaining payload
1077                 if (htbuf->len - htbuf->offset > 0)
1078                     http_buf_enqueue(&hc->oqueue, htbuf);
1079             }
1080             iochan_setflag(hc->iochan, EVENT_OUTPUT);
1081             break;
1082         case EVENT_OUTPUT:
1083             if (!(htbuf = pc->oqueue))
1084             {
1085                 iochan_clearflag(pi, EVENT_OUTPUT);
1086                 return;
1087             }
1088             res = send(iochan_getfd(pi), htbuf->buf + htbuf->offset, htbuf->len, 0);
1089             if (res <= 0)
1090             {
1091                 yaz_log(YLOG_WARN|YLOG_ERRNO, "write");
1092                 http_channel_destroy(hc->iochan);
1093                 return;
1094             }
1095             if (res == htbuf->len)
1096             {
1097                 struct http_buf *np = htbuf->next;
1098                 http_buf_destroy(hc->http_server, htbuf);
1099                 pc->oqueue = np;
1100             }
1101             else
1102             {
1103                 htbuf->len -= res;
1104                 htbuf->offset += res;
1105             }
1106
1107             if (!pc->oqueue) {
1108                 iochan_setflags(pi, EVENT_INPUT); // Turns off output flag
1109             }
1110             break;
1111         default:
1112             yaz_log(YLOG_WARN, "Unexpected event on connection");
1113             http_channel_destroy(hc->iochan);
1114             break;
1115     }
1116 }
1117
1118 static void http_fire_observers(struct http_channel *c);
1119 static void http_destroy_observers(struct http_channel *c);
1120
1121 // Cleanup channel
1122 static void http_channel_destroy(IOCHAN i)
1123 {
1124     struct http_channel *s = iochan_getdata(i);
1125     http_server_t http_server;
1126
1127     if (s->proxy)
1128     {
1129         if (s->proxy->iochan)
1130         {
1131             CLOSESOCKET(iochan_getfd(s->proxy->iochan));
1132             iochan_destroy(s->proxy->iochan);
1133         }
1134         http_buf_destroy_queue(s->http_server, s->proxy->oqueue);
1135         xfree(s->proxy);
1136     }
1137     yaz_timing_destroy(&s->yt);
1138     http_buf_destroy_queue(s->http_server, s->iqueue);
1139     http_buf_destroy_queue(s->http_server, s->oqueue);
1140     http_fire_observers(s);
1141     http_destroy_observers(s);
1142
1143     http_server = s->http_server; /* save it for destroy (decref) */
1144
1145     http_server_destroy(http_server);
1146
1147     CLOSESOCKET(iochan_getfd(i));
1148
1149     iochan_destroy(i);
1150     nmem_destroy(s->nmem);
1151     wrbuf_destroy(s->wrbuf);
1152     xfree(s);
1153 }
1154
1155 static struct http_channel *http_channel_create(http_server_t hs,
1156                                                 const char *addr,
1157                                                 struct conf_server *server)
1158 {
1159     struct http_channel *r;
1160
1161     r = xmalloc(sizeof(struct http_channel));
1162     r->nmem = nmem_create();
1163     r->wrbuf = wrbuf_alloc();
1164
1165     http_server_incref(hs);
1166     r->http_server = hs;
1167     r->http_sessions = hs->http_sessions;
1168     assert(r->http_sessions);
1169     r->server = server;
1170     r->proxy = 0;
1171     r->iochan = 0;
1172     r->iqueue = r->oqueue = 0;
1173     r->state = Http_Idle;
1174     r->keep_alive = 0;
1175     r->request = 0;
1176     r->response = 0;
1177     strcpy(r->version, "1.0");
1178     if (!addr)
1179     {
1180         yaz_log(YLOG_WARN, "Invalid HTTP forward address");
1181         exit(1);
1182     }
1183     strcpy(r->addr, addr);
1184     r->observers = 0;
1185     r->yt = yaz_timing_create();
1186     return r;
1187 }
1188
1189
1190 /* Accept a new command connection */
1191 static void http_accept(IOCHAN i, int event)
1192 {
1193     char host[256];
1194     struct sockaddr_storage addr;
1195     int fd = iochan_getfd(i);
1196     socklen_t len = sizeof addr;
1197     int s;
1198     IOCHAN c;
1199     struct http_channel *ch;
1200     struct conf_server *server = iochan_getdata(i);
1201
1202     if ((s = accept(fd, (struct sockaddr *) &addr, &len)) < 0)
1203     {
1204         yaz_log(YLOG_WARN|YLOG_ERRNO, "accept");
1205         return;
1206     }
1207     if (getnameinfo((struct sockaddr *) &addr, len, host, sizeof(host)-1, 0, 0,
1208         NI_NUMERICHOST))
1209     {
1210         yaz_log(YLOG_WARN|YLOG_ERRNO, "getnameinfo");
1211         CLOSESOCKET(s);
1212         return;
1213     }
1214     enable_nonblock(s);
1215
1216     yaz_log(YLOG_DEBUG, "New command connection");
1217     c = iochan_create(s, http_io, EVENT_INPUT | EVENT_EXCEPT,
1218                       "http_session_socket");
1219
1220
1221     ch = http_channel_create(server->http_server, host, server);
1222     ch->iochan = c;
1223     iochan_setdata(c, ch);
1224     iochan_add(server->iochan_man, c);
1225 }
1226
1227 /* Create a http-channel listener, syntax [host:]port */
1228 int http_init(struct conf_server *server, const char *record_fname)
1229 {
1230     IOCHAN c;
1231     int s = -1;
1232     int one = 1;
1233     FILE *record_file = 0;
1234     struct addrinfo hints, *af = 0, *ai;
1235     int error;
1236     int ipv6_only = -1;
1237
1238     yaz_log(YLOG_LOG, "HTTP listener %s:%s", server->host, server->port);
1239
1240     hints.ai_flags = 0;
1241     hints.ai_family = AF_UNSPEC;
1242     hints.ai_socktype = SOCK_STREAM;
1243     hints.ai_protocol = 0;
1244     hints.ai_addrlen        = 0;
1245     hints.ai_addr           = NULL;
1246     hints.ai_canonname      = NULL;
1247     hints.ai_next           = NULL;
1248
1249     if (!strcmp(server->host, "@"))
1250     {
1251         ipv6_only = 0;
1252         hints.ai_flags = AI_PASSIVE;
1253         error = getaddrinfo(0, server->port, &hints, &af);
1254     }
1255     else
1256         error = getaddrinfo(server->host, server->port, &hints, &af);
1257
1258     if (error)
1259     {
1260         yaz_log(YLOG_FATAL, "Failed to resolve %s: %s", server->host,
1261                 gai_strerror(error));
1262         return 1;
1263     }
1264     for (ai = af; ai; ai = ai->ai_next)
1265     {
1266         if (ai->ai_family == AF_INET6)
1267         {
1268             s = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol);
1269             if (s != -1)
1270                 break;
1271         }
1272     }
1273     if (s == -1)
1274     {
1275         for (ai = af; ai; ai = ai->ai_next)
1276         {
1277             s = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol);
1278             if (s != -1)
1279                 break;
1280         }
1281     }
1282     if (s == -1)
1283     {
1284         yaz_log(YLOG_FATAL|YLOG_ERRNO, "socket");
1285         freeaddrinfo(af);
1286         return 1;
1287     }
1288     if (ipv6_only >= 0 && ai->ai_family == AF_INET6 &&
1289         setsockopt(s, IPPROTO_IPV6, IPV6_V6ONLY, &ipv6_only, sizeof(ipv6_only)))
1290     {
1291         yaz_log(YLOG_FATAL|YLOG_ERRNO, "setsockopt IPV6_V6ONLY %s:%s %d",
1292                 server->host, server->port, ipv6_only);
1293         freeaddrinfo(af);
1294         CLOSESOCKET(s);
1295         return 1;
1296     }
1297     if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one)))
1298     {
1299         yaz_log(YLOG_FATAL|YLOG_ERRNO, "setsockopt SO_REUSEADDR %s:%s",
1300                 server->host, server->port);
1301         freeaddrinfo(af);
1302         CLOSESOCKET(s);
1303         return 1;
1304     }
1305     if (bind(s, ai->ai_addr, ai->ai_addrlen) < 0)
1306     {
1307         yaz_log(YLOG_FATAL|YLOG_ERRNO, "bind %s:%s",
1308                 server->host, server->port);
1309         freeaddrinfo(af);
1310         CLOSESOCKET(s);
1311         return 1;
1312     }
1313     freeaddrinfo(af);
1314     if (listen(s, SOMAXCONN) < 0)
1315     {
1316         yaz_log(YLOG_FATAL|YLOG_ERRNO, "listen %s:%s",
1317                 server->host, server->port);
1318         CLOSESOCKET(s);
1319         return 1;
1320     }
1321
1322     if (record_fname)
1323     {
1324         record_file = fopen(record_fname, "wb");
1325         if (!record_file)
1326         {
1327             yaz_log(YLOG_FATAL|YLOG_ERRNO, "fopen %s", record_fname);
1328             CLOSESOCKET(s);
1329             return 1;
1330         }
1331     }
1332     server->http_server = http_server_create();
1333
1334     server->http_server->record_file = record_file;
1335     server->http_server->listener_socket = s;
1336
1337     c = iochan_create(s, http_accept, EVENT_INPUT | EVENT_EXCEPT, "http_server");
1338     iochan_setdata(c, server);
1339
1340     iochan_add(server->iochan_man, c);
1341     return 0;
1342 }
1343
1344 void http_close_server(struct conf_server *server)
1345 {
1346     /* break the event_loop (select) by closing down the HTTP listener sock */
1347     if (server->http_server->listener_socket)
1348     {
1349 #ifdef WIN32
1350         closesocket(server->http_server->listener_socket);
1351 #else
1352         close(server->http_server->listener_socket);
1353 #endif
1354     }
1355 }
1356
1357 void http_set_proxyaddr(const char *host, struct conf_server *server)
1358 {
1359     const char *p;
1360     short port;
1361     struct hostent *he;
1362     WRBUF w = wrbuf_alloc();
1363
1364     yaz_log(YLOG_LOG, "HTTP backend  %s", host);
1365
1366     p = strchr(host, ':');
1367     if (p)
1368     {
1369         port = atoi(p + 1);
1370         wrbuf_write(w, host, p - host);
1371         wrbuf_puts(w, "");
1372     }
1373     else
1374     {
1375         port = 80;
1376         wrbuf_puts(w, host);
1377     }
1378     if (!(he = gethostbyname(wrbuf_cstr(w))))
1379     {
1380         fprintf(stderr, "Failed to lookup '%s'\n", wrbuf_cstr(w));
1381         exit(1);
1382     }
1383     wrbuf_destroy(w);
1384
1385     server->http_server->proxy_addr = xmalloc(sizeof(struct sockaddr_in));
1386     server->http_server->proxy_addr->sin_family = he->h_addrtype;
1387     memcpy(&server->http_server->proxy_addr->sin_addr.s_addr,
1388            he->h_addr_list[0], he->h_length);
1389     server->http_server->proxy_addr->sin_port = htons(port);
1390 }
1391
1392 static void http_fire_observers(struct http_channel *c)
1393 {
1394     http_channel_observer_t p = c->observers;
1395     while (p)
1396     {
1397         p->destroy(p->data, c, p->data2);
1398         p = p->next;
1399     }
1400 }
1401
1402 static void http_destroy_observers(struct http_channel *c)
1403 {
1404     while (c->observers)
1405     {
1406         http_channel_observer_t obs = c->observers;
1407         c->observers = obs->next;
1408         xfree(obs);
1409     }
1410 }
1411
1412 http_channel_observer_t http_add_observer(struct http_channel *c, void *data,
1413                                           http_channel_destroy_t des)
1414 {
1415     http_channel_observer_t obs = xmalloc(sizeof(*obs));
1416     obs->chan = c;
1417     obs->data = data;
1418     obs->data2 = 0;
1419     obs->destroy= des;
1420     obs->next = c->observers;
1421     c->observers = obs;
1422     return obs;
1423 }
1424
1425 void http_remove_observer(http_channel_observer_t obs)
1426 {
1427     struct http_channel *c = obs->chan;
1428     http_channel_observer_t found, *p = &c->observers;
1429     while (*p != obs)
1430         p = &(*p)->next;
1431     found = *p;
1432     assert(found);
1433     *p = (*p)->next;
1434     xfree(found);
1435 }
1436
1437 struct http_channel *http_channel_observer_chan(http_channel_observer_t obs)
1438 {
1439     return obs->chan;
1440 }
1441
1442 void http_observer_set_data2(http_channel_observer_t obs, void *data2)
1443 {
1444     obs->data2 = data2;
1445 }
1446
1447 http_server_t http_server_create(void)
1448 {
1449     http_server_t hs = xmalloc(sizeof(*hs));
1450     hs->mutex = 0;
1451     hs->proxy_addr = 0;
1452     hs->ref_count = 1;
1453     hs->http_sessions = 0;
1454
1455     hs->record_file = 0;
1456     return hs;
1457 }
1458
1459 void http_server_destroy(http_server_t hs)
1460 {
1461     if (hs)
1462     {
1463         int r;
1464
1465         yaz_mutex_enter(hs->mutex); /* OK: hs->mutex may be NULL */
1466         r = --(hs->ref_count);
1467         yaz_mutex_leave(hs->mutex);
1468
1469         if (r == 0)
1470         {
1471             http_sessions_destroy(hs->http_sessions);
1472             xfree(hs->proxy_addr);
1473             yaz_mutex_destroy(&hs->mutex);
1474             if (hs->record_file)
1475                 fclose(hs->record_file);
1476             xfree(hs);
1477         }
1478     }
1479 }
1480
1481 void http_server_incref(http_server_t hs)
1482 {
1483     assert(hs);
1484     yaz_mutex_enter(hs->mutex);
1485     (hs->ref_count)++;
1486     yaz_mutex_leave(hs->mutex);
1487 }
1488
1489 void http_mutex_init(struct conf_server *server)
1490 {
1491     assert(server);
1492
1493     assert(server->http_server->mutex == 0);
1494     pazpar2_mutex_create(&server->http_server->mutex, "http_server");
1495     server->http_server->http_sessions = http_sessions_create();
1496 }
1497
1498 /*
1499  * Local variables:
1500  * c-basic-offset: 4
1501  * c-file-style: "Stroustrup"
1502  * indent-tabs-mode: nil
1503  * End:
1504  * vim: shiftwidth=4 tabstop=8 expandtab
1505  */
1506