Happy new year
[pazpar2-moved-to-github.git] / src / http.c
1 /* This file is part of Pazpar2.
2    Copyright (C) Index Data
3
4 Pazpar2 is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
7 version.
8
9 Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12 for more details.
13
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
17
18 */
19
20 #if HAVE_CONFIG_H
21 #include <config.h>
22 #endif
23
24 #if HAVE_SYS_TIME_H
25 #include <sys/time.h>
26 #endif
27
28 #include <stdio.h>
29 #ifdef WIN32
30 #include <winsock2.h>
31 #include <ws2tcpip.h>
32 typedef int socklen_t;
33 #endif
34
35 #if HAVE_SYS_SOCKET_H
36 #include <sys/socket.h>
37 #endif
38
39 #include <sys/types.h>
40
41 #include <yaz/snprintf.h>
42 #if HAVE_UNISTD_H
43 #include <unistd.h>
44 #endif
45
46 #include <stdlib.h>
47 #include <string.h>
48 #include <ctype.h>
49 #include <fcntl.h>
50 #if HAVE_NETDB_H
51 #include <netdb.h>
52 #endif
53
54 #include <errno.h>
55 #include <assert.h>
56 #include <string.h>
57
58 #include <yaz/yaz-util.h>
59 #include <yaz/comstack.h>
60 #include <yaz/nmem.h>
61 #include <yaz/mutex.h>
62
63 #include "ppmutex.h"
64 #include "session.h"
65 #include "http.h"
66 #include "parameters.h"
67
68 #define MAX_HTTP_HEADER 4096
69
70 #ifdef WIN32
71 #define strncasecmp _strnicmp
72 #define strcasecmp _stricmp
73 #endif
74
75 struct http_buf
76 {
77 #define HTTP_BUF_SIZE 4096
78     char buf[4096];
79     int offset;
80     int len;
81     struct http_buf *next;
82 };
83
84 static int log_level_post = 0;
85
86 static void proxy_io(IOCHAN i, int event);
87 static struct http_channel *http_channel_create(http_server_t http_server,
88                                                 const char *addr,
89                                                 struct conf_server *server);
90 static void http_channel_destroy(IOCHAN i);
91 static http_server_t http_server_create(void);
92 static void http_server_incref(http_server_t hs);
93
94 #ifdef WIN32
95 #define CLOSESOCKET(x) closesocket(x)
96 #else
97 #define CLOSESOCKET(x) close(x)
98 #endif
99
100 struct http_server
101 {
102     YAZ_MUTEX mutex;
103     int listener_socket;
104     int ref_count;
105     http_sessions_t http_sessions;
106     struct sockaddr_in *proxy_addr;
107     FILE *record_file;
108 };
109
110 struct http_channel_observer_s {
111     void *data;
112     void *data2;
113     http_channel_destroy_t destroy;
114     struct http_channel_observer_s *next;
115     struct http_channel *chan;
116 };
117
118
119 const char *http_lookup_header(struct http_header *header,
120                                const char *name)
121 {
122     for (; header; header = header->next)
123         if (!strcasecmp(name, header->name))
124             return header->value;
125     return 0;
126 }
127
128 static struct http_buf *http_buf_create(http_server_t hs)
129 {
130     struct http_buf *r = xmalloc(sizeof(*r));
131     r->offset = 0;
132     r->len = 0;
133     r->next = 0;
134     return r;
135 }
136
137 static void http_buf_destroy(http_server_t hs, struct http_buf *b)
138 {
139     xfree(b);
140 }
141
142 static void http_buf_destroy_queue(http_server_t hs, struct http_buf *b)
143 {
144     struct http_buf *p;
145     while (b)
146     {
147         p = b->next;
148         http_buf_destroy(hs, b);
149         b = p;
150     }
151 }
152
153 static struct http_buf *http_buf_bybuf(http_server_t hs, char *b, int len)
154 {
155     struct http_buf *res = 0;
156     struct http_buf **p = &res;
157
158     while (len)
159     {
160         int tocopy = len;
161         if (tocopy > HTTP_BUF_SIZE)
162             tocopy = HTTP_BUF_SIZE;
163         *p = http_buf_create(hs);
164         memcpy((*p)->buf, b, tocopy);
165         (*p)->len = tocopy;
166         len -= tocopy;
167         b += tocopy;
168         p = &(*p)->next;
169     }
170     return res;
171 }
172
173 // Add a (chain of) buffers to the end of an existing queue.
174 static void http_buf_enqueue(struct http_buf **queue, struct http_buf *b)
175 {
176     while (*queue)
177         queue = &(*queue)->next;
178     *queue = b;
179 }
180
181 static struct http_buf *http_buf_bywrbuf(http_server_t hs, WRBUF wrbuf)
182 {
183     // Heavens to Betsy (buf)!
184     return http_buf_bybuf(hs, wrbuf_buf(wrbuf), wrbuf_len(wrbuf));
185 }
186
187 // Non-destructively collapse chain of buffers into a string (max *len)
188 // Return
189 static void http_buf_peek(struct http_buf *b, char *buf, int len)
190 {
191     int rd = 0;
192     while (b && rd < len)
193     {
194         int toread = len - rd;
195         if (toread > b->len)
196             toread = b->len;
197         memcpy(buf + rd, b->buf + b->offset, toread);
198         rd += toread;
199         b = b->next;
200     }
201     buf[rd] = '\0';
202 }
203
204 static int http_buf_size(struct http_buf *b)
205 {
206     int sz = 0;
207     for (; b; b = b->next)
208         sz += b->len;
209     return sz;
210 }
211
212 // Ddestructively munch up to len  from head of queue.
213 static int http_buf_read(http_server_t hs,
214                          struct http_buf **b, char *buf, int len)
215 {
216     int rd = 0;
217     while ((*b) && rd < len)
218     {
219         int toread = len - rd;
220         if (toread > (*b)->len)
221             toread = (*b)->len;
222         memcpy(buf + rd, (*b)->buf + (*b)->offset, toread);
223         rd += toread;
224         if (toread < (*b)->len)
225         {
226             (*b)->len -= toread;
227             (*b)->offset += toread;
228             break;
229         }
230         else
231         {
232             struct http_buf *n = (*b)->next;
233             http_buf_destroy(hs, *b);
234             *b = n;
235         }
236     }
237     buf[rd] = '\0';
238     return rd;
239 }
240
241 // Buffers may overlap.
242 static void urldecode(char *i, char *o)
243 {
244     while (*i)
245     {
246         if (*i == '+')
247         {
248             *(o++) = ' ';
249             i++;
250         }
251         else if (*i == '%' && i[1] && i[2])
252         {
253             int v;
254             i++;
255             sscanf(i, "%2x", &v);
256             *o++ = v;
257             i += 2;
258         }
259         else
260             *(o++) = *(i++);
261     }
262     *o = '\0';
263 }
264
265 // Warning: Buffers may not overlap
266 void urlencode(const char *i, char *o)
267 {
268     while (*i)
269     {
270         if (strchr(" /:", *i))
271         {
272             sprintf(o, "%%%.2X", (int) *i);
273             o += 3;
274         }
275         else
276             *(o++) = *i;
277         i++;
278     }
279     *o = '\0';
280 }
281
282 void http_addheader(struct http_response *r, const char *name, const char *value)
283 {
284     struct http_channel *c = r->channel;
285     struct http_header *h = nmem_malloc(c->nmem, sizeof *h);
286     h->name = nmem_strdup(c->nmem, name);
287     h->value = nmem_strdup(c->nmem, value);
288     h->next = r->headers;
289     r->headers = h;
290 }
291
292 const char *http_argbyname(struct http_request *r, const char *name)
293 {
294     struct http_argument *p;
295     if (!name)
296         return 0;
297     for (p = r->arguments; p; p = p->next)
298         if (!strcmp(p->name, name))
299             return p->value;
300     return 0;
301 }
302
303 const char *http_headerbyname(struct http_header *h, const char *name)
304 {
305     for (; h; h = h->next)
306         if (!strcmp(h->name, name))
307             return h->value;
308     return 0;
309 }
310
311 struct http_response *http_create_response(struct http_channel *c)
312 {
313     struct http_response *r = nmem_malloc(c->nmem, sizeof(*r));
314     strcpy(r->code, "200");
315     r->msg = "OK";
316     r->channel = c;
317     r->headers = 0;
318     r->payload = 0;
319     r->content_type = "text/xml";
320     return r;
321 }
322
323
324 static const char *next_crlf(const char *cp, size_t *skipped)
325 {
326     const char *next_cp = strchr(cp, '\n');
327     if (next_cp)
328     {
329         if (next_cp > cp && next_cp[-1] == '\r')
330             *skipped = next_cp - cp - 1;
331         else
332             *skipped = next_cp - cp;
333         next_cp++;
334     }
335     return next_cp;
336 }
337
338 // Check if buf contains a package (minus payload)
339 static int package_check(const char *buf, int sz)
340 {
341     int content_len = 0;
342     int len = 0;
343
344     while (*buf)
345     {
346         size_t skipped = 0;
347         const char *b = next_crlf(buf, &skipped);
348
349         if (!b)
350         {
351             // we did not find CRLF.. See if buffer is too large..
352             if (sz >= MAX_HTTP_HEADER-1)
353                 return MAX_HTTP_HEADER-1; // yes. Return that (will fail later)
354             break;
355         }
356         len += (b - buf);
357         if (skipped == 0)
358         {
359             // CRLF CRLF , i.e. end of header
360             if (len + content_len <= sz)
361                 return len + content_len;
362             break;
363         }
364         buf = b;
365         // following first skip of \r\n so that we don't consider Method
366         if (!strncasecmp(buf, "Content-Length:", 15))
367         {
368             const char *cp = buf+15;
369             while (*cp == ' ')
370                 cp++;
371             content_len = 0;
372             while (*cp && isdigit(*(const unsigned char *)cp))
373                 content_len = content_len*10 + (*cp++ - '0');
374             if (content_len < 0) /* prevent negative offsets */
375                 content_len = 0;
376         }
377     }
378     return 0;     // incomplete request
379 }
380
381 // Check if we have a request. Return 0 or length
382 static int request_check(struct http_buf *queue)
383 {
384     char tmp[MAX_HTTP_HEADER];
385
386     // only peek at the header..
387     http_buf_peek(queue, tmp, MAX_HTTP_HEADER-1);
388     // still we only return non-zero if the complete request is received..
389     return package_check(tmp, http_buf_size(queue));
390 }
391
392 struct http_response *http_parse_response_buf(struct http_channel *c, const char *buf, int len)
393 {
394     char tmp[MAX_HTTP_HEADER];
395     struct http_response *r = http_create_response(c);
396     char *p, *p2;
397     struct http_header **hp = &r->headers;
398
399     if (len >= MAX_HTTP_HEADER)
400         return 0;
401     memcpy(tmp, buf, len);
402     for (p = tmp; *p && *p != ' '; p++) // Skip HTTP version
403         ;
404     p++;
405     // Response code
406     for (p2 = p; *p2 && *p2 != ' ' && p2 - p < 3; p2++)
407         r->code[p2 - p] = *p2;
408     if (!(p = strstr(tmp, "\r\n")))
409         return 0;
410     p += 2;
411     while (*p)
412     {
413         if (!(p2 = strstr(p, "\r\n")))
414             return 0;
415         if (p == p2) // End of headers
416             break;
417         else
418         {
419             struct http_header *h = *hp = nmem_malloc(c->nmem, sizeof(*h));
420             char *value = strchr(p, ':');
421             if (!value)
422                 return 0;
423             *(value++) = '\0';
424             h->name = nmem_strdup(c->nmem, p);
425             while (isspace(*(const unsigned char *) value))
426                 value++;
427             if (value >= p2)  // Empty header;
428             {
429                 h->value = "";
430                 p = p2 + 2;
431                 continue;
432             }
433             *p2 = '\0';
434             h->value = nmem_strdup(c->nmem, value);
435             h->next = 0;
436             hp = &h->next;
437             p = p2 + 2;
438         }
439     }
440     return r;
441 }
442
443 static int http_parse_arguments(struct http_request *r, NMEM nmem,
444                                 const char *args)
445 {
446     const char *p2 = args;
447
448     while (*p2)
449     {
450         struct http_argument *a;
451         const char *equal = strchr(p2, '=');
452         const char *eoa = strchr(p2, '&');
453         if (!equal)
454         {
455             yaz_log(YLOG_WARN, "Expected '=' in argument");
456             return -1;
457         }
458         if (!eoa)
459             eoa = equal + strlen(equal); // last argument
460         else if (equal > eoa)
461         {
462             yaz_log(YLOG_WARN, "Missing '&' in argument");
463             return -1;
464         }
465         a = nmem_malloc(nmem, sizeof(struct http_argument));
466         a->name = nmem_strdupn(nmem, p2, equal - p2);
467         a->value = nmem_strdupn(nmem, equal+1, eoa - equal - 1);
468         urldecode(a->name, a->name);
469         urldecode(a->value, a->value);
470         a->next = r->arguments;
471         r->arguments = a;
472         p2 = eoa;
473         while (*p2 == '&')
474             p2++;
475     }
476     return 0;
477 }
478
479 struct http_request *http_parse_request(struct http_channel *c,
480                                         struct http_buf **queue,
481                                         int len)
482 {
483     struct http_request *r = nmem_malloc(c->nmem, sizeof(*r));
484     char *p, *p2;
485     char *start = nmem_malloc(c->nmem, len+1);
486     char *buf = start;
487
488     if (http_buf_read(c->http_server, queue, buf, len) < len)
489     {
490         yaz_log(YLOG_WARN, "http_buf_read < len (%d)", len);
491         return 0;
492     }
493     r->search = "";
494     r->channel = c;
495     r->arguments = 0;
496     r->headers = 0;
497     r->content_buf = 0;
498     r->content_len = 0;
499     // Parse first line
500     for (p = buf, p2 = r->method; *p && *p != ' ' && p - buf < 19; p++)
501         *(p2++) = *p;
502     if (*p != ' ')
503     {
504         yaz_log(YLOG_WARN, "Unexpected HTTP method in request");
505         return 0;
506     }
507     *p2 = '\0';
508
509     if (!(buf = strchr(buf, ' ')))
510     {
511         yaz_log(YLOG_WARN, "Missing Request-URI in HTTP request");
512         return 0;
513     }
514     buf++;
515     if (!(p = strchr(buf, ' ')))
516     {
517         yaz_log(YLOG_WARN, "HTTP Request-URI not terminated (too long?)");
518         return 0;
519     }
520     *(p++) = '\0';
521     if ((p2 = strchr(buf, '?'))) // Do we have arguments?
522         *(p2++) = '\0';
523     r->path = nmem_strdup(c->nmem, buf);
524     if (p2)
525     {
526         r->search = nmem_strdup(c->nmem, p2);
527         // Parse Arguments
528         http_parse_arguments(r, c->nmem, p2);
529     }
530     buf = p;
531
532     if (strncmp(buf, "HTTP/", 5))
533         strcpy(r->http_version, "1.0");
534     else
535     {
536         size_t skipped;
537         buf += 5; // strlen("HTTP/")
538
539         p = (char*) next_crlf(buf, &skipped);
540         if (!p || skipped < 3 || skipped > 5)
541             return 0;
542
543         memcpy(r->http_version, buf, skipped);
544         r->http_version[skipped] = '\0';
545         buf = p;
546     }
547     strcpy(c->version, r->http_version);
548
549     r->headers = 0;
550     while (*buf)
551     {
552         size_t skipped;
553
554         p = (char *) next_crlf(buf, &skipped);
555         if (!p)
556         {
557             return 0;
558         }
559         else if (skipped == 0)
560         {
561             buf = p;
562             break;
563         }
564         else
565         {
566             char *cp;
567             char *n_v = nmem_malloc(c->nmem, skipped+1);
568             struct http_header *h = nmem_malloc(c->nmem, sizeof(*h));
569
570             memcpy(n_v, buf, skipped);
571             n_v[skipped] = '\0';
572
573             if (!(cp = strchr(n_v, ':')))
574                 return 0;
575             h->name = nmem_strdupn(c->nmem, n_v, cp - n_v);
576             cp++;
577             while (isspace(*cp))
578                 cp++;
579             h->value = nmem_strdup(c->nmem, cp);
580             h->next = r->headers;
581             r->headers = h;
582             buf = p;
583         }
584     }
585
586     // determine if we do keep alive
587     if (!strcmp(c->version, "1.0"))
588     {
589         const char *v = http_lookup_header(r->headers, "Connection");
590         if (v && !strcmp(v, "Keep-Alive"))
591             c->keep_alive = 1;
592         else
593             c->keep_alive = 0;
594     }
595     else
596     {
597         const char *v = http_lookup_header(r->headers, "Connection");
598         if (v && !strcmp(v, "close"))
599             c->keep_alive = 0;
600         else
601             c->keep_alive = 1;
602     }
603     if (buf < start + len)
604     {
605         const char *content_type = http_lookup_header(r->headers,
606                                                       "Content-Type");
607         r->content_len = start + len - buf;
608         r->content_buf = buf;
609
610         if (content_type &&
611             !yaz_strcmp_del("application/x-www-form-urlencoded",
612                             content_type, "; "))
613         {
614             http_parse_arguments(r, c->nmem, r->content_buf);
615         }
616     }
617     return r;
618 }
619
620 static struct http_buf *http_serialize_response(struct http_channel *c,
621         struct http_response *r)
622 {
623     struct http_header *h;
624
625     wrbuf_rewind(c->wrbuf);
626
627     wrbuf_printf(c->wrbuf, "HTTP/%s %s %s\r\n", c->version, r->code, r->msg);
628     for (h = r->headers; h; h = h->next)
629         wrbuf_printf(c->wrbuf, "%s: %s\r\n", h->name, h->value);
630     if (r->payload)
631     {
632         wrbuf_printf(c->wrbuf, "Content-Length: %d\r\n", r->payload ?
633                 (int) strlen(r->payload) : 0);
634         wrbuf_printf(c->wrbuf, "Content-Type: %s\r\n", r->content_type);
635         if (!strcmp(r->content_type, "text/xml"))
636         {
637             xmlDoc *doc = xmlParseMemory(r->payload, strlen(r->payload));
638             if (doc)
639             {
640                 xmlFreeDoc(doc);
641             }
642             else
643             {
644                 yaz_log(YLOG_WARN, "Sending non-wellformed "
645                         "response (bug #1162");
646                 yaz_log(YLOG_WARN, "payload: %s", r->payload);
647             }
648         }
649     }
650     wrbuf_puts(c->wrbuf, "\r\n");
651
652     if (r->payload)
653         wrbuf_puts(c->wrbuf, r->payload);
654
655     if (global_parameters.dump_records > 1)
656     {
657         FILE *lf = yaz_log_file();
658         yaz_log(YLOG_LOG, "Response:");
659         fwrite(wrbuf_buf(c->wrbuf), 1, wrbuf_len(c->wrbuf), lf);
660         fputc('\n', lf);
661     }
662     return http_buf_bywrbuf(c->http_server, c->wrbuf);
663 }
664
665 // Serialize a HTTP request
666 static struct http_buf *http_serialize_request(struct http_request *r)
667 {
668     struct http_channel *c = r->channel;
669     struct http_header *h;
670
671     wrbuf_rewind(c->wrbuf);
672     wrbuf_printf(c->wrbuf, "%s %s%s%s", r->method, r->path,
673                  *r->search ? "?" : "", r->search);
674
675     wrbuf_printf(c->wrbuf, " HTTP/%s\r\n", r->http_version);
676
677     for (h = r->headers; h; h = h->next)
678         wrbuf_printf(c->wrbuf, "%s: %s\r\n", h->name, h->value);
679
680     wrbuf_puts(c->wrbuf, "\r\n");
681
682     if (r->content_buf)
683         wrbuf_write(c->wrbuf, r->content_buf, r->content_len);
684
685 #if 0
686     yaz_log(YLOG_LOG, "WRITING TO PROXY:\n%s\n----",
687             wrbuf_cstr(c->wrbuf));
688 #endif
689     return http_buf_bywrbuf(c->http_server, c->wrbuf);
690 }
691
692
693 static int http_weshouldproxy(struct http_request *rq)
694 {
695     struct http_channel *c = rq->channel;
696     if (c->server->http_server->proxy_addr && !strstr(rq->path, "search.pz2"))
697         return 1;
698     return 0;
699 }
700
701
702 struct http_header * http_header_append(struct http_channel *ch,
703                                         struct http_header * hp,
704                                         const char *name,
705                                         const char *value)
706 {
707     struct http_header *hpnew = 0;
708
709     if (!hp | !ch)
710         return 0;
711
712     while (hp && hp->next)
713         hp = hp->next;
714
715     if(name && strlen(name)&& value && strlen(value)){
716         hpnew = nmem_malloc(ch->nmem, sizeof *hpnew);
717         hpnew->name = nmem_strdup(ch->nmem, name);
718         hpnew->value = nmem_strdup(ch->nmem, value);
719
720         hpnew->next = 0;
721         hp->next = hpnew;
722         hp = hp->next;
723
724         return hpnew;
725     }
726
727     return hp;
728 }
729
730
731 static int is_inprogress(void)
732 {
733 #ifdef WIN32
734     if (WSAGetLastError() == WSAEWOULDBLOCK)
735         return 1;
736 #else
737     if (errno == EINPROGRESS)
738         return 1;
739 #endif
740     return 0;
741 }
742
743 static void enable_nonblock(int sock)
744 {
745     int flags;
746 #ifdef WIN32
747     flags = (flags & CS_FLAGS_BLOCKING) ? 0 : 1;
748     if (ioctlsocket(sock, FIONBIO, &flags) < 0)
749         yaz_log(YLOG_FATAL|YLOG_ERRNO, "ioctlsocket");
750 #else
751     if ((flags = fcntl(sock, F_GETFL, 0)) < 0)
752         yaz_log(YLOG_FATAL|YLOG_ERRNO, "fcntl");
753     if (fcntl(sock, F_SETFL, flags | O_NONBLOCK) < 0)
754         yaz_log(YLOG_FATAL|YLOG_ERRNO, "fcntl2");
755 #endif
756 }
757
758 static int http_proxy(struct http_request *rq)
759 {
760     struct http_channel *c = rq->channel;
761     struct http_proxy *p = c->proxy;
762     struct http_header *hp;
763     struct http_buf *requestbuf;
764     struct conf_server *ser = c->server;
765
766     if (!p) // This is a new connection. Create a proxy channel
767     {
768         int sock;
769         struct protoent *pe;
770         int one = 1;
771
772         if (!(pe = getprotobyname("tcp"))) {
773             abort();
774         }
775         if ((sock = socket(PF_INET, SOCK_STREAM, pe->p_proto)) < 0)
776         {
777             yaz_log(YLOG_WARN|YLOG_ERRNO, "socket");
778             return -1;
779         }
780         if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (char*)
781                         &one, sizeof(one)) < 0)
782             abort();
783         enable_nonblock(sock);
784         if (connect(sock, (struct sockaddr *)
785                     c->server->http_server->proxy_addr,
786                     sizeof(*c->server->http_server->proxy_addr)) < 0)
787         {
788             if (!is_inprogress())
789             {
790                 yaz_log(YLOG_WARN|YLOG_ERRNO, "Proxy connect");
791                 return -1;
792             }
793         }
794         p = xmalloc(sizeof(struct http_proxy));
795         p->oqueue = 0;
796         p->channel = c;
797         p->first_response = 1;
798         c->proxy = p;
799         p->iochan = iochan_create(sock, proxy_io, EVENT_INPUT, "http_proxy");
800         iochan_setdata(p->iochan, p);
801
802         if (iochan_add(ser->iochan_man, p->iochan, 5))
803         {
804             iochan_destroy(p->iochan);
805             xfree(p);
806             return -1;
807         }
808     }
809
810     // Do _not_ modify Host: header, just checking it's existence
811
812     if (!http_lookup_header(rq->headers, "Host"))
813     {
814         yaz_log(YLOG_WARN, "Failed to find Host header in proxy");
815         return -1;
816     }
817
818     // Add new header about paraz2 version, host, remote client address, etc.
819     {
820         char server_via[128];
821
822         hp = rq->headers;
823         hp = http_header_append(c, hp,
824                                 "X-Pazpar2-Version", PACKAGE_VERSION);
825         hp = http_header_append(c, hp,
826                                 "X-Pazpar2-Server-Host", ser->host);
827         hp = http_header_append(c, hp,
828                                 "X-Pazpar2-Server-Port", ser->port);
829         yaz_snprintf(server_via, sizeof(server_via),
830                      "1.1 %s:%s (%s/%s)",
831                      ser->host, ser->port,
832                      PACKAGE_NAME, PACKAGE_VERSION);
833         hp = http_header_append(c, hp, "Via" , server_via);
834         hp = http_header_append(c, hp, "X-Forwarded-For", c->addr);
835     }
836
837     requestbuf = http_serialize_request(rq);
838
839     http_buf_enqueue(&p->oqueue, requestbuf);
840     iochan_setflag(p->iochan, EVENT_OUTPUT);
841     return 0;
842 }
843
844 void http_send_response(struct http_channel *ch)
845 {
846     struct http_response *rs = ch->response;
847     struct http_buf *hb;
848
849     yaz_timing_stop(ch->yt);
850     if (ch->request)
851     {
852         yaz_log(YLOG_LOG, "Response: %6.5f %d %s%s%s ",
853                 yaz_timing_get_real(ch->yt),
854                 iochan_getfd(ch->iochan),
855                 ch->request->path,
856                 *ch->request->search ? "?" : "",
857                 ch->request->search);
858     }
859     assert(rs);
860     hb = http_serialize_response(ch, rs);
861     if (!hb)
862     {
863         yaz_log(YLOG_WARN, "Failed to serialize HTTP response");
864         http_channel_destroy(ch->iochan);
865     }
866     else
867     {
868         http_buf_enqueue(&ch->oqueue, hb);
869         iochan_setflag(ch->iochan, EVENT_OUTPUT);
870         ch->state = Http_Idle;
871     }
872 }
873
874 static void http_error(struct http_channel *hc, int no, const char *msg)
875 {
876     struct http_response *rs = http_create_response(hc);
877
878     hc->response = rs;
879     hc->keep_alive = 0;  // not keeping this HTTP session alive
880
881     sprintf(rs->code, "%d", no);
882
883     rs->msg = nmem_strdup(hc->nmem, msg);
884     rs->payload = nmem_malloc(hc->nmem, 100);
885     yaz_snprintf(rs->payload, 99, "<error>HTTP Error %d: %s</error>\n",
886                  no, msg);
887     http_send_response(hc);
888 }
889
890 static void http_io(IOCHAN i, int event)
891 {
892     struct http_channel *hc = iochan_getdata(i);
893     while (event)
894     {
895         if (event == EVENT_INPUT)
896         {
897             int res, reqlen;
898             struct http_buf *htbuf;
899
900             htbuf = http_buf_create(hc->http_server);
901             res = recv(iochan_getfd(i), htbuf->buf, HTTP_BUF_SIZE -1, 0);
902             if (res == -1 && errno == EAGAIN)
903             {
904                 http_buf_destroy(hc->http_server, htbuf);
905                 return;
906             }
907             if (res <= 0)
908             {
909 #if HAVE_SYS_TIME_H
910                 if (hc->http_server->record_file)
911                 {
912                     struct timeval tv;
913                     gettimeofday(&tv, 0);
914                     fprintf(hc->http_server->record_file, "r %lld %lld %lld 0\n",
915                             (long long) tv.tv_sec, (long long) tv.tv_usec,
916                             (long long) iochan_getfd(i));
917                 }
918 #endif
919                 http_buf_destroy(hc->http_server, htbuf);
920                 fflush(hc->http_server->record_file);
921                 http_channel_destroy(i);
922                 return;
923             }
924             htbuf->buf[res] = '\0';
925             htbuf->len = res;
926             http_buf_enqueue(&hc->iqueue, htbuf);
927
928             while (1)
929             {
930                 if (hc->state == Http_Busy)
931                     return;
932                 reqlen = request_check(hc->iqueue);
933                 if (reqlen <= 2)
934                     return;
935                 // we have a complete HTTP request
936                 nmem_reset(hc->nmem);
937 #if HAVE_SYS_TIME_H
938                 if (hc->http_server->record_file)
939                 {
940                     struct timeval tv;
941                     int sz = 0;
942                     struct http_buf *hb;
943                     for (hb = hc->iqueue; hb; hb = hb->next)
944                         sz += hb->len;
945                     gettimeofday(&tv, 0);
946                     fprintf(hc->http_server->record_file, "r %lld %lld %lld %d\n",
947                             (long long) tv.tv_sec, (long long) tv.tv_usec,
948                             (long long) iochan_getfd(i), sz);
949                     for (hb = hc->iqueue; hb; hb = hb->next)
950                         fwrite(hb->buf, 1, hb->len, hc->http_server->record_file);
951                     fflush(hc->http_server->record_file);
952                 }
953  #endif
954                 yaz_timing_start(hc->yt);
955                 if (!(hc->request = http_parse_request(hc, &hc->iqueue, reqlen)))
956                 {
957                     yaz_log(YLOG_WARN, "Failed to parse request");
958                     http_error(hc, 400, "Bad Request");
959                     return;
960                 }
961                 hc->response = 0;
962                 yaz_log(YLOG_LOG, "Request: - %d %s %s%s%s",
963                         iochan_getfd(i),
964                         hc->request->method,
965                         hc->request->path,
966                         *hc->request->search ? "?" : "",
967                         hc->request->search);
968                 if (hc->request->content_buf && log_level_post)
969                     yaz_log(log_level_post, "%s", hc->request->content_buf);
970                 if (http_weshouldproxy(hc->request))
971                     http_proxy(hc->request);
972                 else
973                 {
974                     // Execute our business logic!
975                     hc->state = Http_Busy;
976                     http_command(hc);
977                 }
978             }
979         }
980         else if (event == EVENT_OUTPUT)
981         {
982             event = 0;
983             if (hc->oqueue)
984             {
985                 struct http_buf *wb = hc->oqueue;
986                 int res;
987                 res = send(iochan_getfd(hc->iochan),
988                            wb->buf + wb->offset, wb->len, 0);
989                 if (res <= 0)
990                 {
991                     yaz_log(YLOG_WARN|YLOG_ERRNO, "write");
992                     http_channel_destroy(i);
993                     return;
994                 }
995                 if (res == wb->len)
996                 {
997 #if HAVE_SYS_TIME_H
998                     if (hc->http_server->record_file)
999                     {
1000                         struct timeval tv;
1001                         int sz = wb->offset + wb->len;
1002                         gettimeofday(&tv, 0);
1003                         fprintf(hc->http_server->record_file, "w %lld %lld %lld %d\n",
1004                                 (long long) tv.tv_sec, (long long) tv.tv_usec,
1005                                 (long long) iochan_getfd(i), sz);
1006                         fwrite(wb->buf, 1, wb->offset + wb->len,
1007                                hc->http_server->record_file);
1008                         fflush(hc->http_server->record_file);
1009                     }
1010  #endif
1011                     hc->oqueue = hc->oqueue->next;
1012                     http_buf_destroy(hc->http_server, wb);
1013                 }
1014                 else
1015                 {
1016                     wb->len -= res;
1017                     wb->offset += res;
1018                 }
1019                 if (!hc->oqueue)
1020                 {
1021                     if (!hc->keep_alive)
1022                     {
1023                         http_channel_destroy(i);
1024                         return;
1025                     }
1026                     else
1027                     {
1028                         iochan_clearflag(i, EVENT_OUTPUT);
1029                         if (hc->iqueue)
1030                             event = EVENT_INPUT;
1031                     }
1032                 }
1033             }
1034             if (!hc->oqueue && hc->proxy && !hc->proxy->iochan)
1035                 http_channel_destroy(i); // Server closed; we're done
1036         }
1037         else
1038         {
1039             yaz_log(YLOG_WARN, "Unexpected event on connection");
1040             http_channel_destroy(i);
1041             event = 0;
1042         }
1043     }
1044 }
1045
1046 // Handles I/O on a client connection to a backend web server (proxy mode)
1047 static void proxy_io(IOCHAN pi, int event)
1048 {
1049     struct http_proxy *pc = iochan_getdata(pi);
1050     struct http_channel *hc = pc->channel;
1051
1052     switch (event)
1053     {
1054         int res;
1055         struct http_buf *htbuf;
1056
1057         case EVENT_INPUT:
1058             htbuf = http_buf_create(hc->http_server);
1059             res = recv(iochan_getfd(pi), htbuf->buf, HTTP_BUF_SIZE -1, 0);
1060             if (res == 0 || (res < 0 && !is_inprogress()))
1061             {
1062                 if (hc->oqueue)
1063                 {
1064                     yaz_log(YLOG_WARN, "Proxy read came up short");
1065                     // Close channel and alert client HTTP channel that we're gone
1066                     http_buf_destroy(hc->http_server, htbuf);
1067                     CLOSESOCKET(iochan_getfd(pi));
1068                     iochan_destroy(pi);
1069                     pc->iochan = 0;
1070                 }
1071                 else
1072                 {
1073                     http_channel_destroy(hc->iochan);
1074                     return;
1075                 }
1076             }
1077             else
1078             {
1079                 htbuf->buf[res] = '\0';
1080                 htbuf->offset = 0;
1081                 htbuf->len = res;
1082                 // Write any remaining payload
1083                 if (htbuf->len - htbuf->offset > 0)
1084                     http_buf_enqueue(&hc->oqueue, htbuf);
1085             }
1086             iochan_setflag(hc->iochan, EVENT_OUTPUT);
1087             break;
1088         case EVENT_OUTPUT:
1089             if (!(htbuf = pc->oqueue))
1090             {
1091                 iochan_clearflag(pi, EVENT_OUTPUT);
1092                 return;
1093             }
1094             res = send(iochan_getfd(pi), htbuf->buf + htbuf->offset, htbuf->len, 0);
1095             if (res <= 0)
1096             {
1097                 yaz_log(YLOG_WARN|YLOG_ERRNO, "write");
1098                 http_channel_destroy(hc->iochan);
1099                 return;
1100             }
1101             if (res == htbuf->len)
1102             {
1103                 struct http_buf *np = htbuf->next;
1104                 http_buf_destroy(hc->http_server, htbuf);
1105                 pc->oqueue = np;
1106             }
1107             else
1108             {
1109                 htbuf->len -= res;
1110                 htbuf->offset += res;
1111             }
1112
1113             if (!pc->oqueue) {
1114                 iochan_setflags(pi, EVENT_INPUT); // Turns off output flag
1115             }
1116             break;
1117         default:
1118             yaz_log(YLOG_WARN, "Unexpected event on connection");
1119             http_channel_destroy(hc->iochan);
1120             break;
1121     }
1122 }
1123
1124 static void http_fire_observers(struct http_channel *c);
1125 static void http_destroy_observers(struct http_channel *c);
1126
1127 // Cleanup channel
1128 static void http_channel_destroy(IOCHAN i)
1129 {
1130     struct http_channel *s = iochan_getdata(i);
1131     http_server_t http_server;
1132
1133     if (s->proxy)
1134     {
1135         if (s->proxy->iochan)
1136         {
1137             CLOSESOCKET(iochan_getfd(s->proxy->iochan));
1138             iochan_destroy(s->proxy->iochan);
1139         }
1140         http_buf_destroy_queue(s->http_server, s->proxy->oqueue);
1141         xfree(s->proxy);
1142     }
1143     yaz_timing_destroy(&s->yt);
1144     http_buf_destroy_queue(s->http_server, s->iqueue);
1145     http_buf_destroy_queue(s->http_server, s->oqueue);
1146     http_fire_observers(s);
1147     http_destroy_observers(s);
1148
1149     http_server = s->http_server; /* save it for destroy (decref) */
1150
1151     http_server_destroy(http_server);
1152
1153     CLOSESOCKET(iochan_getfd(i));
1154
1155     iochan_destroy(i);
1156     nmem_destroy(s->nmem);
1157     wrbuf_destroy(s->wrbuf);
1158     xfree(s);
1159 }
1160
1161 static struct http_channel *http_channel_create(http_server_t hs,
1162                                                 const char *addr,
1163                                                 struct conf_server *server)
1164 {
1165     struct http_channel *r;
1166
1167     r = xmalloc(sizeof(struct http_channel));
1168     r->nmem = nmem_create();
1169     r->wrbuf = wrbuf_alloc();
1170
1171     http_server_incref(hs);
1172     r->http_server = hs;
1173     r->http_sessions = hs->http_sessions;
1174     assert(r->http_sessions);
1175     r->server = server;
1176     r->proxy = 0;
1177     r->iochan = 0;
1178     r->iqueue = r->oqueue = 0;
1179     r->state = Http_Idle;
1180     r->keep_alive = 0;
1181     r->request = 0;
1182     r->response = 0;
1183     strcpy(r->version, "1.0");
1184     if (!addr)
1185     {
1186         yaz_log(YLOG_WARN, "Invalid HTTP forward address");
1187         exit(1);
1188     }
1189     strcpy(r->addr, addr);
1190     r->observers = 0;
1191     r->yt = yaz_timing_create();
1192     return r;
1193 }
1194
1195
1196 /* Accept a new command connection */
1197 static void http_accept(IOCHAN i, int event)
1198 {
1199     char host[256];
1200     struct sockaddr_storage addr;
1201     int fd = iochan_getfd(i);
1202     socklen_t len = sizeof addr;
1203     int s;
1204     IOCHAN c;
1205     struct http_channel *ch;
1206     struct conf_server *server = iochan_getdata(i);
1207
1208     if ((s = accept(fd, (struct sockaddr *) &addr, &len)) < 0)
1209     {
1210         yaz_log(YLOG_WARN|YLOG_ERRNO, "accept");
1211         return;
1212     }
1213     if (getnameinfo((struct sockaddr *) &addr, len, host, sizeof(host)-1, 0, 0,
1214         NI_NUMERICHOST))
1215     {
1216         yaz_log(YLOG_WARN|YLOG_ERRNO, "getnameinfo");
1217         CLOSESOCKET(s);
1218         return;
1219     }
1220     enable_nonblock(s);
1221
1222     yaz_log(YLOG_DEBUG, "New command connection");
1223     c = iochan_create(s, http_io, EVENT_INPUT | EVENT_EXCEPT,
1224                       "http_session_socket");
1225
1226     ch = http_channel_create(server->http_server, host, server);
1227     ch->iochan = c;
1228     iochan_setdata(c, ch);
1229     if (iochan_add(server->iochan_man, c, 0))
1230     {
1231         yaz_log(YLOG_WARN, "Refusing incoming HTTP connection");
1232         http_channel_destroy(c);
1233     }
1234 }
1235
1236 /* Create a http-channel listener, syntax [host:]port */
1237 int http_init(struct conf_server *server, const char *record_fname)
1238 {
1239     IOCHAN c;
1240     int s = -1;
1241     int one = 1;
1242     FILE *record_file = 0;
1243     struct addrinfo hints, *af = 0, *ai;
1244     int error;
1245     int ipv6_only = -1;
1246
1247     yaz_log(YLOG_LOG, "HTTP listener %s:%s", server->host, server->port);
1248
1249     hints.ai_flags = 0;
1250     hints.ai_family = AF_UNSPEC;
1251     hints.ai_socktype = SOCK_STREAM;
1252     hints.ai_protocol = 0;
1253     hints.ai_addrlen        = 0;
1254     hints.ai_addr           = NULL;
1255     hints.ai_canonname      = NULL;
1256     hints.ai_next           = NULL;
1257
1258     if (!strcmp(server->host, "@"))
1259     {
1260         ipv6_only = 0;
1261         hints.ai_flags = AI_PASSIVE;
1262         error = getaddrinfo(0, server->port, &hints, &af);
1263     }
1264     else
1265         error = getaddrinfo(server->host, server->port, &hints, &af);
1266
1267     if (error)
1268     {
1269         yaz_log(YLOG_FATAL, "Failed to resolve %s: %s", server->host,
1270                 gai_strerror(error));
1271         return 1;
1272     }
1273     for (ai = af; ai; ai = ai->ai_next)
1274     {
1275         if (ai->ai_family == AF_INET6)
1276         {
1277             s = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol);
1278             if (s != -1)
1279                 break;
1280         }
1281     }
1282     if (s == -1)
1283     {
1284         for (ai = af; ai; ai = ai->ai_next)
1285         {
1286             s = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol);
1287             if (s != -1)
1288                 break;
1289         }
1290     }
1291     if (s == -1)
1292     {
1293         yaz_log(YLOG_FATAL|YLOG_ERRNO, "socket");
1294         freeaddrinfo(af);
1295         return 1;
1296     }
1297     if (ipv6_only >= 0 && ai->ai_family == AF_INET6 &&
1298         setsockopt(s, IPPROTO_IPV6, IPV6_V6ONLY, &ipv6_only, sizeof(ipv6_only)))
1299     {
1300         yaz_log(YLOG_FATAL|YLOG_ERRNO, "setsockopt IPV6_V6ONLY %s:%s %d",
1301                 server->host, server->port, ipv6_only);
1302         freeaddrinfo(af);
1303         CLOSESOCKET(s);
1304         return 1;
1305     }
1306     if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one)))
1307     {
1308         yaz_log(YLOG_FATAL|YLOG_ERRNO, "setsockopt SO_REUSEADDR %s:%s",
1309                 server->host, server->port);
1310         freeaddrinfo(af);
1311         CLOSESOCKET(s);
1312         return 1;
1313     }
1314     if (bind(s, ai->ai_addr, ai->ai_addrlen) < 0)
1315     {
1316         yaz_log(YLOG_FATAL|YLOG_ERRNO, "bind %s:%s",
1317                 server->host, server->port);
1318         freeaddrinfo(af);
1319         CLOSESOCKET(s);
1320         return 1;
1321     }
1322     freeaddrinfo(af);
1323     if (listen(s, SOMAXCONN) < 0)
1324     {
1325         yaz_log(YLOG_FATAL|YLOG_ERRNO, "listen %s:%s",
1326                 server->host, server->port);
1327         CLOSESOCKET(s);
1328         return 1;
1329     }
1330
1331     if (record_fname)
1332     {
1333         record_file = fopen(record_fname, "wb");
1334         if (!record_file)
1335         {
1336             yaz_log(YLOG_FATAL|YLOG_ERRNO, "fopen %s", record_fname);
1337             CLOSESOCKET(s);
1338             return 1;
1339         }
1340     }
1341
1342     c = iochan_create(s, http_accept, EVENT_INPUT|EVENT_EXCEPT, "http_server");
1343     if (iochan_add(server->iochan_man, c, 0))
1344     {
1345         yaz_log(YLOG_WARN, "Can not create HTTP binding socket");
1346         iochan_destroy(c);
1347         return -1;
1348     }
1349
1350     server->http_server = http_server_create();
1351     server->http_server->record_file = record_file;
1352     server->http_server->listener_socket = s;
1353     iochan_setdata(c, server);
1354
1355     return 0;
1356 }
1357
1358 void http_close_server(struct conf_server *server)
1359 {
1360     /* break the event_loop (select) by closing down the HTTP listener sock */
1361     if (server->http_server->listener_socket)
1362     {
1363 #ifdef WIN32
1364         closesocket(server->http_server->listener_socket);
1365 #else
1366         close(server->http_server->listener_socket);
1367 #endif
1368     }
1369 }
1370
1371 void http_set_proxyaddr(const char *host, struct conf_server *server)
1372 {
1373     const char *p;
1374     short port;
1375     struct hostent *he;
1376     WRBUF w = wrbuf_alloc();
1377
1378     yaz_log(YLOG_LOG, "HTTP backend  %s", host);
1379
1380     p = strchr(host, ':');
1381     if (p)
1382     {
1383         port = atoi(p + 1);
1384         wrbuf_write(w, host, p - host);
1385         wrbuf_puts(w, "");
1386     }
1387     else
1388     {
1389         port = 80;
1390         wrbuf_puts(w, host);
1391     }
1392     if (!(he = gethostbyname(wrbuf_cstr(w))))
1393     {
1394         fprintf(stderr, "Failed to lookup '%s'\n", wrbuf_cstr(w));
1395         exit(1);
1396     }
1397     wrbuf_destroy(w);
1398
1399     server->http_server->proxy_addr = xmalloc(sizeof(struct sockaddr_in));
1400     server->http_server->proxy_addr->sin_family = he->h_addrtype;
1401     memcpy(&server->http_server->proxy_addr->sin_addr.s_addr,
1402            he->h_addr_list[0], he->h_length);
1403     server->http_server->proxy_addr->sin_port = htons(port);
1404 }
1405
1406 static void http_fire_observers(struct http_channel *c)
1407 {
1408     http_channel_observer_t p = c->observers;
1409     while (p)
1410     {
1411         p->destroy(p->data, c, p->data2);
1412         p = p->next;
1413     }
1414 }
1415
1416 static void http_destroy_observers(struct http_channel *c)
1417 {
1418     while (c->observers)
1419     {
1420         http_channel_observer_t obs = c->observers;
1421         c->observers = obs->next;
1422         xfree(obs);
1423     }
1424 }
1425
1426 http_channel_observer_t http_add_observer(struct http_channel *c, void *data,
1427                                           http_channel_destroy_t des)
1428 {
1429     http_channel_observer_t obs = xmalloc(sizeof(*obs));
1430     obs->chan = c;
1431     obs->data = data;
1432     obs->data2 = 0;
1433     obs->destroy= des;
1434     obs->next = c->observers;
1435     c->observers = obs;
1436     return obs;
1437 }
1438
1439 void http_remove_observer(http_channel_observer_t obs)
1440 {
1441     struct http_channel *c = obs->chan;
1442     http_channel_observer_t found, *p = &c->observers;
1443     while (*p != obs)
1444         p = &(*p)->next;
1445     found = *p;
1446     assert(found);
1447     *p = (*p)->next;
1448     xfree(found);
1449 }
1450
1451 struct http_channel *http_channel_observer_chan(http_channel_observer_t obs)
1452 {
1453     return obs->chan;
1454 }
1455
1456 void http_observer_set_data2(http_channel_observer_t obs, void *data2)
1457 {
1458     obs->data2 = data2;
1459 }
1460
1461 http_server_t http_server_create(void)
1462 {
1463     http_server_t hs = xmalloc(sizeof(*hs));
1464     hs->mutex = 0;
1465     hs->proxy_addr = 0;
1466     hs->ref_count = 1;
1467     hs->http_sessions = 0;
1468     hs->record_file = 0;
1469
1470     log_level_post = yaz_log_module_level("post");
1471
1472     return hs;
1473 }
1474
1475 void http_server_destroy(http_server_t hs)
1476 {
1477     if (hs)
1478     {
1479         int r;
1480
1481         yaz_mutex_enter(hs->mutex); /* OK: hs->mutex may be NULL */
1482         r = --(hs->ref_count);
1483         yaz_mutex_leave(hs->mutex);
1484
1485         if (r == 0)
1486         {
1487             http_sessions_destroy(hs->http_sessions);
1488             xfree(hs->proxy_addr);
1489             yaz_mutex_destroy(&hs->mutex);
1490             if (hs->record_file)
1491                 fclose(hs->record_file);
1492             xfree(hs);
1493         }
1494     }
1495 }
1496
1497 void http_server_incref(http_server_t hs)
1498 {
1499     assert(hs);
1500     yaz_mutex_enter(hs->mutex);
1501     (hs->ref_count)++;
1502     yaz_mutex_leave(hs->mutex);
1503 }
1504
1505 void http_mutex_init(struct conf_server *server)
1506 {
1507     assert(server);
1508
1509     assert(server->http_server->mutex == 0);
1510     pazpar2_mutex_create(&server->http_server->mutex, "http_server");
1511     server->http_server->http_sessions = http_sessions_create();
1512 }
1513
1514 /*
1515  * Local variables:
1516  * c-basic-offset: 4
1517  * c-file-style: "Stroustrup"
1518  * indent-tabs-mode: nil
1519  * End:
1520  * vim: shiftwidth=4 tabstop=8 expandtab
1521  */
1522