Add facility to record HTTP requests to Pazpar2
[pazpar2-moved-to-github.git] / src / http.c
index 0f5da4e..1ce0aa4 100644 (file)
@@ -1,5 +1,5 @@
 /* This file is part of Pazpar2.
-   Copyright (C) 2006-2010 Index Data
+   Copyright (C) 2006-2011 Index Data
 
 Pazpar2 is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free
@@ -21,6 +21,10 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 #include <config.h>
 #endif
 
+#if HAVE_SYS_TIME_H
+#include <sys/time.h>
+#endif
+
 #include <stdio.h>
 #ifdef WIN32
 #include <winsock.h>
@@ -63,7 +67,8 @@ typedef int socklen_t;
 #include <yaz/nmem.h>
 #include <yaz/mutex.h>
 
-#include "pazpar2.h"
+#include "ppmutex.h"
+#include "session.h"
 #include "http.h"
 
 #define MAX_HTTP_HEADER 4096
@@ -94,12 +99,18 @@ static void http_server_incref(http_server_t hs);
 struct http_server
 {
     struct http_buf *http_buf_freelist;
+    int http_buf_freelist_count;
+    int http_buf_freelist_max;
+
     struct http_channel *http_channel_freelist;
+    int http_channel_freelist_count;
+    int http_channel_freelist_max;
     YAZ_MUTEX mutex;
     int listener_socket;
     int ref_count;
     http_sessions_t http_sessions;
     struct sockaddr_in *proxy_addr;
+    FILE *record_file;
 };
 
 struct http_channel_observer_s {
@@ -129,6 +140,7 @@ static struct http_buf *http_buf_create(http_server_t hs)
     {
         r = hs->http_buf_freelist;
         hs->http_buf_freelist = hs->http_buf_freelist->next;
+        hs->http_buf_freelist_count--;
     }
     yaz_mutex_leave(hs->mutex);
     if (!r)
@@ -142,8 +154,22 @@ static struct http_buf *http_buf_create(http_server_t hs)
 static void http_buf_destroy(http_server_t hs, struct http_buf *b)
 {
     yaz_mutex_enter(hs->mutex);
-    b->next = hs->http_buf_freelist;
-    hs->http_buf_freelist = b;
+    if (hs->http_buf_freelist_max > 0 && hs->http_buf_freelist_count >= hs->http_buf_freelist_max) {
+        xfree(b);
+        while ((b = hs->http_buf_freelist)) {
+            xfree(b);
+            hs->http_buf_freelist = hs->http_buf_freelist->next;
+        }
+        hs->http_buf_freelist_count = 0;
+    }
+    else {
+        b->next = hs->http_buf_freelist;
+        hs->http_buf_freelist = b;
+        hs->http_buf_freelist_count++;
+#if 0 
+        yaz_log(YLOG_DEBUG, "Free %d http buffers on server.", hs->http_buf_freelist_count);
+#endif
+    }
     yaz_mutex_leave(hs->mutex);
 }
 
@@ -797,9 +823,10 @@ static int http_proxy(struct http_request *rq)
         p->first_response = 1;
         c->proxy = p;
         // We will add EVENT_OUTPUT below
-        p->iochan = iochan_create(sock, proxy_io, EVENT_INPUT);
+        p->iochan = iochan_create(sock, proxy_io, EVENT_INPUT, "http_proxy");
         iochan_setdata(p->iochan, p);
-        pazpar2_add_channel(p->iochan);
+
+        iochan_add(ser->iochan_man, p->iochan);
     }
 
     // Do _not_ modify Host: header, just checking it's existence
@@ -876,13 +903,13 @@ static void http_error(struct http_channel *hc, int no, const char *msg)
 static void http_io(IOCHAN i, int event)
 {
     struct http_channel *hc = iochan_getdata(i);
-
-    switch (event)
+    while (event)
     {
-        int res, reqlen;
-        struct http_buf *htbuf;
-
-        case EVENT_INPUT:
+        if (event == EVENT_INPUT)
+        {
+            int res, reqlen;
+            struct http_buf *htbuf;
+            
             htbuf = http_buf_create(hc->http_server);
             res = recv(iochan_getfd(i), htbuf->buf, HTTP_BUF_SIZE -1, 0);
             if (res == -1 && errno == EAGAIN)
@@ -892,7 +919,18 @@ static void http_io(IOCHAN i, int event)
             }
             if (res <= 0)
             {
+#if HAVE_SYS_TIME_H
+                if (hc->http_server->record_file)
+                {
+                    struct timeval tv;
+                    gettimeofday(&tv, 0);
+                    fprintf(hc->http_server->record_file, "%lld %lld %lld 0\n",
+                            (long long) tv.tv_sec, (long long) tv.tv_usec,
+                            (long long) iochan_getfd(i));
+                }
+#endif
                 http_buf_destroy(hc->http_server, htbuf);
+                fflush(hc->http_server->record_file);
                 http_channel_destroy(i);
                 return;
             }
@@ -909,6 +947,22 @@ static void http_io(IOCHAN i, int event)
                     return;
                 // we have a complete HTTP request
                 nmem_reset(hc->nmem);
+#if HAVE_SYS_TIME_H
+                if (hc->http_server->record_file)
+                {
+                    struct timeval tv;
+                    int sz = 0;
+                    struct http_buf *hb;
+                    for (hb = hc->iqueue; hb; hb = hb->next)
+                        sz += hb->len;
+                    gettimeofday(&tv, 0);
+                    fprintf(hc->http_server->record_file, "%lld %lld %lld %d\n",
+                            (long long) tv.tv_sec, (long long) tv.tv_usec,
+                            (long long) iochan_getfd(i), sz);
+                    for (hb = hc->iqueue; hb; hb = hb->next)
+                        fwrite(hb->buf, 1, hb->len, hc->http_server->record_file);
+                }
+ #endif
                 if (!(hc->request = http_parse_request(hc, &hc->iqueue, reqlen)))
                 {
                     yaz_log(YLOG_WARN, "Failed to parse request");
@@ -931,12 +985,16 @@ static void http_io(IOCHAN i, int event)
                     http_command(hc);
                 }
             }
-            break;
-        case EVENT_OUTPUT:
+        }
+        else if (event == EVENT_OUTPUT)
+        {
+            event = 0;
             if (hc->oqueue)
             {
                 struct http_buf *wb = hc->oqueue;
-                res = send(iochan_getfd(hc->iochan), wb->buf + wb->offset, wb->len, 0);
+                int res;
+                res = send(iochan_getfd(hc->iochan),
+                           wb->buf + wb->offset, wb->len, 0);
                 if (res <= 0)
                 {
                     yaz_log(YLOG_WARN|YLOG_ERRNO, "write");
@@ -953,7 +1011,8 @@ static void http_io(IOCHAN i, int event)
                     wb->len -= res;
                     wb->offset += res;
                 }
-                if (!hc->oqueue) {
+                if (!hc->oqueue)
+                {
                     if (!hc->keep_alive)
                     {
                         http_channel_destroy(i);
@@ -963,17 +1022,19 @@ static void http_io(IOCHAN i, int event)
                     {
                         iochan_clearflag(i, EVENT_OUTPUT);
                         if (hc->iqueue)
-                            iochan_setevent(hc->iochan, EVENT_INPUT);
+                            event = EVENT_INPUT;
                     }
                 }
             }
-
             if (!hc->oqueue && hc->proxy && !hc->proxy->iochan) 
                 http_channel_destroy(i); // Server closed; we're done
-            break;
-        default:
+        }
+        else
+        {
             yaz_log(YLOG_WARN, "Unexpected event on connection");
             http_channel_destroy(i);
+            event = 0;
+        }
     }
 }
 
@@ -1089,8 +1150,21 @@ static void http_channel_destroy(IOCHAN i)
     http_server = s->http_server; /* save it for destroy (decref) */
 
     yaz_mutex_enter(s->http_server->mutex);
-    s->next = s->http_server->http_channel_freelist;
-    s->http_server->http_channel_freelist = s;
+    if (s->http_server->http_channel_freelist_max > 0 && s->http_server->http_channel_freelist_count >= s->http_server->http_channel_freelist_max) {
+        while ((s->next = s->http_server->http_channel_freelist)) {
+            nmem_destroy(s->next->nmem);
+            wrbuf_destroy(s->next->wrbuf);
+            xfree(s->next);
+            s->http_server->http_channel_freelist = s->http_server->http_channel_freelist->next;
+        }
+        s->http_server->http_channel_freelist_count = 0;
+    }
+    else {
+        s->next = s->http_server->http_channel_freelist;
+        s->http_server->http_channel_freelist = s;
+        s->http_server->http_channel_freelist_count++;
+        yaz_log(YLOG_DEBUG, "Free %d channels on server.", s->http_server->http_channel_freelist_count);
+    }
     yaz_mutex_leave(s->http_server->mutex);
 
     http_server_destroy(http_server);
@@ -1111,8 +1185,10 @@ static struct http_channel *http_channel_create(http_server_t hs,
 
     yaz_mutex_enter(hs->mutex);
     r = hs->http_channel_freelist;
-    if (r)
+    if (r) {
         hs->http_channel_freelist = r->next;
+        hs->http_channel_freelist_count--;
+    }
     yaz_mutex_leave(hs->mutex);
 
     if (r)
@@ -1129,6 +1205,7 @@ static struct http_channel *http_channel_create(http_server_t hs,
     http_server_incref(hs);
     r->http_server = hs;
     r->http_sessions = hs->http_sessions;
+    assert(r->http_sessions);
     r->server = server;
     r->proxy = 0;
     r->iochan = 0;
@@ -1168,18 +1245,18 @@ static void http_accept(IOCHAN i, int event)
     enable_nonblock(s);
 
     yaz_log(YLOG_DEBUG, "New command connection");
-    c = iochan_create(s, http_io, EVENT_INPUT | EVENT_EXCEPT);
+    c = iochan_create(s, http_io, EVENT_INPUT | EVENT_EXCEPT, "http_session_socket");
     
     ch = http_channel_create(server->http_server, inet_ntoa(addr.sin_addr),
                              server);
     ch->iochan = c;
     iochan_setdata(c, ch);
-
-    pazpar2_add_channel(c);
+    iochan_add(server->iochan_man, c);
 }
 
 /* Create a http-channel listener, syntax [host:]port */
-int http_init(const char *addr, struct conf_server *server)
+int http_init(const char *addr, struct conf_server *server,
+              const char *record_fname)
 {
     IOCHAN c;
     int l;
@@ -1188,9 +1265,21 @@ int http_init(const char *addr, struct conf_server *server)
     int one = 1;
     const char *pp;
     short port;
+    FILE *record_file = 0;
 
     yaz_log(YLOG_LOG, "HTTP listener %s", addr);
 
+
+    if (record_fname)
+    {
+        record_file = fopen(record_fname, "wb");
+        if (!record_file)
+        {
+            yaz_log(YLOG_FATAL|YLOG_ERRNO, "fopen %s", record_fname);
+            return 1;
+        }
+    }
+
     memset(&myaddr, 0, sizeof myaddr);
     myaddr.sin_family = AF_INET;
     pp = strchr(addr, ':');
@@ -1242,11 +1331,13 @@ int http_init(const char *addr, struct conf_server *server)
 
     server->http_server = http_server_create();
 
+    server->http_server->record_file = record_file;
     server->http_server->listener_socket = l;
 
-    c = iochan_create(l, http_accept, EVENT_INPUT | EVENT_EXCEPT);
+    c = iochan_create(l, http_accept, EVENT_INPUT | EVENT_EXCEPT, "http_server");
     iochan_setdata(c, server);
-    pazpar2_add_channel(c);
+
+    iochan_add(server->iochan_man, c);
     return 0;
 }
 
@@ -1359,9 +1450,18 @@ http_server_t http_server_create(void)
     hs->mutex = 0;
     hs->proxy_addr = 0;
     hs->ref_count = 1;
-    hs->http_buf_freelist = 0;
-    hs->http_channel_freelist = 0;
     hs->http_sessions = 0;
+
+    hs->http_channel_freelist = 0;
+    hs->http_channel_freelist_count = 0;
+    /* Disable max check */
+    hs->http_channel_freelist_max   = 0;
+
+    hs->http_buf_freelist = 0;
+    hs->http_buf_freelist_count = 0;
+    /* Disable max check */
+    hs->http_buf_freelist_max = 0;
+    hs->record_file = 0;
     return hs;
 }
 
@@ -1371,14 +1471,9 @@ void http_server_destroy(http_server_t hs)
     {
         int r;
 
-        if (hs->mutex)
-        {
-            yaz_mutex_enter(hs->mutex);
-            r = --(hs->ref_count);
-            yaz_mutex_leave(hs->mutex);
-        }
-        else
-            r = --(hs->ref_count);
+        yaz_mutex_enter(hs->mutex); /* OK: hs->mutex may be NULL */
+        r = --(hs->ref_count);
+        yaz_mutex_leave(hs->mutex);
 
         if (r == 0)
         {
@@ -1401,6 +1496,8 @@ void http_server_destroy(http_server_t hs)
             http_sessions_destroy(hs->http_sessions);
             xfree(hs->proxy_addr);
             yaz_mutex_destroy(&hs->mutex);
+            if (hs->record_file)
+                fclose(hs->record_file);
             xfree(hs);
         }
     }
@@ -1419,7 +1516,7 @@ void http_mutex_init(struct conf_server *server)
     assert(server);
 
     assert(server->http_server->mutex == 0);
-    yaz_mutex_create(&server->http_server->mutex);
+    pazpar2_mutex_create(&server->http_server->mutex, "http_server");
     server->http_server->http_sessions = http_sessions_create();
 }