From 7b13bb9165493bdef71e929ebcdc02b6a6605aca Mon Sep 17 00:00:00 2001 From: Adam Dickmeiss Date: Wed, 9 Nov 2011 16:29:31 +0100 Subject: [PATCH] Re-enable threaded host resolving And do it properly for HTTP connections as well; Pazpar2 1.6.3 and earlier versions only performed threaded resolving for Z39.50 connections. Requires YAZ 4.2.20 that offers the "tproxy" setting for ZOOM connections. --- configure.ac | 2 +- debian/control | 2 +- src/Makefile.am | 3 +- src/connection.c | 76 ++++++++++++++--- src/connection.h | 1 + src/database.c | 74 ---------------- src/database.h | 5 -- src/getaddrinfo.c | 228 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/host.c | 153 +++++++++++++++++++++++++++++++++ src/host.h | 14 +++- src/pazpar2_config.h | 1 + src/session.h | 1 + win/makefile | 2 + 13 files changed, 469 insertions(+), 93 deletions(-) create mode 100644 src/getaddrinfo.c create mode 100644 src/host.c diff --git a/configure.ac b/configure.ac index 2580281..2739cf7 100644 --- a/configure.ac +++ b/configure.ac @@ -22,7 +22,7 @@ AC_LANG(C) AC_C_INLINE -YAZ_INIT([static icu],[4.2.18]) +YAZ_INIT([static icu],[4.2.20]) if test -z "$YAZLIB"; then AC_MSG_ERROR([YAZ development libraries missing]) fi diff --git a/debian/control b/debian/control index 21e3fd7..9f04b67 100644 --- a/debian/control +++ b/debian/control @@ -4,7 +4,7 @@ Priority: extra Maintainer: Adam Dickmeiss Build-Depends: debhelper (>= 5), autotools-dev, - libyaz4-dev (>= 4.2.18), + libyaz4-dev (>= 4.2.20), docbook-xsl, libgnutls-dev Standards-Version: 3.7.2 diff --git a/src/Makefile.am b/src/Makefile.am index 55d3ed5..156d677 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -22,7 +22,8 @@ libpazpar2_a_SOURCES = \ database.c database.h \ eventl.c eventl.h \ facet_limit.c facet_limit.h \ - host.h \ + getaddrinfo.c \ + host.c host.h \ http.c http.h http_command.c \ incref.c incref.h \ jenkins_hash.c jenkins_hash.h \ diff --git a/src/connection.c b/src/connection.c index c5a8ab5..a3b0c65 100644 --- a/src/connection.c +++ b/src/connection.c @@ -150,7 +150,7 @@ static void connection_destroy(struct connection *co) ZOOM_connection_destroy(co->link); iochan_destroy(co->iochan); } - yaz_log(YLOG_DEBUG, "%p Connection destroy %s", co, co->host->hostport); + yaz_log(YLOG_DEBUG, "%p Connection destroy %s", co, co->host->url); if (co->client) { @@ -183,7 +183,8 @@ static struct connection *connection_create(struct client *cl, co->operation_timeout = operation_timeout; co->session_timeout = session_timeout; - connection_connect(co, iochan_man); + if (host->ipport) + connection_connect(co, iochan_man); yaz_mutex_enter(host->mutex); co->next = co->host->connections; @@ -367,6 +368,50 @@ static void connection_release(struct connection *co) co->client = 0; } +void connect_resolver_host(struct host *host, iochan_man_t iochan_man) +{ + struct connection *con; + +start: + yaz_mutex_enter(host->mutex); + con = host->connections; + while (con) + { + if (con->state == Conn_Closed) + { + if (!host->ipport) /* unresolved */ + { + remove_connection_from_host(con); + yaz_mutex_leave(host->mutex); + connection_destroy(con); + goto start; + /* start all over .. at some point it will be NULL */ + } + else if (!con->client) + { + remove_connection_from_host(con); + yaz_mutex_leave(host->mutex); + connection_destroy(con); + /* start all over .. at some point it will be NULL */ + goto start; + } + else + { + yaz_mutex_leave(host->mutex); + connection_connect(con, iochan_man); + client_start_search(con->client, 0, 0); /* TODO */ + goto start; + } + } + else + { + yaz_log(YLOG_LOG, "connect_resolver_host: state=%d", con->state); + con = con->next; + } + } + yaz_mutex_leave(host->mutex); +} + static struct host *connection_get_host(struct connection *con) { return con->host; @@ -382,7 +427,6 @@ static int connection_connect(struct connection *con, iochan_man_t iochan_man) const char *sru_version = 0; struct session_database *sdb = client_get_database(con->client); - const char *zproxy = session_setting_oneval(sdb, PZ_ZPROXY); const char *apdulog = session_setting_oneval(sdb, PZ_APDULOG); assert(con); @@ -394,11 +438,19 @@ static int connection_connect(struct connection *con, iochan_man_t iochan_man) if ((charset = session_setting_oneval(sdb, PZ_NEGOTIATION_CHARSET))) ZOOM_options_set(zoptions, "charset", charset); - if (zproxy && *zproxy) + assert(host->ipport); + if (host->proxy) { - con->zproxy = xstrdup(zproxy); - ZOOM_options_set(zoptions, "proxy", zproxy); + yaz_log(YLOG_LOG, "proxy=%s", host->ipport); + ZOOM_options_set(zoptions, "proxy", host->ipport); } + else + { + assert(host->tproxy); + yaz_log(YLOG_LOG, "tproxy=%s", host->ipport); + ZOOM_options_set(zoptions, "tproxy", host->ipport); + } + if (apdulog && *apdulog) ZOOM_options_set(zoptions, "apdulog", apdulog); @@ -420,14 +472,14 @@ static int connection_connect(struct connection *con, iochan_man_t iochan_man) { char http_hostport[512]; strcpy(http_hostport, "http://"); - strcat(http_hostport, host->hostport); + strcat(http_hostport, host->url); + yaz_log(YLOG_LOG, "SRU connect to : %s", http_hostport); ZOOM_connection_connect(con->link, http_hostport, 0); } else { - ZOOM_connection_connect(con->link, host->hostport, 0); + ZOOM_connection_connect(con->link, host->url, 0); } - con->iochan = iochan_create(-1, connection_handler, 0, "connection_socket"); con->state = Conn_Connecting; iochan_settimeout(con->iochan, con->operation_timeout); @@ -449,7 +501,9 @@ int client_prep_connection(struct client *cl, struct session_database *sdb = client_get_database(cl); const char *zproxy = session_setting_oneval(sdb, PZ_ZPROXY); const char *url = session_setting_oneval(sdb, PZ_URL); + const char *sru = session_setting_oneval(sdb, PZ_SRU); struct host *host = 0; + int default_port = *sru ? 80 : 210; if (zproxy && zproxy[0] == '\0') zproxy = 0; @@ -458,10 +512,12 @@ int client_prep_connection(struct client *cl, url = sdb->database->id; host = find_host(client_get_session(cl)->service->server->database_hosts, - url); + url, zproxy, default_port, iochan_man); yaz_log(YLOG_DEBUG, "client_prep_connection: target=%s url=%s", client_get_id(cl), url); + if (!host) + return 0; co = client_get_connection(cl); diff --git a/src/connection.h b/src/connection.h index 6b61047..0aa0f4c 100644 --- a/src/connection.h +++ b/src/connection.h @@ -33,6 +33,7 @@ struct host; void connect_resolver_host(struct host *host, iochan_man_t iochan); ZOOM_connection connection_get_link(struct connection *co); void connection_continue(struct connection *co); +void connect_resolver_host(struct host *host, iochan_man_t iochan_man); #endif diff --git a/src/database.c b/src/database.c index 41e0f06..c933181 100644 --- a/src/database.c +++ b/src/database.c @@ -54,52 +54,6 @@ struct database_criterion { struct database_criterion *next; }; -struct database_hosts { - struct host *hosts; - YAZ_MUTEX mutex; -}; - -// Create a new host structure for hostport -static struct host *create_host(const char *hostport) -{ - struct host *host; - char *db_comment; - - host = xmalloc(sizeof(struct host)); - host->hostport = xstrdup(hostport); - db_comment = strchr(host->hostport, '#'); - if (db_comment) - *db_comment = '\0'; - host->connections = 0; - host->mutex = 0; - - pazpar2_mutex_create(&host->mutex, "host"); - - yaz_cond_create(&host->cond_ready); - - return host; -} - -struct host *find_host(database_hosts_t hosts, const char *hostport) -{ - struct host *p; - yaz_mutex_enter(hosts->mutex); - for (p = hosts->hosts; p; p = p->next) - if (!strcmp(p->hostport, hostport)) - break; - if (!p) - { - p = create_host(hostport); - if (p) - { - p->next = hosts->hosts; - hosts->hosts = p; - } - } - yaz_mutex_leave(hosts->mutex); - return p; -} - struct database *new_database(const char *id, NMEM nmem) { struct database *db; @@ -313,34 +267,6 @@ int predef_grep_databases(void *context, struct conf_service *service, return i; } -database_hosts_t database_hosts_create(void) -{ - database_hosts_t p = xmalloc(sizeof(*p)); - p->hosts = 0; - p->mutex = 0; - pazpar2_mutex_create(&p->mutex, "database"); - return p; -} - -void database_hosts_destroy(database_hosts_t *pp) -{ - if (*pp) - { - struct host *p = (*pp)->hosts; - while (p) - { - struct host *p_next = p->next; - yaz_mutex_destroy(&p->mutex); - yaz_cond_destroy(&p->cond_ready); - xfree(p->hostport); - xfree(p); - p = p_next; - } - yaz_mutex_destroy(&(*pp)->mutex); - xfree(*pp); - } -} - /* * Local variables: * c-basic-offset: 4 diff --git a/src/database.h b/src/database.h index 1394732..2b0a3fc 100644 --- a/src/database.h +++ b/src/database.h @@ -20,7 +20,6 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA #ifndef DATABASE_H #define DATABASE_H -typedef struct database_hosts *database_hosts_t; struct session_database; struct session; struct conf_service; @@ -33,9 +32,5 @@ int predef_grep_databases(void *context, struct conf_service *service, int match_zurl(const char *zurl, const char *pattern); struct database *new_database(const char *id, NMEM nmem); -database_hosts_t database_hosts_create(void); -void database_hosts_destroy(database_hosts_t *); - -struct host *find_host(database_hosts_t hosts, const char *hostport); #endif diff --git a/src/getaddrinfo.c b/src/getaddrinfo.c new file mode 100644 index 0000000..3ca2f4e --- /dev/null +++ b/src/getaddrinfo.c @@ -0,0 +1,228 @@ +/* This file is part of Pazpar2. + Copyright (C) 2006-2011 Index Data + +Pazpar2 is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. + +Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#if HAVE_CONFIG_H +#include +#endif + +#include "sel_thread.h" + +#if HAVE_UNISTD_H +#include +#endif +#include + +#include +#include +#if HAVE_SYS_SOCKET_H +#include +#endif +#ifdef WIN32 +#include +#endif +#if HAVE_NETDB_H +#include +#endif +#if HAVE_NETINET_IN_H +#include +#endif + +#include +#include +#include + +#include "session.h" +#include "connection.h" +#include "host.h" + +/* Only use a threaded resolver on Unix that offers getaddrinfo. + gethostbyname is NOT reentrant. + */ +#if HAVE_GETADDRINFO +#ifndef WIN32 +#define USE_THREADED_RESOLVER 1 +#endif +#endif + +struct work { + char *hostport; /* hostport to be resolved in separate thread */ + char *ipport; /* result or NULL if it could not be resolved */ + struct host *host; /* host that we're dealing with - mother thread */ + iochan_man_t iochan_man; /* iochan manager */ +}; + +static int log_level = YLOG_LOG; + +void perform_getaddrinfo(struct work *w) +{ + int res = 0; + char *port; +#if HAVE_GETADDRINFO + struct addrinfo *addrinfo, hints; +#else + struct hostent *hp; +#endif + char *hostport = xstrdup(w->hostport); + if ((port = strchr(hostport, ':'))) + *(port++) = '\0'; + else + { + port = "210"; + } + +#if HAVE_GETADDRINFO + hints.ai_flags = 0; + hints.ai_family = PF_INET; + hints.ai_socktype = SOCK_STREAM; + hints.ai_protocol = IPPROTO_TCP; + hints.ai_addrlen = 0; + hints.ai_addr = 0; + hints.ai_canonname = 0; + hints.ai_next = 0; + // This is not robust code. It assumes that getaddrinfo always + // returns AF_INET address. + if ((res = getaddrinfo(hostport, port, &hints, &addrinfo))) + { + yaz_log(YLOG_WARN, "Failed to resolve %s %s", + w->hostport, gai_strerror(res)); + } + else + { + char ipport[128]; + unsigned char addrbuf[4]; + assert(addrinfo->ai_family == PF_INET); + memcpy(addrbuf, + &((struct sockaddr_in*)addrinfo->ai_addr)->sin_addr.s_addr, 4); + sprintf(ipport, "%u.%u.%u.%u:%s", + addrbuf[0], addrbuf[1], addrbuf[2], addrbuf[3], port); + freeaddrinfo(addrinfo); + w->ipport = xstrdup(ipport); + yaz_log(log_level, "Resolved %s -> %s", hostport, ipport); + } +#else + hp = gethostbyname(hostport); + if (!hp) + { + yaz_log(YLOG_WARN|YLOG_ERRNO, "Failed to resolve %s", hostport); + } + else + { + char ipport[128]; + unsigned char addrbuf[4]; + + memcpy(addrbuf, *hp->h_addr_list, 4 * sizeof(unsigned char)); + sprintf(ipport, "%u.%u.%u.%u:%s", + addrbuf[0], addrbuf[1], addrbuf[2], addrbuf[3], port); + w->ipport = xstrdup(ipport); + yaz_log(log_level, "Resolved %s -> %s", hostport, ipport); + } +#endif + xfree(hostport); +} + +static void work_handler(void *vp) +{ + struct work *w = vp; + + int sec = 0; /* >0 for debugging/testing purposes */ + if (sec) + { + yaz_log(log_level, "waiting %d seconds", sec); +#if HAVE_UNISTD_H + sleep(sec); +#endif + } + perform_getaddrinfo(w); +} + +#if USE_THREADED_RESOLVER +void iochan_handler(struct iochan *i, int event) +{ + sel_thread_t p = iochan_getdata(i); + + if (event & EVENT_INPUT) + { + struct work *w = sel_thread_result(p); + w->host->ipport = w->ipport; + connect_resolver_host(w->host, w->iochan_man); + xfree(w); + } +} + +static sel_thread_t resolver_thread = 0; + +static void getaddrinfo_start(iochan_man_t iochan_man) +{ + int fd; + sel_thread_t p = resolver_thread = + sel_thread_create(work_handler, 0 /* work_destroy */, &fd, + 3 /* no of resolver threads */); + if (!p) + { + yaz_log(YLOG_FATAL|YLOG_ERRNO, "sel_create_create failed"); + exit(1); + } + else + { + IOCHAN chan = iochan_create(fd, iochan_handler, EVENT_INPUT, + "getaddrinfo_socket"); + iochan_setdata(chan, p); + iochan_add(iochan_man, chan); + } + yaz_log(log_level, "resolver start"); + resolver_thread = p; +} +#endif + +int host_getaddrinfo(struct host *host, iochan_man_t iochan_man) +{ + struct work *w = xmalloc(sizeof(*w)); + int use_thread = 0; /* =0 to disable threading entirely */ + + w->hostport = host->tproxy ? host->tproxy : host->proxy; + w->ipport = 0; + w->host = host; + w->iochan_man = iochan_man; +#if USE_THREADED_RESOLVER + if (use_thread) + { + if (resolver_thread == 0) + getaddrinfo_start(iochan_man); + assert(resolver_thread); + sel_thread_add(resolver_thread, w); + return 0; + } +#endif + perform_getaddrinfo(w); + host->ipport = w->ipport; + xfree(w); + if (!host->ipport) + return -1; + return 0; +} + +/* + * Local variables: + * c-basic-offset: 4 + * c-file-style: "Stroustrup" + * indent-tabs-mode: nil + * End: + * vim: shiftwidth=4 tabstop=8 expandtab + */ + diff --git a/src/host.c b/src/host.c new file mode 100644 index 0000000..57c6eb6 --- /dev/null +++ b/src/host.c @@ -0,0 +1,153 @@ +/* This file is part of Pazpar2. + Copyright (C) 2006-2011 Index Data + +Pazpar2 is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. + +Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*/ + +#if HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "ppmutex.h" +#include "session.h" +#include "host.h" + +#include + +struct database_hosts { + struct host *hosts; + YAZ_MUTEX mutex; +}; + +// Create a new host structure for hostport +static struct host *create_host(const char *url, const char *proxy, + int default_port, + iochan_man_t iochan_man) +{ + struct host *host; + char *db_comment; + + host = xmalloc(sizeof(struct host)); + host->url = xstrdup(url); + host->proxy = 0; + host->tproxy = 0; + if (proxy && *proxy) + host->proxy = xstrdup(proxy); + else + { + char *cp; + + host->tproxy = xmalloc (strlen(url) + 10); /* so we can add :port */ + strcpy(host->tproxy, url); + for (cp = host->tproxy; *cp; cp++) + if (strchr("/?#~", *cp)) + { + *cp = '\0'; + break; + } + if (!strchr(host->tproxy, ':')) + sprintf(cp, ":%d", default_port); /* no port given, add it */ + } + + db_comment = strchr(host->url, '#'); + if (db_comment) + *db_comment = '\0'; + host->connections = 0; + host->ipport = 0; + host->mutex = 0; + + if (host_getaddrinfo(host, iochan_man)) + { + xfree(host->ipport); + xfree(host->tproxy); + xfree(host->proxy); + xfree(host->url); + xfree(host); + return 0; + } + pazpar2_mutex_create(&host->mutex, "host"); + + yaz_cond_create(&host->cond_ready); + + return host; +} + +struct host *find_host(database_hosts_t hosts, const char *url, + const char *proxy, int port, + iochan_man_t iochan_man) +{ + struct host *p; + yaz_mutex_enter(hosts->mutex); + for (p = hosts->hosts; p; p = p->next) + if (!strcmp(p->url, url)) + break; + if (!p) + { + p = create_host(url, proxy, port, iochan_man); + if (p) + { + p->next = hosts->hosts; + hosts->hosts = p; + } + } + yaz_mutex_leave(hosts->mutex); + return p; +} + +database_hosts_t database_hosts_create(void) +{ + database_hosts_t p = xmalloc(sizeof(*p)); + p->hosts = 0; + p->mutex = 0; + pazpar2_mutex_create(&p->mutex, "database"); + return p; +} + +void database_hosts_destroy(database_hosts_t *pp) +{ + if (*pp) + { + struct host *p = (*pp)->hosts; + while (p) + { + struct host *p_next = p->next; + yaz_mutex_destroy(&p->mutex); + yaz_cond_destroy(&p->cond_ready); + xfree(p->url); + xfree(p->ipport); + xfree(p); + p = p_next; + } + yaz_mutex_destroy(&(*pp)->mutex); + xfree(*pp); + } +} + +/* + * Local variables: + * c-basic-offset: 4 + * c-file-style: "Stroustrup" + * indent-tabs-mode: nil + * End: + * vim: shiftwidth=4 tabstop=8 expandtab + */ + diff --git a/src/host.h b/src/host.h index 752ca09..d30632f 100644 --- a/src/host.h +++ b/src/host.h @@ -22,15 +22,27 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA #include +typedef struct database_hosts *database_hosts_t; + /** \brief Represents a host (irrespective of databases) */ struct host { - char *hostport; + char *url; // logical host/database?args .. + char *tproxy; // tproxy address (no Z39.50 UI) + char *proxy; // logical proxy address + char *ipport; // tproxy or proxy resolved struct connection *connections; // All connections to this struct host *next; YAZ_MUTEX mutex; YAZ_COND cond_ready; }; +database_hosts_t database_hosts_create(void); +void database_hosts_destroy(database_hosts_t *); + +struct host *find_host(database_hosts_t hosts, const char *hostport, + const char *proxy, int port, iochan_man_t iochan_man); + +int host_getaddrinfo(struct host *host, iochan_man_t iochan_man); #endif diff --git a/src/pazpar2_config.h b/src/pazpar2_config.h index 403ec16..f5fb27d 100644 --- a/src/pazpar2_config.h +++ b/src/pazpar2_config.h @@ -27,6 +27,7 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA #include "charsets.h" #include "http.h" #include "database.h" +#include "host.h" enum conf_metadata_type { Metadata_type_generic, // Generic text field diff --git a/src/session.h b/src/session.h index 44bd94c..54ccd6c 100644 --- a/src/session.h +++ b/src/session.h @@ -181,6 +181,7 @@ int ingest_record(struct client *cl, const char *rec, int record_no, NMEM nmem); void session_alert_watch(struct session *s, int what); void add_facet(struct session *s, const char *type, const char *value, int count); + void perform_termlist(struct http_channel *c, struct session *se, const char *name, int num); void session_log(struct session *s, int level, const char *fmt, ...) diff --git a/win/makefile b/win/makefile index 7e6b3ea..8caf4e3 100644 --- a/win/makefile +++ b/win/makefile @@ -178,6 +178,8 @@ LNKOPT= $(COMMON_LNK_OPTIONS) $(RELEASE_LNK_OPTIONS) $(LNK_LIBS) # Source and object modules PAZPAR2_OBJS = \ + "$(OBJDIR)\getaddrinfo.obj" \ + "$(OBJDIR)\host.obj" \ "$(OBJDIR)\pazpar2.obj" \ "$(OBJDIR)\pazpar2_config.obj" \ "$(OBJDIR)\http.obj" \ -- 1.7.10.4