Use pthread_atfork to lock/unlock yaz_log system
[yaz-moved-to-github.git] / src / zoom-c.c
1 /* This file is part of the YAZ toolkit.
2  * Copyright (C) Index Data
3  * See the file LICENSE for details.
4  */
5 /**
6  * \file zoom-c.c
7  * \brief Implements ZOOM C interface.
8  */
9 #if HAVE_CONFIG_H
10 #include <config.h>
11 #endif
12
13 #include <assert.h>
14 #include <string.h>
15 #include <errno.h>
16 #include "zoom-p.h"
17
18 #include <yaz/yaz-util.h>
19 #include <yaz/xmalloc.h>
20 #include <yaz/otherinfo.h>
21 #include <yaz/log.h>
22 #include <yaz/diagbib1.h>
23 #include <yaz/charneg.h>
24 #include <yaz/query-charset.h>
25 #include <yaz/snprintf.h>
26 #include <yaz/facet.h>
27
28 #include <yaz/shptr.h>
29
30 #if SHPTR
31 YAZ_SHPTR_TYPE(WRBUF)
32 #endif
33
34 static int log_api0 = 0;
35 static int log_details0 = 0;
36
37 static void resultset_destroy(ZOOM_resultset r);
38 static zoom_ret do_write_ex(ZOOM_connection c, char *buf_out, int len_out);
39
40 static void initlog(void)
41 {
42     static int log_level_initialized = 0;
43
44     if (!log_level_initialized)
45     {
46         log_api0 = yaz_log_module_level("zoom");
47         log_details0 = yaz_log_module_level("zoomdetails");
48         log_level_initialized = 1;
49     }
50 }
51
52 void ZOOM_connection_remove_tasks(ZOOM_connection c);
53
54 void ZOOM_set_dset_error(ZOOM_connection c, int error,
55                          const char *dset,
56                          const char *addinfo, const char *addinfo2)
57 {
58     char *cp;
59
60     xfree(c->addinfo);
61     c->addinfo = 0;
62     c->error = error;
63     if (!c->diagset || strcmp(dset, c->diagset))
64     {
65         xfree(c->diagset);
66         c->diagset = xstrdup(dset);
67         /* remove integer part from SRW diagset .. */
68         if ((cp = strrchr(c->diagset, '/')))
69             *cp = '\0';
70     }
71     if (addinfo && addinfo2)
72     {
73         c->addinfo = (char*) xmalloc(strlen(addinfo) + strlen(addinfo2) + 3);
74         strcpy(c->addinfo, addinfo);
75         strcat(c->addinfo, ": ");
76         strcat(c->addinfo, addinfo2);
77     }
78     else if (addinfo)
79         c->addinfo = xstrdup(addinfo);
80     if (error != ZOOM_ERROR_NONE)
81     {
82         yaz_log(c->log_api, "%p set_dset_error %s %s:%d %s %s",
83                 c, c->host_port ? c->host_port : "<>", dset, error,
84                 addinfo ? addinfo : "",
85                 addinfo2 ? addinfo2 : "");
86         ZOOM_connection_remove_tasks(c);
87     }
88 }
89
90 int ZOOM_uri_to_code(const char *uri)
91 {
92     int code = 0;
93     const char *cp;
94     if ((cp = strrchr(uri, '/')))
95         code = atoi(cp+1);
96     return code;
97 }
98
99 void ZOOM_set_error(ZOOM_connection c, int error, const char *addinfo)
100 {
101     ZOOM_set_dset_error(c, error, "ZOOM", addinfo, 0);
102 }
103
104 static void clear_error(ZOOM_connection c)
105 {
106     /*
107      * If an error is tied to an operation then it's ok to clear: for
108      * example, a diagnostic returned from a search is cleared by a
109      * subsequent search.  However, problems such as Connection Lost
110      * or Init Refused are not cleared, because they are not
111      * recoverable: doing another search doesn't help.
112      */
113
114     ZOOM_connection_remove_events(c);
115     switch (c->error)
116     {
117     case ZOOM_ERROR_CONNECT:
118     case ZOOM_ERROR_MEMORY:
119     case ZOOM_ERROR_DECODE:
120     case ZOOM_ERROR_CONNECTION_LOST:
121     case ZOOM_ERROR_INIT:
122     case ZOOM_ERROR_INTERNAL:
123     case ZOOM_ERROR_UNSUPPORTED_PROTOCOL:
124         break;
125     default:
126         ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
127     }
128 }
129
130 void ZOOM_connection_show_task(ZOOM_task task)
131 {
132     switch(task->which)
133     {
134     case ZOOM_TASK_SEARCH:
135         yaz_log(YLOG_LOG, "search p=%p", task);
136         break;
137     case ZOOM_TASK_CONNECT:
138         yaz_log(YLOG_LOG, "connect p=%p", task);
139         break;
140     case ZOOM_TASK_SCAN:
141         yaz_log(YLOG_LOG, "scan p=%p", task);
142         break;
143     }
144 }
145
146 void ZOOM_connection_show_tasks(ZOOM_connection c)
147 {
148     ZOOM_task task;
149     yaz_log(YLOG_LOG, "connection p=%p tasks", c);
150     for (task = c->tasks; task; task = task->next)
151         ZOOM_connection_show_task(task);
152 }
153
154 ZOOM_task ZOOM_connection_add_task(ZOOM_connection c, int which)
155 {
156     ZOOM_task *taskp = &c->tasks;
157     while (*taskp)
158         taskp = &(*taskp)->next;
159     *taskp = (ZOOM_task) xmalloc(sizeof(**taskp));
160     (*taskp)->running = 0;
161     (*taskp)->which = which;
162     (*taskp)->next = 0;
163     clear_error(c);
164     return *taskp;
165 }
166
167 ZOOM_API(int) ZOOM_connection_is_idle(ZOOM_connection c)
168 {
169     return c->tasks ? 0 : 1;
170 }
171
172 ZOOM_task ZOOM_connection_insert_task(ZOOM_connection c, int which)
173 {
174     ZOOM_task task = (ZOOM_task) xmalloc(sizeof(*task));
175
176     task->next = c->tasks;
177     c->tasks = task;
178
179     task->running = 0;
180     task->which = which;
181     return task;
182 }
183
184 void ZOOM_connection_remove_task(ZOOM_connection c)
185 {
186     ZOOM_task task = c->tasks;
187
188     if (task)
189     {
190         c->tasks = task->next;
191         switch (task->which)
192         {
193         case ZOOM_TASK_SEARCH:
194             resultset_destroy(task->u.search.resultset);
195             xfree(task->u.search.syntax);
196             xfree(task->u.search.elementSetName);
197             xfree(task->u.search.schema);
198             break;
199         case ZOOM_TASK_CONNECT:
200             break;
201         case ZOOM_TASK_SCAN:
202             ZOOM_scanset_destroy(task->u.scan.scan);
203             break;
204         case ZOOM_TASK_PACKAGE:
205             ZOOM_package_destroy(task->u.package);
206             break;
207         case ZOOM_TASK_SORT:
208             resultset_destroy(task->u.sort.resultset);
209             ZOOM_query_destroy(task->u.sort.q);
210             break;
211         default:
212             assert(0);
213         }
214         xfree(task);
215
216         if (!c->tasks)
217         {
218             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_END);
219             ZOOM_connection_put_event(c, event);
220         }
221     }
222 }
223
224 void ZOOM_connection_remove_tasks(ZOOM_connection c)
225 {
226     while (c->tasks)
227         ZOOM_connection_remove_task(c);
228 }
229
230 static void odr_wrbuf_write(ODR o, void *handle, int type,
231                             const char *buf, int len)
232 {
233     WRBUF w = (WRBUF) handle;
234     wrbuf_write(w, buf, len);
235 }
236
237 ZOOM_API(ZOOM_connection)
238     ZOOM_connection_create(ZOOM_options options)
239 {
240     ZOOM_connection c = (ZOOM_connection) xmalloc(sizeof(*c));
241
242     initlog();
243
244     c->log_api = log_api0;
245     c->log_details = log_details0;
246
247     yaz_log(c->log_api, "%p ZOOM_connection_create", c);
248
249     c->proto = PROTO_Z3950;
250     c->cs = 0;
251     ZOOM_connection_set_mask(c, 0);
252     c->reconnect_ok = 0;
253     c->state = STATE_IDLE;
254     c->addinfo = 0;
255     c->diagset = 0;
256     ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
257     c->buf_in = 0;
258     c->len_in = 0;
259     c->buf_out = 0;
260     c->len_out = 0;
261     c->resultsets = 0;
262
263     c->options = ZOOM_options_create_with_parent(options);
264
265     c->host_port = 0;
266     c->proxy = 0;
267     c->tproxy = 0;
268
269     c->charset = c->lang = 0;
270
271     c->cookie_out = 0;
272     c->cookie_in = 0;
273     c->client_IP = 0;
274     c->tasks = 0;
275
276     c->user = 0;
277     c->group = 0;
278     c->password = 0;
279     c->url_authentication = 0;
280
281     c->maximum_record_size = 0;
282     c->preferred_message_size = 0;
283
284     c->odr_in = odr_createmem(ODR_DECODE);
285     c->odr_out = odr_createmem(ODR_ENCODE);
286     c->odr_print = 0;
287     c->odr_save = 0;
288
289     c->async = 0;
290     c->support_named_resultsets = 0;
291     c->last_event = ZOOM_EVENT_NONE;
292
293     c->m_queue_front = 0;
294     c->m_queue_back = 0;
295
296     c->sru_version = 0;
297     c->no_redirects = 0;
298     c->cookies = 0;
299     c->saveAPDU_wrbuf = 0;
300
301     ZOOM_memcached_init(c);
302     return c;
303 }
304
305 ZOOM_API(void) ZOOM_connection_save_apdu_wrbuf(ZOOM_connection c, WRBUF w)
306 {
307     if (c->odr_save)
308     {
309         odr_destroy(c->odr_save);
310         c->odr_save = 0;
311     }
312     if (w)
313     {
314         c->odr_save = odr_createmem(ODR_PRINT);
315         odr_set_stream(c->odr_save, w, odr_wrbuf_write, 0);
316     }
317 }
318
319 /* set database names. Take local databases (if set); otherwise
320    take databases given in ZURL (if set); otherwise use Default */
321 char **ZOOM_connection_get_databases(ZOOM_connection con, ZOOM_options options,
322                                      int *num, ODR odr)
323 {
324     char **databaseNames;
325     const char *cp = ZOOM_options_get(options, "databaseName");
326
327     if ((!cp || !*cp) && con->host_port)
328         cs_get_host_args(con->host_port, &cp);
329     if (!cp || !*cp)
330         cp = "Default";
331     nmem_strsplit(odr_getmem(odr), "+", cp,  &databaseNames, num);
332     return databaseNames;
333 }
334
335 ZOOM_API(ZOOM_connection)
336     ZOOM_connection_new(const char *host, int portnum)
337 {
338     ZOOM_connection c = ZOOM_connection_create(0);
339
340     ZOOM_connection_connect(c, host, portnum);
341     return c;
342 }
343
344 static zoom_sru_mode get_sru_mode_from_string(const char *s)
345 {
346     if (!s || !*s)
347         return zoom_sru_soap;
348     if (!yaz_matchstr(s, "soap"))
349         return zoom_sru_soap;
350     else if (!yaz_matchstr(s, "get"))
351         return zoom_sru_get;
352     else if (!yaz_matchstr(s, "post"))
353         return zoom_sru_post;
354     else if (!yaz_matchstr(s, "solr"))
355         return zoom_sru_solr;
356     return zoom_sru_error;
357 }
358
359 ZOOM_API(void)
360     ZOOM_connection_connect(ZOOM_connection c,
361                             const char *host, int portnum)
362 {
363     const char *val;
364     const char *http_lead;
365
366     initlog();
367
368     yaz_log(c->log_api, "%p ZOOM_connection_connect host=%s portnum=%d",
369             c, host ? host : "null", portnum);
370
371     ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
372     ZOOM_connection_remove_tasks(c);
373
374     if (c->odr_print)
375     {
376         odr_setprint(c->odr_print, 0); /* prevent destroy from fclose'ing */
377         odr_destroy(c->odr_print);
378     }
379     if (ZOOM_options_get_bool(c->options, "apdulog", 0))
380     {
381         c->odr_print = odr_createmem(ODR_PRINT);
382         odr_setprint(c->odr_print, yaz_log_file());
383     }
384     else
385         c->odr_print = 0;
386
387     if (c->cs)
388     {
389         yaz_log(c->log_details, "%p ZOOM_connection_connect reconnect ok", c);
390         c->reconnect_ok = 1;
391         return;
392     }
393     yaz_log(c->log_details, "%p ZOOM_connection_connect connect", c);
394     xfree(c->proxy);
395     c->proxy = 0;
396     val = ZOOM_options_get(c->options, "proxy");
397     if (val && *val)
398     {
399         yaz_log(c->log_details, "%p ZOOM_connection_connect proxy=%s", c, val);
400         c->proxy = xstrdup(val);
401     }
402
403     xfree(c->tproxy);
404     c->tproxy = 0;
405     val = ZOOM_options_get(c->options, "tproxy");
406     if (val && *val)
407     {
408         yaz_log(c->log_details, "%p ZOOM_connection_connect tproxy=%s", c, val);
409         c->tproxy = xstrdup(val);
410     }
411
412     xfree(c->charset);
413     c->charset = 0;
414     val = ZOOM_options_get(c->options, "charset");
415     if (val && *val)
416     {
417         yaz_log(c->log_details, "%p ZOOM_connection_connect charset=%s", c, val);
418         c->charset = xstrdup(val);
419     }
420
421     xfree(c->lang);
422     val = ZOOM_options_get(c->options, "lang");
423     if (val && *val)
424     {
425         yaz_log(c->log_details, "%p ZOOM_connection_connect lang=%s", c, val);
426         c->lang = xstrdup(val);
427     }
428     else
429         c->lang = 0;
430
431     val = ZOOM_options_get(c->options, "sru");
432     if (val && *val && !strstr(host, "://"))
433         http_lead = "http://";
434     else
435         http_lead = "";
436     c->sru_mode = get_sru_mode_from_string(val);
437
438     if (host)
439     {
440         char hostn[128];
441         xfree(c->host_port);
442         if (portnum)
443         {
444             sprintf(hostn, "%.80s:%d", host, portnum);
445             host = hostn;
446         }
447         c->host_port = xmalloc(strlen(host) + strlen(http_lead) + 1);
448         strcpy(c->host_port, http_lead);
449         strcat(c->host_port, host);
450     }
451
452     {
453         /*
454          * If the "<scheme>:" part of the host string is preceded by one
455          * or more comma-separated <name>=<value> pairs, these are taken
456          * to be options to be set on the connection object.  Among other
457          * applications, this facility can be used to embed authentication
458          * in a host string:
459          *          user=admin,password=secret,tcp:localhost:9999
460          */
461         char *remainder = c->host_port;
462         char *pcolon = strchr(remainder, ':');
463         char *pcomma;
464         char *pequals;
465         while ((pcomma = strchr(remainder, ',')) != 0 &&
466                (pcolon == 0 || pcomma < pcolon))
467         {
468             *pcomma = '\0';
469             if ((pequals = strchr(remainder, '=')) != 0)
470             {
471                 *pequals = '\0';
472                 ZOOM_connection_option_set(c, remainder, pequals+1);
473             }
474             remainder = pcomma+1;
475         }
476
477         if (remainder != c->host_port)
478         {
479             remainder = xstrdup(remainder);
480             xfree(c->host_port);
481             c->host_port = remainder;
482         }
483     }
484
485     xfree(c->sru_version);
486     val = ZOOM_options_get(c->options, "sru_version");
487     c->sru_version = xstrdup(val ? val : "1.2");
488
489     ZOOM_options_set(c->options, "host", c->host_port);
490
491     xfree(c->cookie_out);
492     c->cookie_out = 0;
493     val = ZOOM_options_get(c->options, "cookie");
494     if (val && *val)
495     {
496         yaz_log(c->log_details, "%p ZOOM_connection_connect cookie=%s", c, val);
497         c->cookie_out = xstrdup(val);
498     }
499
500     xfree(c->client_IP);
501     c->client_IP = 0;
502     val = ZOOM_options_get(c->options, "clientIP");
503     if (val && *val)
504     {
505         yaz_log(c->log_details, "%p ZOOM_connection_connect clientIP=%s",
506                 c, val);
507         c->client_IP = xstrdup(val);
508     }
509
510     xfree(c->group);
511     c->group = 0;
512     val = ZOOM_options_get(c->options, "group");
513     if (val && *val)
514         c->group = xstrdup(val);
515
516     xfree(c->user);
517     c->user = 0;
518     val = ZOOM_options_get(c->options, "user");
519     if (val && *val)
520         c->user = xstrdup(val);
521
522     xfree(c->password);
523     c->password = 0;
524     val = ZOOM_options_get(c->options, "password");
525     if (!val)
526         val = ZOOM_options_get(c->options, "pass");
527     if (val && *val)
528         c->password = xstrdup(val);
529
530     val = ZOOM_options_get(c->options, "authenticationMode");
531     if (val && !strcmp(val, "url"))
532         c->url_authentication = 1;
533     else
534         c->url_authentication = 0;
535
536     c->maximum_record_size =
537         ZOOM_options_get_int(c->options, "maximumRecordSize", 64*1024*1024);
538     c->preferred_message_size =
539         ZOOM_options_get_int(c->options, "preferredMessageSize", 64*1024*1024);
540
541     c->async = ZOOM_options_get_bool(c->options, "async", 0);
542
543     yaz_cookies_destroy(c->cookies);
544     c->cookies = yaz_cookies_create();
545
546     if (ZOOM_memcached_configure(c))
547     {
548         ZOOM_connection_remove_tasks(c);
549         return;
550     }
551     if (c->sru_mode == zoom_sru_error)
552     {
553         ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, val);
554         ZOOM_connection_remove_tasks(c);
555         return;
556     }
557
558     yaz_log(c->log_details, "%p ZOOM_connection_connect async=%d", c, c->async);
559     ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
560
561     if (!c->async)
562     {
563         while (ZOOM_event(1, &c))
564             ;
565     }
566 }
567
568 ZOOM_API(void) ZOOM_resultset_release(ZOOM_resultset r)
569 {
570     if (r->connection)
571     {
572         /* remove ourselves from the resultsets in connection */
573         ZOOM_resultset *rp = &r->connection->resultsets;
574         while (1)
575         {
576             assert(*rp);   /* we must be in this list!! */
577             if (*rp == r)
578             {   /* OK, we're here - take us out of it */
579                 *rp = (*rp)->next;
580                 break;
581             }
582             rp = &(*rp)->next;
583         }
584         r->connection = 0;
585     }
586 }
587
588 ZOOM_API(void)
589     ZOOM_connection_destroy(ZOOM_connection c)
590 {
591     ZOOM_resultset r;
592     if (!c)
593         return;
594     yaz_log(c->log_api, "%p ZOOM_connection_destroy", c);
595
596     ZOOM_memcached_destroy(c);
597     if (c->cs)
598         cs_close(c->cs);
599
600     for (r = c->resultsets; r; r = r->next)
601         r->connection = 0;
602
603     xfree(c->buf_in);
604     xfree(c->addinfo);
605     xfree(c->diagset);
606     odr_destroy(c->odr_in);
607     odr_destroy(c->odr_out);
608     if (c->odr_save)
609         odr_destroy(c->odr_save);
610     if (c->odr_print)
611     {
612         odr_setprint(c->odr_print, 0); /* prevent destroy from fclose'ing */
613         odr_destroy(c->odr_print);
614     }
615     ZOOM_options_destroy(c->options);
616     ZOOM_connection_remove_tasks(c);
617     ZOOM_connection_remove_events(c);
618     xfree(c->host_port);
619     xfree(c->proxy);
620     xfree(c->tproxy);
621     xfree(c->charset);
622     xfree(c->lang);
623     xfree(c->cookie_out);
624     xfree(c->cookie_in);
625     xfree(c->client_IP);
626     xfree(c->user);
627     xfree(c->group);
628     xfree(c->password);
629     xfree(c->sru_version);
630     yaz_cookies_destroy(c->cookies);
631     wrbuf_destroy(c->saveAPDU_wrbuf);
632     xfree(c);
633 }
634
635 void ZOOM_resultset_addref(ZOOM_resultset r)
636 {
637     if (r)
638     {
639         yaz_mutex_enter(r->mutex);
640         (r->refcount)++;
641         yaz_log(log_details0, "%p ZOOM_resultset_addref count=%d",
642                 r, r->refcount);
643         yaz_mutex_leave(r->mutex);
644     }
645 }
646
647 static int g_resultsets = 0;
648 static YAZ_MUTEX g_resultset_mutex = 0;
649
650 /* TODO We need to initialize this before running threaded:
651  * call resultset_use(0)  */
652
653 static int resultset_use(int delta) {
654     int resultset_count;
655     if (g_resultset_mutex == 0)
656         yaz_mutex_create(&g_resultset_mutex);
657     yaz_mutex_enter(g_resultset_mutex);
658     g_resultsets += delta;
659     resultset_count = g_resultsets;
660     yaz_mutex_leave(g_resultset_mutex);
661     return resultset_count;
662 }
663
664 int resultsets_count(void) {
665     return resultset_use(0);
666 }
667
668 ZOOM_resultset ZOOM_resultset_create(void)
669 {
670     int i;
671     ZOOM_resultset r = (ZOOM_resultset) xmalloc(sizeof(*r));
672
673     initlog();
674
675     yaz_log(log_details0, "%p ZOOM_resultset_create", r);
676     r->refcount = 1;
677     r->size = 0;
678     r->odr = odr_createmem(ODR_DECODE);
679     r->piggyback = 1;
680     r->setname = 0;
681     r->step = 0;
682     for (i = 0; i<RECORD_HASH_SIZE; i++)
683         r->record_hash[i] = 0;
684     r->r_sort_spec = 0;
685     r->query = 0;
686     r->connection = 0;
687     r->databaseNames = 0;
688     r->num_databaseNames = 0;
689     r->req_facets = 0;
690     r->res_facets = 0;
691     r->num_res_facets = 0;
692     r->facets_names = 0;
693     r->mutex = 0;
694     yaz_mutex_create(&r->mutex);
695 #if SHPTR
696     {
697         WRBUF w = wrbuf_alloc();
698         YAZ_SHPTR_INIT(r->record_wrbuf, w);
699     }
700 #endif
701     resultset_use(1);
702     r->mc_key = 0;
703     r->live_set = 0;
704     return r;
705 }
706
707 ZOOM_API(ZOOM_resultset)
708     ZOOM_connection_search_pqf(ZOOM_connection c, const char *q)
709 {
710     ZOOM_resultset r;
711     ZOOM_query s = ZOOM_query_create();
712
713     ZOOM_query_prefix(s, q);
714
715     r = ZOOM_connection_search(c, s);
716     ZOOM_query_destroy(s);
717     return r;
718 }
719
720 ZOOM_API(ZOOM_resultset)
721     ZOOM_connection_search(ZOOM_connection c, ZOOM_query q)
722 {
723     ZOOM_resultset r = ZOOM_resultset_create();
724     ZOOM_task task;
725     int start, count;
726     const char *syntax, *elementSetName, *schema;
727     yaz_log(c->log_api, "%p ZOOM_connection_search set %p query %p", c, r, q);
728     r->r_sort_spec = ZOOM_query_get_sortspec(q);
729     r->query = q;
730     ZOOM_query_addref(q);
731
732     r->options = ZOOM_options_create_with_parent(c->options);
733
734     r->req_facets = odr_strdup_null(r->odr,
735                                     ZOOM_options_get(r->options, "facets"));
736     start = ZOOM_options_get_int(r->options, "start", 0);
737     count = ZOOM_options_get_int(r->options, "count", 0);
738     {
739         /* If "presentChunk" is defined use that; otherwise "step" */
740         const char *cp = ZOOM_options_get(r->options, "presentChunk");
741         r->step = ZOOM_options_get_int(r->options,
742                                        (cp != 0 ? "presentChunk": "step"), 0);
743     }
744     r->piggyback = ZOOM_options_get_bool(r->options, "piggyback", 1);
745     r->setname = odr_strdup_null(r->odr,
746                                  ZOOM_options_get(r->options, "setname"));
747     r->databaseNames = ZOOM_connection_get_databases(c, c->options,
748                                                      &r->num_databaseNames,
749                                                      r->odr);
750     r->connection = c;
751     r->next = c->resultsets;
752     c->resultsets = r;
753
754     ZOOM_memcached_resultset(r, q);
755
756     if (c->host_port && c->proto == PROTO_HTTP)
757     {
758         if (!c->cs)
759         {
760             yaz_log(c->log_details, "ZOOM_connection_search: no comstack");
761             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
762         }
763         else
764         {
765             yaz_log(c->log_details, "ZOOM_connection_search: reconnect");
766             c->reconnect_ok = 1;
767         }
768     }
769
770     task = ZOOM_connection_add_task(c, ZOOM_TASK_SEARCH);
771     task->u.search.resultset = r;
772     task->u.search.start = start;
773     task->u.search.count = count;
774
775     syntax = ZOOM_options_get(r->options, "preferredRecordSyntax");
776     task->u.search.syntax = syntax ? xstrdup(syntax) : 0;
777     elementSetName = ZOOM_options_get(r->options, "elementSetName");
778     task->u.search.elementSetName = elementSetName ?
779         xstrdup(elementSetName) : 0;
780     schema = ZOOM_options_get(r->options, "schema");
781     task->u.search.schema = schema ? xstrdup(schema) : 0;
782
783     ZOOM_resultset_addref(r);
784
785     if (!c->async)
786     {
787         while (ZOOM_event(1, &c))
788             ;
789     }
790     return r;
791 }
792
793 ZOOM_API(void)
794     ZOOM_resultset_sort(ZOOM_resultset r,
795                          const char *sort_type, const char *sort_spec)
796 {
797     (void) ZOOM_resultset_sort1(r, sort_type, sort_spec);
798 }
799
800 ZOOM_API(int)
801     ZOOM_resultset_sort1(ZOOM_resultset r,
802                          const char *sort_type, const char *sort_spec)
803 {
804     ZOOM_connection c = r->connection;
805     ZOOM_task task;
806     ZOOM_query newq;
807
808     newq = ZOOM_query_create();
809     if (ZOOM_query_sortby(newq, sort_spec) < 0)
810         return -1;
811
812     yaz_log(c->log_api, "%p ZOOM_resultset_sort r=%p sort_type=%s sort_spec=%s",
813             r, r, sort_type, sort_spec);
814     if (!c)
815         return 0;
816
817     if (c->host_port && c->proto == PROTO_HTTP)
818     {
819         if (!c->cs)
820         {
821             yaz_log(c->log_details, "%p ZOOM_resultset_sort: no comstack", r);
822             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
823         }
824         else
825         {
826             yaz_log(c->log_details, "%p ZOOM_resultset_sort: prepare reconnect",
827                     r);
828             c->reconnect_ok = 1;
829         }
830     }
831
832     ZOOM_resultset_cache_reset(r);
833     task = ZOOM_connection_add_task(c, ZOOM_TASK_SORT);
834     task->u.sort.resultset = r;
835     task->u.sort.q = newq;
836
837     ZOOM_resultset_addref(r);
838
839     if (!c->async)
840     {
841         while (ZOOM_event(1, &c))
842             ;
843     }
844
845     return 0;
846 }
847
848 ZOOM_API(void)
849     ZOOM_resultset_destroy(ZOOM_resultset r)
850 {
851     resultset_destroy(r);
852 }
853
854 static void resultset_destroy(ZOOM_resultset r)
855 {
856     if (!r)
857         return;
858     yaz_mutex_enter(r->mutex);
859     (r->refcount)--;
860     yaz_log(log_details0, "%p ZOOM_resultset_destroy r=%p count=%d",
861             r, r, r->refcount);
862     if (r->refcount == 0)
863     {
864         yaz_mutex_leave(r->mutex);
865
866         yaz_log(log_details0, "%p ZOOM_connection resultset_destroy: Deleting resultset (%p) ", r->connection, r);
867         ZOOM_resultset_cache_reset(r);
868         ZOOM_resultset_release(r);
869         ZOOM_query_destroy(r->query);
870         ZOOM_options_destroy(r->options);
871         odr_destroy(r->odr);
872         yaz_mutex_destroy(&r->mutex);
873 #if SHPTR
874         YAZ_SHPTR_DEC(r->record_wrbuf, wrbuf_destroy);
875 #endif
876         wrbuf_destroy(r->mc_key);
877         resultset_use(-1);
878         xfree(r);
879     }
880     else
881         yaz_mutex_leave(r->mutex);
882 }
883
884 ZOOM_API(size_t)
885     ZOOM_resultset_size(ZOOM_resultset r)
886 {
887     return r->size;
888 }
889
890 int ZOOM_test_reconnect(ZOOM_connection c)
891 {
892     ZOOM_Event event;
893
894     if (!c->reconnect_ok)
895         return 0;
896     ZOOM_connection_close(c);
897     c->reconnect_ok = 0;
898     c->tasks->running = 0;
899     ZOOM_connection_insert_task(c, ZOOM_TASK_CONNECT);
900
901     event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
902     ZOOM_connection_put_event(c, event);
903
904     return 1;
905 }
906
907 static void ZOOM_resultset_retrieve(ZOOM_resultset r,
908                                     int force_sync, int start, int count)
909 {
910     ZOOM_task task;
911     ZOOM_connection c;
912     const char *cp;
913     const char *syntax, *elementSetName;
914
915     if (!r)
916         return;
917     yaz_log(log_details0, "%p ZOOM_resultset_retrieve force_sync=%d start=%d"
918             " count=%d", r, force_sync, start, count);
919     c = r->connection;
920     if (!c)
921         return;
922
923     if (c->host_port && c->proto == PROTO_HTTP)
924     {
925         if (!c->cs)
926         {
927             yaz_log(log_details0, "%p ZOOM_resultset_retrieve: no comstack", r);
928             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
929         }
930         else
931         {
932             yaz_log(log_details0, "%p ZOOM_resultset_retrieve: prepare "
933                     "reconnect", r);
934             c->reconnect_ok = 1;
935         }
936     }
937     task = ZOOM_connection_add_task(c, ZOOM_TASK_SEARCH);
938     task->u.search.resultset = r;
939     task->u.search.start = start;
940     task->u.search.count = count;
941
942     syntax = ZOOM_options_get(r->options, "preferredRecordSyntax");
943     task->u.search.syntax = syntax ? xstrdup(syntax) : 0;
944     elementSetName = ZOOM_options_get(r->options, "elementSetName");
945     task->u.search.elementSetName = elementSetName
946         ? xstrdup(elementSetName) : 0;
947
948     cp = ZOOM_options_get(r->options, "schema");
949     task->u.search.schema = cp ? xstrdup(cp) : 0;
950
951     ZOOM_resultset_addref(r);
952
953     if (!r->connection->async || force_sync)
954         while (r->connection && ZOOM_event(1, &r->connection))
955             ;
956 }
957
958 ZOOM_API(void)
959     ZOOM_resultset_records(ZOOM_resultset r, ZOOM_record *recs,
960                            size_t start, size_t count)
961 {
962     int force_present = 0;
963
964     if (!r)
965         return ;
966     yaz_log(log_api0, "%p ZOOM_resultset_records r=%p start=%ld count=%ld",
967             r, r, (long) start, (long) count);
968     if (count && recs)
969         force_present = 1;
970     ZOOM_resultset_retrieve(r, force_present, start, count);
971     if (force_present)
972     {
973         size_t i;
974         for (i = 0; i< count; i++)
975             recs[i] = ZOOM_resultset_record_immediate(r, i+start);
976     }
977 }
978
979 ZOOM_API(size_t)
980     ZOOM_resultset_facets_size(ZOOM_resultset r)
981 {
982     return r->num_res_facets;
983 }
984
985 ZOOM_API(ZOOM_facet_field)
986     ZOOM_resultset_get_facet_field(ZOOM_resultset r, const char *name)
987 {
988     int num = r->num_res_facets;
989     ZOOM_facet_field *facets = r->res_facets;
990     int i;
991     for (i = 0; i < num; i++)
992         if (!strcmp(facets[i]->facet_name, name))
993             return facets[i];
994     return 0;
995 }
996
997 ZOOM_API(ZOOM_facet_field)
998     ZOOM_resultset_get_facet_field_by_index(ZOOM_resultset r, int idx)
999 {
1000     int num = r->num_res_facets;
1001     ZOOM_facet_field *facets = r->res_facets;
1002     if (idx >= 0 && idx < num)
1003         return facets[idx];
1004     return 0;
1005 }
1006
1007 ZOOM_API(ZOOM_facet_field *)
1008     ZOOM_resultset_facets(ZOOM_resultset r)
1009 {
1010     return r->res_facets;
1011 }
1012
1013 ZOOM_API(const char**)
1014     ZOOM_resultset_facets_names(ZOOM_resultset r)
1015 {
1016     return (const char **) r->facets_names;
1017 }
1018
1019 ZOOM_API(const char*)
1020     ZOOM_facet_field_name(ZOOM_facet_field field)
1021 {
1022     return field->facet_name;
1023 }
1024
1025 ZOOM_API(size_t)
1026     ZOOM_facet_field_term_count(ZOOM_facet_field field)
1027 {
1028     return field->num_terms;
1029 }
1030
1031 ZOOM_API(const char*)
1032     ZOOM_facet_field_get_term(ZOOM_facet_field field, size_t idx, int *freq)
1033 {
1034     *freq = field->facet_terms[idx].frequency;
1035     return field->facet_terms[idx].term;
1036 }
1037
1038
1039 static void get_cert(ZOOM_connection c)
1040 {
1041     char *cert_buf;
1042     int cert_len;
1043
1044     if (cs_get_peer_certificate_x509(c->cs, &cert_buf, &cert_len))
1045     {
1046         ZOOM_connection_option_setl(c, "sslPeerCert",
1047                                     cert_buf, cert_len);
1048         xfree(cert_buf);
1049     }
1050 }
1051
1052 static zoom_ret do_connect_host(ZOOM_connection c,
1053                                 const char *logical_url);
1054
1055 static zoom_ret do_connect(ZOOM_connection c)
1056 {
1057     return do_connect_host(c, c->host_port);
1058 }
1059
1060 static zoom_ret do_connect_host(ZOOM_connection c, const char *logical_url)
1061 {
1062     void *add;
1063
1064     if (c->cs)
1065         cs_close(c->cs);
1066     c->cs = cs_create_host_proxy(logical_url, 0, &add,
1067                                  c->tproxy ? c->tproxy : c->proxy);
1068
1069     if (c->cs && c->cs->protocol == PROTO_HTTP)
1070     {
1071 #if YAZ_HAVE_XML2
1072         c->proto = PROTO_HTTP;
1073 #else
1074         ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, "SRW");
1075         ZOOM_connection_close(c);
1076         return zoom_complete;
1077 #endif
1078     }
1079     if (c->cs)
1080     {
1081         int ret = cs_connect(c->cs, add);
1082         if (ret == 0)
1083         {
1084             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1085             ZOOM_connection_put_event(c, event);
1086             get_cert(c);
1087             if (c->proto == PROTO_Z3950)
1088                 ZOOM_connection_Z3950_send_init(c);
1089             else
1090             {
1091                 /* no init request for SRW .. */
1092                 assert(c->tasks->which == ZOOM_TASK_CONNECT);
1093                 ZOOM_connection_remove_task(c);
1094                 ZOOM_connection_set_mask(c, 0);
1095                 ZOOM_connection_exec_task(c);
1096             }
1097             c->state = STATE_ESTABLISHED;
1098             return zoom_pending;
1099         }
1100         else if (ret > 0)
1101         {
1102             int mask = ZOOM_SELECT_EXCEPT;
1103             if (c->cs->io_pending & CS_WANT_WRITE)
1104                 mask += ZOOM_SELECT_WRITE;
1105             if (c->cs->io_pending & CS_WANT_READ)
1106                 mask += ZOOM_SELECT_READ;
1107             ZOOM_connection_set_mask(c, mask);
1108             c->state = STATE_CONNECTING;
1109             return zoom_pending;
1110         }
1111     }
1112     c->state = STATE_IDLE;
1113     ZOOM_set_error(c, ZOOM_ERROR_CONNECT, logical_url);
1114     return zoom_complete;
1115 }
1116
1117 /* returns 1 if PDU was sent OK (still pending )
1118    0 if PDU was not sent OK (nothing to wait for)
1119 */
1120
1121 ZOOM_API(ZOOM_record)
1122     ZOOM_resultset_record_immediate(ZOOM_resultset s,size_t pos)
1123 {
1124     const char *syntax =
1125         ZOOM_options_get(s->options, "preferredRecordSyntax");
1126     const char *elementSetName =
1127         ZOOM_options_get(s->options, "elementSetName");
1128     const char *schema =
1129         ZOOM_options_get(s->options, "schema");
1130
1131     return ZOOM_record_cache_lookup_i(s, pos, syntax, elementSetName, schema);
1132 }
1133
1134 ZOOM_API(ZOOM_record)
1135     ZOOM_resultset_record(ZOOM_resultset r, size_t pos)
1136 {
1137     ZOOM_record rec = ZOOM_resultset_record_immediate(r, pos);
1138
1139     if (!rec)
1140     {
1141         /*
1142          * MIKE: I think force_sync should always be zero, but I don't
1143          * want to make this change until I get the go-ahead from
1144          * Adam, in case something depends on the old synchronous
1145          * behaviour.
1146          */
1147         int force_sync = 1;
1148         if (getenv("ZOOM_RECORD_NO_FORCE_SYNC")) force_sync = 0;
1149         ZOOM_resultset_retrieve(r, force_sync, pos, 1);
1150         rec = ZOOM_resultset_record_immediate(r, pos);
1151     }
1152     return rec;
1153 }
1154
1155 ZOOM_API(ZOOM_scanset)
1156     ZOOM_connection_scan(ZOOM_connection c, const char *start)
1157 {
1158     ZOOM_scanset s;
1159     ZOOM_query q = ZOOM_query_create();
1160
1161     ZOOM_query_prefix(q, start);
1162
1163     s = ZOOM_connection_scan1(c, q);
1164     ZOOM_query_destroy(q);
1165     return s;
1166
1167 }
1168
1169 ZOOM_API(ZOOM_scanset)
1170     ZOOM_connection_scan1(ZOOM_connection c, ZOOM_query q)
1171 {
1172     ZOOM_scanset scan = 0;
1173     Z_Query *z_query = ZOOM_query_get_Z_Query(q);
1174
1175     if (!z_query)
1176         return 0;
1177     scan = (ZOOM_scanset) xmalloc(sizeof(*scan));
1178     scan->connection = c;
1179     scan->odr = odr_createmem(ODR_DECODE);
1180     scan->options = ZOOM_options_create_with_parent(c->options);
1181     scan->refcount = 1;
1182     scan->scan_response = 0;
1183     scan->srw_scan_response = 0;
1184
1185     scan->query = q;
1186     ZOOM_query_addref(q);
1187     scan->databaseNames = ZOOM_connection_get_databases(c, c->options,
1188                                             &scan->num_databaseNames,
1189                                             scan->odr);
1190
1191     if (1)
1192     {
1193         ZOOM_task task = ZOOM_connection_add_task(c, ZOOM_TASK_SCAN);
1194         task->u.scan.scan = scan;
1195
1196         (scan->refcount)++;
1197         if (!c->async)
1198         {
1199             while (ZOOM_event(1, &c))
1200                 ;
1201         }
1202     }
1203     return scan;
1204 }
1205
1206 ZOOM_API(void)
1207     ZOOM_scanset_destroy(ZOOM_scanset scan)
1208 {
1209     if (!scan)
1210         return;
1211     (scan->refcount)--;
1212     if (scan->refcount == 0)
1213     {
1214         ZOOM_query_destroy(scan->query);
1215
1216         odr_destroy(scan->odr);
1217
1218         ZOOM_options_destroy(scan->options);
1219         xfree(scan);
1220     }
1221 }
1222
1223 static zoom_ret send_package(ZOOM_connection c)
1224 {
1225     ZOOM_Event event;
1226
1227     yaz_log(c->log_details, "%p send_package", c);
1228     if (!c->tasks)
1229         return zoom_complete;
1230     assert (c->tasks->which == ZOOM_TASK_PACKAGE);
1231
1232     event = ZOOM_Event_create(ZOOM_EVENT_SEND_APDU);
1233     ZOOM_connection_put_event(c, event);
1234
1235     c->buf_out = c->tasks->u.package->buf_out;
1236     c->len_out = c->tasks->u.package->len_out;
1237
1238     return ZOOM_send_buf(c);
1239 }
1240
1241 ZOOM_API(size_t)
1242     ZOOM_scanset_size(ZOOM_scanset scan)
1243 {
1244     if (!scan)
1245         return 0;
1246
1247     if (scan->scan_response && scan->scan_response->entries)
1248         return scan->scan_response->entries->num_entries;
1249     else if (scan->srw_scan_response)
1250         return scan->srw_scan_response->num_terms;
1251     return 0;
1252 }
1253
1254 static void ZOOM_scanset_term_x(ZOOM_scanset scan, size_t pos,
1255                                 size_t *occ,
1256                                 const char **value_term, size_t *value_len,
1257                                 const char **disp_term, size_t *disp_len)
1258 {
1259     size_t noent = ZOOM_scanset_size(scan);
1260
1261     *value_term = 0;
1262     *value_len = 0;
1263
1264     *disp_term = 0;
1265     *disp_len = 0;
1266
1267     *occ = 0;
1268     if (pos >= noent)
1269         return;
1270     if (scan->scan_response)
1271     {
1272         Z_ScanResponse *res = scan->scan_response;
1273         if (res->entries->entries[pos]->which == Z_Entry_termInfo)
1274         {
1275             Z_TermInfo *t = res->entries->entries[pos]->u.termInfo;
1276
1277             *value_term = (const char *) t->term->u.general->buf;
1278             *value_len = t->term->u.general->len;
1279             if (t->displayTerm)
1280             {
1281                 *disp_term = t->displayTerm;
1282                 *disp_len = strlen(*disp_term);
1283             }
1284             else if (t->term->which == Z_Term_general)
1285             {
1286                 *disp_term = (const char *) t->term->u.general->buf;
1287                 *disp_len = t->term->u.general->len;
1288             }
1289             *occ = t->globalOccurrences ? *t->globalOccurrences : 0;
1290         }
1291     }
1292     if (scan->srw_scan_response)
1293     {
1294         Z_SRW_scanResponse *res = scan->srw_scan_response;
1295         Z_SRW_scanTerm *t = res->terms + pos;
1296         if (t)
1297         {
1298             *value_term = t->value;
1299             *value_len = strlen(*value_term);
1300
1301             if (t->displayTerm)
1302                 *disp_term = t->displayTerm;
1303             else
1304                 *disp_term = t->value;
1305             *disp_len = strlen(*disp_term);
1306             *occ = t->numberOfRecords ? *t->numberOfRecords : 0;
1307         }
1308     }
1309 }
1310
1311 ZOOM_API(const char *)
1312     ZOOM_scanset_term(ZOOM_scanset scan, size_t pos,
1313                       size_t *occ, size_t *len)
1314 {
1315     const char *value_term = 0;
1316     size_t value_len = 0;
1317     const char *disp_term = 0;
1318     size_t disp_len = 0;
1319
1320     ZOOM_scanset_term_x(scan, pos, occ, &value_term, &value_len,
1321                         &disp_term, &disp_len);
1322
1323     *len = value_len;
1324     return value_term;
1325 }
1326
1327 ZOOM_API(const char *)
1328     ZOOM_scanset_display_term(ZOOM_scanset scan, size_t pos,
1329                               size_t *occ, size_t *len)
1330 {
1331     const char *value_term = 0;
1332     size_t value_len = 0;
1333     const char *disp_term = 0;
1334     size_t disp_len = 0;
1335
1336     ZOOM_scanset_term_x(scan, pos, occ, &value_term, &value_len,
1337                         &disp_term, &disp_len);
1338
1339     *len = disp_len;
1340     return disp_term;
1341 }
1342
1343 ZOOM_API(const char *)
1344     ZOOM_scanset_option_get(ZOOM_scanset scan, const char *key)
1345 {
1346     return ZOOM_options_get(scan->options, key);
1347 }
1348
1349 ZOOM_API(void)
1350     ZOOM_scanset_option_set(ZOOM_scanset scan, const char *key,
1351                             const char *val)
1352 {
1353     ZOOM_options_set(scan->options, key, val);
1354 }
1355
1356
1357 ZOOM_API(ZOOM_package)
1358     ZOOM_connection_package(ZOOM_connection c, ZOOM_options options)
1359 {
1360     ZOOM_package p = (ZOOM_package) xmalloc(sizeof(*p));
1361
1362     p->connection = c;
1363     p->odr_out = odr_createmem(ODR_ENCODE);
1364     p->options = ZOOM_options_create_with_parent2(options, c->options);
1365     p->refcount = 1;
1366     p->buf_out = 0;
1367     p->len_out = 0;
1368     return p;
1369 }
1370
1371 ZOOM_API(void)
1372     ZOOM_package_destroy(ZOOM_package p)
1373 {
1374     if (!p)
1375         return;
1376     (p->refcount)--;
1377     if (p->refcount == 0)
1378     {
1379         odr_destroy(p->odr_out);
1380         xfree(p->buf_out);
1381
1382         ZOOM_options_destroy(p->options);
1383         xfree(p);
1384     }
1385 }
1386
1387 ZOOM_API(const char *)
1388     ZOOM_package_option_get(ZOOM_package p, const char *key)
1389 {
1390     return ZOOM_options_get(p->options, key);
1391 }
1392
1393 ZOOM_API(const char *)
1394     ZOOM_package_option_getl(ZOOM_package p, const char *key, int *lenp)
1395 {
1396     return ZOOM_options_getl(p->options, key, lenp);
1397 }
1398
1399 ZOOM_API(void)
1400     ZOOM_package_option_set(ZOOM_package p, const char *key,
1401                             const char *val)
1402 {
1403     ZOOM_options_set(p->options, key, val);
1404 }
1405
1406 ZOOM_API(void)
1407     ZOOM_package_option_setl(ZOOM_package p, const char *key,
1408                              const char *val, int len)
1409 {
1410     ZOOM_options_setl(p->options, key, val, len);
1411 }
1412
1413 ZOOM_API(int)
1414     ZOOM_connection_exec_task(ZOOM_connection c)
1415 {
1416     ZOOM_task task = c->tasks;
1417     zoom_ret ret = zoom_complete;
1418
1419     if (!task)
1420         return 0;
1421     yaz_log(c->log_details, "%p ZOOM_connection_exec_task type=%d run=%d",
1422             c, task->which, task->running);
1423     if (c->error != ZOOM_ERROR_NONE)
1424     {
1425         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1426                 "removing tasks because of error = %d", c, c->error);
1427         ZOOM_connection_remove_tasks(c);
1428         return 0;
1429     }
1430     if (task->running)
1431     {
1432         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1433                 "task already running", c);
1434         return 0;
1435     }
1436     task->running = 1;
1437     ret = zoom_complete;
1438     if (c->cs || task->which == ZOOM_TASK_CONNECT)
1439     {
1440         switch (task->which)
1441         {
1442         case ZOOM_TASK_SEARCH:
1443             if (c->proto == PROTO_HTTP)
1444                 ret = ZOOM_connection_srw_send_search(c);
1445             else
1446                 ret = ZOOM_connection_Z3950_search(c);
1447             break;
1448         case ZOOM_TASK_CONNECT:
1449             ret = do_connect(c);
1450             break;
1451         case ZOOM_TASK_SCAN:
1452             if (c->proto == PROTO_HTTP)
1453                 ret = ZOOM_connection_srw_send_scan(c);
1454             else
1455                 ret = ZOOM_connection_Z3950_send_scan(c);
1456             break;
1457         case ZOOM_TASK_PACKAGE:
1458             ret = send_package(c);
1459             break;
1460         case ZOOM_TASK_SORT:
1461             c->tasks->u.sort.resultset->r_sort_spec =
1462                 ZOOM_query_get_sortspec(c->tasks->u.sort.q);
1463             ret = send_Z3950_sort(c, c->tasks->u.sort.resultset);
1464             break;
1465         }
1466     }
1467     else
1468     {
1469         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1470                 "remove tasks because no connection exist", c);
1471         ZOOM_connection_remove_tasks(c);
1472     }
1473     if (ret == zoom_complete)
1474     {
1475         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1476                 "task removed (complete)", c);
1477         ZOOM_connection_remove_task(c);
1478         return 0;
1479     }
1480     yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1481             "task pending", c);
1482     return 1;
1483 }
1484
1485 #if YAZ_HAVE_XML2
1486
1487 static zoom_ret send_HTTP_redirect(ZOOM_connection c, const char *uri)
1488 {
1489     Z_GDU *gdu = z_get_HTTP_Request_uri(c->odr_out, uri, 0, c->proxy ? 1 : 0);
1490
1491     gdu->u.HTTP_Request->method = odr_strdup(c->odr_out, "GET");
1492     z_HTTP_header_add(c->odr_out, &gdu->u.HTTP_Request->headers, "Accept",
1493                       "text/xml");
1494     yaz_cookies_request(c->cookies, c->odr_out, gdu->u.HTTP_Request);
1495     if (c->user && c->password)
1496     {
1497         z_HTTP_header_add_basic_auth(c->odr_out, &gdu->u.HTTP_Request->headers,
1498                                      c->user, c->password);
1499     }
1500     return ZOOM_send_GDU(c, gdu);
1501 }
1502
1503 zoom_ret ZOOM_send_GDU(ZOOM_connection c, Z_GDU *gdu)
1504 {
1505     ZOOM_Event event;
1506
1507     int r = z_GDU(c->odr_out, &gdu, 0, 0);
1508     if (!r)
1509         return zoom_complete;
1510     if (c->odr_print)
1511         z_GDU(c->odr_print, &gdu, 0, 0);
1512     if (c->odr_save)
1513         z_GDU(c->odr_save, &gdu, 0, 0);
1514     c->buf_out = odr_getbuf(c->odr_out, &c->len_out, 0);
1515     odr_reset(c->odr_out);
1516
1517     event = ZOOM_Event_create(ZOOM_EVENT_SEND_APDU);
1518     ZOOM_connection_put_event(c, event);
1519
1520     return ZOOM_send_buf(c);
1521 }
1522
1523 #if YAZ_HAVE_XML2
1524 void ZOOM_set_HTTP_error(ZOOM_connection c, int error,
1525                          const char *addinfo, const char *addinfo2)
1526 {
1527     ZOOM_set_dset_error(c, error, "HTTP", addinfo, addinfo2);
1528 }
1529 #endif
1530
1531
1532 static void handle_http(ZOOM_connection c, Z_HTTP_Response *hres)
1533 {
1534     zoom_ret cret = zoom_complete;
1535     int ret = -1;
1536     char *addinfo = 0;
1537     const char *connection_head = z_HTTP_header_lookup(hres->headers,
1538                                                        "Connection");
1539     const char *location;
1540
1541     ZOOM_connection_set_mask(c, 0);
1542     yaz_log(c->log_details, "%p handle_http", c);
1543
1544     yaz_cookies_response(c->cookies, hres);
1545     if ((hres->code == 301 || hres->code == 302) && c->sru_mode == zoom_sru_get
1546         && (location = z_HTTP_header_lookup(hres->headers, "Location")))
1547     {
1548         c->no_redirects++;
1549         if (c->no_redirects > 10)
1550         {
1551             ZOOM_set_HTTP_error(c, hres->code, 0, 0);
1552             c->no_redirects = 0;
1553             ZOOM_connection_close(c);
1554         }
1555         else
1556         {
1557             /* since redirect may change host we just reconnect. A smarter
1558                implementation might check whether it's the same server */
1559
1560             int host_change = 0;
1561             location = yaz_check_location(c->odr_in, c->host_port,
1562                                           location, &host_change);
1563             if (do_connect_host(c, location) == zoom_complete)
1564                 return;  /* connect failed.. */
1565             send_HTTP_redirect(c, location);
1566             return;
1567         }
1568     }
1569     else
1570     {
1571         ret = ZOOM_handle_sru(c, hres, &cret, &addinfo);
1572         if (ret == 0)
1573         {
1574             if (c->no_redirects) /* end of redirect. change hosts again */
1575                 ZOOM_connection_close(c);
1576         }
1577         c->no_redirects = 0;
1578     }
1579     if (ret)
1580     {
1581         if (hres->code != 200)
1582             ZOOM_set_HTTP_error(c, hres->code, 0, 0);
1583         else
1584         {
1585             yaz_log(YLOG_LOG, "set error... addinfo=%s", addinfo ?
1586                     addinfo : "NULL");
1587             ZOOM_set_error(c, ZOOM_ERROR_DECODE, addinfo);
1588         }
1589         ZOOM_connection_close(c);
1590     }
1591     if (cret == zoom_complete)
1592     {
1593         yaz_log(c->log_details, "removing tasks in handle_http");
1594         ZOOM_connection_remove_task(c);
1595     }
1596     {
1597         int must_close = 0;
1598         if (!strcmp(hres->version, "1.0"))
1599         {
1600             /* HTTP 1.0: only if Keep-Alive we stay alive.. */
1601             if (!connection_head || strcmp(connection_head, "Keep-Alive"))
1602                 must_close = 1;
1603         }
1604         else
1605         {
1606             /* HTTP 1.1: only if no close we stay alive.. */
1607             if (connection_head && !strcmp(connection_head, "close"))
1608                 must_close = 1;
1609         }
1610         if (must_close)
1611         {
1612             ZOOM_connection_close(c);
1613             if (c->tasks)
1614             {
1615                 c->tasks->running = 0;
1616                 ZOOM_connection_insert_task(c, ZOOM_TASK_CONNECT);
1617                 c->reconnect_ok = 0;
1618             }
1619         }
1620         else
1621             c->reconnect_ok = 1; /* if the server closes anyway */
1622     }
1623 }
1624 #endif
1625
1626 static int do_read(ZOOM_connection c)
1627 {
1628     int r, more;
1629     ZOOM_Event event;
1630
1631     event = ZOOM_Event_create(ZOOM_EVENT_RECV_DATA);
1632     ZOOM_connection_put_event(c, event);
1633
1634     r = cs_get(c->cs, &c->buf_in, &c->len_in);
1635     more = cs_more(c->cs);
1636     yaz_log(c->log_details, "%p do_read len=%d more=%d", c, r, more);
1637     if (r == 1)
1638         return 0;
1639     if (r <= 0)
1640     {
1641         if (!ZOOM_test_reconnect(c))
1642         {
1643             ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
1644             ZOOM_connection_close(c);
1645         }
1646     }
1647     else
1648     {
1649         Z_GDU *gdu;
1650         ZOOM_Event event;
1651
1652         odr_reset(c->odr_in);
1653         odr_setbuf(c->odr_in, c->buf_in, r, 0);
1654         event = ZOOM_Event_create(ZOOM_EVENT_RECV_APDU);
1655         ZOOM_connection_put_event(c, event);
1656
1657         if (!z_GDU(c->odr_in, &gdu, 0, 0))
1658         {
1659             int x;
1660             int err = odr_geterrorx(c->odr_in, &x);
1661             char msg[100];
1662             const char *element = odr_getelement(c->odr_in);
1663             yaz_snprintf(msg, sizeof(msg),
1664                     "ODR code %d:%d element=%s offset=%d",
1665                     err, x, element ? element : "<unknown>",
1666                     odr_offset(c->odr_in));
1667             ZOOM_set_error(c, ZOOM_ERROR_DECODE, msg);
1668             if (c->log_api)
1669             {
1670                 FILE *ber_file = yaz_log_file();
1671                 if (ber_file)
1672                     odr_dumpBER(ber_file, c->buf_in, r);
1673             }
1674             ZOOM_connection_close(c);
1675         }
1676         else
1677         {
1678             if (c->odr_print)
1679                 z_GDU(c->odr_print, &gdu, 0, 0);
1680             if (c->odr_save)
1681                 z_GDU(c->odr_save, &gdu, 0, 0);
1682             if (gdu->which == Z_GDU_Z3950)
1683                 ZOOM_handle_Z3950_apdu(c, gdu->u.z3950);
1684             else if (gdu->which == Z_GDU_HTTP_Response)
1685             {
1686 #if YAZ_HAVE_XML2
1687                 handle_http(c, gdu->u.HTTP_Response);
1688 #else
1689                 ZOOM_set_error(c, ZOOM_ERROR_DECODE, 0);
1690                 ZOOM_connection_close(c);
1691 #endif
1692             }
1693         }
1694     }
1695     return 1;
1696 }
1697
1698 static zoom_ret do_write_ex(ZOOM_connection c, char *buf_out, int len_out)
1699 {
1700     int r;
1701     ZOOM_Event event;
1702
1703     event = ZOOM_Event_create(ZOOM_EVENT_SEND_DATA);
1704     ZOOM_connection_put_event(c, event);
1705
1706     yaz_log(c->log_details, "%p do_write_ex len=%d", c, len_out);
1707     if ((r = cs_put(c->cs, buf_out, len_out)) < 0)
1708     {
1709         yaz_log(c->log_details, "%p do_write_ex write failed", c);
1710         if (ZOOM_test_reconnect(c))
1711         {
1712             return zoom_pending;
1713         }
1714         if (c->state == STATE_CONNECTING)
1715             ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
1716         else
1717             ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
1718         ZOOM_connection_close(c);
1719         return zoom_complete;
1720     }
1721     else if (r == 1)
1722     {
1723         int mask = ZOOM_SELECT_EXCEPT;
1724         if (c->cs->io_pending & CS_WANT_WRITE)
1725             mask += ZOOM_SELECT_WRITE;
1726         if (c->cs->io_pending & CS_WANT_READ)
1727             mask += ZOOM_SELECT_READ;
1728         ZOOM_connection_set_mask(c, mask);
1729         yaz_log(c->log_details, "%p do_write_ex write incomplete mask=%d",
1730                 c, c->mask);
1731     }
1732     else
1733     {
1734         ZOOM_connection_set_mask(c, ZOOM_SELECT_READ|ZOOM_SELECT_EXCEPT);
1735         yaz_log(c->log_details, "%p do_write_ex write complete mask=%d",
1736                 c, c->mask);
1737     }
1738     return zoom_pending;
1739 }
1740
1741 zoom_ret ZOOM_send_buf(ZOOM_connection c)
1742 {
1743     return do_write_ex(c, c->buf_out, c->len_out);
1744 }
1745
1746
1747 ZOOM_API(const char *)
1748     ZOOM_connection_option_get(ZOOM_connection c, const char *key)
1749 {
1750     if (!strcmp(key, "APDU"))
1751     {
1752         return c->saveAPDU_wrbuf ? wrbuf_cstr(c->saveAPDU_wrbuf) : "";
1753     }
1754     else
1755         return ZOOM_options_get(c->options, key);
1756 }
1757
1758 ZOOM_API(const char *)
1759     ZOOM_connection_option_getl(ZOOM_connection c, const char *key, int *lenp)
1760 {
1761     if (!strcmp(key, "APDU"))
1762     {
1763         if (c->saveAPDU_wrbuf)
1764         {
1765             *lenp = wrbuf_len(c->saveAPDU_wrbuf);
1766             return wrbuf_cstr(c->saveAPDU_wrbuf);
1767         }
1768         else
1769         {
1770             *lenp = 0;
1771             return "";
1772         }
1773     }
1774     else
1775         return ZOOM_options_getl(c->options, key, lenp);
1776 }
1777
1778 ZOOM_API(void)
1779     ZOOM_connection_option_set(ZOOM_connection c, const char *key,
1780                                const char *val)
1781 {
1782     if (!strcmp(key, "saveAPDU"))
1783     {
1784         if (val && strcmp(val, "0"))
1785         {
1786             if (!c->saveAPDU_wrbuf)
1787                 c->saveAPDU_wrbuf = wrbuf_alloc();
1788             else
1789                 wrbuf_rewind(c->saveAPDU_wrbuf);
1790         }
1791         else
1792         {
1793             wrbuf_destroy(c->saveAPDU_wrbuf);
1794             c->saveAPDU_wrbuf = 0;
1795         }
1796         ZOOM_connection_save_apdu_wrbuf(c, c->saveAPDU_wrbuf);
1797     }
1798     else
1799         ZOOM_options_set(c->options, key, val);
1800 }
1801
1802 ZOOM_API(void)
1803     ZOOM_connection_option_setl(ZOOM_connection c, const char *key,
1804                                 const char *val, int len)
1805 {
1806     ZOOM_options_setl(c->options, key, val, len);
1807 }
1808
1809 ZOOM_API(const char *)
1810     ZOOM_resultset_option_get(ZOOM_resultset r, const char *key)
1811 {
1812     return ZOOM_options_get(r->options, key);
1813 }
1814
1815 ZOOM_API(void)
1816     ZOOM_resultset_option_set(ZOOM_resultset r, const char *key,
1817                               const char *val)
1818 {
1819     ZOOM_options_set(r->options, key, val);
1820 }
1821
1822
1823 ZOOM_API(int)
1824     ZOOM_connection_errcode(ZOOM_connection c)
1825 {
1826     return ZOOM_connection_error(c, 0, 0);
1827 }
1828
1829 ZOOM_API(const char *)
1830     ZOOM_connection_errmsg(ZOOM_connection c)
1831 {
1832     const char *msg;
1833     ZOOM_connection_error(c, &msg, 0);
1834     return msg;
1835 }
1836
1837 ZOOM_API(const char *)
1838     ZOOM_connection_addinfo(ZOOM_connection c)
1839 {
1840     const char *addinfo;
1841     ZOOM_connection_error(c, 0, &addinfo);
1842     return addinfo;
1843 }
1844
1845 ZOOM_API(const char *)
1846     ZOOM_connection_diagset(ZOOM_connection c)
1847 {
1848     const char *diagset;
1849     ZOOM_connection_error_x(c, 0, 0, &diagset);
1850     return diagset;
1851 }
1852
1853 ZOOM_API(const char *)
1854     ZOOM_diag_str(int error)
1855 {
1856     switch (error)
1857     {
1858     case ZOOM_ERROR_NONE:
1859         return "No error";
1860     case ZOOM_ERROR_CONNECT:
1861         return "Connect failed";
1862     case ZOOM_ERROR_MEMORY:
1863         return "Out of memory";
1864     case ZOOM_ERROR_ENCODE:
1865         return "Encoding failed";
1866     case ZOOM_ERROR_DECODE:
1867         return "Decoding failed";
1868     case ZOOM_ERROR_CONNECTION_LOST:
1869         return "Connection lost";
1870     case ZOOM_ERROR_INIT:
1871         return "Init rejected";
1872     case ZOOM_ERROR_INTERNAL:
1873         return "Internal failure";
1874     case ZOOM_ERROR_TIMEOUT:
1875         return "Timeout";
1876     case ZOOM_ERROR_UNSUPPORTED_PROTOCOL:
1877         return "Unsupported protocol";
1878     case ZOOM_ERROR_UNSUPPORTED_QUERY:
1879         return "Unsupported query type";
1880     case ZOOM_ERROR_INVALID_QUERY:
1881         return "Invalid query";
1882     case ZOOM_ERROR_CQL_PARSE:
1883         return "CQL parsing error";
1884     case ZOOM_ERROR_CQL_TRANSFORM:
1885         return "CQL transformation error";
1886     case ZOOM_ERROR_CCL_CONFIG:
1887         return "CCL configuration error";
1888     case ZOOM_ERROR_CCL_PARSE:
1889         return "CCL parsing error";
1890     case ZOOM_ERROR_ES_INVALID_ACTION:
1891         return "Extended Service. invalid action";
1892     case ZOOM_ERROR_ES_INVALID_VERSION:
1893         return "Extended Service. invalid version";
1894     case ZOOM_ERROR_ES_INVALID_SYNTAX:
1895         return "Extended Service. invalid syntax";
1896     case ZOOM_ERROR_MEMCACHED:
1897         return "Memcached";
1898     default:
1899         return diagbib1_str(error);
1900     }
1901 }
1902
1903 ZOOM_API(int)
1904     ZOOM_connection_error_x(ZOOM_connection c, const char **cp,
1905                             const char **addinfo, const char **diagset)
1906 {
1907     int error = c->error;
1908     if (cp)
1909     {
1910         if (!c->diagset || !strcmp(c->diagset, "ZOOM"))
1911             *cp = ZOOM_diag_str(error);
1912         else if (!strcmp(c->diagset, "HTTP"))
1913             *cp = z_HTTP_errmsg(c->error);
1914         else if (!strcmp(c->diagset, "Bib-1"))
1915             *cp = ZOOM_diag_str(error);
1916         else if (!strcmp(c->diagset, "info:srw/diagnostic/1"))
1917             *cp = yaz_diag_srw_str(c->error);
1918         else
1919             *cp = "Unknown error and diagnostic set";
1920     }
1921     if (addinfo)
1922         *addinfo = c->addinfo ? c->addinfo : "";
1923     if (diagset)
1924         *diagset = c->diagset ? c->diagset : "";
1925     return c->error;
1926 }
1927
1928 ZOOM_API(int)
1929     ZOOM_connection_error(ZOOM_connection c, const char **cp,
1930                           const char **addinfo)
1931 {
1932     return ZOOM_connection_error_x(c, cp, addinfo, 0);
1933 }
1934
1935 static void ZOOM_connection_do_io(ZOOM_connection c, int mask)
1936 {
1937     ZOOM_Event event = 0;
1938     int r = cs_look(c->cs);
1939     yaz_log(c->log_details, "%p ZOOM_connection_do_io mask=%d cs_look=%d",
1940             c, mask, r);
1941
1942     if (r == CS_NONE)
1943     {
1944         event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1945         ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
1946         ZOOM_connection_close(c);
1947         ZOOM_connection_put_event(c, event);
1948     }
1949     else if (r == CS_CONNECT)
1950     {
1951         int ret = ret = cs_rcvconnect(c->cs);
1952         yaz_log(c->log_details, "%p ZOOM_connection_do_io "
1953                 "cs_rcvconnect returned %d", c, ret);
1954         if (ret == 1)
1955         {
1956             int mask = ZOOM_SELECT_EXCEPT;
1957             if (c->cs->io_pending & CS_WANT_WRITE)
1958                 mask += ZOOM_SELECT_WRITE;
1959             if (c->cs->io_pending & CS_WANT_READ)
1960                 mask += ZOOM_SELECT_READ;
1961             ZOOM_connection_set_mask(c, mask);
1962             event = ZOOM_Event_create(ZOOM_EVENT_NONE);
1963             ZOOM_connection_put_event(c, event);
1964         }
1965         else if (ret == 0)
1966         {
1967             event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1968             ZOOM_connection_put_event(c, event);
1969             get_cert(c);
1970             if (c->proto == PROTO_Z3950)
1971                 ZOOM_connection_Z3950_send_init(c);
1972             else
1973             {
1974                 /* no init request for SRW .. */
1975                 assert(c->tasks->which == ZOOM_TASK_CONNECT);
1976                 ZOOM_connection_remove_task(c);
1977                 ZOOM_connection_set_mask(c, 0);
1978                 ZOOM_connection_exec_task(c);
1979             }
1980             c->state = STATE_ESTABLISHED;
1981         }
1982         else
1983         {
1984             ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
1985             ZOOM_connection_close(c);
1986         }
1987     }
1988     else
1989     {
1990         if (mask & ZOOM_SELECT_EXCEPT)
1991         {
1992             if (!ZOOM_test_reconnect(c))
1993             {
1994                 ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
1995                 ZOOM_connection_close(c);
1996             }
1997             return;
1998         }
1999         if (mask & ZOOM_SELECT_READ)
2000             do_read(c);
2001         if (c->cs && (mask & ZOOM_SELECT_WRITE))
2002             ZOOM_send_buf(c);
2003     }
2004 }
2005
2006 ZOOM_API(int)
2007     ZOOM_connection_last_event(ZOOM_connection cs)
2008 {
2009     if (!cs)
2010         return ZOOM_EVENT_NONE;
2011     return cs->last_event;
2012 }
2013
2014
2015 ZOOM_API(int) ZOOM_connection_fire_event_timeout(ZOOM_connection c)
2016 {
2017     if (c->mask)
2018     {
2019         ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_TIMEOUT);
2020         /* timeout and this connection was waiting */
2021         ZOOM_set_error(c, ZOOM_ERROR_TIMEOUT, 0);
2022         ZOOM_connection_close(c);
2023         ZOOM_connection_put_event(c, event);
2024     }
2025     return 0;
2026 }
2027
2028 ZOOM_API(int)
2029     ZOOM_connection_process(ZOOM_connection c)
2030 {
2031     ZOOM_Event event;
2032     if (!c)
2033         return 0;
2034
2035     event = ZOOM_connection_get_event(c);
2036     if (event)
2037     {
2038         ZOOM_Event_destroy(event);
2039         return 1;
2040     }
2041     ZOOM_connection_exec_task(c);
2042     event = ZOOM_connection_get_event(c);
2043     if (event)
2044     {
2045         ZOOM_Event_destroy(event);
2046         return 1;
2047     }
2048     return 0;
2049 }
2050
2051 ZOOM_API(int)
2052     ZOOM_event_nonblock(int no, ZOOM_connection *cs)
2053 {
2054     int i;
2055
2056     yaz_log(log_details0, "ZOOM_process_event(no=%d,cs=%p)", no, cs);
2057
2058     for (i = 0; i<no; i++)
2059     {
2060         ZOOM_connection c = cs[i];
2061
2062         if (c && ZOOM_connection_process(c))
2063             return i+1;
2064     }
2065     return 0;
2066 }
2067
2068 ZOOM_API(int) ZOOM_connection_fire_event_socket(ZOOM_connection c, int mask)
2069 {
2070     if (c->mask && mask)
2071         ZOOM_connection_do_io(c, mask);
2072     return 0;
2073 }
2074
2075 ZOOM_API(int) ZOOM_connection_get_socket(ZOOM_connection c)
2076 {
2077     if (c->cs)
2078         return cs_fileno(c->cs);
2079     return -1;
2080 }
2081
2082 ZOOM_API(int) ZOOM_connection_set_mask(ZOOM_connection c, int mask)
2083 {
2084     c->mask = mask;
2085     if (!c->cs)
2086         return -1;
2087     return 0;
2088 }
2089
2090 ZOOM_API(int) ZOOM_connection_get_mask(ZOOM_connection c)
2091 {
2092     if (c->cs)
2093         return c->mask;
2094     return 0;
2095 }
2096
2097 ZOOM_API(int) ZOOM_connection_get_timeout(ZOOM_connection c)
2098 {
2099     return ZOOM_options_get_int(c->options, "timeout", 30);
2100 }
2101
2102 ZOOM_API(void) ZOOM_connection_close(ZOOM_connection c)
2103 {
2104     if (c->cs)
2105         cs_close(c->cs);
2106     c->cs = 0;
2107     ZOOM_connection_set_mask(c, 0);
2108     c->state = STATE_IDLE;
2109 }
2110
2111 /*
2112  * Local variables:
2113  * c-basic-offset: 4
2114  * c-file-style: "Stroustrup"
2115  * indent-tabs-mode: nil
2116  * End:
2117  * vim: shiftwidth=4 tabstop=8 expandtab
2118  */
2119