383b344f3e8d72a034af63df7ee9cb636794e238
[yaz-moved-to-github.git] / src / zoom-c.c
1 /* This file is part of the YAZ toolkit.
2  * Copyright (C) Index Data
3  * See the file LICENSE for details.
4  */
5 /**
6  * \file zoom-c.c
7  * \brief Implements ZOOM C interface.
8  */
9 #if HAVE_CONFIG_H
10 #include <config.h>
11 #endif
12
13 #include <assert.h>
14 #include <string.h>
15 #include <errno.h>
16 #include "zoom-p.h"
17
18 #include <yaz/yaz-util.h>
19 #include <yaz/xmalloc.h>
20 #include <yaz/otherinfo.h>
21 #include <yaz/log.h>
22 #include <yaz/diagbib1.h>
23 #include <yaz/charneg.h>
24 #include <yaz/query-charset.h>
25 #include <yaz/snprintf.h>
26 #include <yaz/facet.h>
27
28 #include <yaz/shptr.h>
29
30 #if SHPTR
31 YAZ_SHPTR_TYPE(WRBUF)
32 #endif
33
34 static int log_api0 = 0;
35 static int log_details0 = 0;
36
37 static void resultset_destroy(ZOOM_resultset r);
38 static zoom_ret do_write_ex(ZOOM_connection c, char *buf_out, int len_out);
39
40 static void initlog(void)
41 {
42     static int log_level_initialized = 0;
43
44     if (!log_level_initialized)
45     {
46         log_api0 = yaz_log_module_level("zoom");
47         log_details0 = yaz_log_module_level("zoomdetails");
48         log_level_initialized = 1;
49     }
50 }
51
52 void ZOOM_connection_remove_tasks(ZOOM_connection c);
53
54 void ZOOM_set_dset_error(ZOOM_connection c, int error,
55                          const char *dset,
56                          const char *addinfo, const char *addinfo2)
57 {
58     char *cp;
59
60     xfree(c->addinfo);
61     c->addinfo = 0;
62     c->error = error;
63     if (!c->diagset || strcmp(dset, c->diagset))
64     {
65         xfree(c->diagset);
66         c->diagset = xstrdup(dset);
67         /* remove integer part from SRW diagset .. */
68         if ((cp = strrchr(c->diagset, '/')))
69             *cp = '\0';
70     }
71     if (addinfo && addinfo2)
72     {
73         c->addinfo = (char*) xmalloc(strlen(addinfo) + strlen(addinfo2) + 3);
74         strcpy(c->addinfo, addinfo);
75         strcat(c->addinfo, ": ");
76         strcat(c->addinfo, addinfo2);
77     }
78     else if (addinfo)
79         c->addinfo = xstrdup(addinfo);
80     if (error != ZOOM_ERROR_NONE)
81     {
82         yaz_log(c->log_api, "%p set_dset_error %s %s:%d %s %s",
83                 c, c->host_port ? c->host_port : "<>", dset, error,
84                 addinfo ? addinfo : "",
85                 addinfo2 ? addinfo2 : "");
86         ZOOM_connection_remove_tasks(c);
87     }
88 }
89
90 int ZOOM_uri_to_code(const char *uri)
91 {
92     int code = 0;
93     const char *cp;
94     if ((cp = strrchr(uri, '/')))
95         code = atoi(cp+1);
96     return code;
97 }
98
99 void ZOOM_set_error(ZOOM_connection c, int error, const char *addinfo)
100 {
101     ZOOM_set_dset_error(c, error, "ZOOM", addinfo, 0);
102 }
103
104 static void clear_error(ZOOM_connection c)
105 {
106     /*
107      * If an error is tied to an operation then it's ok to clear: for
108      * example, a diagnostic returned from a search is cleared by a
109      * subsequent search.  However, problems such as Connection Lost
110      * or Init Refused are not cleared, because they are not
111      * recoverable: doing another search doesn't help.
112      */
113
114     ZOOM_connection_remove_events(c);
115     switch (c->error)
116     {
117     case ZOOM_ERROR_CONNECT:
118     case ZOOM_ERROR_MEMORY:
119     case ZOOM_ERROR_DECODE:
120     case ZOOM_ERROR_CONNECTION_LOST:
121     case ZOOM_ERROR_INIT:
122     case ZOOM_ERROR_INTERNAL:
123     case ZOOM_ERROR_UNSUPPORTED_PROTOCOL:
124         break;
125     default:
126         ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
127     }
128 }
129
130 void ZOOM_connection_show_task(ZOOM_task task)
131 {
132     switch(task->which)
133     {
134     case ZOOM_TASK_SEARCH:
135         yaz_log(YLOG_LOG, "search p=%p", task);
136         break;
137     case ZOOM_TASK_CONNECT:
138         yaz_log(YLOG_LOG, "connect p=%p", task);
139         break;
140     case ZOOM_TASK_SCAN:
141         yaz_log(YLOG_LOG, "scan p=%p", task);
142         break;
143     }
144 }
145
146 void ZOOM_connection_show_tasks(ZOOM_connection c)
147 {
148     ZOOM_task task;
149     yaz_log(YLOG_LOG, "connection p=%p tasks", c);
150     for (task = c->tasks; task; task = task->next)
151         ZOOM_connection_show_task(task);
152 }
153
154 ZOOM_task ZOOM_connection_add_task(ZOOM_connection c, int which)
155 {
156     ZOOM_task *taskp = &c->tasks;
157     while (*taskp)
158         taskp = &(*taskp)->next;
159     *taskp = (ZOOM_task) xmalloc(sizeof(**taskp));
160     (*taskp)->running = 0;
161     (*taskp)->which = which;
162     (*taskp)->next = 0;
163     clear_error(c);
164     return *taskp;
165 }
166
167 ZOOM_API(int) ZOOM_connection_is_idle(ZOOM_connection c)
168 {
169     return c->tasks ? 0 : 1;
170 }
171
172 ZOOM_task ZOOM_connection_insert_task(ZOOM_connection c, int which)
173 {
174     ZOOM_task task = (ZOOM_task) xmalloc(sizeof(*task));
175
176     task->next = c->tasks;
177     c->tasks = task;
178
179     task->running = 0;
180     task->which = which;
181     return task;
182 }
183
184 void ZOOM_connection_remove_task(ZOOM_connection c)
185 {
186     ZOOM_task task = c->tasks;
187
188     if (task)
189     {
190         c->tasks = task->next;
191         switch (task->which)
192         {
193         case ZOOM_TASK_SEARCH:
194             resultset_destroy(task->u.search.resultset);
195             xfree(task->u.search.syntax);
196             xfree(task->u.search.elementSetName);
197             xfree(task->u.search.schema);
198             break;
199         case ZOOM_TASK_CONNECT:
200             break;
201         case ZOOM_TASK_SCAN:
202             ZOOM_scanset_destroy(task->u.scan.scan);
203             break;
204         case ZOOM_TASK_PACKAGE:
205             ZOOM_package_destroy(task->u.package);
206             break;
207         case ZOOM_TASK_SORT:
208             resultset_destroy(task->u.sort.resultset);
209             ZOOM_query_destroy(task->u.sort.q);
210             break;
211         default:
212             assert(0);
213         }
214         xfree(task);
215
216         if (!c->tasks)
217         {
218             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_END);
219             ZOOM_connection_put_event(c, event);
220         }
221     }
222 }
223
224 void ZOOM_connection_remove_tasks(ZOOM_connection c)
225 {
226     while (c->tasks)
227         ZOOM_connection_remove_task(c);
228 }
229
230 static void odr_wrbuf_write(ODR o, void *handle, int type,
231                             const char *buf, int len)
232 {
233     WRBUF w = (WRBUF) handle;
234     wrbuf_write(w, buf, len);
235 }
236
237 ZOOM_API(ZOOM_connection)
238     ZOOM_connection_create(ZOOM_options options)
239 {
240     ZOOM_connection c = (ZOOM_connection) xmalloc(sizeof(*c));
241
242     initlog();
243
244     c->log_api = log_api0;
245     c->log_details = log_details0;
246
247     yaz_log(c->log_api, "%p ZOOM_connection_create", c);
248
249     c->proto = PROTO_Z3950;
250     c->cs = 0;
251     ZOOM_connection_set_mask(c, 0);
252     c->reconnect_ok = 0;
253     c->state = STATE_IDLE;
254     c->addinfo = 0;
255     c->diagset = 0;
256     ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
257     c->buf_in = 0;
258     c->len_in = 0;
259     c->buf_out = 0;
260     c->len_out = 0;
261     c->resultsets = 0;
262
263     c->options = ZOOM_options_create_with_parent(options);
264
265     c->host_port = 0;
266     c->proxy = 0;
267     c->tproxy = 0;
268
269     c->charset = c->lang = 0;
270
271     c->cookie_out = 0;
272     c->cookie_in = 0;
273     c->client_IP = 0;
274     c->tasks = 0;
275
276     c->user = 0;
277     c->group = 0;
278     c->password = 0;
279     c->url_authentication = 0;
280
281     c->maximum_record_size = 0;
282     c->preferred_message_size = 0;
283
284     c->odr_in = odr_createmem(ODR_DECODE);
285     c->odr_out = odr_createmem(ODR_ENCODE);
286     c->odr_print = 0;
287     c->odr_save = 0;
288
289     c->async = 0;
290     c->support_named_resultsets = 0;
291     c->last_event = ZOOM_EVENT_NONE;
292
293     c->m_queue_front = 0;
294     c->m_queue_back = 0;
295
296     c->sru_version = 0;
297     c->no_redirects = 0;
298     c->cookies = 0;
299     c->saveAPDU_wrbuf = 0;
300
301     ZOOM_memcached_init(c);
302     return c;
303 }
304
305 ZOOM_API(void) ZOOM_connection_save_apdu_wrbuf(ZOOM_connection c, WRBUF w)
306 {
307     if (c->odr_save)
308     {
309         odr_destroy(c->odr_save);
310         c->odr_save = 0;
311     }
312     if (w)
313     {
314         c->odr_save = odr_createmem(ODR_PRINT);
315         odr_set_stream(c->odr_save, w, odr_wrbuf_write, 0);
316     }
317 }
318
319 /* set database names. Take local databases (if set); otherwise
320    take databases given in ZURL (if set); otherwise use Default */
321 char **ZOOM_connection_get_databases(ZOOM_connection con, ZOOM_options options,
322                                      int *num, ODR odr)
323 {
324     char **databaseNames;
325     const char *cp = ZOOM_options_get(options, "databaseName");
326
327     if ((!cp || !*cp) && con->host_port)
328         cs_get_host_args(con->host_port, &cp);
329     if (!cp || !*cp)
330         cp = "Default";
331     nmem_strsplit(odr_getmem(odr), "+", cp,  &databaseNames, num);
332     return databaseNames;
333 }
334
335 ZOOM_API(ZOOM_connection)
336     ZOOM_connection_new(const char *host, int portnum)
337 {
338     ZOOM_connection c = ZOOM_connection_create(0);
339
340     ZOOM_connection_connect(c, host, portnum);
341     return c;
342 }
343
344 static zoom_sru_mode get_sru_mode_from_string(const char *s)
345 {
346     if (!s || !*s)
347         return zoom_sru_soap;
348     if (!yaz_matchstr(s, "soap"))
349         return zoom_sru_soap;
350     else if (!yaz_matchstr(s, "get"))
351         return zoom_sru_get;
352     else if (!yaz_matchstr(s, "post"))
353         return zoom_sru_post;
354     else if (!yaz_matchstr(s, "solr"))
355         return zoom_sru_solr;
356     return zoom_sru_error;
357 }
358
359 ZOOM_API(void)
360     ZOOM_connection_connect(ZOOM_connection c,
361                             const char *host, int portnum)
362 {
363     const char *val;
364     const char *http_lead;
365
366     initlog();
367
368     yaz_log(c->log_api, "%p ZOOM_connection_connect host=%s portnum=%d",
369             c, host ? host : "null", portnum);
370
371     ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
372     ZOOM_connection_remove_tasks(c);
373
374     if (c->odr_print)
375     {
376         odr_setprint(c->odr_print, 0); /* prevent destroy from fclose'ing */
377         odr_destroy(c->odr_print);
378     }
379     if (ZOOM_options_get_bool(c->options, "apdulog", 0))
380     {
381         c->odr_print = odr_createmem(ODR_PRINT);
382         odr_setprint(c->odr_print, yaz_log_file());
383     }
384     else
385         c->odr_print = 0;
386
387     if (c->cs)
388     {
389         yaz_log(c->log_details, "%p ZOOM_connection_connect reconnect ok", c);
390         c->reconnect_ok = 1;
391         return;
392     }
393     yaz_log(c->log_details, "%p ZOOM_connection_connect connect", c);
394     xfree(c->proxy);
395     c->proxy = 0;
396     val = ZOOM_options_get(c->options, "proxy");
397     if (val && *val)
398     {
399         yaz_log(c->log_details, "%p ZOOM_connection_connect proxy=%s", c, val);
400         c->proxy = xstrdup(val);
401     }
402
403     xfree(c->tproxy);
404     c->tproxy = 0;
405     val = ZOOM_options_get(c->options, "tproxy");
406     if (val && *val)
407     {
408         yaz_log(c->log_details, "%p ZOOM_connection_connect tproxy=%s", c, val);
409         c->tproxy = xstrdup(val);
410     }
411
412     xfree(c->charset);
413     c->charset = 0;
414     val = ZOOM_options_get(c->options, "charset");
415     if (val && *val)
416     {
417         yaz_log(c->log_details, "%p ZOOM_connection_connect charset=%s", c, val);
418         c->charset = xstrdup(val);
419     }
420
421     xfree(c->lang);
422     val = ZOOM_options_get(c->options, "lang");
423     if (val && *val)
424     {
425         yaz_log(c->log_details, "%p ZOOM_connection_connect lang=%s", c, val);
426         c->lang = xstrdup(val);
427     }
428     else
429         c->lang = 0;
430
431     val = ZOOM_options_get(c->options, "sru");
432     if (val && *val && !strstr(host, "://"))
433         http_lead = "http://";
434     else
435         http_lead = "";
436     c->sru_mode = get_sru_mode_from_string(val);
437
438     if (host)
439     {
440         char hostn[128];
441         xfree(c->host_port);
442         if (portnum)
443         {
444             sprintf(hostn, "%.80s:%d", host, portnum);
445             host = hostn;
446         }
447         c->host_port = xmalloc(strlen(host) + strlen(http_lead) + 1);
448         strcpy(c->host_port, http_lead);
449         strcat(c->host_port, host);
450     }
451
452     {
453         /*
454          * If the "<scheme>:" part of the host string is preceded by one
455          * or more comma-separated <name>=<value> pairs, these are taken
456          * to be options to be set on the connection object.  Among other
457          * applications, this facility can be used to embed authentication
458          * in a host string:
459          *          user=admin,password=secret,tcp:localhost:9999
460          */
461         char *remainder = c->host_port;
462         char *pcolon = strchr(remainder, ':');
463         char *pcomma;
464         char *pequals;
465         while ((pcomma = strchr(remainder, ',')) != 0 &&
466                (pcolon == 0 || pcomma < pcolon))
467         {
468             *pcomma = '\0';
469             if ((pequals = strchr(remainder, '=')) != 0)
470             {
471                 *pequals = '\0';
472                 ZOOM_connection_option_set(c, remainder, pequals+1);
473             }
474             remainder = pcomma+1;
475         }
476
477         if (remainder != c->host_port)
478         {
479             remainder = xstrdup(remainder);
480             xfree(c->host_port);
481             c->host_port = remainder;
482         }
483     }
484
485     xfree(c->sru_version);
486     val = ZOOM_options_get(c->options, "sru_version");
487     c->sru_version = xstrdup(val ? val : "1.2");
488
489     ZOOM_options_set(c->options, "host", c->host_port);
490
491     xfree(c->cookie_out);
492     c->cookie_out = 0;
493     val = ZOOM_options_get(c->options, "cookie");
494     if (val && *val)
495     {
496         yaz_log(c->log_details, "%p ZOOM_connection_connect cookie=%s", c, val);
497         c->cookie_out = xstrdup(val);
498     }
499
500     xfree(c->client_IP);
501     c->client_IP = 0;
502     val = ZOOM_options_get(c->options, "clientIP");
503     if (val && *val)
504     {
505         yaz_log(c->log_details, "%p ZOOM_connection_connect clientIP=%s",
506                 c, val);
507         c->client_IP = xstrdup(val);
508     }
509
510     xfree(c->group);
511     c->group = 0;
512     val = ZOOM_options_get(c->options, "group");
513     if (val && *val)
514         c->group = xstrdup(val);
515
516     xfree(c->user);
517     c->user = 0;
518     val = ZOOM_options_get(c->options, "user");
519     if (val && *val)
520         c->user = xstrdup(val);
521
522     xfree(c->password);
523     c->password = 0;
524     val = ZOOM_options_get(c->options, "password");
525     if (!val)
526         val = ZOOM_options_get(c->options, "pass");
527     if (val && *val)
528         c->password = xstrdup(val);
529
530     val = ZOOM_options_get(c->options, "authenticationMode");
531     if (val && !strcmp(val, "url"))
532         c->url_authentication = 1;
533     else
534         c->url_authentication = 0;
535
536     c->maximum_record_size =
537         ZOOM_options_get_int(c->options, "maximumRecordSize", 64*1024*1024);
538     c->preferred_message_size =
539         ZOOM_options_get_int(c->options, "preferredMessageSize", 64*1024*1024);
540
541     c->async = ZOOM_options_get_bool(c->options, "async", 0);
542
543     yaz_cookies_destroy(c->cookies);
544     c->cookies = yaz_cookies_create();
545
546     if (ZOOM_memcached_configure(c))
547     {
548         ZOOM_connection_remove_tasks(c);
549         return;
550     }
551     if (c->sru_mode == zoom_sru_error)
552     {
553         ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, val);
554         ZOOM_connection_remove_tasks(c);
555         return;
556     }
557
558     yaz_log(c->log_details, "%p ZOOM_connection_connect async=%d", c, c->async);
559     ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
560
561     if (!c->async)
562     {
563         while (ZOOM_event(1, &c))
564             ;
565     }
566 }
567
568 ZOOM_API(void) ZOOM_resultset_release(ZOOM_resultset r)
569 {
570     if (r->connection)
571     {
572         /* remove ourselves from the resultsets in connection */
573         ZOOM_resultset *rp = &r->connection->resultsets;
574         while (1)
575         {
576             assert(*rp);   /* we must be in this list!! */
577             if (*rp == r)
578             {   /* OK, we're here - take us out of it */
579                 *rp = (*rp)->next;
580                 break;
581             }
582             rp = &(*rp)->next;
583         }
584         r->connection = 0;
585     }
586 }
587
588 ZOOM_API(void)
589     ZOOM_connection_destroy(ZOOM_connection c)
590 {
591     ZOOM_resultset r;
592     if (!c)
593         return;
594     yaz_log(c->log_api, "%p ZOOM_connection_destroy", c);
595
596     ZOOM_memcached_destroy(c);
597     if (c->cs)
598         cs_close(c->cs);
599
600     for (r = c->resultsets; r; r = r->next)
601         r->connection = 0;
602
603     xfree(c->buf_in);
604     xfree(c->addinfo);
605     xfree(c->diagset);
606     odr_destroy(c->odr_in);
607     odr_destroy(c->odr_out);
608     if (c->odr_save)
609         odr_destroy(c->odr_save);
610     if (c->odr_print)
611     {
612         odr_setprint(c->odr_print, 0); /* prevent destroy from fclose'ing */
613         odr_destroy(c->odr_print);
614     }
615     ZOOM_options_destroy(c->options);
616     ZOOM_connection_remove_tasks(c);
617     ZOOM_connection_remove_events(c);
618     xfree(c->host_port);
619     xfree(c->proxy);
620     xfree(c->tproxy);
621     xfree(c->charset);
622     xfree(c->lang);
623     xfree(c->cookie_out);
624     xfree(c->cookie_in);
625     xfree(c->client_IP);
626     xfree(c->user);
627     xfree(c->group);
628     xfree(c->password);
629     xfree(c->sru_version);
630     yaz_cookies_destroy(c->cookies);
631     wrbuf_destroy(c->saveAPDU_wrbuf);
632     xfree(c);
633 }
634
635 void ZOOM_resultset_addref(ZOOM_resultset r)
636 {
637     if (r)
638     {
639         yaz_mutex_enter(r->mutex);
640         (r->refcount)++;
641         yaz_log(log_details0, "%p ZOOM_resultset_addref count=%d",
642                 r, r->refcount);
643         yaz_mutex_leave(r->mutex);
644     }
645 }
646
647 static int g_resultsets = 0;
648 static YAZ_MUTEX g_resultset_mutex = 0;
649
650 /* TODO We need to initialize this before running threaded:
651  * call resultset_use(0)  */
652
653 static int resultset_use(int delta) {
654     int resultset_count;
655     if (g_resultset_mutex == 0)
656         yaz_mutex_create(&g_resultset_mutex);
657     yaz_mutex_enter(g_resultset_mutex);
658     g_resultsets += delta;
659     resultset_count = g_resultsets;
660     yaz_mutex_leave(g_resultset_mutex);
661     return resultset_count;
662 }
663
664 int resultsets_count(void) {
665     return resultset_use(0);
666 }
667
668 ZOOM_resultset ZOOM_resultset_create(void)
669 {
670     int i;
671     ZOOM_resultset r = (ZOOM_resultset) xmalloc(sizeof(*r));
672
673     initlog();
674
675     yaz_log(log_details0, "%p ZOOM_resultset_create", r);
676     r->refcount = 1;
677     r->size = 0;
678     r->odr = odr_createmem(ODR_DECODE);
679     r->piggyback = 1;
680     r->setname = 0;
681     r->step = 0;
682     for (i = 0; i<RECORD_HASH_SIZE; i++)
683         r->record_hash[i] = 0;
684     r->r_sort_spec = 0;
685     r->query = 0;
686     r->connection = 0;
687     r->databaseNames = 0;
688     r->num_databaseNames = 0;
689     r->req_facets = 0;
690     r->res_facets = 0;
691     r->num_res_facets = 0;
692     r->facets_names = 0;
693     r->mutex = 0;
694     yaz_mutex_create(&r->mutex);
695 #if SHPTR
696     {
697         WRBUF w = wrbuf_alloc();
698         YAZ_SHPTR_INIT(r->record_wrbuf, w);
699     }
700 #endif
701     resultset_use(1);
702     r->mc_key = 0;
703     r->live_set = 0;
704     return r;
705 }
706
707 ZOOM_API(ZOOM_resultset)
708     ZOOM_connection_search_pqf(ZOOM_connection c, const char *q)
709 {
710     ZOOM_resultset r;
711     ZOOM_query s = ZOOM_query_create();
712
713     ZOOM_query_prefix(s, q);
714
715     r = ZOOM_connection_search(c, s);
716     ZOOM_query_destroy(s);
717     return r;
718 }
719
720 ZOOM_API(ZOOM_resultset)
721     ZOOM_connection_search(ZOOM_connection c, ZOOM_query q)
722 {
723     ZOOM_resultset r = ZOOM_resultset_create();
724     const char *cp;
725     ZOOM_task task;
726     int start, count;
727     const char *syntax, *elementSetName, *schema, *facets;
728     yaz_log(c->log_api, "%p ZOOM_connection_search set %p query %p", c, r, q);
729     r->r_sort_spec = ZOOM_query_get_sortspec(q);
730     r->query = q;
731     ZOOM_query_addref(q);
732
733     r->options = ZOOM_options_create_with_parent(c->options);
734
735     r->req_facets = odr_strdup_null(r->odr, 
736                                     ZOOM_options_get(r->options, "facets"));
737     start = ZOOM_options_get_int(r->options, "start", 0);
738     count = ZOOM_options_get_int(r->options, "count", 0);
739     {
740         /* If "presentChunk" is defined use that; otherwise "step" */
741         const char *cp = ZOOM_options_get(r->options, "presentChunk");
742         r->step = ZOOM_options_get_int(r->options,
743                                        (cp != 0 ? "presentChunk": "step"), 0);
744     }
745     r->piggyback = ZOOM_options_get_bool(r->options, "piggyback", 1);
746     cp = ZOOM_options_get(r->options, "setname");
747     if (cp)
748         r->setname = xstrdup(cp);
749
750     r->databaseNames = ZOOM_connection_get_databases(c, c->options, &r->num_databaseNames,
751                                          r->odr);
752
753     r->connection = c;
754     r->next = c->resultsets;
755     c->resultsets = r;
756
757     ZOOM_memcached_resultset(r, q);
758
759     if (c->host_port && c->proto == PROTO_HTTP)
760     {
761         if (!c->cs)
762         {
763             yaz_log(c->log_details, "ZOOM_connection_search: no comstack");
764             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
765         }
766         else
767         {
768             yaz_log(c->log_details, "ZOOM_connection_search: reconnect");
769             c->reconnect_ok = 1;
770         }
771     }
772
773     task = ZOOM_connection_add_task(c, ZOOM_TASK_SEARCH);
774     task->u.search.resultset = r;
775     task->u.search.start = start;
776     task->u.search.count = count;
777
778     syntax = ZOOM_options_get(r->options, "preferredRecordSyntax");
779     task->u.search.syntax = syntax ? xstrdup(syntax) : 0;
780     elementSetName = ZOOM_options_get(r->options, "elementSetName");
781     task->u.search.elementSetName = elementSetName ?
782         xstrdup(elementSetName) : 0;
783     schema = ZOOM_options_get(r->options, "schema");
784     task->u.search.schema = schema ? xstrdup(schema) : 0;
785
786     ZOOM_resultset_addref(r);
787
788     if (!c->async)
789     {
790         while (ZOOM_event(1, &c))
791             ;
792     }
793     return r;
794 }
795
796 ZOOM_API(void)
797     ZOOM_resultset_sort(ZOOM_resultset r,
798                          const char *sort_type, const char *sort_spec)
799 {
800     (void) ZOOM_resultset_sort1(r, sort_type, sort_spec);
801 }
802
803 ZOOM_API(int)
804     ZOOM_resultset_sort1(ZOOM_resultset r,
805                          const char *sort_type, const char *sort_spec)
806 {
807     ZOOM_connection c = r->connection;
808     ZOOM_task task;
809     ZOOM_query newq;
810
811     newq = ZOOM_query_create();
812     if (ZOOM_query_sortby(newq, sort_spec) < 0)
813         return -1;
814
815     yaz_log(c->log_api, "%p ZOOM_resultset_sort r=%p sort_type=%s sort_spec=%s",
816             r, r, sort_type, sort_spec);
817     if (!c)
818         return 0;
819
820     if (c->host_port && c->proto == PROTO_HTTP)
821     {
822         if (!c->cs)
823         {
824             yaz_log(c->log_details, "%p ZOOM_resultset_sort: no comstack", r);
825             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
826         }
827         else
828         {
829             yaz_log(c->log_details, "%p ZOOM_resultset_sort: prepare reconnect",
830                     r);
831             c->reconnect_ok = 1;
832         }
833     }
834
835     ZOOM_resultset_cache_reset(r);
836     task = ZOOM_connection_add_task(c, ZOOM_TASK_SORT);
837     task->u.sort.resultset = r;
838     task->u.sort.q = newq;
839
840     ZOOM_resultset_addref(r);
841
842     if (!c->async)
843     {
844         while (ZOOM_event(1, &c))
845             ;
846     }
847
848     return 0;
849 }
850
851 ZOOM_API(void)
852     ZOOM_resultset_destroy(ZOOM_resultset r)
853 {
854     resultset_destroy(r);
855 }
856
857 static void resultset_destroy(ZOOM_resultset r)
858 {
859     if (!r)
860         return;
861     yaz_mutex_enter(r->mutex);
862     (r->refcount)--;
863     yaz_log(log_details0, "%p ZOOM_resultset_destroy r=%p count=%d",
864             r, r, r->refcount);
865     if (r->refcount == 0)
866     {
867         yaz_mutex_leave(r->mutex);
868
869         yaz_log(log_details0, "%p ZOOM_connection resultset_destroy: Deleting resultset (%p) ", r->connection, r);
870         ZOOM_resultset_cache_reset(r);
871         ZOOM_resultset_release(r);
872         ZOOM_query_destroy(r->query);
873         ZOOM_options_destroy(r->options);
874         odr_destroy(r->odr);
875         xfree(r->setname);
876         yaz_mutex_destroy(&r->mutex);
877 #if SHPTR
878         YAZ_SHPTR_DEC(r->record_wrbuf, wrbuf_destroy);
879 #endif
880         wrbuf_destroy(r->mc_key);
881         resultset_use(-1);
882         xfree(r);
883     }
884     else
885         yaz_mutex_leave(r->mutex);
886 }
887
888 ZOOM_API(size_t)
889     ZOOM_resultset_size(ZOOM_resultset r)
890 {
891     return r->size;
892 }
893
894 int ZOOM_test_reconnect(ZOOM_connection c)
895 {
896     ZOOM_Event event;
897
898     if (!c->reconnect_ok)
899         return 0;
900     ZOOM_connection_close(c);
901     c->reconnect_ok = 0;
902     c->tasks->running = 0;
903     ZOOM_connection_insert_task(c, ZOOM_TASK_CONNECT);
904
905     event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
906     ZOOM_connection_put_event(c, event);
907
908     return 1;
909 }
910
911 static void ZOOM_resultset_retrieve(ZOOM_resultset r,
912                                     int force_sync, int start, int count)
913 {
914     ZOOM_task task;
915     ZOOM_connection c;
916     const char *cp;
917     const char *syntax, *elementSetName;
918
919     if (!r)
920         return;
921     yaz_log(log_details0, "%p ZOOM_resultset_retrieve force_sync=%d start=%d"
922             " count=%d", r, force_sync, start, count);
923     c = r->connection;
924     if (!c)
925         return;
926
927     if (c->host_port && c->proto == PROTO_HTTP)
928     {
929         if (!c->cs)
930         {
931             yaz_log(log_details0, "%p ZOOM_resultset_retrieve: no comstack", r);
932             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
933         }
934         else
935         {
936             yaz_log(log_details0, "%p ZOOM_resultset_retrieve: prepare "
937                     "reconnect", r);
938             c->reconnect_ok = 1;
939         }
940     }
941     task = ZOOM_connection_add_task(c, ZOOM_TASK_SEARCH);
942     task->u.search.resultset = r;
943     task->u.search.start = start;
944     task->u.search.count = count;
945
946     syntax = ZOOM_options_get(r->options, "preferredRecordSyntax");
947     task->u.search.syntax = syntax ? xstrdup(syntax) : 0;
948     elementSetName = ZOOM_options_get(r->options, "elementSetName");
949     task->u.search.elementSetName = elementSetName
950         ? xstrdup(elementSetName) : 0;
951
952     cp = ZOOM_options_get(r->options, "schema");
953     task->u.search.schema = cp ? xstrdup(cp) : 0;
954
955     ZOOM_resultset_addref(r);
956
957     if (!r->connection->async || force_sync)
958         while (r->connection && ZOOM_event(1, &r->connection))
959             ;
960 }
961
962 ZOOM_API(void)
963     ZOOM_resultset_records(ZOOM_resultset r, ZOOM_record *recs,
964                            size_t start, size_t count)
965 {
966     int force_present = 0;
967
968     if (!r)
969         return ;
970     yaz_log(log_api0, "%p ZOOM_resultset_records r=%p start=%ld count=%ld",
971             r, r, (long) start, (long) count);
972     if (count && recs)
973         force_present = 1;
974     ZOOM_resultset_retrieve(r, force_present, start, count);
975     if (force_present)
976     {
977         size_t i;
978         for (i = 0; i< count; i++)
979             recs[i] = ZOOM_resultset_record_immediate(r, i+start);
980     }
981 }
982
983 ZOOM_API(size_t)
984     ZOOM_resultset_facets_size(ZOOM_resultset r)
985 {
986     return r->num_res_facets;
987 }
988
989 ZOOM_API(ZOOM_facet_field)
990     ZOOM_resultset_get_facet_field(ZOOM_resultset r, const char *name)
991 {
992     int num = r->num_res_facets;
993     ZOOM_facet_field *facets = r->res_facets;
994     int i;
995     for (i = 0; i < num; i++)
996         if (!strcmp(facets[i]->facet_name, name))
997             return facets[i];
998     return 0;
999 }
1000
1001 ZOOM_API(ZOOM_facet_field)
1002     ZOOM_resultset_get_facet_field_by_index(ZOOM_resultset r, int idx)
1003 {
1004     int num = r->num_res_facets;
1005     ZOOM_facet_field *facets = r->res_facets;
1006     if (idx >= 0 && idx < num)
1007         return facets[idx];
1008     return 0;
1009 }
1010
1011 ZOOM_API(ZOOM_facet_field *)
1012     ZOOM_resultset_facets(ZOOM_resultset r)
1013 {
1014     return r->res_facets;
1015 }
1016
1017 ZOOM_API(const char**)
1018     ZOOM_resultset_facets_names(ZOOM_resultset r)
1019 {
1020     return (const char **) r->facets_names;
1021 }
1022
1023 ZOOM_API(const char*)
1024     ZOOM_facet_field_name(ZOOM_facet_field field)
1025 {
1026     return field->facet_name;
1027 }
1028
1029 ZOOM_API(size_t)
1030     ZOOM_facet_field_term_count(ZOOM_facet_field field)
1031 {
1032     return field->num_terms;
1033 }
1034
1035 ZOOM_API(const char*)
1036     ZOOM_facet_field_get_term(ZOOM_facet_field field, size_t idx, int *freq)
1037 {
1038     *freq = field->facet_terms[idx].frequency;
1039     return field->facet_terms[idx].term;
1040 }
1041
1042
1043 static void get_cert(ZOOM_connection c)
1044 {
1045     char *cert_buf;
1046     int cert_len;
1047
1048     if (cs_get_peer_certificate_x509(c->cs, &cert_buf, &cert_len))
1049     {
1050         ZOOM_connection_option_setl(c, "sslPeerCert",
1051                                     cert_buf, cert_len);
1052         xfree(cert_buf);
1053     }
1054 }
1055
1056 static zoom_ret do_connect_host(ZOOM_connection c,
1057                                 const char *logical_url);
1058
1059 static zoom_ret do_connect(ZOOM_connection c)
1060 {
1061     return do_connect_host(c, c->host_port);
1062 }
1063
1064 static zoom_ret do_connect_host(ZOOM_connection c, const char *logical_url)
1065 {
1066     void *add;
1067
1068     if (c->cs)
1069         cs_close(c->cs);
1070     c->cs = cs_create_host_proxy(logical_url, 0, &add,
1071                                  c->tproxy ? c->tproxy : c->proxy);
1072
1073     if (c->cs && c->cs->protocol == PROTO_HTTP)
1074     {
1075 #if YAZ_HAVE_XML2
1076         c->proto = PROTO_HTTP;
1077 #else
1078         ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, "SRW");
1079         ZOOM_connection_close(c);
1080         return zoom_complete;
1081 #endif
1082     }
1083     if (c->cs)
1084     {
1085         int ret = cs_connect(c->cs, add);
1086         if (ret == 0)
1087         {
1088             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1089             ZOOM_connection_put_event(c, event);
1090             get_cert(c);
1091             if (c->proto == PROTO_Z3950)
1092                 ZOOM_connection_Z3950_send_init(c);
1093             else
1094             {
1095                 /* no init request for SRW .. */
1096                 assert(c->tasks->which == ZOOM_TASK_CONNECT);
1097                 ZOOM_connection_remove_task(c);
1098                 ZOOM_connection_set_mask(c, 0);
1099                 ZOOM_connection_exec_task(c);
1100             }
1101             c->state = STATE_ESTABLISHED;
1102             return zoom_pending;
1103         }
1104         else if (ret > 0)
1105         {
1106             int mask = ZOOM_SELECT_EXCEPT;
1107             if (c->cs->io_pending & CS_WANT_WRITE)
1108                 mask += ZOOM_SELECT_WRITE;
1109             if (c->cs->io_pending & CS_WANT_READ)
1110                 mask += ZOOM_SELECT_READ;
1111             ZOOM_connection_set_mask(c, mask);
1112             c->state = STATE_CONNECTING;
1113             return zoom_pending;
1114         }
1115     }
1116     c->state = STATE_IDLE;
1117     ZOOM_set_error(c, ZOOM_ERROR_CONNECT, logical_url);
1118     return zoom_complete;
1119 }
1120
1121 /* returns 1 if PDU was sent OK (still pending )
1122    0 if PDU was not sent OK (nothing to wait for)
1123 */
1124
1125 ZOOM_API(ZOOM_record)
1126     ZOOM_resultset_record_immediate(ZOOM_resultset s,size_t pos)
1127 {
1128     const char *syntax =
1129         ZOOM_options_get(s->options, "preferredRecordSyntax");
1130     const char *elementSetName =
1131         ZOOM_options_get(s->options, "elementSetName");
1132     const char *schema =
1133         ZOOM_options_get(s->options, "schema");
1134
1135     return ZOOM_record_cache_lookup(s, pos, syntax, elementSetName, schema);
1136 }
1137
1138 ZOOM_API(ZOOM_record)
1139     ZOOM_resultset_record(ZOOM_resultset r, size_t pos)
1140 {
1141     ZOOM_record rec = ZOOM_resultset_record_immediate(r, pos);
1142
1143     if (!rec)
1144     {
1145         /*
1146          * MIKE: I think force_sync should always be zero, but I don't
1147          * want to make this change until I get the go-ahead from
1148          * Adam, in case something depends on the old synchronous
1149          * behaviour.
1150          */
1151         int force_sync = 1;
1152         if (getenv("ZOOM_RECORD_NO_FORCE_SYNC")) force_sync = 0;
1153         ZOOM_resultset_retrieve(r, force_sync, pos, 1);
1154         rec = ZOOM_resultset_record_immediate(r, pos);
1155     }
1156     return rec;
1157 }
1158
1159 ZOOM_API(ZOOM_scanset)
1160     ZOOM_connection_scan(ZOOM_connection c, const char *start)
1161 {
1162     ZOOM_scanset s;
1163     ZOOM_query q = ZOOM_query_create();
1164
1165     ZOOM_query_prefix(q, start);
1166
1167     s = ZOOM_connection_scan1(c, q);
1168     ZOOM_query_destroy(q);
1169     return s;
1170
1171 }
1172
1173 ZOOM_API(ZOOM_scanset)
1174     ZOOM_connection_scan1(ZOOM_connection c, ZOOM_query q)
1175 {
1176     ZOOM_scanset scan = 0;
1177     Z_Query *z_query = ZOOM_query_get_Z_Query(q);
1178
1179     if (!z_query)
1180         return 0;
1181     scan = (ZOOM_scanset) xmalloc(sizeof(*scan));
1182     scan->connection = c;
1183     scan->odr = odr_createmem(ODR_DECODE);
1184     scan->options = ZOOM_options_create_with_parent(c->options);
1185     scan->refcount = 1;
1186     scan->scan_response = 0;
1187     scan->srw_scan_response = 0;
1188
1189     scan->query = q;
1190     ZOOM_query_addref(q);
1191     scan->databaseNames = ZOOM_connection_get_databases(c, c->options,
1192                                             &scan->num_databaseNames,
1193                                             scan->odr);
1194
1195     if (1)
1196     {
1197         ZOOM_task task = ZOOM_connection_add_task(c, ZOOM_TASK_SCAN);
1198         task->u.scan.scan = scan;
1199
1200         (scan->refcount)++;
1201         if (!c->async)
1202         {
1203             while (ZOOM_event(1, &c))
1204                 ;
1205         }
1206     }
1207     return scan;
1208 }
1209
1210 ZOOM_API(void)
1211     ZOOM_scanset_destroy(ZOOM_scanset scan)
1212 {
1213     if (!scan)
1214         return;
1215     (scan->refcount)--;
1216     if (scan->refcount == 0)
1217     {
1218         ZOOM_query_destroy(scan->query);
1219
1220         odr_destroy(scan->odr);
1221
1222         ZOOM_options_destroy(scan->options);
1223         xfree(scan);
1224     }
1225 }
1226
1227 static zoom_ret send_package(ZOOM_connection c)
1228 {
1229     ZOOM_Event event;
1230
1231     yaz_log(c->log_details, "%p send_package", c);
1232     if (!c->tasks)
1233         return zoom_complete;
1234     assert (c->tasks->which == ZOOM_TASK_PACKAGE);
1235
1236     event = ZOOM_Event_create(ZOOM_EVENT_SEND_APDU);
1237     ZOOM_connection_put_event(c, event);
1238
1239     c->buf_out = c->tasks->u.package->buf_out;
1240     c->len_out = c->tasks->u.package->len_out;
1241
1242     return ZOOM_send_buf(c);
1243 }
1244
1245 ZOOM_API(size_t)
1246     ZOOM_scanset_size(ZOOM_scanset scan)
1247 {
1248     if (!scan)
1249         return 0;
1250
1251     if (scan->scan_response && scan->scan_response->entries)
1252         return scan->scan_response->entries->num_entries;
1253     else if (scan->srw_scan_response)
1254         return scan->srw_scan_response->num_terms;
1255     return 0;
1256 }
1257
1258 static void ZOOM_scanset_term_x(ZOOM_scanset scan, size_t pos,
1259                                 size_t *occ,
1260                                 const char **value_term, size_t *value_len,
1261                                 const char **disp_term, size_t *disp_len)
1262 {
1263     size_t noent = ZOOM_scanset_size(scan);
1264
1265     *value_term = 0;
1266     *value_len = 0;
1267
1268     *disp_term = 0;
1269     *disp_len = 0;
1270
1271     *occ = 0;
1272     if (pos >= noent)
1273         return;
1274     if (scan->scan_response)
1275     {
1276         Z_ScanResponse *res = scan->scan_response;
1277         if (res->entries->entries[pos]->which == Z_Entry_termInfo)
1278         {
1279             Z_TermInfo *t = res->entries->entries[pos]->u.termInfo;
1280
1281             *value_term = (const char *) t->term->u.general->buf;
1282             *value_len = t->term->u.general->len;
1283             if (t->displayTerm)
1284             {
1285                 *disp_term = t->displayTerm;
1286                 *disp_len = strlen(*disp_term);
1287             }
1288             else if (t->term->which == Z_Term_general)
1289             {
1290                 *disp_term = (const char *) t->term->u.general->buf;
1291                 *disp_len = t->term->u.general->len;
1292             }
1293             *occ = t->globalOccurrences ? *t->globalOccurrences : 0;
1294         }
1295     }
1296     if (scan->srw_scan_response)
1297     {
1298         Z_SRW_scanResponse *res = scan->srw_scan_response;
1299         Z_SRW_scanTerm *t = res->terms + pos;
1300         if (t)
1301         {
1302             *value_term = t->value;
1303             *value_len = strlen(*value_term);
1304
1305             if (t->displayTerm)
1306                 *disp_term = t->displayTerm;
1307             else
1308                 *disp_term = t->value;
1309             *disp_len = strlen(*disp_term);
1310             *occ = t->numberOfRecords ? *t->numberOfRecords : 0;
1311         }
1312     }
1313 }
1314
1315 ZOOM_API(const char *)
1316     ZOOM_scanset_term(ZOOM_scanset scan, size_t pos,
1317                       size_t *occ, size_t *len)
1318 {
1319     const char *value_term = 0;
1320     size_t value_len = 0;
1321     const char *disp_term = 0;
1322     size_t disp_len = 0;
1323
1324     ZOOM_scanset_term_x(scan, pos, occ, &value_term, &value_len,
1325                         &disp_term, &disp_len);
1326
1327     *len = value_len;
1328     return value_term;
1329 }
1330
1331 ZOOM_API(const char *)
1332     ZOOM_scanset_display_term(ZOOM_scanset scan, size_t pos,
1333                               size_t *occ, size_t *len)
1334 {
1335     const char *value_term = 0;
1336     size_t value_len = 0;
1337     const char *disp_term = 0;
1338     size_t disp_len = 0;
1339
1340     ZOOM_scanset_term_x(scan, pos, occ, &value_term, &value_len,
1341                         &disp_term, &disp_len);
1342
1343     *len = disp_len;
1344     return disp_term;
1345 }
1346
1347 ZOOM_API(const char *)
1348     ZOOM_scanset_option_get(ZOOM_scanset scan, const char *key)
1349 {
1350     return ZOOM_options_get(scan->options, key);
1351 }
1352
1353 ZOOM_API(void)
1354     ZOOM_scanset_option_set(ZOOM_scanset scan, const char *key,
1355                             const char *val)
1356 {
1357     ZOOM_options_set(scan->options, key, val);
1358 }
1359
1360
1361 ZOOM_API(ZOOM_package)
1362     ZOOM_connection_package(ZOOM_connection c, ZOOM_options options)
1363 {
1364     ZOOM_package p = (ZOOM_package) xmalloc(sizeof(*p));
1365
1366     p->connection = c;
1367     p->odr_out = odr_createmem(ODR_ENCODE);
1368     p->options = ZOOM_options_create_with_parent2(options, c->options);
1369     p->refcount = 1;
1370     p->buf_out = 0;
1371     p->len_out = 0;
1372     return p;
1373 }
1374
1375 ZOOM_API(void)
1376     ZOOM_package_destroy(ZOOM_package p)
1377 {
1378     if (!p)
1379         return;
1380     (p->refcount)--;
1381     if (p->refcount == 0)
1382     {
1383         odr_destroy(p->odr_out);
1384         xfree(p->buf_out);
1385
1386         ZOOM_options_destroy(p->options);
1387         xfree(p);
1388     }
1389 }
1390
1391 ZOOM_API(const char *)
1392     ZOOM_package_option_get(ZOOM_package p, const char *key)
1393 {
1394     return ZOOM_options_get(p->options, key);
1395 }
1396
1397 ZOOM_API(const char *)
1398     ZOOM_package_option_getl(ZOOM_package p, const char *key, int *lenp)
1399 {
1400     return ZOOM_options_getl(p->options, key, lenp);
1401 }
1402
1403 ZOOM_API(void)
1404     ZOOM_package_option_set(ZOOM_package p, const char *key,
1405                             const char *val)
1406 {
1407     ZOOM_options_set(p->options, key, val);
1408 }
1409
1410 ZOOM_API(void)
1411     ZOOM_package_option_setl(ZOOM_package p, const char *key,
1412                              const char *val, int len)
1413 {
1414     ZOOM_options_setl(p->options, key, val, len);
1415 }
1416
1417 ZOOM_API(int)
1418     ZOOM_connection_exec_task(ZOOM_connection c)
1419 {
1420     ZOOM_task task = c->tasks;
1421     zoom_ret ret = zoom_complete;
1422
1423     if (!task)
1424         return 0;
1425     yaz_log(c->log_details, "%p ZOOM_connection_exec_task type=%d run=%d",
1426             c, task->which, task->running);
1427     if (c->error != ZOOM_ERROR_NONE)
1428     {
1429         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1430                 "removing tasks because of error = %d", c, c->error);
1431         ZOOM_connection_remove_tasks(c);
1432         return 0;
1433     }
1434     if (task->running)
1435     {
1436         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1437                 "task already running", c);
1438         return 0;
1439     }
1440     task->running = 1;
1441     ret = zoom_complete;
1442     if (c->cs || task->which == ZOOM_TASK_CONNECT)
1443     {
1444         switch (task->which)
1445         {
1446         case ZOOM_TASK_SEARCH:
1447             if (c->proto == PROTO_HTTP)
1448                 ret = ZOOM_connection_srw_send_search(c);
1449             else
1450                 ret = ZOOM_connection_Z3950_search(c);
1451             break;
1452         case ZOOM_TASK_CONNECT:
1453             ret = do_connect(c);
1454             break;
1455         case ZOOM_TASK_SCAN:
1456             if (c->proto == PROTO_HTTP)
1457                 ret = ZOOM_connection_srw_send_scan(c);
1458             else
1459                 ret = ZOOM_connection_Z3950_send_scan(c);
1460             break;
1461         case ZOOM_TASK_PACKAGE:
1462             ret = send_package(c);
1463             break;
1464         case ZOOM_TASK_SORT:
1465             c->tasks->u.sort.resultset->r_sort_spec =
1466                 ZOOM_query_get_sortspec(c->tasks->u.sort.q);
1467             ret = send_Z3950_sort(c, c->tasks->u.sort.resultset);
1468             break;
1469         }
1470     }
1471     else
1472     {
1473         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1474                 "remove tasks because no connection exist", c);
1475         ZOOM_connection_remove_tasks(c);
1476     }
1477     if (ret == zoom_complete)
1478     {
1479         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1480                 "task removed (complete)", c);
1481         ZOOM_connection_remove_task(c);
1482         return 0;
1483     }
1484     yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1485             "task pending", c);
1486     return 1;
1487 }
1488
1489 #if YAZ_HAVE_XML2
1490
1491 static zoom_ret send_HTTP_redirect(ZOOM_connection c, const char *uri)
1492 {
1493     Z_GDU *gdu = z_get_HTTP_Request_uri(c->odr_out, uri, 0, c->proxy ? 1 : 0);
1494
1495     gdu->u.HTTP_Request->method = odr_strdup(c->odr_out, "GET");
1496     z_HTTP_header_add(c->odr_out, &gdu->u.HTTP_Request->headers, "Accept",
1497                       "text/xml");
1498     yaz_cookies_request(c->cookies, c->odr_out, gdu->u.HTTP_Request);
1499     if (c->user && c->password)
1500     {
1501         z_HTTP_header_add_basic_auth(c->odr_out, &gdu->u.HTTP_Request->headers,
1502                                      c->user, c->password);
1503     }
1504     return ZOOM_send_GDU(c, gdu);
1505 }
1506
1507 zoom_ret ZOOM_send_GDU(ZOOM_connection c, Z_GDU *gdu)
1508 {
1509     ZOOM_Event event;
1510
1511     int r = z_GDU(c->odr_out, &gdu, 0, 0);
1512     if (!r)
1513         return zoom_complete;
1514     if (c->odr_print)
1515         z_GDU(c->odr_print, &gdu, 0, 0);
1516     if (c->odr_save)
1517         z_GDU(c->odr_save, &gdu, 0, 0);
1518     c->buf_out = odr_getbuf(c->odr_out, &c->len_out, 0);
1519     odr_reset(c->odr_out);
1520
1521     event = ZOOM_Event_create(ZOOM_EVENT_SEND_APDU);
1522     ZOOM_connection_put_event(c, event);
1523
1524     return ZOOM_send_buf(c);
1525 }
1526
1527 #if YAZ_HAVE_XML2
1528 void ZOOM_set_HTTP_error(ZOOM_connection c, int error,
1529                          const char *addinfo, const char *addinfo2)
1530 {
1531     ZOOM_set_dset_error(c, error, "HTTP", addinfo, addinfo2);
1532 }
1533 #endif
1534
1535
1536 static void handle_http(ZOOM_connection c, Z_HTTP_Response *hres)
1537 {
1538     zoom_ret cret = zoom_complete;
1539     int ret = -1;
1540     char *addinfo = 0;
1541     const char *connection_head = z_HTTP_header_lookup(hres->headers,
1542                                                        "Connection");
1543     const char *location;
1544
1545     ZOOM_connection_set_mask(c, 0);
1546     yaz_log(c->log_details, "%p handle_http", c);
1547
1548     yaz_cookies_response(c->cookies, hres);
1549     if ((hres->code == 301 || hres->code == 302) && c->sru_mode == zoom_sru_get
1550         && (location = z_HTTP_header_lookup(hres->headers, "Location")))
1551     {
1552         c->no_redirects++;
1553         if (c->no_redirects > 10)
1554         {
1555             ZOOM_set_HTTP_error(c, hres->code, 0, 0);
1556             c->no_redirects = 0;
1557             ZOOM_connection_close(c);
1558         }
1559         else
1560         {
1561             /* since redirect may change host we just reconnect. A smarter
1562                implementation might check whether it's the same server */
1563
1564             int host_change = 0;
1565             location = yaz_check_location(c->odr_in, c->host_port,
1566                                           location, &host_change);
1567             if (do_connect_host(c, location) == zoom_complete)
1568                 return;  /* connect failed.. */
1569             send_HTTP_redirect(c, location);
1570             return;
1571         }
1572     }
1573     else
1574     {
1575         ret = ZOOM_handle_sru(c, hres, &cret, &addinfo);
1576         if (ret == 0)
1577         {
1578             if (c->no_redirects) /* end of redirect. change hosts again */
1579                 ZOOM_connection_close(c);
1580         }
1581         c->no_redirects = 0;
1582     }
1583     if (ret)
1584     {
1585         if (hres->code != 200)
1586             ZOOM_set_HTTP_error(c, hres->code, 0, 0);
1587         else
1588         {
1589             yaz_log(YLOG_LOG, "set error... addinfo=%s", addinfo ?
1590                     addinfo : "NULL");
1591             ZOOM_set_error(c, ZOOM_ERROR_DECODE, addinfo);
1592         }
1593         ZOOM_connection_close(c);
1594     }
1595     if (cret == zoom_complete)
1596     {
1597         yaz_log(c->log_details, "removing tasks in handle_http");
1598         ZOOM_connection_remove_task(c);
1599     }
1600     {
1601         int must_close = 0;
1602         if (!strcmp(hres->version, "1.0"))
1603         {
1604             /* HTTP 1.0: only if Keep-Alive we stay alive.. */
1605             if (!connection_head || strcmp(connection_head, "Keep-Alive"))
1606                 must_close = 1;
1607         }
1608         else
1609         {
1610             /* HTTP 1.1: only if no close we stay alive.. */
1611             if (connection_head && !strcmp(connection_head, "close"))
1612                 must_close = 1;
1613         }
1614         if (must_close)
1615         {
1616             ZOOM_connection_close(c);
1617             if (c->tasks)
1618             {
1619                 c->tasks->running = 0;
1620                 ZOOM_connection_insert_task(c, ZOOM_TASK_CONNECT);
1621                 c->reconnect_ok = 0;
1622             }
1623         }
1624         else
1625             c->reconnect_ok = 1; /* if the server closes anyway */
1626     }
1627 }
1628 #endif
1629
1630 static int do_read(ZOOM_connection c)
1631 {
1632     int r, more;
1633     ZOOM_Event event;
1634
1635     event = ZOOM_Event_create(ZOOM_EVENT_RECV_DATA);
1636     ZOOM_connection_put_event(c, event);
1637
1638     r = cs_get(c->cs, &c->buf_in, &c->len_in);
1639     more = cs_more(c->cs);
1640     yaz_log(c->log_details, "%p do_read len=%d more=%d", c, r, more);
1641     if (r == 1)
1642         return 0;
1643     if (r <= 0)
1644     {
1645         if (!ZOOM_test_reconnect(c))
1646         {
1647             ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
1648             ZOOM_connection_close(c);
1649         }
1650     }
1651     else
1652     {
1653         Z_GDU *gdu;
1654         ZOOM_Event event;
1655
1656         odr_reset(c->odr_in);
1657         odr_setbuf(c->odr_in, c->buf_in, r, 0);
1658         event = ZOOM_Event_create(ZOOM_EVENT_RECV_APDU);
1659         ZOOM_connection_put_event(c, event);
1660
1661         if (!z_GDU(c->odr_in, &gdu, 0, 0))
1662         {
1663             int x;
1664             int err = odr_geterrorx(c->odr_in, &x);
1665             char msg[100];
1666             const char *element = odr_getelement(c->odr_in);
1667             yaz_snprintf(msg, sizeof(msg),
1668                     "ODR code %d:%d element=%s offset=%d",
1669                     err, x, element ? element : "<unknown>",
1670                     odr_offset(c->odr_in));
1671             ZOOM_set_error(c, ZOOM_ERROR_DECODE, msg);
1672             if (c->log_api)
1673             {
1674                 FILE *ber_file = yaz_log_file();
1675                 if (ber_file)
1676                     odr_dumpBER(ber_file, c->buf_in, r);
1677             }
1678             ZOOM_connection_close(c);
1679         }
1680         else
1681         {
1682             if (c->odr_print)
1683                 z_GDU(c->odr_print, &gdu, 0, 0);
1684             if (c->odr_save)
1685                 z_GDU(c->odr_save, &gdu, 0, 0);
1686             if (gdu->which == Z_GDU_Z3950)
1687                 ZOOM_handle_Z3950_apdu(c, gdu->u.z3950);
1688             else if (gdu->which == Z_GDU_HTTP_Response)
1689             {
1690 #if YAZ_HAVE_XML2
1691                 handle_http(c, gdu->u.HTTP_Response);
1692 #else
1693                 ZOOM_set_error(c, ZOOM_ERROR_DECODE, 0);
1694                 ZOOM_connection_close(c);
1695 #endif
1696             }
1697         }
1698     }
1699     return 1;
1700 }
1701
1702 static zoom_ret do_write_ex(ZOOM_connection c, char *buf_out, int len_out)
1703 {
1704     int r;
1705     ZOOM_Event event;
1706
1707     event = ZOOM_Event_create(ZOOM_EVENT_SEND_DATA);
1708     ZOOM_connection_put_event(c, event);
1709
1710     yaz_log(c->log_details, "%p do_write_ex len=%d", c, len_out);
1711     if ((r = cs_put(c->cs, buf_out, len_out)) < 0)
1712     {
1713         yaz_log(c->log_details, "%p do_write_ex write failed", c);
1714         if (ZOOM_test_reconnect(c))
1715         {
1716             return zoom_pending;
1717         }
1718         if (c->state == STATE_CONNECTING)
1719             ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
1720         else
1721             ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
1722         ZOOM_connection_close(c);
1723         return zoom_complete;
1724     }
1725     else if (r == 1)
1726     {
1727         int mask = ZOOM_SELECT_EXCEPT;
1728         if (c->cs->io_pending & CS_WANT_WRITE)
1729             mask += ZOOM_SELECT_WRITE;
1730         if (c->cs->io_pending & CS_WANT_READ)
1731             mask += ZOOM_SELECT_READ;
1732         ZOOM_connection_set_mask(c, mask);
1733         yaz_log(c->log_details, "%p do_write_ex write incomplete mask=%d",
1734                 c, c->mask);
1735     }
1736     else
1737     {
1738         ZOOM_connection_set_mask(c, ZOOM_SELECT_READ|ZOOM_SELECT_EXCEPT);
1739         yaz_log(c->log_details, "%p do_write_ex write complete mask=%d",
1740                 c, c->mask);
1741     }
1742     return zoom_pending;
1743 }
1744
1745 zoom_ret ZOOM_send_buf(ZOOM_connection c)
1746 {
1747     return do_write_ex(c, c->buf_out, c->len_out);
1748 }
1749
1750
1751 ZOOM_API(const char *)
1752     ZOOM_connection_option_get(ZOOM_connection c, const char *key)
1753 {
1754     if (!strcmp(key, "APDU"))
1755     {
1756         return c->saveAPDU_wrbuf ? wrbuf_cstr(c->saveAPDU_wrbuf) : "";
1757     }
1758     else
1759         return ZOOM_options_get(c->options, key);
1760 }
1761
1762 ZOOM_API(const char *)
1763     ZOOM_connection_option_getl(ZOOM_connection c, const char *key, int *lenp)
1764 {
1765     if (!strcmp(key, "APDU"))
1766     {
1767         if (c->saveAPDU_wrbuf)
1768         {
1769             *lenp = wrbuf_len(c->saveAPDU_wrbuf);
1770             return wrbuf_cstr(c->saveAPDU_wrbuf);
1771         }
1772         else
1773         {
1774             *lenp = 0;
1775             return "";
1776         }
1777     }
1778     else
1779         return ZOOM_options_getl(c->options, key, lenp);
1780 }
1781
1782 ZOOM_API(void)
1783     ZOOM_connection_option_set(ZOOM_connection c, const char *key,
1784                                const char *val)
1785 {
1786     if (!strcmp(key, "saveAPDU"))
1787     {
1788         if (val && strcmp(val, "0"))
1789         {
1790             if (!c->saveAPDU_wrbuf)
1791                 c->saveAPDU_wrbuf = wrbuf_alloc();
1792             else
1793                 wrbuf_rewind(c->saveAPDU_wrbuf);
1794         }
1795         else
1796         {
1797             wrbuf_destroy(c->saveAPDU_wrbuf);
1798             c->saveAPDU_wrbuf = 0;
1799         }
1800         ZOOM_connection_save_apdu_wrbuf(c, c->saveAPDU_wrbuf);
1801     }
1802     else
1803         ZOOM_options_set(c->options, key, val);
1804 }
1805
1806 ZOOM_API(void)
1807     ZOOM_connection_option_setl(ZOOM_connection c, const char *key,
1808                                 const char *val, int len)
1809 {
1810     ZOOM_options_setl(c->options, key, val, len);
1811 }
1812
1813 ZOOM_API(const char *)
1814     ZOOM_resultset_option_get(ZOOM_resultset r, const char *key)
1815 {
1816     return ZOOM_options_get(r->options, key);
1817 }
1818
1819 ZOOM_API(void)
1820     ZOOM_resultset_option_set(ZOOM_resultset r, const char *key,
1821                               const char *val)
1822 {
1823     ZOOM_options_set(r->options, key, val);
1824 }
1825
1826
1827 ZOOM_API(int)
1828     ZOOM_connection_errcode(ZOOM_connection c)
1829 {
1830     return ZOOM_connection_error(c, 0, 0);
1831 }
1832
1833 ZOOM_API(const char *)
1834     ZOOM_connection_errmsg(ZOOM_connection c)
1835 {
1836     const char *msg;
1837     ZOOM_connection_error(c, &msg, 0);
1838     return msg;
1839 }
1840
1841 ZOOM_API(const char *)
1842     ZOOM_connection_addinfo(ZOOM_connection c)
1843 {
1844     const char *addinfo;
1845     ZOOM_connection_error(c, 0, &addinfo);
1846     return addinfo;
1847 }
1848
1849 ZOOM_API(const char *)
1850     ZOOM_connection_diagset(ZOOM_connection c)
1851 {
1852     const char *diagset;
1853     ZOOM_connection_error_x(c, 0, 0, &diagset);
1854     return diagset;
1855 }
1856
1857 ZOOM_API(const char *)
1858     ZOOM_diag_str(int error)
1859 {
1860     switch (error)
1861     {
1862     case ZOOM_ERROR_NONE:
1863         return "No error";
1864     case ZOOM_ERROR_CONNECT:
1865         return "Connect failed";
1866     case ZOOM_ERROR_MEMORY:
1867         return "Out of memory";
1868     case ZOOM_ERROR_ENCODE:
1869         return "Encoding failed";
1870     case ZOOM_ERROR_DECODE:
1871         return "Decoding failed";
1872     case ZOOM_ERROR_CONNECTION_LOST:
1873         return "Connection lost";
1874     case ZOOM_ERROR_INIT:
1875         return "Init rejected";
1876     case ZOOM_ERROR_INTERNAL:
1877         return "Internal failure";
1878     case ZOOM_ERROR_TIMEOUT:
1879         return "Timeout";
1880     case ZOOM_ERROR_UNSUPPORTED_PROTOCOL:
1881         return "Unsupported protocol";
1882     case ZOOM_ERROR_UNSUPPORTED_QUERY:
1883         return "Unsupported query type";
1884     case ZOOM_ERROR_INVALID_QUERY:
1885         return "Invalid query";
1886     case ZOOM_ERROR_CQL_PARSE:
1887         return "CQL parsing error";
1888     case ZOOM_ERROR_CQL_TRANSFORM:
1889         return "CQL transformation error";
1890     case ZOOM_ERROR_CCL_CONFIG:
1891         return "CCL configuration error";
1892     case ZOOM_ERROR_CCL_PARSE:
1893         return "CCL parsing error";
1894     case ZOOM_ERROR_ES_INVALID_ACTION:
1895         return "Extended Service. invalid action";
1896     case ZOOM_ERROR_ES_INVALID_VERSION:
1897         return "Extended Service. invalid version";
1898     case ZOOM_ERROR_ES_INVALID_SYNTAX:
1899         return "Extended Service. invalid syntax";
1900     case ZOOM_ERROR_MEMCACHED:
1901         return "Memcached";
1902     default:
1903         return diagbib1_str(error);
1904     }
1905 }
1906
1907 ZOOM_API(int)
1908     ZOOM_connection_error_x(ZOOM_connection c, const char **cp,
1909                             const char **addinfo, const char **diagset)
1910 {
1911     int error = c->error;
1912     if (cp)
1913     {
1914         if (!c->diagset || !strcmp(c->diagset, "ZOOM"))
1915             *cp = ZOOM_diag_str(error);
1916         else if (!strcmp(c->diagset, "HTTP"))
1917             *cp = z_HTTP_errmsg(c->error);
1918         else if (!strcmp(c->diagset, "Bib-1"))
1919             *cp = ZOOM_diag_str(error);
1920         else if (!strcmp(c->diagset, "info:srw/diagnostic/1"))
1921             *cp = yaz_diag_srw_str(c->error);
1922         else
1923             *cp = "Unknown error and diagnostic set";
1924     }
1925     if (addinfo)
1926         *addinfo = c->addinfo ? c->addinfo : "";
1927     if (diagset)
1928         *diagset = c->diagset ? c->diagset : "";
1929     return c->error;
1930 }
1931
1932 ZOOM_API(int)
1933     ZOOM_connection_error(ZOOM_connection c, const char **cp,
1934                           const char **addinfo)
1935 {
1936     return ZOOM_connection_error_x(c, cp, addinfo, 0);
1937 }
1938
1939 static void ZOOM_connection_do_io(ZOOM_connection c, int mask)
1940 {
1941     ZOOM_Event event = 0;
1942     int r = cs_look(c->cs);
1943     yaz_log(c->log_details, "%p ZOOM_connection_do_io mask=%d cs_look=%d",
1944             c, mask, r);
1945
1946     if (r == CS_NONE)
1947     {
1948         event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1949         ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
1950         ZOOM_connection_close(c);
1951         ZOOM_connection_put_event(c, event);
1952     }
1953     else if (r == CS_CONNECT)
1954     {
1955         int ret = ret = cs_rcvconnect(c->cs);
1956         yaz_log(c->log_details, "%p ZOOM_connection_do_io "
1957                 "cs_rcvconnect returned %d", c, ret);
1958         if (ret == 1)
1959         {
1960             int mask = ZOOM_SELECT_EXCEPT;
1961             if (c->cs->io_pending & CS_WANT_WRITE)
1962                 mask += ZOOM_SELECT_WRITE;
1963             if (c->cs->io_pending & CS_WANT_READ)
1964                 mask += ZOOM_SELECT_READ;
1965             ZOOM_connection_set_mask(c, mask);
1966             event = ZOOM_Event_create(ZOOM_EVENT_NONE);
1967             ZOOM_connection_put_event(c, event);
1968         }
1969         else if (ret == 0)
1970         {
1971             event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1972             ZOOM_connection_put_event(c, event);
1973             get_cert(c);
1974             if (c->proto == PROTO_Z3950)
1975                 ZOOM_connection_Z3950_send_init(c);
1976             else
1977             {
1978                 /* no init request for SRW .. */
1979                 assert(c->tasks->which == ZOOM_TASK_CONNECT);
1980                 ZOOM_connection_remove_task(c);
1981                 ZOOM_connection_set_mask(c, 0);
1982                 ZOOM_connection_exec_task(c);
1983             }
1984             c->state = STATE_ESTABLISHED;
1985         }
1986         else
1987         {
1988             ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
1989             ZOOM_connection_close(c);
1990         }
1991     }
1992     else
1993     {
1994         if (mask & ZOOM_SELECT_EXCEPT)
1995         {
1996             if (!ZOOM_test_reconnect(c))
1997             {
1998                 ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
1999                 ZOOM_connection_close(c);
2000             }
2001             return;
2002         }
2003         if (mask & ZOOM_SELECT_READ)
2004             do_read(c);
2005         if (c->cs && (mask & ZOOM_SELECT_WRITE))
2006             ZOOM_send_buf(c);
2007     }
2008 }
2009
2010 ZOOM_API(int)
2011     ZOOM_connection_last_event(ZOOM_connection cs)
2012 {
2013     if (!cs)
2014         return ZOOM_EVENT_NONE;
2015     return cs->last_event;
2016 }
2017
2018
2019 ZOOM_API(int) ZOOM_connection_fire_event_timeout(ZOOM_connection c)
2020 {
2021     if (c->mask)
2022     {
2023         ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_TIMEOUT);
2024         /* timeout and this connection was waiting */
2025         ZOOM_set_error(c, ZOOM_ERROR_TIMEOUT, 0);
2026         ZOOM_connection_close(c);
2027         ZOOM_connection_put_event(c, event);
2028     }
2029     return 0;
2030 }
2031
2032 ZOOM_API(int)
2033     ZOOM_connection_process(ZOOM_connection c)
2034 {
2035     ZOOM_Event event;
2036     if (!c)
2037         return 0;
2038
2039     event = ZOOM_connection_get_event(c);
2040     if (event)
2041     {
2042         ZOOM_Event_destroy(event);
2043         return 1;
2044     }
2045     ZOOM_connection_exec_task(c);
2046     event = ZOOM_connection_get_event(c);
2047     if (event)
2048     {
2049         ZOOM_Event_destroy(event);
2050         return 1;
2051     }
2052     return 0;
2053 }
2054
2055 ZOOM_API(int)
2056     ZOOM_event_nonblock(int no, ZOOM_connection *cs)
2057 {
2058     int i;
2059
2060     yaz_log(log_details0, "ZOOM_process_event(no=%d,cs=%p)", no, cs);
2061
2062     for (i = 0; i<no; i++)
2063     {
2064         ZOOM_connection c = cs[i];
2065
2066         if (c && ZOOM_connection_process(c))
2067             return i+1;
2068     }
2069     return 0;
2070 }
2071
2072 ZOOM_API(int) ZOOM_connection_fire_event_socket(ZOOM_connection c, int mask)
2073 {
2074     if (c->mask && mask)
2075         ZOOM_connection_do_io(c, mask);
2076     return 0;
2077 }
2078
2079 ZOOM_API(int) ZOOM_connection_get_socket(ZOOM_connection c)
2080 {
2081     if (c->cs)
2082         return cs_fileno(c->cs);
2083     return -1;
2084 }
2085
2086 ZOOM_API(int) ZOOM_connection_set_mask(ZOOM_connection c, int mask)
2087 {
2088     c->mask = mask;
2089     if (!c->cs)
2090         return -1;
2091     return 0;
2092 }
2093
2094 ZOOM_API(int) ZOOM_connection_get_mask(ZOOM_connection c)
2095 {
2096     if (c->cs)
2097         return c->mask;
2098     return 0;
2099 }
2100
2101 ZOOM_API(int) ZOOM_connection_get_timeout(ZOOM_connection c)
2102 {
2103     return ZOOM_options_get_int(c->options, "timeout", 30);
2104 }
2105
2106 ZOOM_API(void) ZOOM_connection_close(ZOOM_connection c)
2107 {
2108     if (c->cs)
2109         cs_close(c->cs);
2110     c->cs = 0;
2111     ZOOM_connection_set_mask(c, 0);
2112     c->state = STATE_IDLE;
2113 }
2114
2115 /*
2116  * Local variables:
2117  * c-basic-offset: 4
2118  * c-file-style: "Stroustrup"
2119  * indent-tabs-mode: nil
2120  * End:
2121  * vim: shiftwidth=4 tabstop=8 expandtab
2122  */
2123