37b87f969f2a39e2c492a2ace346e1b50f9dbe11
[yaz-moved-to-github.git] / src / zoom-c.c
1 /* This file is part of the YAZ toolkit.
2  * Copyright (C) Index Data
3  * See the file LICENSE for details.
4  */
5 /**
6  * \file zoom-c.c
7  * \brief Implements ZOOM C interface.
8  */
9 #if HAVE_CONFIG_H
10 #include <config.h>
11 #endif
12
13 #include <assert.h>
14 #include <string.h>
15 #include <errno.h>
16 #include "zoom-p.h"
17
18 #include <yaz/yaz-util.h>
19 #include <yaz/xmalloc.h>
20 #include <yaz/otherinfo.h>
21 #include <yaz/log.h>
22 #include <yaz/diagbib1.h>
23 #include <yaz/charneg.h>
24 #include <yaz/query-charset.h>
25 #include <yaz/snprintf.h>
26 #include <yaz/facet.h>
27
28 #include <yaz/shptr.h>
29
30 #if SHPTR
31 YAZ_SHPTR_TYPE(WRBUF)
32 #endif
33
34 static int log_api0 = 0;
35 static int log_details0 = 0;
36
37 static void resultset_destroy(ZOOM_resultset r);
38 static zoom_ret do_write_ex(ZOOM_connection c, char *buf_out, int len_out);
39
40 static void initlog(void)
41 {
42     static int log_level_initialized = 0;
43
44     if (!log_level_initialized)
45     {
46         log_api0 = yaz_log_module_level("zoom");
47         log_details0 = yaz_log_module_level("zoomdetails");
48         log_level_initialized = 1;
49     }
50 }
51
52 void ZOOM_connection_remove_tasks(ZOOM_connection c);
53
54 void ZOOM_set_dset_error(ZOOM_connection c, int error,
55                          const char *dset,
56                          const char *addinfo, const char *addinfo2)
57 {
58     char *cp;
59
60     xfree(c->addinfo);
61     c->addinfo = 0;
62     c->error = error;
63     if (!c->diagset || strcmp(dset, c->diagset))
64     {
65         xfree(c->diagset);
66         c->diagset = xstrdup(dset);
67         /* remove integer part from SRW diagset .. */
68         if ((cp = strrchr(c->diagset, '/')))
69             *cp = '\0';
70     }
71     if (addinfo && addinfo2)
72     {
73         c->addinfo = (char*) xmalloc(strlen(addinfo) + strlen(addinfo2) + 3);
74         strcpy(c->addinfo, addinfo);
75         strcat(c->addinfo, ": ");
76         strcat(c->addinfo, addinfo2);
77     }
78     else if (addinfo)
79         c->addinfo = xstrdup(addinfo);
80     if (error != ZOOM_ERROR_NONE)
81     {
82         yaz_log(c->log_api, "%p set_dset_error %s %s:%d %s %s",
83                 c, c->host_port ? c->host_port : "<>", dset, error,
84                 addinfo ? addinfo : "",
85                 addinfo2 ? addinfo2 : "");
86         ZOOM_connection_remove_tasks(c);
87     }
88 }
89
90 int ZOOM_uri_to_code(const char *uri)
91 {
92     int code = 0;
93     const char *cp;
94     if ((cp = strrchr(uri, '/')))
95         code = atoi(cp+1);
96     return code;
97 }
98
99 void ZOOM_set_error(ZOOM_connection c, int error, const char *addinfo)
100 {
101     ZOOM_set_dset_error(c, error, "ZOOM", addinfo, 0);
102 }
103
104 static void clear_error(ZOOM_connection c)
105 {
106     /*
107      * If an error is tied to an operation then it's ok to clear: for
108      * example, a diagnostic returned from a search is cleared by a
109      * subsequent search.  However, problems such as Connection Lost
110      * or Init Refused are not cleared, because they are not
111      * recoverable: doing another search doesn't help.
112      */
113
114     ZOOM_connection_remove_events(c);
115     switch (c->error)
116     {
117     case ZOOM_ERROR_CONNECT:
118     case ZOOM_ERROR_MEMORY:
119     case ZOOM_ERROR_DECODE:
120     case ZOOM_ERROR_CONNECTION_LOST:
121     case ZOOM_ERROR_INIT:
122     case ZOOM_ERROR_INTERNAL:
123     case ZOOM_ERROR_UNSUPPORTED_PROTOCOL:
124         break;
125     default:
126         ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
127     }
128 }
129
130 void ZOOM_connection_show_task(ZOOM_task task)
131 {
132     switch(task->which)
133     {
134     case ZOOM_TASK_SEARCH:
135         yaz_log(YLOG_LOG, "search p=%p", task);
136         break;
137     case ZOOM_TASK_CONNECT:
138         yaz_log(YLOG_LOG, "connect p=%p", task);
139         break;
140     case ZOOM_TASK_SCAN:
141         yaz_log(YLOG_LOG, "scan p=%p", task);
142         break;
143     }
144 }
145
146 void ZOOM_connection_show_tasks(ZOOM_connection c)
147 {
148     ZOOM_task task;
149     yaz_log(YLOG_LOG, "connection p=%p tasks", c);
150     for (task = c->tasks; task; task = task->next)
151         ZOOM_connection_show_task(task);
152 }
153
154 ZOOM_task ZOOM_connection_add_task(ZOOM_connection c, int which)
155 {
156     ZOOM_task *taskp = &c->tasks;
157     while (*taskp)
158         taskp = &(*taskp)->next;
159     *taskp = (ZOOM_task) xmalloc(sizeof(**taskp));
160     (*taskp)->running = 0;
161     (*taskp)->which = which;
162     (*taskp)->next = 0;
163     clear_error(c);
164     return *taskp;
165 }
166
167 ZOOM_API(int) ZOOM_connection_is_idle(ZOOM_connection c)
168 {
169     return c->tasks ? 0 : 1;
170 }
171
172 ZOOM_task ZOOM_connection_insert_task(ZOOM_connection c, int which)
173 {
174     ZOOM_task task = (ZOOM_task) xmalloc(sizeof(*task));
175
176     task->next = c->tasks;
177     c->tasks = task;
178
179     task->running = 0;
180     task->which = which;
181     return task;
182 }
183
184 void ZOOM_connection_remove_task(ZOOM_connection c)
185 {
186     ZOOM_task task = c->tasks;
187
188     if (task)
189     {
190         c->tasks = task->next;
191         switch (task->which)
192         {
193         case ZOOM_TASK_SEARCH:
194             resultset_destroy(task->u.search.resultset);
195             xfree(task->u.search.syntax);
196             xfree(task->u.search.elementSetName);
197             xfree(task->u.search.schema);
198             break;
199         case ZOOM_TASK_CONNECT:
200             break;
201         case ZOOM_TASK_SCAN:
202             ZOOM_scanset_destroy(task->u.scan.scan);
203             break;
204         case ZOOM_TASK_PACKAGE:
205             ZOOM_package_destroy(task->u.package);
206             break;
207         case ZOOM_TASK_SORT:
208             resultset_destroy(task->u.sort.resultset);
209             ZOOM_query_destroy(task->u.sort.q);
210             break;
211         default:
212             assert(0);
213         }
214         xfree(task);
215
216         if (!c->tasks)
217         {
218             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_END);
219             ZOOM_connection_put_event(c, event);
220         }
221     }
222 }
223
224 void ZOOM_connection_remove_tasks(ZOOM_connection c)
225 {
226     while (c->tasks)
227         ZOOM_connection_remove_task(c);
228 }
229
230 static void odr_wrbuf_write(ODR o, void *handle, int type,
231                             const char *buf, int len)
232 {
233     WRBUF w = (WRBUF) handle;
234     wrbuf_write(w, buf, len);
235 }
236
237 ZOOM_API(ZOOM_connection)
238     ZOOM_connection_create(ZOOM_options options)
239 {
240     ZOOM_connection c = (ZOOM_connection) xmalloc(sizeof(*c));
241
242     initlog();
243
244     c->log_api = log_api0;
245     c->log_details = log_details0;
246
247     yaz_log(c->log_api, "%p ZOOM_connection_create", c);
248
249     c->proto = PROTO_Z3950;
250     c->cs = 0;
251     ZOOM_connection_set_mask(c, 0);
252     c->reconnect_ok = 0;
253     c->state = STATE_IDLE;
254     c->addinfo = 0;
255     c->diagset = 0;
256     ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
257     c->buf_in = 0;
258     c->len_in = 0;
259     c->buf_out = 0;
260     c->len_out = 0;
261     c->resultsets = 0;
262
263     c->options = ZOOM_options_create_with_parent(options);
264
265     c->host_port = 0;
266     c->proxy = 0;
267     c->tproxy = 0;
268
269     c->charset = c->lang = 0;
270
271     c->cookie_out = 0;
272     c->cookie_in = 0;
273     c->client_IP = 0;
274     c->tasks = 0;
275
276     c->user = 0;
277     c->group = 0;
278     c->password = 0;
279     c->url_authentication = 0;
280
281     c->maximum_record_size = 0;
282     c->preferred_message_size = 0;
283
284     c->odr_in = odr_createmem(ODR_DECODE);
285     c->odr_out = odr_createmem(ODR_ENCODE);
286     c->odr_print = 0;
287     c->odr_save = 0;
288
289     c->async = 0;
290     c->support_named_resultsets = 0;
291     c->last_event = ZOOM_EVENT_NONE;
292
293     c->m_queue_front = 0;
294     c->m_queue_back = 0;
295
296     c->sru_version = 0;
297     c->no_redirects = 0;
298     c->cookies = 0;
299     c->saveAPDU_wrbuf = 0;
300
301     ZOOM_memcached_init(c);
302     return c;
303 }
304
305 ZOOM_API(void) ZOOM_connection_save_apdu_wrbuf(ZOOM_connection c, WRBUF w)
306 {
307     if (c->odr_save)
308     {
309         odr_destroy(c->odr_save);
310         c->odr_save = 0;
311     }
312     if (w)
313     {
314         c->odr_save = odr_createmem(ODR_PRINT);
315         odr_set_stream(c->odr_save, w, odr_wrbuf_write, 0);
316     }
317 }
318
319 /* set database names. Take local databases (if set); otherwise
320    take databases given in ZURL (if set); otherwise use Default */
321 char **ZOOM_connection_get_databases(ZOOM_connection con, ZOOM_options options,
322                                      int *num, ODR odr)
323 {
324     char **databaseNames;
325     const char *cp = ZOOM_options_get(options, "databaseName");
326
327     if ((!cp || !*cp) && con->host_port)
328         cs_get_host_args(con->host_port, &cp);
329     if (!cp || !*cp)
330         cp = "Default";
331     nmem_strsplit(odr_getmem(odr), "+", cp,  &databaseNames, num);
332     return databaseNames;
333 }
334
335 ZOOM_API(ZOOM_connection)
336     ZOOM_connection_new(const char *host, int portnum)
337 {
338     ZOOM_connection c = ZOOM_connection_create(0);
339
340     ZOOM_connection_connect(c, host, portnum);
341     return c;
342 }
343
344 static zoom_sru_mode get_sru_mode_from_string(const char *s)
345 {
346     if (!s || !*s)
347         return zoom_sru_soap;
348     if (!yaz_matchstr(s, "soap"))
349         return zoom_sru_soap;
350     else if (!yaz_matchstr(s, "get"))
351         return zoom_sru_get;
352     else if (!yaz_matchstr(s, "post"))
353         return zoom_sru_post;
354     else if (!yaz_matchstr(s, "solr"))
355         return zoom_sru_solr;
356     return zoom_sru_error;
357 }
358
359 ZOOM_API(void)
360     ZOOM_connection_connect(ZOOM_connection c,
361                             const char *host, int portnum)
362 {
363     const char *val;
364
365     initlog();
366
367     yaz_log(c->log_api, "%p ZOOM_connection_connect host=%s portnum=%d",
368             c, host ? host : "null", portnum);
369
370     ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
371     ZOOM_connection_remove_tasks(c);
372
373     if (c->odr_print)
374     {
375         odr_setprint(c->odr_print, 0); /* prevent destroy from fclose'ing */
376         odr_destroy(c->odr_print);
377     }
378     if (ZOOM_options_get_bool(c->options, "apdulog", 0))
379     {
380         c->odr_print = odr_createmem(ODR_PRINT);
381         odr_setprint(c->odr_print, yaz_log_file());
382     }
383     else
384         c->odr_print = 0;
385
386     if (c->cs)
387     {
388         yaz_log(c->log_details, "%p ZOOM_connection_connect reconnect ok", c);
389         c->reconnect_ok = 1;
390         return;
391     }
392     yaz_log(c->log_details, "%p ZOOM_connection_connect connect", c);
393     xfree(c->proxy);
394     c->proxy = 0;
395     val = ZOOM_options_get(c->options, "proxy");
396     if (val && *val)
397     {
398         yaz_log(c->log_details, "%p ZOOM_connection_connect proxy=%s", c, val);
399         c->proxy = xstrdup(val);
400     }
401
402     xfree(c->tproxy);
403     c->tproxy = 0;
404     val = ZOOM_options_get(c->options, "tproxy");
405     if (val && *val)
406     {
407         yaz_log(c->log_details, "%p ZOOM_connection_connect tproxy=%s", c, val);
408         c->tproxy = xstrdup(val);
409     }
410
411     xfree(c->charset);
412     c->charset = 0;
413     val = ZOOM_options_get(c->options, "charset");
414     if (val && *val)
415     {
416         yaz_log(c->log_details, "%p ZOOM_connection_connect charset=%s", c, val);
417         c->charset = xstrdup(val);
418     }
419
420     xfree(c->lang);
421     val = ZOOM_options_get(c->options, "lang");
422     if (val && *val)
423     {
424         yaz_log(c->log_details, "%p ZOOM_connection_connect lang=%s", c, val);
425         c->lang = xstrdup(val);
426     }
427     else
428         c->lang = 0;
429
430     if (host)
431     {
432         char hostn[128];
433         const char *http_lead;
434
435         val = ZOOM_options_get(c->options, "sru");
436         if (val && *val && !strstr(host, "://"))
437             http_lead = "http://";
438         else
439             http_lead = "";
440         c->sru_mode = get_sru_mode_from_string(val);
441
442         xfree(c->host_port);
443         if (portnum)
444         {
445             sprintf(hostn, "%.80s:%d", host, portnum);
446             host = hostn;
447         }
448         c->host_port = xmalloc(strlen(host) + strlen(http_lead) + 1);
449         strcpy(c->host_port, http_lead);
450         strcat(c->host_port, host);
451     }
452
453     {
454         /*
455          * If the "<scheme>:" part of the host string is preceded by one
456          * or more comma-separated <name>=<value> pairs, these are taken
457          * to be options to be set on the connection object.  Among other
458          * applications, this facility can be used to embed authentication
459          * in a host string:
460          *          user=admin,password=secret,tcp:localhost:9999
461          */
462         char *remainder = c->host_port;
463         char *pcolon = strchr(remainder, ':');
464         char *pcomma;
465         char *pequals;
466         while ((pcomma = strchr(remainder, ',')) != 0 &&
467                (pcolon == 0 || pcomma < pcolon))
468         {
469             *pcomma = '\0';
470             if ((pequals = strchr(remainder, '=')) != 0)
471             {
472                 *pequals = '\0';
473                 ZOOM_connection_option_set(c, remainder, pequals+1);
474             }
475             remainder = pcomma+1;
476         }
477
478         if (remainder != c->host_port)
479         {
480             remainder = xstrdup(remainder);
481             xfree(c->host_port);
482             c->host_port = remainder;
483         }
484     }
485
486     xfree(c->sru_version);
487     val = ZOOM_options_get(c->options, "sru_version");
488     c->sru_version = xstrdup(val ? val : "1.2");
489
490     ZOOM_options_set(c->options, "host", c->host_port);
491
492     xfree(c->cookie_out);
493     c->cookie_out = 0;
494     val = ZOOM_options_get(c->options, "cookie");
495     if (val && *val)
496     {
497         yaz_log(c->log_details, "%p ZOOM_connection_connect cookie=%s", c, val);
498         c->cookie_out = xstrdup(val);
499     }
500
501     xfree(c->client_IP);
502     c->client_IP = 0;
503     val = ZOOM_options_get(c->options, "clientIP");
504     if (val && *val)
505     {
506         yaz_log(c->log_details, "%p ZOOM_connection_connect clientIP=%s",
507                 c, val);
508         c->client_IP = xstrdup(val);
509     }
510
511     xfree(c->group);
512     c->group = 0;
513     val = ZOOM_options_get(c->options, "group");
514     if (val && *val)
515         c->group = xstrdup(val);
516
517     xfree(c->user);
518     c->user = 0;
519     val = ZOOM_options_get(c->options, "user");
520     if (val && *val)
521         c->user = xstrdup(val);
522
523     xfree(c->password);
524     c->password = 0;
525     val = ZOOM_options_get(c->options, "password");
526     if (!val)
527         val = ZOOM_options_get(c->options, "pass");
528     if (val && *val)
529         c->password = xstrdup(val);
530
531     val = ZOOM_options_get(c->options, "authenticationMode");
532     if (val && !strcmp(val, "url"))
533         c->url_authentication = 1;
534     else
535         c->url_authentication = 0;
536
537     c->maximum_record_size =
538         ZOOM_options_get_int(c->options, "maximumRecordSize", 64*1024*1024);
539     c->preferred_message_size =
540         ZOOM_options_get_int(c->options, "preferredMessageSize", 64*1024*1024);
541
542     c->async = ZOOM_options_get_bool(c->options, "async", 0);
543
544     yaz_cookies_destroy(c->cookies);
545     c->cookies = yaz_cookies_create();
546
547     if (ZOOM_memcached_configure(c))
548     {
549         ZOOM_connection_remove_tasks(c);
550         return;
551     }
552     if (c->sru_mode == zoom_sru_error)
553     {
554         ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, val);
555         ZOOM_connection_remove_tasks(c);
556         return;
557     }
558
559     yaz_log(c->log_details, "%p ZOOM_connection_connect async=%d", c, c->async);
560     ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
561
562     if (!c->async)
563     {
564         while (ZOOM_event(1, &c))
565             ;
566     }
567 }
568
569 ZOOM_API(void) ZOOM_resultset_release(ZOOM_resultset r)
570 {
571     if (r->connection)
572     {
573         /* remove ourselves from the resultsets in connection */
574         ZOOM_resultset *rp = &r->connection->resultsets;
575         while (1)
576         {
577             assert(*rp);   /* we must be in this list!! */
578             if (*rp == r)
579             {   /* OK, we're here - take us out of it */
580                 *rp = (*rp)->next;
581                 break;
582             }
583             rp = &(*rp)->next;
584         }
585         r->connection = 0;
586     }
587 }
588
589 ZOOM_API(void)
590     ZOOM_connection_destroy(ZOOM_connection c)
591 {
592     ZOOM_resultset r;
593     if (!c)
594         return;
595     yaz_log(c->log_api, "%p ZOOM_connection_destroy", c);
596
597     ZOOM_memcached_destroy(c);
598     if (c->cs)
599         cs_close(c->cs);
600
601     for (r = c->resultsets; r; r = r->next)
602         r->connection = 0;
603
604     xfree(c->buf_in);
605     xfree(c->addinfo);
606     xfree(c->diagset);
607     odr_destroy(c->odr_in);
608     odr_destroy(c->odr_out);
609     if (c->odr_save)
610         odr_destroy(c->odr_save);
611     if (c->odr_print)
612     {
613         odr_setprint(c->odr_print, 0); /* prevent destroy from fclose'ing */
614         odr_destroy(c->odr_print);
615     }
616     ZOOM_options_destroy(c->options);
617     ZOOM_connection_remove_tasks(c);
618     ZOOM_connection_remove_events(c);
619     xfree(c->host_port);
620     xfree(c->proxy);
621     xfree(c->tproxy);
622     xfree(c->charset);
623     xfree(c->lang);
624     xfree(c->cookie_out);
625     xfree(c->cookie_in);
626     xfree(c->client_IP);
627     xfree(c->user);
628     xfree(c->group);
629     xfree(c->password);
630     xfree(c->sru_version);
631     yaz_cookies_destroy(c->cookies);
632     wrbuf_destroy(c->saveAPDU_wrbuf);
633     xfree(c);
634 }
635
636 void ZOOM_resultset_addref(ZOOM_resultset r)
637 {
638     if (r)
639     {
640         yaz_mutex_enter(r->mutex);
641         (r->refcount)++;
642         yaz_log(log_details0, "%p ZOOM_resultset_addref count=%d",
643                 r, r->refcount);
644         yaz_mutex_leave(r->mutex);
645     }
646 }
647
648 static int g_resultsets = 0;
649 static YAZ_MUTEX g_resultset_mutex = 0;
650
651 /* TODO We need to initialize this before running threaded:
652  * call resultset_use(0)  */
653
654 static int resultset_use(int delta) {
655     int resultset_count;
656     if (g_resultset_mutex == 0)
657         yaz_mutex_create(&g_resultset_mutex);
658     yaz_mutex_enter(g_resultset_mutex);
659     g_resultsets += delta;
660     resultset_count = g_resultsets;
661     yaz_mutex_leave(g_resultset_mutex);
662     return resultset_count;
663 }
664
665 int resultsets_count(void) {
666     return resultset_use(0);
667 }
668
669 ZOOM_resultset ZOOM_resultset_create(void)
670 {
671     int i;
672     ZOOM_resultset r = (ZOOM_resultset) xmalloc(sizeof(*r));
673
674     initlog();
675
676     yaz_log(log_details0, "%p ZOOM_resultset_create", r);
677     r->refcount = 1;
678     r->size = 0;
679     r->odr = odr_createmem(ODR_DECODE);
680     r->piggyback = 1;
681     r->setname = 0;
682     r->step = 0;
683     for (i = 0; i<RECORD_HASH_SIZE; i++)
684         r->record_hash[i] = 0;
685     r->r_sort_spec = 0;
686     r->query = 0;
687     r->connection = 0;
688     r->databaseNames = 0;
689     r->num_databaseNames = 0;
690     r->req_facets = 0;
691     r->res_facets = 0;
692     r->num_res_facets = 0;
693     r->facets_names = 0;
694     r->mutex = 0;
695     yaz_mutex_create(&r->mutex);
696 #if SHPTR
697     {
698         WRBUF w = wrbuf_alloc();
699         YAZ_SHPTR_INIT(r->record_wrbuf, w);
700     }
701 #endif
702     resultset_use(1);
703     r->mc_key = 0;
704     r->live_set = 0;
705     return r;
706 }
707
708 ZOOM_API(ZOOM_resultset)
709     ZOOM_connection_search_pqf(ZOOM_connection c, const char *q)
710 {
711     ZOOM_resultset r;
712     ZOOM_query s = ZOOM_query_create();
713
714     ZOOM_query_prefix(s, q);
715
716     r = ZOOM_connection_search(c, s);
717     ZOOM_query_destroy(s);
718     return r;
719 }
720
721 ZOOM_API(ZOOM_resultset)
722     ZOOM_connection_search(ZOOM_connection c, ZOOM_query q)
723 {
724     ZOOM_resultset r = ZOOM_resultset_create();
725     ZOOM_task task;
726     int start, count;
727     const char *syntax, *elementSetName, *schema;
728     yaz_log(c->log_api, "%p ZOOM_connection_search set %p query %p", c, r, q);
729     r->r_sort_spec = ZOOM_query_get_sortspec(q);
730     r->query = q;
731     ZOOM_query_addref(q);
732
733     r->options = ZOOM_options_create_with_parent(c->options);
734
735     r->req_facets = odr_strdup_null(r->odr,
736                                     ZOOM_options_get(r->options, "facets"));
737     start = ZOOM_options_get_int(r->options, "start", 0);
738     count = ZOOM_options_get_int(r->options, "count", 0);
739     {
740         /* If "presentChunk" is defined use that; otherwise "step" */
741         const char *cp = ZOOM_options_get(r->options, "presentChunk");
742         r->step = ZOOM_options_get_int(r->options,
743                                        (cp != 0 ? "presentChunk": "step"), 0);
744     }
745     r->piggyback = ZOOM_options_get_bool(r->options, "piggyback", 1);
746     r->setname = odr_strdup_null(r->odr,
747                                  ZOOM_options_get(r->options, "setname"));
748     r->databaseNames = ZOOM_connection_get_databases(c, c->options,
749                                                      &r->num_databaseNames,
750                                                      r->odr);
751     r->connection = c;
752     r->next = c->resultsets;
753     c->resultsets = r;
754
755     ZOOM_memcached_resultset(r, q);
756
757     if (c->host_port && c->proto == PROTO_HTTP)
758     {
759         if (!c->cs)
760         {
761             yaz_log(c->log_details, "ZOOM_connection_search: no comstack");
762             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
763         }
764         else
765         {
766             yaz_log(c->log_details, "ZOOM_connection_search: reconnect");
767             c->reconnect_ok = 1;
768         }
769     }
770
771     task = ZOOM_connection_add_task(c, ZOOM_TASK_SEARCH);
772     task->u.search.resultset = r;
773     task->u.search.start = start;
774     task->u.search.count = count;
775
776     syntax = ZOOM_options_get(r->options, "preferredRecordSyntax");
777     task->u.search.syntax = syntax ? xstrdup(syntax) : 0;
778     elementSetName = ZOOM_options_get(r->options, "elementSetName");
779     task->u.search.elementSetName = elementSetName ?
780         xstrdup(elementSetName) : 0;
781     schema = ZOOM_options_get(r->options, "schema");
782     task->u.search.schema = schema ? xstrdup(schema) : 0;
783
784     ZOOM_resultset_addref(r);
785
786     if (!c->async)
787     {
788         while (ZOOM_event(1, &c))
789             ;
790     }
791     return r;
792 }
793
794 ZOOM_API(void)
795     ZOOM_resultset_sort(ZOOM_resultset r,
796                          const char *sort_type, const char *sort_spec)
797 {
798     (void) ZOOM_resultset_sort1(r, sort_type, sort_spec);
799 }
800
801 ZOOM_API(int)
802     ZOOM_resultset_sort1(ZOOM_resultset r,
803                          const char *sort_type, const char *sort_spec)
804 {
805     ZOOM_connection c = r->connection;
806     ZOOM_task task;
807     ZOOM_query newq;
808
809     newq = ZOOM_query_create();
810     if (ZOOM_query_sortby(newq, sort_spec) < 0)
811         return -1;
812
813     yaz_log(c->log_api, "%p ZOOM_resultset_sort r=%p sort_type=%s sort_spec=%s",
814             r, r, sort_type, sort_spec);
815     if (!c)
816         return 0;
817
818     if (c->host_port && c->proto == PROTO_HTTP)
819     {
820         if (!c->cs)
821         {
822             yaz_log(c->log_details, "%p ZOOM_resultset_sort: no comstack", r);
823             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
824         }
825         else
826         {
827             yaz_log(c->log_details, "%p ZOOM_resultset_sort: prepare reconnect",
828                     r);
829             c->reconnect_ok = 1;
830         }
831     }
832
833     ZOOM_resultset_cache_reset(r);
834     task = ZOOM_connection_add_task(c, ZOOM_TASK_SORT);
835     task->u.sort.resultset = r;
836     task->u.sort.q = newq;
837
838     ZOOM_resultset_addref(r);
839
840     if (!c->async)
841     {
842         while (ZOOM_event(1, &c))
843             ;
844     }
845
846     return 0;
847 }
848
849 ZOOM_API(void)
850     ZOOM_resultset_destroy(ZOOM_resultset r)
851 {
852     resultset_destroy(r);
853 }
854
855 static void resultset_destroy(ZOOM_resultset r)
856 {
857     if (!r)
858         return;
859     yaz_mutex_enter(r->mutex);
860     (r->refcount)--;
861     yaz_log(log_details0, "%p ZOOM_resultset_destroy r=%p count=%d",
862             r, r, r->refcount);
863     if (r->refcount == 0)
864     {
865         yaz_mutex_leave(r->mutex);
866
867         yaz_log(log_details0, "%p ZOOM_connection resultset_destroy: Deleting resultset (%p) ", r->connection, r);
868         ZOOM_resultset_cache_reset(r);
869         ZOOM_resultset_release(r);
870         ZOOM_query_destroy(r->query);
871         ZOOM_options_destroy(r->options);
872         odr_destroy(r->odr);
873         yaz_mutex_destroy(&r->mutex);
874 #if SHPTR
875         YAZ_SHPTR_DEC(r->record_wrbuf, wrbuf_destroy);
876 #endif
877         wrbuf_destroy(r->mc_key);
878         resultset_use(-1);
879         xfree(r);
880     }
881     else
882         yaz_mutex_leave(r->mutex);
883 }
884
885 ZOOM_API(size_t)
886     ZOOM_resultset_size(ZOOM_resultset r)
887 {
888     return r->size;
889 }
890
891 int ZOOM_test_reconnect(ZOOM_connection c)
892 {
893     ZOOM_Event event;
894
895     if (!c->reconnect_ok)
896         return 0;
897     ZOOM_connection_close(c);
898     c->reconnect_ok = 0;
899     c->tasks->running = 0;
900     ZOOM_connection_insert_task(c, ZOOM_TASK_CONNECT);
901
902     event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
903     ZOOM_connection_put_event(c, event);
904
905     return 1;
906 }
907
908 static void ZOOM_resultset_retrieve(ZOOM_resultset r,
909                                     int force_sync, int start, int count)
910 {
911     ZOOM_task task;
912     ZOOM_connection c;
913     const char *cp;
914     const char *syntax, *elementSetName;
915
916     if (!r)
917         return;
918     yaz_log(log_details0, "%p ZOOM_resultset_retrieve force_sync=%d start=%d"
919             " count=%d", r, force_sync, start, count);
920     c = r->connection;
921     if (!c)
922         return;
923
924     if (c->host_port && c->proto == PROTO_HTTP)
925     {
926         if (!c->cs)
927         {
928             yaz_log(log_details0, "%p ZOOM_resultset_retrieve: no comstack", r);
929             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
930         }
931         else
932         {
933             yaz_log(log_details0, "%p ZOOM_resultset_retrieve: prepare "
934                     "reconnect", r);
935             c->reconnect_ok = 1;
936         }
937     }
938     task = ZOOM_connection_add_task(c, ZOOM_TASK_SEARCH);
939     task->u.search.resultset = r;
940     task->u.search.start = start;
941     task->u.search.count = count;
942
943     syntax = ZOOM_options_get(r->options, "preferredRecordSyntax");
944     task->u.search.syntax = syntax ? xstrdup(syntax) : 0;
945     elementSetName = ZOOM_options_get(r->options, "elementSetName");
946     task->u.search.elementSetName = elementSetName
947         ? xstrdup(elementSetName) : 0;
948
949     cp = ZOOM_options_get(r->options, "schema");
950     task->u.search.schema = cp ? xstrdup(cp) : 0;
951
952     ZOOM_resultset_addref(r);
953
954     if (!r->connection->async || force_sync)
955         while (r->connection && ZOOM_event(1, &r->connection))
956             ;
957 }
958
959 ZOOM_API(void)
960     ZOOM_resultset_records(ZOOM_resultset r, ZOOM_record *recs,
961                            size_t start, size_t count)
962 {
963     int force_present = 0;
964
965     if (!r)
966         return ;
967     yaz_log(log_api0, "%p ZOOM_resultset_records r=%p start=%ld count=%ld",
968             r, r, (long) start, (long) count);
969     if (count && recs)
970         force_present = 1;
971     ZOOM_resultset_retrieve(r, force_present, start, count);
972     if (force_present)
973     {
974         size_t i;
975         for (i = 0; i< count; i++)
976             recs[i] = ZOOM_resultset_record_immediate(r, i+start);
977     }
978 }
979
980 ZOOM_API(size_t)
981     ZOOM_resultset_facets_size(ZOOM_resultset r)
982 {
983     return r->num_res_facets;
984 }
985
986 ZOOM_API(ZOOM_facet_field)
987     ZOOM_resultset_get_facet_field(ZOOM_resultset r, const char *name)
988 {
989     int num = r->num_res_facets;
990     ZOOM_facet_field *facets = r->res_facets;
991     int i;
992     for (i = 0; i < num; i++)
993         if (!strcmp(facets[i]->facet_name, name))
994             return facets[i];
995     return 0;
996 }
997
998 ZOOM_API(ZOOM_facet_field)
999     ZOOM_resultset_get_facet_field_by_index(ZOOM_resultset r, int idx)
1000 {
1001     int num = r->num_res_facets;
1002     ZOOM_facet_field *facets = r->res_facets;
1003     if (idx >= 0 && idx < num)
1004         return facets[idx];
1005     return 0;
1006 }
1007
1008 ZOOM_API(ZOOM_facet_field *)
1009     ZOOM_resultset_facets(ZOOM_resultset r)
1010 {
1011     return r->res_facets;
1012 }
1013
1014 ZOOM_API(const char**)
1015     ZOOM_resultset_facets_names(ZOOM_resultset r)
1016 {
1017     return (const char **) r->facets_names;
1018 }
1019
1020 ZOOM_API(const char*)
1021     ZOOM_facet_field_name(ZOOM_facet_field field)
1022 {
1023     return field->facet_name;
1024 }
1025
1026 ZOOM_API(size_t)
1027     ZOOM_facet_field_term_count(ZOOM_facet_field field)
1028 {
1029     return field->num_terms;
1030 }
1031
1032 ZOOM_API(const char*)
1033     ZOOM_facet_field_get_term(ZOOM_facet_field field, size_t idx, int *freq)
1034 {
1035     *freq = field->facet_terms[idx].frequency;
1036     return field->facet_terms[idx].term;
1037 }
1038
1039
1040 static void get_cert(ZOOM_connection c)
1041 {
1042     char *cert_buf;
1043     int cert_len;
1044
1045     if (cs_get_peer_certificate_x509(c->cs, &cert_buf, &cert_len))
1046     {
1047         ZOOM_connection_option_setl(c, "sslPeerCert",
1048                                     cert_buf, cert_len);
1049         xfree(cert_buf);
1050     }
1051 }
1052
1053 static zoom_ret do_connect_host(ZOOM_connection c,
1054                                 const char *logical_url);
1055
1056 static zoom_ret do_connect(ZOOM_connection c)
1057 {
1058     return do_connect_host(c, c->host_port);
1059 }
1060
1061 static zoom_ret do_connect_host(ZOOM_connection c, const char *logical_url)
1062 {
1063     void *add;
1064
1065     if (c->cs)
1066         cs_close(c->cs);
1067     c->cs = cs_create_host_proxy(logical_url, CS_FLAGS_DNS_NO_BLOCK, &add,
1068                                  c->tproxy ? c->tproxy : c->proxy);
1069
1070     if (c->cs && c->cs->protocol == PROTO_HTTP)
1071     {
1072 #if YAZ_HAVE_XML2
1073         c->proto = PROTO_HTTP;
1074 #else
1075         ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, "SRW");
1076         ZOOM_connection_close(c);
1077         return zoom_complete;
1078 #endif
1079     }
1080     if (c->cs)
1081     {
1082         int ret = cs_connect(c->cs, add);
1083         if (ret == 0)
1084         {
1085             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1086             ZOOM_connection_put_event(c, event);
1087             get_cert(c);
1088             if (c->proto == PROTO_Z3950)
1089                 ZOOM_connection_Z3950_send_init(c);
1090             else
1091             {
1092                 /* no init request for SRW .. */
1093                 assert(c->tasks->which == ZOOM_TASK_CONNECT);
1094                 ZOOM_connection_remove_task(c);
1095                 ZOOM_connection_set_mask(c, 0);
1096                 ZOOM_connection_exec_task(c);
1097             }
1098             c->state = STATE_ESTABLISHED;
1099             return zoom_pending;
1100         }
1101         else if (ret > 0)
1102         {
1103             int mask = ZOOM_SELECT_EXCEPT;
1104             if (c->cs->io_pending & CS_WANT_WRITE)
1105                 mask += ZOOM_SELECT_WRITE;
1106             if (c->cs->io_pending & CS_WANT_READ)
1107                 mask += ZOOM_SELECT_READ;
1108             ZOOM_connection_set_mask(c, mask);
1109             c->state = STATE_CONNECTING;
1110             return zoom_pending;
1111         }
1112     }
1113     c->state = STATE_IDLE;
1114     ZOOM_set_error(c, ZOOM_ERROR_CONNECT, logical_url);
1115     return zoom_complete;
1116 }
1117
1118 /* returns 1 if PDU was sent OK (still pending )
1119    0 if PDU was not sent OK (nothing to wait for)
1120 */
1121
1122 ZOOM_API(ZOOM_record)
1123     ZOOM_resultset_record_immediate(ZOOM_resultset s,size_t pos)
1124 {
1125     const char *syntax =
1126         ZOOM_options_get(s->options, "preferredRecordSyntax");
1127     const char *elementSetName =
1128         ZOOM_options_get(s->options, "elementSetName");
1129     const char *schema =
1130         ZOOM_options_get(s->options, "schema");
1131
1132     return ZOOM_record_cache_lookup_i(s, pos, syntax, elementSetName, schema);
1133 }
1134
1135 ZOOM_API(ZOOM_record)
1136     ZOOM_resultset_record(ZOOM_resultset r, size_t pos)
1137 {
1138     ZOOM_record rec = ZOOM_resultset_record_immediate(r, pos);
1139
1140     if (!rec)
1141     {
1142         /*
1143          * MIKE: I think force_sync should always be zero, but I don't
1144          * want to make this change until I get the go-ahead from
1145          * Adam, in case something depends on the old synchronous
1146          * behaviour.
1147          */
1148         int force_sync = 1;
1149         if (getenv("ZOOM_RECORD_NO_FORCE_SYNC")) force_sync = 0;
1150         ZOOM_resultset_retrieve(r, force_sync, pos, 1);
1151         rec = ZOOM_resultset_record_immediate(r, pos);
1152     }
1153     return rec;
1154 }
1155
1156 ZOOM_API(ZOOM_scanset)
1157     ZOOM_connection_scan(ZOOM_connection c, const char *start)
1158 {
1159     ZOOM_scanset s;
1160     ZOOM_query q = ZOOM_query_create();
1161
1162     ZOOM_query_prefix(q, start);
1163
1164     s = ZOOM_connection_scan1(c, q);
1165     ZOOM_query_destroy(q);
1166     return s;
1167
1168 }
1169
1170 ZOOM_API(ZOOM_scanset)
1171     ZOOM_connection_scan1(ZOOM_connection c, ZOOM_query q)
1172 {
1173     ZOOM_scanset scan = 0;
1174     Z_Query *z_query = ZOOM_query_get_Z_Query(q);
1175
1176     if (!z_query)
1177         return 0;
1178     scan = (ZOOM_scanset) xmalloc(sizeof(*scan));
1179     scan->connection = c;
1180     scan->odr = odr_createmem(ODR_DECODE);
1181     scan->options = ZOOM_options_create_with_parent(c->options);
1182     scan->refcount = 1;
1183     scan->scan_response = 0;
1184     scan->srw_scan_response = 0;
1185
1186     scan->query = q;
1187     ZOOM_query_addref(q);
1188     scan->databaseNames = ZOOM_connection_get_databases(c, c->options,
1189                                             &scan->num_databaseNames,
1190                                             scan->odr);
1191
1192     if (1)
1193     {
1194         ZOOM_task task = ZOOM_connection_add_task(c, ZOOM_TASK_SCAN);
1195         task->u.scan.scan = scan;
1196
1197         (scan->refcount)++;
1198         if (!c->async)
1199         {
1200             while (ZOOM_event(1, &c))
1201                 ;
1202         }
1203     }
1204     return scan;
1205 }
1206
1207 ZOOM_API(void)
1208     ZOOM_scanset_destroy(ZOOM_scanset scan)
1209 {
1210     if (!scan)
1211         return;
1212     (scan->refcount)--;
1213     if (scan->refcount == 0)
1214     {
1215         ZOOM_query_destroy(scan->query);
1216
1217         odr_destroy(scan->odr);
1218
1219         ZOOM_options_destroy(scan->options);
1220         xfree(scan);
1221     }
1222 }
1223
1224 static zoom_ret send_package(ZOOM_connection c)
1225 {
1226     ZOOM_Event event;
1227
1228     yaz_log(c->log_details, "%p send_package", c);
1229     if (!c->tasks)
1230         return zoom_complete;
1231     assert (c->tasks->which == ZOOM_TASK_PACKAGE);
1232
1233     event = ZOOM_Event_create(ZOOM_EVENT_SEND_APDU);
1234     ZOOM_connection_put_event(c, event);
1235
1236     c->buf_out = c->tasks->u.package->buf_out;
1237     c->len_out = c->tasks->u.package->len_out;
1238
1239     return ZOOM_send_buf(c);
1240 }
1241
1242 ZOOM_API(size_t)
1243     ZOOM_scanset_size(ZOOM_scanset scan)
1244 {
1245     if (!scan)
1246         return 0;
1247
1248     if (scan->scan_response && scan->scan_response->entries)
1249         return scan->scan_response->entries->num_entries;
1250     else if (scan->srw_scan_response)
1251         return scan->srw_scan_response->num_terms;
1252     return 0;
1253 }
1254
1255 static void ZOOM_scanset_term_x(ZOOM_scanset scan, size_t pos,
1256                                 size_t *occ,
1257                                 const char **value_term, size_t *value_len,
1258                                 const char **disp_term, size_t *disp_len)
1259 {
1260     size_t noent = ZOOM_scanset_size(scan);
1261
1262     *value_term = 0;
1263     *value_len = 0;
1264
1265     *disp_term = 0;
1266     *disp_len = 0;
1267
1268     *occ = 0;
1269     if (pos >= noent)
1270         return;
1271     if (scan->scan_response)
1272     {
1273         Z_ScanResponse *res = scan->scan_response;
1274         if (res->entries->entries[pos]->which == Z_Entry_termInfo)
1275         {
1276             Z_TermInfo *t = res->entries->entries[pos]->u.termInfo;
1277
1278             *value_term = (const char *) t->term->u.general->buf;
1279             *value_len = t->term->u.general->len;
1280             if (t->displayTerm)
1281             {
1282                 *disp_term = t->displayTerm;
1283                 *disp_len = strlen(*disp_term);
1284             }
1285             else if (t->term->which == Z_Term_general)
1286             {
1287                 *disp_term = (const char *) t->term->u.general->buf;
1288                 *disp_len = t->term->u.general->len;
1289             }
1290             *occ = t->globalOccurrences ? *t->globalOccurrences : 0;
1291         }
1292     }
1293     if (scan->srw_scan_response)
1294     {
1295         Z_SRW_scanResponse *res = scan->srw_scan_response;
1296         Z_SRW_scanTerm *t = res->terms + pos;
1297         if (t)
1298         {
1299             *value_term = t->value;
1300             *value_len = strlen(*value_term);
1301
1302             if (t->displayTerm)
1303                 *disp_term = t->displayTerm;
1304             else
1305                 *disp_term = t->value;
1306             *disp_len = strlen(*disp_term);
1307             *occ = t->numberOfRecords ? *t->numberOfRecords : 0;
1308         }
1309     }
1310 }
1311
1312 ZOOM_API(const char *)
1313     ZOOM_scanset_term(ZOOM_scanset scan, size_t pos,
1314                       size_t *occ, size_t *len)
1315 {
1316     const char *value_term = 0;
1317     size_t value_len = 0;
1318     const char *disp_term = 0;
1319     size_t disp_len = 0;
1320
1321     ZOOM_scanset_term_x(scan, pos, occ, &value_term, &value_len,
1322                         &disp_term, &disp_len);
1323
1324     *len = value_len;
1325     return value_term;
1326 }
1327
1328 ZOOM_API(const char *)
1329     ZOOM_scanset_display_term(ZOOM_scanset scan, size_t pos,
1330                               size_t *occ, size_t *len)
1331 {
1332     const char *value_term = 0;
1333     size_t value_len = 0;
1334     const char *disp_term = 0;
1335     size_t disp_len = 0;
1336
1337     ZOOM_scanset_term_x(scan, pos, occ, &value_term, &value_len,
1338                         &disp_term, &disp_len);
1339
1340     *len = disp_len;
1341     return disp_term;
1342 }
1343
1344 ZOOM_API(const char *)
1345     ZOOM_scanset_option_get(ZOOM_scanset scan, const char *key)
1346 {
1347     return ZOOM_options_get(scan->options, key);
1348 }
1349
1350 ZOOM_API(void)
1351     ZOOM_scanset_option_set(ZOOM_scanset scan, const char *key,
1352                             const char *val)
1353 {
1354     ZOOM_options_set(scan->options, key, val);
1355 }
1356
1357
1358 ZOOM_API(ZOOM_package)
1359     ZOOM_connection_package(ZOOM_connection c, ZOOM_options options)
1360 {
1361     ZOOM_package p = (ZOOM_package) xmalloc(sizeof(*p));
1362
1363     p->connection = c;
1364     p->odr_out = odr_createmem(ODR_ENCODE);
1365     p->options = ZOOM_options_create_with_parent2(options, c->options);
1366     p->refcount = 1;
1367     p->buf_out = 0;
1368     p->len_out = 0;
1369     return p;
1370 }
1371
1372 ZOOM_API(void)
1373     ZOOM_package_destroy(ZOOM_package p)
1374 {
1375     if (!p)
1376         return;
1377     (p->refcount)--;
1378     if (p->refcount == 0)
1379     {
1380         odr_destroy(p->odr_out);
1381         xfree(p->buf_out);
1382
1383         ZOOM_options_destroy(p->options);
1384         xfree(p);
1385     }
1386 }
1387
1388 ZOOM_API(const char *)
1389     ZOOM_package_option_get(ZOOM_package p, const char *key)
1390 {
1391     return ZOOM_options_get(p->options, key);
1392 }
1393
1394 ZOOM_API(const char *)
1395     ZOOM_package_option_getl(ZOOM_package p, const char *key, int *lenp)
1396 {
1397     return ZOOM_options_getl(p->options, key, lenp);
1398 }
1399
1400 ZOOM_API(void)
1401     ZOOM_package_option_set(ZOOM_package p, const char *key,
1402                             const char *val)
1403 {
1404     ZOOM_options_set(p->options, key, val);
1405 }
1406
1407 ZOOM_API(void)
1408     ZOOM_package_option_setl(ZOOM_package p, const char *key,
1409                              const char *val, int len)
1410 {
1411     ZOOM_options_setl(p->options, key, val, len);
1412 }
1413
1414 ZOOM_API(int)
1415     ZOOM_connection_exec_task(ZOOM_connection c)
1416 {
1417     ZOOM_task task = c->tasks;
1418     zoom_ret ret = zoom_complete;
1419
1420     if (!task)
1421         return 0;
1422     yaz_log(c->log_details, "%p ZOOM_connection_exec_task type=%d run=%d",
1423             c, task->which, task->running);
1424     if (c->error != ZOOM_ERROR_NONE)
1425     {
1426         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1427                 "removing tasks because of error = %d", c, c->error);
1428         ZOOM_connection_remove_tasks(c);
1429         return 0;
1430     }
1431     if (task->running)
1432     {
1433         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1434                 "task already running", c);
1435         return 0;
1436     }
1437     task->running = 1;
1438     ret = zoom_complete;
1439     if (c->cs || task->which == ZOOM_TASK_CONNECT)
1440     {
1441         switch (task->which)
1442         {
1443         case ZOOM_TASK_SEARCH:
1444             if (c->proto == PROTO_HTTP)
1445                 ret = ZOOM_connection_srw_send_search(c);
1446             else
1447                 ret = ZOOM_connection_Z3950_search(c);
1448             break;
1449         case ZOOM_TASK_CONNECT:
1450             ret = do_connect(c);
1451             break;
1452         case ZOOM_TASK_SCAN:
1453             if (c->proto == PROTO_HTTP)
1454                 ret = ZOOM_connection_srw_send_scan(c);
1455             else
1456                 ret = ZOOM_connection_Z3950_send_scan(c);
1457             break;
1458         case ZOOM_TASK_PACKAGE:
1459             ret = send_package(c);
1460             break;
1461         case ZOOM_TASK_SORT:
1462             c->tasks->u.sort.resultset->r_sort_spec =
1463                 ZOOM_query_get_sortspec(c->tasks->u.sort.q);
1464             ret = send_Z3950_sort(c, c->tasks->u.sort.resultset);
1465             break;
1466         }
1467     }
1468     else
1469     {
1470         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1471                 "remove tasks because no connection exist", c);
1472         ZOOM_connection_remove_tasks(c);
1473     }
1474     if (ret == zoom_complete)
1475     {
1476         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1477                 "task removed (complete)", c);
1478         ZOOM_connection_remove_task(c);
1479         return 0;
1480     }
1481     yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1482             "task pending", c);
1483     return 1;
1484 }
1485
1486 #if YAZ_HAVE_XML2
1487
1488 static zoom_ret send_HTTP_redirect(ZOOM_connection c, const char *uri)
1489 {
1490     Z_GDU *gdu = z_get_HTTP_Request_uri(c->odr_out, uri, 0, c->proxy ? 1 : 0);
1491
1492     gdu->u.HTTP_Request->method = odr_strdup(c->odr_out, "GET");
1493     z_HTTP_header_add(c->odr_out, &gdu->u.HTTP_Request->headers, "Accept",
1494                       "text/xml");
1495     yaz_cookies_request(c->cookies, c->odr_out, gdu->u.HTTP_Request);
1496     if (c->user && c->password)
1497     {
1498         z_HTTP_header_add_basic_auth(c->odr_out, &gdu->u.HTTP_Request->headers,
1499                                      c->user, c->password);
1500     }
1501     return ZOOM_send_GDU(c, gdu);
1502 }
1503
1504 zoom_ret ZOOM_send_GDU(ZOOM_connection c, Z_GDU *gdu)
1505 {
1506     ZOOM_Event event;
1507
1508     int r = z_GDU(c->odr_out, &gdu, 0, 0);
1509     if (!r)
1510         return zoom_complete;
1511     if (c->odr_print)
1512         z_GDU(c->odr_print, &gdu, 0, 0);
1513     if (c->odr_save)
1514         z_GDU(c->odr_save, &gdu, 0, 0);
1515     c->buf_out = odr_getbuf(c->odr_out, &c->len_out, 0);
1516     odr_reset(c->odr_out);
1517
1518     event = ZOOM_Event_create(ZOOM_EVENT_SEND_APDU);
1519     ZOOM_connection_put_event(c, event);
1520
1521     return ZOOM_send_buf(c);
1522 }
1523
1524 #if YAZ_HAVE_XML2
1525 void ZOOM_set_HTTP_error(ZOOM_connection c, int error,
1526                          const char *addinfo, const char *addinfo2)
1527 {
1528     ZOOM_set_dset_error(c, error, "HTTP", addinfo, addinfo2);
1529 }
1530 #endif
1531
1532
1533 static void handle_http(ZOOM_connection c, Z_HTTP_Response *hres)
1534 {
1535     zoom_ret cret = zoom_complete;
1536     int ret = -1;
1537     char *addinfo = 0;
1538     const char *connection_head = z_HTTP_header_lookup(hres->headers,
1539                                                        "Connection");
1540     const char *location;
1541
1542     ZOOM_connection_set_mask(c, 0);
1543     yaz_log(c->log_details, "%p handle_http", c);
1544
1545     yaz_cookies_response(c->cookies, hres);
1546     if ((hres->code == 301 || hres->code == 302) && c->sru_mode == zoom_sru_get
1547         && (location = z_HTTP_header_lookup(hres->headers, "Location")))
1548     {
1549         c->no_redirects++;
1550         if (c->no_redirects > 10)
1551         {
1552             ZOOM_set_HTTP_error(c, hres->code, 0, 0);
1553             c->no_redirects = 0;
1554             ZOOM_connection_close(c);
1555         }
1556         else
1557         {
1558             /* since redirect may change host we just reconnect. A smarter
1559                implementation might check whether it's the same server */
1560
1561             int host_change = 0;
1562             location = yaz_check_location(c->odr_in, c->host_port,
1563                                           location, &host_change);
1564             if (do_connect_host(c, location) == zoom_complete)
1565                 return;  /* connect failed.. */
1566             cs_rcvconnect(c->cs);
1567             send_HTTP_redirect(c, location);
1568             return;
1569         }
1570     }
1571     else
1572     {
1573         ret = ZOOM_handle_sru(c, hres, &cret, &addinfo);
1574         if (ret == 0)
1575         {
1576             if (c->no_redirects) /* end of redirect. change hosts again */
1577                 ZOOM_connection_close(c);
1578         }
1579         c->no_redirects = 0;
1580     }
1581     if (ret)
1582     {
1583         if (hres->code != 200)
1584             ZOOM_set_HTTP_error(c, hres->code, 0, 0);
1585         else
1586         {
1587             yaz_log(YLOG_LOG, "set error... addinfo=%s", addinfo ?
1588                     addinfo : "NULL");
1589             ZOOM_set_error(c, ZOOM_ERROR_DECODE, addinfo);
1590         }
1591         ZOOM_connection_close(c);
1592     }
1593     if (cret == zoom_complete)
1594     {
1595         yaz_log(c->log_details, "removing tasks in handle_http");
1596         ZOOM_connection_remove_task(c);
1597     }
1598     {
1599         int must_close = 0;
1600         if (!strcmp(hres->version, "1.0"))
1601         {
1602             /* HTTP 1.0: only if Keep-Alive we stay alive.. */
1603             if (!connection_head || strcmp(connection_head, "Keep-Alive"))
1604                 must_close = 1;
1605         }
1606         else
1607         {
1608             /* HTTP 1.1: only if no close we stay alive.. */
1609             if (connection_head && !strcmp(connection_head, "close"))
1610                 must_close = 1;
1611         }
1612         if (must_close)
1613         {
1614             ZOOM_connection_close(c);
1615             if (c->tasks)
1616             {
1617                 c->tasks->running = 0;
1618                 ZOOM_connection_insert_task(c, ZOOM_TASK_CONNECT);
1619                 c->reconnect_ok = 0;
1620             }
1621         }
1622         else
1623             c->reconnect_ok = 1; /* if the server closes anyway */
1624     }
1625 }
1626 #endif
1627
1628 static int do_read(ZOOM_connection c)
1629 {
1630     int r, more;
1631     ZOOM_Event event;
1632
1633     event = ZOOM_Event_create(ZOOM_EVENT_RECV_DATA);
1634     ZOOM_connection_put_event(c, event);
1635
1636     r = cs_get(c->cs, &c->buf_in, &c->len_in);
1637     more = cs_more(c->cs);
1638     yaz_log(c->log_details, "%p do_read len=%d more=%d", c, r, more);
1639     if (r == 1)
1640         return 0;
1641     if (r <= 0)
1642     {
1643         if (!ZOOM_test_reconnect(c))
1644         {
1645             ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
1646             ZOOM_connection_close(c);
1647         }
1648     }
1649     else
1650     {
1651         Z_GDU *gdu;
1652         ZOOM_Event event;
1653
1654         odr_reset(c->odr_in);
1655         odr_setbuf(c->odr_in, c->buf_in, r, 0);
1656         event = ZOOM_Event_create(ZOOM_EVENT_RECV_APDU);
1657         ZOOM_connection_put_event(c, event);
1658
1659         if (!z_GDU(c->odr_in, &gdu, 0, 0))
1660         {
1661             int x;
1662             int err = odr_geterrorx(c->odr_in, &x);
1663             char msg[100];
1664             const char *element = odr_getelement(c->odr_in);
1665             yaz_snprintf(msg, sizeof(msg),
1666                     "ODR code %d:%d element=%s offset=%d",
1667                     err, x, element ? element : "<unknown>",
1668                     odr_offset(c->odr_in));
1669             ZOOM_set_error(c, ZOOM_ERROR_DECODE, msg);
1670             if (c->log_api)
1671             {
1672                 FILE *ber_file = yaz_log_file();
1673                 if (ber_file)
1674                     odr_dumpBER(ber_file, c->buf_in, r);
1675             }
1676             ZOOM_connection_close(c);
1677         }
1678         else
1679         {
1680             if (c->odr_print)
1681                 z_GDU(c->odr_print, &gdu, 0, 0);
1682             if (c->odr_save)
1683                 z_GDU(c->odr_save, &gdu, 0, 0);
1684             if (gdu->which == Z_GDU_Z3950)
1685                 ZOOM_handle_Z3950_apdu(c, gdu->u.z3950);
1686             else if (gdu->which == Z_GDU_HTTP_Response)
1687             {
1688 #if YAZ_HAVE_XML2
1689                 handle_http(c, gdu->u.HTTP_Response);
1690 #else
1691                 ZOOM_set_error(c, ZOOM_ERROR_DECODE, 0);
1692                 ZOOM_connection_close(c);
1693 #endif
1694             }
1695         }
1696     }
1697     return 1;
1698 }
1699
1700 static zoom_ret do_write_ex(ZOOM_connection c, char *buf_out, int len_out)
1701 {
1702     int r;
1703     ZOOM_Event event;
1704
1705     event = ZOOM_Event_create(ZOOM_EVENT_SEND_DATA);
1706     ZOOM_connection_put_event(c, event);
1707
1708     yaz_log(c->log_details, "%p do_write_ex len=%d", c, len_out);
1709     if ((r = cs_put(c->cs, buf_out, len_out)) < 0)
1710     {
1711         yaz_log(c->log_details, "%p do_write_ex write failed", c);
1712         if (ZOOM_test_reconnect(c))
1713         {
1714             return zoom_pending;
1715         }
1716         if (c->state == STATE_CONNECTING)
1717             ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
1718         else
1719             ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
1720         ZOOM_connection_close(c);
1721         return zoom_complete;
1722     }
1723     else if (r == 1)
1724     {
1725         int mask = ZOOM_SELECT_EXCEPT;
1726         if (c->cs->io_pending & CS_WANT_WRITE)
1727             mask += ZOOM_SELECT_WRITE;
1728         if (c->cs->io_pending & CS_WANT_READ)
1729             mask += ZOOM_SELECT_READ;
1730         ZOOM_connection_set_mask(c, mask);
1731         yaz_log(c->log_details, "%p do_write_ex write incomplete mask=%d",
1732                 c, c->mask);
1733     }
1734     else
1735     {
1736         ZOOM_connection_set_mask(c, ZOOM_SELECT_READ|ZOOM_SELECT_EXCEPT);
1737         yaz_log(c->log_details, "%p do_write_ex write complete mask=%d",
1738                 c, c->mask);
1739     }
1740     return zoom_pending;
1741 }
1742
1743 zoom_ret ZOOM_send_buf(ZOOM_connection c)
1744 {
1745     return do_write_ex(c, c->buf_out, c->len_out);
1746 }
1747
1748
1749 ZOOM_API(const char *)
1750     ZOOM_connection_option_get(ZOOM_connection c, const char *key)
1751 {
1752     if (!strcmp(key, "APDU"))
1753     {
1754         return c->saveAPDU_wrbuf ? wrbuf_cstr(c->saveAPDU_wrbuf) : "";
1755     }
1756     else
1757         return ZOOM_options_get(c->options, key);
1758 }
1759
1760 ZOOM_API(const char *)
1761     ZOOM_connection_option_getl(ZOOM_connection c, const char *key, int *lenp)
1762 {
1763     if (!strcmp(key, "APDU"))
1764     {
1765         if (c->saveAPDU_wrbuf)
1766         {
1767             *lenp = wrbuf_len(c->saveAPDU_wrbuf);
1768             return wrbuf_cstr(c->saveAPDU_wrbuf);
1769         }
1770         else
1771         {
1772             *lenp = 0;
1773             return "";
1774         }
1775     }
1776     else
1777         return ZOOM_options_getl(c->options, key, lenp);
1778 }
1779
1780 ZOOM_API(void)
1781     ZOOM_connection_option_set(ZOOM_connection c, const char *key,
1782                                const char *val)
1783 {
1784     if (!strcmp(key, "saveAPDU"))
1785     {
1786         if (val && strcmp(val, "0"))
1787         {
1788             if (!c->saveAPDU_wrbuf)
1789                 c->saveAPDU_wrbuf = wrbuf_alloc();
1790             else
1791                 wrbuf_rewind(c->saveAPDU_wrbuf);
1792         }
1793         else
1794         {
1795             wrbuf_destroy(c->saveAPDU_wrbuf);
1796             c->saveAPDU_wrbuf = 0;
1797         }
1798         ZOOM_connection_save_apdu_wrbuf(c, c->saveAPDU_wrbuf);
1799     }
1800     else
1801         ZOOM_options_set(c->options, key, val);
1802 }
1803
1804 ZOOM_API(void)
1805     ZOOM_connection_option_setl(ZOOM_connection c, const char *key,
1806                                 const char *val, int len)
1807 {
1808     ZOOM_options_setl(c->options, key, val, len);
1809 }
1810
1811 ZOOM_API(const char *)
1812     ZOOM_resultset_option_get(ZOOM_resultset r, const char *key)
1813 {
1814     return ZOOM_options_get(r->options, key);
1815 }
1816
1817 ZOOM_API(void)
1818     ZOOM_resultset_option_set(ZOOM_resultset r, const char *key,
1819                               const char *val)
1820 {
1821     ZOOM_options_set(r->options, key, val);
1822 }
1823
1824
1825 ZOOM_API(int)
1826     ZOOM_connection_errcode(ZOOM_connection c)
1827 {
1828     return ZOOM_connection_error(c, 0, 0);
1829 }
1830
1831 ZOOM_API(const char *)
1832     ZOOM_connection_errmsg(ZOOM_connection c)
1833 {
1834     const char *msg;
1835     ZOOM_connection_error(c, &msg, 0);
1836     return msg;
1837 }
1838
1839 ZOOM_API(const char *)
1840     ZOOM_connection_addinfo(ZOOM_connection c)
1841 {
1842     const char *addinfo;
1843     ZOOM_connection_error(c, 0, &addinfo);
1844     return addinfo;
1845 }
1846
1847 ZOOM_API(const char *)
1848     ZOOM_connection_diagset(ZOOM_connection c)
1849 {
1850     const char *diagset;
1851     ZOOM_connection_error_x(c, 0, 0, &diagset);
1852     return diagset;
1853 }
1854
1855 ZOOM_API(const char *)
1856     ZOOM_diag_str(int error)
1857 {
1858     switch (error)
1859     {
1860     case ZOOM_ERROR_NONE:
1861         return "No error";
1862     case ZOOM_ERROR_CONNECT:
1863         return "Connect failed";
1864     case ZOOM_ERROR_MEMORY:
1865         return "Out of memory";
1866     case ZOOM_ERROR_ENCODE:
1867         return "Encoding failed";
1868     case ZOOM_ERROR_DECODE:
1869         return "Decoding failed";
1870     case ZOOM_ERROR_CONNECTION_LOST:
1871         return "Connection lost";
1872     case ZOOM_ERROR_INIT:
1873         return "Init rejected";
1874     case ZOOM_ERROR_INTERNAL:
1875         return "Internal failure";
1876     case ZOOM_ERROR_TIMEOUT:
1877         return "Timeout";
1878     case ZOOM_ERROR_UNSUPPORTED_PROTOCOL:
1879         return "Unsupported protocol";
1880     case ZOOM_ERROR_UNSUPPORTED_QUERY:
1881         return "Unsupported query type";
1882     case ZOOM_ERROR_INVALID_QUERY:
1883         return "Invalid query";
1884     case ZOOM_ERROR_CQL_PARSE:
1885         return "CQL parsing error";
1886     case ZOOM_ERROR_CQL_TRANSFORM:
1887         return "CQL transformation error";
1888     case ZOOM_ERROR_CCL_CONFIG:
1889         return "CCL configuration error";
1890     case ZOOM_ERROR_CCL_PARSE:
1891         return "CCL parsing error";
1892     case ZOOM_ERROR_ES_INVALID_ACTION:
1893         return "Extended Service. invalid action";
1894     case ZOOM_ERROR_ES_INVALID_VERSION:
1895         return "Extended Service. invalid version";
1896     case ZOOM_ERROR_ES_INVALID_SYNTAX:
1897         return "Extended Service. invalid syntax";
1898     case ZOOM_ERROR_MEMCACHED:
1899         return "Memcached";
1900     default:
1901         return diagbib1_str(error);
1902     }
1903 }
1904
1905 ZOOM_API(int)
1906     ZOOM_connection_error_x(ZOOM_connection c, const char **cp,
1907                             const char **addinfo, const char **diagset)
1908 {
1909     int error = c->error;
1910     if (cp)
1911     {
1912         if (!c->diagset || !strcmp(c->diagset, "ZOOM"))
1913             *cp = ZOOM_diag_str(error);
1914         else if (!strcmp(c->diagset, "HTTP"))
1915             *cp = z_HTTP_errmsg(c->error);
1916         else if (!strcmp(c->diagset, "Bib-1"))
1917             *cp = ZOOM_diag_str(error);
1918         else if (!strcmp(c->diagset, "info:srw/diagnostic/1"))
1919             *cp = yaz_diag_srw_str(c->error);
1920         else
1921             *cp = "Unknown error and diagnostic set";
1922     }
1923     if (addinfo)
1924         *addinfo = c->addinfo ? c->addinfo : "";
1925     if (diagset)
1926         *diagset = c->diagset ? c->diagset : "";
1927     return c->error;
1928 }
1929
1930 ZOOM_API(int)
1931     ZOOM_connection_error(ZOOM_connection c, const char **cp,
1932                           const char **addinfo)
1933 {
1934     return ZOOM_connection_error_x(c, cp, addinfo, 0);
1935 }
1936
1937 static void ZOOM_connection_do_io(ZOOM_connection c, int mask)
1938 {
1939     ZOOM_Event event = 0;
1940     int r = cs_look(c->cs);
1941     yaz_log(c->log_details, "%p ZOOM_connection_do_io mask=%d cs_look=%d",
1942             c, mask, r);
1943
1944     if (r == CS_NONE)
1945     {
1946         event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1947         ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
1948         ZOOM_connection_close(c);
1949         ZOOM_connection_put_event(c, event);
1950     }
1951     else if (r == CS_CONNECT)
1952     {
1953         int ret = ret = cs_rcvconnect(c->cs);
1954         yaz_log(c->log_details, "%p ZOOM_connection_do_io "
1955                 "cs_rcvconnect returned %d", c, ret);
1956         if (ret == 1)
1957         {
1958             int mask = ZOOM_SELECT_EXCEPT;
1959             if (c->cs->io_pending & CS_WANT_WRITE)
1960                 mask += ZOOM_SELECT_WRITE;
1961             if (c->cs->io_pending & CS_WANT_READ)
1962                 mask += ZOOM_SELECT_READ;
1963             ZOOM_connection_set_mask(c, mask);
1964             event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1965             ZOOM_connection_put_event(c, event);
1966         }
1967         else if (ret == 0)
1968         {
1969             event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1970             ZOOM_connection_put_event(c, event);
1971             get_cert(c);
1972             if (c->proto == PROTO_Z3950)
1973                 ZOOM_connection_Z3950_send_init(c);
1974             else
1975             {
1976                 /* no init request for SRW .. */
1977                 assert(c->tasks->which == ZOOM_TASK_CONNECT);
1978                 ZOOM_connection_remove_task(c);
1979                 ZOOM_connection_set_mask(c, 0);
1980                 ZOOM_connection_exec_task(c);
1981             }
1982             c->state = STATE_ESTABLISHED;
1983         }
1984         else
1985         {
1986             ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
1987             ZOOM_connection_close(c);
1988         }
1989     }
1990     else
1991     {
1992         if (mask & ZOOM_SELECT_EXCEPT)
1993         {
1994             if (!ZOOM_test_reconnect(c))
1995             {
1996                 ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
1997                 ZOOM_connection_close(c);
1998             }
1999             return;
2000         }
2001         if (mask & ZOOM_SELECT_READ)
2002             do_read(c);
2003         if (c->cs && (mask & ZOOM_SELECT_WRITE))
2004             ZOOM_send_buf(c);
2005     }
2006 }
2007
2008 ZOOM_API(int)
2009     ZOOM_connection_last_event(ZOOM_connection cs)
2010 {
2011     if (!cs)
2012         return ZOOM_EVENT_NONE;
2013     return cs->last_event;
2014 }
2015
2016
2017 ZOOM_API(int) ZOOM_connection_fire_event_timeout(ZOOM_connection c)
2018 {
2019     if (c->mask)
2020     {
2021         ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_TIMEOUT);
2022         /* timeout and this connection was waiting */
2023         ZOOM_set_error(c, ZOOM_ERROR_TIMEOUT, 0);
2024         ZOOM_connection_close(c);
2025         ZOOM_connection_put_event(c, event);
2026     }
2027     return 0;
2028 }
2029
2030 ZOOM_API(int)
2031     ZOOM_connection_process(ZOOM_connection c)
2032 {
2033     ZOOM_Event event;
2034     if (!c)
2035         return 0;
2036
2037     event = ZOOM_connection_get_event(c);
2038     if (event)
2039     {
2040         ZOOM_Event_destroy(event);
2041         return 1;
2042     }
2043     ZOOM_connection_exec_task(c);
2044     event = ZOOM_connection_get_event(c);
2045     if (event)
2046     {
2047         ZOOM_Event_destroy(event);
2048         return 1;
2049     }
2050     return 0;
2051 }
2052
2053 ZOOM_API(int)
2054     ZOOM_event_nonblock(int no, ZOOM_connection *cs)
2055 {
2056     int i;
2057
2058     yaz_log(log_details0, "ZOOM_process_event(no=%d,cs=%p)", no, cs);
2059
2060     for (i = 0; i<no; i++)
2061     {
2062         ZOOM_connection c = cs[i];
2063
2064         if (c && ZOOM_connection_process(c))
2065             return i+1;
2066     }
2067     return 0;
2068 }
2069
2070 ZOOM_API(int) ZOOM_connection_fire_event_socket(ZOOM_connection c, int mask)
2071 {
2072     if (c->mask && mask)
2073         ZOOM_connection_do_io(c, mask);
2074     return 0;
2075 }
2076
2077 ZOOM_API(int) ZOOM_connection_get_socket(ZOOM_connection c)
2078 {
2079     if (c->cs)
2080         return cs_fileno(c->cs);
2081     return -1;
2082 }
2083
2084 ZOOM_API(int) ZOOM_connection_set_mask(ZOOM_connection c, int mask)
2085 {
2086     c->mask = mask;
2087     if (!c->cs)
2088         return -1;
2089     return 0;
2090 }
2091
2092 ZOOM_API(int) ZOOM_connection_get_mask(ZOOM_connection c)
2093 {
2094     if (c->cs)
2095         return c->mask;
2096     return 0;
2097 }
2098
2099 ZOOM_API(int) ZOOM_connection_get_timeout(ZOOM_connection c)
2100 {
2101     return ZOOM_options_get_int(c->options, "timeout", 30);
2102 }
2103
2104 ZOOM_API(void) ZOOM_connection_close(ZOOM_connection c)
2105 {
2106     if (c->cs)
2107         cs_close(c->cs);
2108     c->cs = 0;
2109     ZOOM_connection_set_mask(c, 0);
2110     c->state = STATE_IDLE;
2111 }
2112
2113 /*
2114  * Local variables:
2115  * c-basic-offset: 4
2116  * c-file-style: "Stroustrup"
2117  * indent-tabs-mode: nil
2118  * End:
2119  * vim: shiftwidth=4 tabstop=8 expandtab
2120  */
2121