Add test for YAZ-760
[yaz-moved-to-github.git] / src / zoom-c.c
1 /* This file is part of the YAZ toolkit.
2  * Copyright (C) Index Data
3  * See the file LICENSE for details.
4  */
5 /**
6  * \file zoom-c.c
7  * \brief Implements ZOOM C interface.
8  */
9 #if HAVE_CONFIG_H
10 #include <config.h>
11 #endif
12
13 #include <assert.h>
14 #include <string.h>
15 #include <errno.h>
16 #include "zoom-p.h"
17
18 #include <yaz/yaz-util.h>
19 #include <yaz/xmalloc.h>
20 #include <yaz/otherinfo.h>
21 #include <yaz/log.h>
22 #include <yaz/diagbib1.h>
23 #include <yaz/charneg.h>
24 #include <yaz/query-charset.h>
25 #include <yaz/snprintf.h>
26 #include <yaz/facet.h>
27
28 #include <yaz/shptr.h>
29
30 #if SHPTR
31 YAZ_SHPTR_TYPE(WRBUF)
32 #endif
33
34 static int log_api0 = 0;
35 static int log_details0 = 0;
36
37 static void resultset_destroy(ZOOM_resultset r);
38 static zoom_ret do_write_ex(ZOOM_connection c, char *buf_out, int len_out);
39
40 static void initlog(void)
41 {
42     static int log_level_initialized = 0;
43
44     if (!log_level_initialized)
45     {
46         log_api0 = yaz_log_module_level("zoom");
47         log_details0 = yaz_log_module_level("zoomdetails");
48         log_level_initialized = 1;
49     }
50 }
51
52 void ZOOM_connection_remove_tasks(ZOOM_connection c);
53
54 void ZOOM_set_dset_error(ZOOM_connection c, int error,
55                          const char *dset,
56                          const char *addinfo, const char *addinfo2)
57 {
58     char *cp;
59
60     xfree(c->addinfo);
61     c->addinfo = 0;
62     c->error = error;
63     if (!c->diagset || strcmp(dset, c->diagset))
64     {
65         xfree(c->diagset);
66         c->diagset = xstrdup(dset);
67         /* remove integer part from SRW diagset .. */
68         if ((cp = strrchr(c->diagset, '/')))
69             *cp = '\0';
70     }
71     if (addinfo && addinfo2)
72     {
73         c->addinfo = (char*) xmalloc(strlen(addinfo) + strlen(addinfo2) + 3);
74         strcpy(c->addinfo, addinfo);
75         strcat(c->addinfo, ": ");
76         strcat(c->addinfo, addinfo2);
77     }
78     else if (addinfo)
79         c->addinfo = xstrdup(addinfo);
80     if (error != ZOOM_ERROR_NONE)
81     {
82         yaz_log(c->log_api, "%p set_dset_error %s %s:%d %s %s",
83                 c, c->host_port ? c->host_port : "<>", dset, error,
84                 addinfo ? addinfo : "",
85                 addinfo2 ? addinfo2 : "");
86         ZOOM_connection_remove_tasks(c);
87     }
88 }
89
90 int ZOOM_uri_to_code(const char *uri)
91 {
92     int code = 0;
93     const char *cp;
94     if ((cp = strrchr(uri, '/')))
95         code = atoi(cp+1);
96     return code;
97 }
98
99 void ZOOM_set_error(ZOOM_connection c, int error, const char *addinfo)
100 {
101     ZOOM_set_dset_error(c, error, "ZOOM", addinfo, 0);
102 }
103
104 static void clear_error(ZOOM_connection c)
105 {
106     /*
107      * If an error is tied to an operation then it's ok to clear: for
108      * example, a diagnostic returned from a search is cleared by a
109      * subsequent search.  However, problems such as Connection Lost
110      * or Init Refused are not cleared, because they are not
111      * recoverable: doing another search doesn't help.
112      */
113
114     ZOOM_connection_remove_events(c);
115     switch (c->error)
116     {
117     case ZOOM_ERROR_CONNECT:
118     case ZOOM_ERROR_MEMORY:
119     case ZOOM_ERROR_DECODE:
120     case ZOOM_ERROR_CONNECTION_LOST:
121     case ZOOM_ERROR_INIT:
122     case ZOOM_ERROR_INTERNAL:
123     case ZOOM_ERROR_UNSUPPORTED_PROTOCOL:
124         break;
125     default:
126         ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
127     }
128 }
129
130 void ZOOM_connection_show_task(ZOOM_task task)
131 {
132     switch(task->which)
133     {
134     case ZOOM_TASK_SEARCH:
135         yaz_log(YLOG_LOG, "search p=%p", task);
136         break;
137     case ZOOM_TASK_CONNECT:
138         yaz_log(YLOG_LOG, "connect p=%p", task);
139         break;
140     case ZOOM_TASK_SCAN:
141         yaz_log(YLOG_LOG, "scan p=%p", task);
142         break;
143     }
144 }
145
146 void ZOOM_connection_show_tasks(ZOOM_connection c)
147 {
148     ZOOM_task task;
149     yaz_log(YLOG_LOG, "connection p=%p tasks", c);
150     for (task = c->tasks; task; task = task->next)
151         ZOOM_connection_show_task(task);
152 }
153
154 ZOOM_task ZOOM_connection_add_task(ZOOM_connection c, int which)
155 {
156     ZOOM_task *taskp = &c->tasks;
157     while (*taskp)
158         taskp = &(*taskp)->next;
159     *taskp = (ZOOM_task) xmalloc(sizeof(**taskp));
160     (*taskp)->running = 0;
161     (*taskp)->which = which;
162     (*taskp)->next = 0;
163     clear_error(c);
164     return *taskp;
165 }
166
167 ZOOM_API(int) ZOOM_connection_is_idle(ZOOM_connection c)
168 {
169     return c->tasks ? 0 : 1;
170 }
171
172 ZOOM_task ZOOM_connection_insert_task(ZOOM_connection c, int which)
173 {
174     ZOOM_task task = (ZOOM_task) xmalloc(sizeof(*task));
175
176     task->next = c->tasks;
177     c->tasks = task;
178
179     task->running = 0;
180     task->which = which;
181     return task;
182 }
183
184 void ZOOM_connection_remove_task(ZOOM_connection c)
185 {
186     ZOOM_task task = c->tasks;
187
188     if (task)
189     {
190         c->tasks = task->next;
191         switch (task->which)
192         {
193         case ZOOM_TASK_SEARCH:
194             resultset_destroy(task->u.search.resultset);
195             xfree(task->u.search.syntax);
196             xfree(task->u.search.elementSetName);
197             xfree(task->u.search.schema);
198             break;
199         case ZOOM_TASK_CONNECT:
200             break;
201         case ZOOM_TASK_SCAN:
202             ZOOM_scanset_destroy(task->u.scan.scan);
203             break;
204         case ZOOM_TASK_PACKAGE:
205             ZOOM_package_destroy(task->u.package);
206             break;
207         case ZOOM_TASK_SORT:
208             resultset_destroy(task->u.sort.resultset);
209             ZOOM_query_destroy(task->u.sort.q);
210             break;
211         default:
212             assert(0);
213         }
214         xfree(task);
215
216         if (!c->tasks)
217         {
218             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_END);
219             ZOOM_connection_put_event(c, event);
220         }
221     }
222 }
223
224 void ZOOM_connection_remove_tasks(ZOOM_connection c)
225 {
226     while (c->tasks)
227         ZOOM_connection_remove_task(c);
228 }
229
230 static void odr_wrbuf_write(ODR o, void *handle, int type,
231                             const char *buf, int len)
232 {
233     WRBUF w = (WRBUF) handle;
234     wrbuf_write(w, buf, len);
235 }
236
237 ZOOM_API(ZOOM_connection)
238     ZOOM_connection_create(ZOOM_options options)
239 {
240     ZOOM_connection c = (ZOOM_connection) xmalloc(sizeof(*c));
241
242     initlog();
243
244     c->log_api = log_api0;
245     c->log_details = log_details0;
246
247     yaz_log(c->log_api, "%p ZOOM_connection_create", c);
248
249     c->proto = PROTO_Z3950;
250     c->cs = 0;
251     ZOOM_connection_set_mask(c, 0);
252     c->reconnect_ok = 0;
253     c->state = STATE_IDLE;
254     c->addinfo = 0;
255     c->diagset = 0;
256     ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
257     c->buf_in = 0;
258     c->len_in = 0;
259     c->buf_out = 0;
260     c->len_out = 0;
261     c->resultsets = 0;
262
263     c->options = ZOOM_options_create_with_parent(options);
264
265     c->host_port = 0;
266     c->proxy = 0;
267     c->tproxy = 0;
268
269     c->charset = c->lang = 0;
270
271     c->cookie_out = 0;
272     c->cookie_in = 0;
273     c->client_IP = 0;
274     c->tasks = 0;
275
276     c->user = 0;
277     c->group = 0;
278     c->password = 0;
279     c->url_authentication = 0;
280
281     c->maximum_record_size = 0;
282     c->preferred_message_size = 0;
283
284     c->odr_in = odr_createmem(ODR_DECODE);
285     c->odr_out = odr_createmem(ODR_ENCODE);
286     c->odr_print = 0;
287     c->odr_save = 0;
288
289     c->async = 0;
290     c->support_named_resultsets = 0;
291     c->last_event = ZOOM_EVENT_NONE;
292
293     c->m_queue_front = 0;
294     c->m_queue_back = 0;
295
296     c->sru_version = 0;
297     c->no_redirects = 0;
298     c->cookies = 0;
299     c->saveAPDU_wrbuf = 0;
300
301     ZOOM_memcached_init(c);
302     return c;
303 }
304
305 ZOOM_API(void) ZOOM_connection_save_apdu_wrbuf(ZOOM_connection c, WRBUF w)
306 {
307     if (c->odr_save)
308     {
309         odr_destroy(c->odr_save);
310         c->odr_save = 0;
311     }
312     if (w)
313     {
314         c->odr_save = odr_createmem(ODR_PRINT);
315         odr_set_stream(c->odr_save, w, odr_wrbuf_write, 0);
316     }
317 }
318
319 /* set database names. Take local databases (if set); otherwise
320    take databases given in ZURL (if set); otherwise use Default */
321 char **ZOOM_connection_get_databases(ZOOM_connection con, ZOOM_options options,
322                                      int *num, ODR odr)
323 {
324     char **databaseNames;
325     const char *cp = ZOOM_options_get(options, "databaseName");
326
327     if ((!cp || !*cp) && con->host_port)
328         cs_get_host_args(con->host_port, &cp);
329     if (!cp || !*cp)
330         cp = "Default";
331     nmem_strsplit(odr_getmem(odr), "+", cp,  &databaseNames, num);
332     return databaseNames;
333 }
334
335 ZOOM_API(ZOOM_connection)
336     ZOOM_connection_new(const char *host, int portnum)
337 {
338     ZOOM_connection c = ZOOM_connection_create(0);
339
340     ZOOM_connection_connect(c, host, portnum);
341     return c;
342 }
343
344 static zoom_sru_mode get_sru_mode_from_string(const char *s)
345 {
346     if (!s || !*s)
347         return zoom_sru_soap;
348     if (!yaz_matchstr(s, "soap"))
349         return zoom_sru_soap;
350     else if (!yaz_matchstr(s, "get"))
351         return zoom_sru_get;
352     else if (!yaz_matchstr(s, "post"))
353         return zoom_sru_post;
354     else if (!yaz_matchstr(s, "solr"))
355         return zoom_sru_solr;
356     return zoom_sru_error;
357 }
358
359 ZOOM_API(void)
360     ZOOM_connection_connect(ZOOM_connection c,
361                             const char *host, int portnum)
362 {
363     const char *val;
364
365     initlog();
366
367     yaz_log(c->log_api, "%p ZOOM_connection_connect host=%s portnum=%d",
368             c, host ? host : "null", portnum);
369
370     ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
371     ZOOM_connection_remove_tasks(c);
372
373     if (c->cs)
374     {
375         yaz_log(c->log_details, "%p ZOOM_connection_connect reconnect ok", c);
376         c->reconnect_ok = 1;
377         return;
378     }
379     yaz_log(c->log_details, "%p ZOOM_connection_connect connect", c);
380     xfree(c->proxy);
381     c->proxy = 0;
382     val = ZOOM_options_get(c->options, "proxy");
383     if (val && *val)
384     {
385         yaz_log(c->log_details, "%p ZOOM_connection_connect proxy=%s", c, val);
386         c->proxy = xstrdup(val);
387     }
388
389     xfree(c->tproxy);
390     c->tproxy = 0;
391     val = ZOOM_options_get(c->options, "tproxy");
392     if (val && *val)
393     {
394         yaz_log(c->log_details, "%p ZOOM_connection_connect tproxy=%s", c, val);
395         c->tproxy = xstrdup(val);
396     }
397
398     xfree(c->charset);
399     c->charset = 0;
400     val = ZOOM_options_get(c->options, "charset");
401     if (val && *val)
402     {
403         yaz_log(c->log_details, "%p ZOOM_connection_connect charset=%s", c, val);
404         c->charset = xstrdup(val);
405     }
406
407     xfree(c->lang);
408     val = ZOOM_options_get(c->options, "lang");
409     if (val && *val)
410     {
411         yaz_log(c->log_details, "%p ZOOM_connection_connect lang=%s", c, val);
412         c->lang = xstrdup(val);
413     }
414     else
415         c->lang = 0;
416
417     if (host)
418     {
419         char hostn[128];
420         const char *http_lead;
421
422         val = ZOOM_options_get(c->options, "sru");
423         if (val && *val && !strstr(host, "://"))
424             http_lead = "http://";
425         else
426             http_lead = "";
427         c->sru_mode = get_sru_mode_from_string(val);
428
429         xfree(c->host_port);
430         if (portnum)
431         {
432             sprintf(hostn, "%.80s:%d", host, portnum);
433             host = hostn;
434         }
435         c->host_port = xmalloc(strlen(host) + strlen(http_lead) + 1);
436         strcpy(c->host_port, http_lead);
437         strcat(c->host_port, host);
438     }
439
440     {
441         /*
442          * If the "<scheme>:" part of the host string is preceded by one
443          * or more comma-separated <name>=<value> pairs, these are taken
444          * to be options to be set on the connection object.  Among other
445          * applications, this facility can be used to embed authentication
446          * in a host string:
447          *          user=admin,password=secret,tcp:localhost:9999
448          */
449         char *remainder = c->host_port;
450         char *pcolon = strchr(remainder, ':');
451         char *pcomma;
452         char *pequals;
453         while ((pcomma = strchr(remainder, ',')) != 0 &&
454                (pcolon == 0 || pcomma < pcolon))
455         {
456             *pcomma = '\0';
457             if ((pequals = strchr(remainder, '=')) != 0)
458             {
459                 *pequals = '\0';
460                 ZOOM_connection_option_set(c, remainder, pequals+1);
461             }
462             remainder = pcomma+1;
463         }
464
465         if (remainder != c->host_port)
466         {
467             remainder = xstrdup(remainder);
468             xfree(c->host_port);
469             c->host_port = remainder;
470         }
471     }
472
473     xfree(c->sru_version);
474     val = ZOOM_options_get(c->options, "sru_version");
475     c->sru_version = xstrdup(val ? val : "1.2");
476
477     ZOOM_options_set(c->options, "host", c->host_port);
478
479     xfree(c->cookie_out);
480     c->cookie_out = 0;
481     val = ZOOM_options_get(c->options, "cookie");
482     if (val && *val)
483     {
484         yaz_log(c->log_details, "%p ZOOM_connection_connect cookie=%s", c, val);
485         c->cookie_out = xstrdup(val);
486     }
487
488     xfree(c->client_IP);
489     c->client_IP = 0;
490     val = ZOOM_options_get(c->options, "clientIP");
491     if (val && *val)
492     {
493         yaz_log(c->log_details, "%p ZOOM_connection_connect clientIP=%s",
494                 c, val);
495         c->client_IP = xstrdup(val);
496     }
497
498     xfree(c->group);
499     c->group = 0;
500     val = ZOOM_options_get(c->options, "group");
501     if (val && *val)
502         c->group = xstrdup(val);
503
504     xfree(c->user);
505     c->user = 0;
506     val = ZOOM_options_get(c->options, "user");
507     if (val && *val)
508         c->user = xstrdup(val);
509
510     xfree(c->password);
511     c->password = 0;
512     val = ZOOM_options_get(c->options, "password");
513     if (!val)
514         val = ZOOM_options_get(c->options, "pass");
515     if (val && *val)
516         c->password = xstrdup(val);
517
518     val = ZOOM_options_get(c->options, "authenticationMode");
519     if (val && !strcmp(val, "url"))
520         c->url_authentication = 1;
521     else
522         c->url_authentication = 0;
523
524     c->maximum_record_size =
525         ZOOM_options_get_int(c->options, "maximumRecordSize", 64*1024*1024);
526     c->preferred_message_size =
527         ZOOM_options_get_int(c->options, "preferredMessageSize", 64*1024*1024);
528
529     c->async = ZOOM_options_get_bool(c->options, "async", 0);
530
531     yaz_cookies_destroy(c->cookies);
532     c->cookies = yaz_cookies_create();
533
534     if (ZOOM_memcached_configure(c))
535     {
536         ZOOM_connection_remove_tasks(c);
537         return;
538     }
539     if (c->sru_mode == zoom_sru_error)
540     {
541         ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, val);
542         ZOOM_connection_remove_tasks(c);
543         return;
544     }
545
546     if (c->odr_print)
547         odr_destroy(c->odr_print);
548     c->odr_print = 0;
549     val = ZOOM_options_get(c->options, "apdufile");
550     if (val)
551     {
552         c->odr_print = odr_createmem(ODR_PRINT);
553         if (strcmp(val, "-"))
554         {
555             FILE *f = fopen(val, "a");
556             if (!f)
557             {
558                 WRBUF w = wrbuf_alloc();
559                 wrbuf_printf(w, "fopen: %s", val);
560                 ZOOM_set_error(c, ZOOM_ERROR_INTERNAL, wrbuf_cstr(w));
561                 wrbuf_destroy(w);
562                 return;
563             }
564             odr_setprint(c->odr_print, f);
565         }
566     }
567     else if (ZOOM_options_get_bool(c->options, "apdulog", 0))
568     {
569         c->odr_print = odr_createmem(ODR_PRINT);
570         odr_setprint_noclose(c->odr_print, yaz_log_file());
571     }
572
573     yaz_log(c->log_details, "%p ZOOM_connection_connect async=%d", c, c->async);
574     ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
575
576     if (!c->async)
577     {
578         while (ZOOM_event(1, &c))
579             ;
580     }
581 }
582
583 ZOOM_API(void) ZOOM_resultset_release(ZOOM_resultset r)
584 {
585     if (r->connection)
586     {
587         /* remove ourselves from the resultsets in connection */
588         ZOOM_resultset *rp = &r->connection->resultsets;
589         while (1)
590         {
591             assert(*rp);   /* we must be in this list!! */
592             if (*rp == r)
593             {   /* OK, we're here - take us out of it */
594                 *rp = (*rp)->next;
595                 break;
596             }
597             rp = &(*rp)->next;
598         }
599         r->connection = 0;
600     }
601 }
602
603 ZOOM_API(void)
604     ZOOM_connection_destroy(ZOOM_connection c)
605 {
606     ZOOM_resultset r;
607     if (!c)
608         return;
609     yaz_log(c->log_api, "%p ZOOM_connection_destroy", c);
610
611     ZOOM_memcached_destroy(c);
612     if (c->cs)
613         cs_close(c->cs);
614
615     for (r = c->resultsets; r; r = r->next)
616         r->connection = 0;
617
618     xfree(c->buf_in);
619     xfree(c->addinfo);
620     xfree(c->diagset);
621     odr_destroy(c->odr_in);
622     odr_destroy(c->odr_out);
623     if (c->odr_save)
624         odr_destroy(c->odr_save);
625     if (c->odr_print)
626         odr_destroy(c->odr_print);
627     ZOOM_options_destroy(c->options);
628     ZOOM_connection_remove_tasks(c);
629     ZOOM_connection_remove_events(c);
630     xfree(c->host_port);
631     xfree(c->proxy);
632     xfree(c->tproxy);
633     xfree(c->charset);
634     xfree(c->lang);
635     xfree(c->cookie_out);
636     xfree(c->cookie_in);
637     xfree(c->client_IP);
638     xfree(c->user);
639     xfree(c->group);
640     xfree(c->password);
641     xfree(c->sru_version);
642     yaz_cookies_destroy(c->cookies);
643     wrbuf_destroy(c->saveAPDU_wrbuf);
644     xfree(c);
645 }
646
647 void ZOOM_resultset_addref(ZOOM_resultset r)
648 {
649     if (r)
650     {
651         yaz_mutex_enter(r->mutex);
652         (r->refcount)++;
653         yaz_log(log_details0, "%p ZOOM_resultset_addref count=%d",
654                 r, r->refcount);
655         yaz_mutex_leave(r->mutex);
656     }
657 }
658
659 static int g_resultsets = 0;
660 static YAZ_MUTEX g_resultset_mutex = 0;
661
662 /* TODO We need to initialize this before running threaded:
663  * call resultset_use(0)  */
664
665 static int resultset_use(int delta) {
666     int resultset_count;
667     if (g_resultset_mutex == 0)
668         yaz_mutex_create(&g_resultset_mutex);
669     yaz_mutex_enter(g_resultset_mutex);
670     g_resultsets += delta;
671     resultset_count = g_resultsets;
672     yaz_mutex_leave(g_resultset_mutex);
673     return resultset_count;
674 }
675
676 int resultsets_count(void) {
677     return resultset_use(0);
678 }
679
680 ZOOM_resultset ZOOM_resultset_create(void)
681 {
682     int i;
683     ZOOM_resultset r = (ZOOM_resultset) xmalloc(sizeof(*r));
684
685     initlog();
686
687     yaz_log(log_details0, "%p ZOOM_resultset_create", r);
688     r->refcount = 1;
689     r->size = 0;
690     r->odr = odr_createmem(ODR_DECODE);
691     r->piggyback = 1;
692     r->setname = 0;
693     r->step = 0;
694     for (i = 0; i<RECORD_HASH_SIZE; i++)
695         r->record_hash[i] = 0;
696     r->r_sort_spec = 0;
697     r->query = 0;
698     r->connection = 0;
699     r->databaseNames = 0;
700     r->num_databaseNames = 0;
701     r->req_facets = 0;
702     r->res_facets = 0;
703     r->num_res_facets = 0;
704     r->facets_names = 0;
705     r->mutex = 0;
706     yaz_mutex_create(&r->mutex);
707 #if SHPTR
708     {
709         WRBUF w = wrbuf_alloc();
710         YAZ_SHPTR_INIT(r->record_wrbuf, w);
711     }
712 #endif
713     resultset_use(1);
714     r->mc_key = 0;
715     r->live_set = 0;
716     return r;
717 }
718
719 ZOOM_API(ZOOM_resultset)
720     ZOOM_connection_search_pqf(ZOOM_connection c, const char *q)
721 {
722     ZOOM_resultset r;
723     ZOOM_query s = ZOOM_query_create();
724
725     ZOOM_query_prefix(s, q);
726
727     r = ZOOM_connection_search(c, s);
728     ZOOM_query_destroy(s);
729     return r;
730 }
731
732 ZOOM_API(ZOOM_resultset)
733     ZOOM_connection_search(ZOOM_connection c, ZOOM_query q)
734 {
735     ZOOM_resultset r = ZOOM_resultset_create();
736     ZOOM_task task;
737     int start, count;
738     const char *syntax, *elementSetName, *schema;
739     yaz_log(c->log_api, "%p ZOOM_connection_search set %p query %p", c, r, q);
740     r->r_sort_spec = ZOOM_query_get_sortspec(q);
741     r->query = q;
742     ZOOM_query_addref(q);
743
744     r->options = ZOOM_options_create_with_parent(c->options);
745
746     r->req_facets = odr_strdup_null(r->odr,
747                                     ZOOM_options_get(r->options, "facets"));
748     start = ZOOM_options_get_int(r->options, "start", 0);
749     count = ZOOM_options_get_int(r->options, "count", 0);
750     {
751         /* If "presentChunk" is defined use that; otherwise "step" */
752         const char *cp = ZOOM_options_get(r->options, "presentChunk");
753         r->step = ZOOM_options_get_int(r->options,
754                                        (cp != 0 ? "presentChunk": "step"), 0);
755     }
756     r->piggyback = ZOOM_options_get_bool(r->options, "piggyback", 1);
757     r->setname = odr_strdup_null(r->odr,
758                                  ZOOM_options_get(r->options, "setname"));
759     r->databaseNames = ZOOM_connection_get_databases(c, c->options,
760                                                      &r->num_databaseNames,
761                                                      r->odr);
762     r->connection = c;
763     r->next = c->resultsets;
764     c->resultsets = r;
765
766     ZOOM_memcached_resultset(r, q);
767
768     if (c->host_port && c->proto == PROTO_HTTP)
769     {
770         if (!c->cs)
771         {
772             yaz_log(c->log_details, "ZOOM_connection_search: no comstack");
773             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
774         }
775         else
776         {
777             yaz_log(c->log_details, "ZOOM_connection_search: reconnect");
778             c->reconnect_ok = 1;
779         }
780     }
781
782     task = ZOOM_connection_add_task(c, ZOOM_TASK_SEARCH);
783     task->u.search.resultset = r;
784     task->u.search.start = start;
785     task->u.search.count = count;
786
787     syntax = ZOOM_options_get(r->options, "preferredRecordSyntax");
788     task->u.search.syntax = syntax ? xstrdup(syntax) : 0;
789     elementSetName = ZOOM_options_get(r->options, "elementSetName");
790     task->u.search.elementSetName = elementSetName ?
791         xstrdup(elementSetName) : 0;
792     schema = ZOOM_options_get(r->options, "schema");
793     task->u.search.schema = schema ? xstrdup(schema) : 0;
794
795     ZOOM_resultset_addref(r);
796
797     if (!c->async)
798     {
799         while (ZOOM_event(1, &c))
800             ;
801     }
802     return r;
803 }
804
805 ZOOM_API(void)
806     ZOOM_resultset_sort(ZOOM_resultset r,
807                          const char *sort_type, const char *sort_spec)
808 {
809     (void) ZOOM_resultset_sort1(r, sort_type, sort_spec);
810 }
811
812 ZOOM_API(int)
813     ZOOM_resultset_sort1(ZOOM_resultset r,
814                          const char *sort_type, const char *sort_spec)
815 {
816     ZOOM_connection c = r->connection;
817     ZOOM_task task;
818     ZOOM_query newq;
819
820     newq = ZOOM_query_create();
821     if (ZOOM_query_sortby(newq, sort_spec) < 0)
822         return -1;
823
824     yaz_log(c->log_api, "%p ZOOM_resultset_sort r=%p sort_type=%s sort_spec=%s",
825             r, r, sort_type, sort_spec);
826     if (!c)
827         return 0;
828
829     if (c->host_port && c->proto == PROTO_HTTP)
830     {
831         if (!c->cs)
832         {
833             yaz_log(c->log_details, "%p ZOOM_resultset_sort: no comstack", r);
834             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
835         }
836         else
837         {
838             yaz_log(c->log_details, "%p ZOOM_resultset_sort: prepare reconnect",
839                     r);
840             c->reconnect_ok = 1;
841         }
842     }
843
844     ZOOM_resultset_cache_reset(r);
845     task = ZOOM_connection_add_task(c, ZOOM_TASK_SORT);
846     task->u.sort.resultset = r;
847     task->u.sort.q = newq;
848
849     ZOOM_resultset_addref(r);
850
851     if (!c->async)
852     {
853         while (ZOOM_event(1, &c))
854             ;
855     }
856
857     return 0;
858 }
859
860 ZOOM_API(void)
861     ZOOM_resultset_destroy(ZOOM_resultset r)
862 {
863     resultset_destroy(r);
864 }
865
866 static void resultset_destroy(ZOOM_resultset r)
867 {
868     if (!r)
869         return;
870     yaz_mutex_enter(r->mutex);
871     (r->refcount)--;
872     yaz_log(log_details0, "%p ZOOM_resultset_destroy r=%p count=%d",
873             r, r, r->refcount);
874     if (r->refcount == 0)
875     {
876         yaz_mutex_leave(r->mutex);
877
878         yaz_log(log_details0, "%p ZOOM_connection resultset_destroy: Deleting resultset (%p) ", r->connection, r);
879         ZOOM_resultset_cache_reset(r);
880         ZOOM_resultset_release(r);
881         ZOOM_query_destroy(r->query);
882         ZOOM_options_destroy(r->options);
883         odr_destroy(r->odr);
884         yaz_mutex_destroy(&r->mutex);
885 #if SHPTR
886         YAZ_SHPTR_DEC(r->record_wrbuf, wrbuf_destroy);
887 #endif
888         wrbuf_destroy(r->mc_key);
889         resultset_use(-1);
890         xfree(r);
891     }
892     else
893         yaz_mutex_leave(r->mutex);
894 }
895
896 ZOOM_API(size_t)
897     ZOOM_resultset_size(ZOOM_resultset r)
898 {
899     return r->size;
900 }
901
902 int ZOOM_test_reconnect(ZOOM_connection c)
903 {
904     ZOOM_Event event;
905
906     if (!c->reconnect_ok)
907         return 0;
908     ZOOM_connection_close(c);
909     c->reconnect_ok = 0;
910     c->tasks->running = 0;
911     ZOOM_connection_insert_task(c, ZOOM_TASK_CONNECT);
912
913     event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
914     ZOOM_connection_put_event(c, event);
915
916     return 1;
917 }
918
919 static void ZOOM_resultset_retrieve(ZOOM_resultset r,
920                                     int force_sync, int start, int count)
921 {
922     ZOOM_task task;
923     ZOOM_connection c;
924     const char *cp;
925     const char *syntax, *elementSetName;
926
927     if (!r)
928         return;
929     yaz_log(log_details0, "%p ZOOM_resultset_retrieve force_sync=%d start=%d"
930             " count=%d", r, force_sync, start, count);
931     c = r->connection;
932     if (!c)
933         return;
934
935     if (c->host_port && c->proto == PROTO_HTTP)
936     {
937         if (!c->cs)
938         {
939             yaz_log(log_details0, "%p ZOOM_resultset_retrieve: no comstack", r);
940             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
941         }
942         else
943         {
944             yaz_log(log_details0, "%p ZOOM_resultset_retrieve: prepare "
945                     "reconnect", r);
946             c->reconnect_ok = 1;
947         }
948     }
949     task = ZOOM_connection_add_task(c, ZOOM_TASK_SEARCH);
950     task->u.search.resultset = r;
951     task->u.search.start = start;
952     task->u.search.count = count;
953
954     syntax = ZOOM_options_get(r->options, "preferredRecordSyntax");
955     task->u.search.syntax = syntax ? xstrdup(syntax) : 0;
956     elementSetName = ZOOM_options_get(r->options, "elementSetName");
957     task->u.search.elementSetName = elementSetName
958         ? xstrdup(elementSetName) : 0;
959
960     cp = ZOOM_options_get(r->options, "schema");
961     task->u.search.schema = cp ? xstrdup(cp) : 0;
962
963     ZOOM_resultset_addref(r);
964
965     if (!r->connection->async || force_sync)
966         while (r->connection && ZOOM_event(1, &r->connection))
967             ;
968 }
969
970 ZOOM_API(void)
971     ZOOM_resultset_records(ZOOM_resultset r, ZOOM_record *recs,
972                            size_t start, size_t count)
973 {
974     int force_present = 0;
975
976     if (!r)
977         return ;
978     yaz_log(log_api0, "%p ZOOM_resultset_records r=%p start=%ld count=%ld",
979             r, r, (long) start, (long) count);
980     if (count && recs)
981         force_present = 1;
982     ZOOM_resultset_retrieve(r, force_present, start, count);
983     if (force_present)
984     {
985         size_t i;
986         for (i = 0; i< count; i++)
987             recs[i] = ZOOM_resultset_record_immediate(r, i+start);
988     }
989 }
990
991 ZOOM_API(size_t)
992     ZOOM_resultset_facets_size(ZOOM_resultset r)
993 {
994     return r->num_res_facets;
995 }
996
997 ZOOM_API(ZOOM_facet_field)
998     ZOOM_resultset_get_facet_field(ZOOM_resultset r, const char *name)
999 {
1000     int num = r->num_res_facets;
1001     ZOOM_facet_field *facets = r->res_facets;
1002     int i;
1003     for (i = 0; i < num; i++)
1004         if (!strcmp(facets[i]->facet_name, name))
1005             return facets[i];
1006     return 0;
1007 }
1008
1009 ZOOM_API(ZOOM_facet_field)
1010     ZOOM_resultset_get_facet_field_by_index(ZOOM_resultset r, int idx)
1011 {
1012     int num = r->num_res_facets;
1013     ZOOM_facet_field *facets = r->res_facets;
1014     if (idx >= 0 && idx < num)
1015         return facets[idx];
1016     return 0;
1017 }
1018
1019 ZOOM_API(ZOOM_facet_field *)
1020     ZOOM_resultset_facets(ZOOM_resultset r)
1021 {
1022     return r->res_facets;
1023 }
1024
1025 ZOOM_API(const char**)
1026     ZOOM_resultset_facets_names(ZOOM_resultset r)
1027 {
1028     return (const char **) r->facets_names;
1029 }
1030
1031 ZOOM_API(const char*)
1032     ZOOM_facet_field_name(ZOOM_facet_field field)
1033 {
1034     return field->facet_name;
1035 }
1036
1037 ZOOM_API(size_t)
1038     ZOOM_facet_field_term_count(ZOOM_facet_field field)
1039 {
1040     return field->num_terms;
1041 }
1042
1043 ZOOM_API(const char*)
1044     ZOOM_facet_field_get_term(ZOOM_facet_field field, size_t idx, int *freq)
1045 {
1046     *freq = field->facet_terms[idx].frequency;
1047     return field->facet_terms[idx].term;
1048 }
1049
1050
1051 static void get_cert(ZOOM_connection c)
1052 {
1053     char *cert_buf;
1054     int cert_len;
1055
1056     if (cs_get_peer_certificate_x509(c->cs, &cert_buf, &cert_len))
1057     {
1058         ZOOM_connection_option_setl(c, "sslPeerCert",
1059                                     cert_buf, cert_len);
1060         xfree(cert_buf);
1061     }
1062 }
1063
1064 static zoom_ret do_connect_host(ZOOM_connection c,
1065                                 const char *logical_url);
1066
1067 static zoom_ret do_connect(ZOOM_connection c)
1068 {
1069     return do_connect_host(c, c->host_port);
1070 }
1071
1072 static zoom_ret do_connect_host(ZOOM_connection c, const char *logical_url)
1073 {
1074     void *add;
1075
1076     if (c->cs)
1077         cs_close(c->cs);
1078     c->cs = cs_create_host_proxy(logical_url, CS_FLAGS_DNS_NO_BLOCK, &add,
1079                                  c->tproxy ? c->tproxy : c->proxy);
1080
1081     if (c->cs && c->cs->protocol == PROTO_HTTP)
1082     {
1083 #if YAZ_HAVE_XML2
1084         c->proto = PROTO_HTTP;
1085 #else
1086         ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, "SRW");
1087         ZOOM_connection_close(c);
1088         return zoom_complete;
1089 #endif
1090     }
1091     if (c->cs)
1092     {
1093         int ret = cs_connect(c->cs, add);
1094         if (ret == 0)
1095         {
1096             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1097             ZOOM_connection_put_event(c, event);
1098             get_cert(c);
1099             if (c->proto == PROTO_Z3950)
1100                 ZOOM_connection_Z3950_send_init(c);
1101             else
1102             {
1103                 /* no init request for SRW .. */
1104                 assert(c->tasks->which == ZOOM_TASK_CONNECT);
1105                 ZOOM_connection_remove_task(c);
1106                 ZOOM_connection_set_mask(c, 0);
1107                 ZOOM_connection_exec_task(c);
1108             }
1109             c->state = STATE_ESTABLISHED;
1110             return zoom_pending;
1111         }
1112         else if (ret > 0)
1113         {
1114             int mask = ZOOM_SELECT_EXCEPT;
1115             if (c->cs->io_pending & CS_WANT_WRITE)
1116                 mask += ZOOM_SELECT_WRITE;
1117             if (c->cs->io_pending & CS_WANT_READ)
1118                 mask += ZOOM_SELECT_READ;
1119             ZOOM_connection_set_mask(c, mask);
1120             c->state = STATE_CONNECTING;
1121             return zoom_pending;
1122         }
1123     }
1124     c->state = STATE_IDLE;
1125     ZOOM_set_error(c, ZOOM_ERROR_CONNECT, logical_url);
1126     return zoom_complete;
1127 }
1128
1129 /* returns 1 if PDU was sent OK (still pending )
1130    0 if PDU was not sent OK (nothing to wait for)
1131 */
1132
1133 ZOOM_API(ZOOM_record)
1134     ZOOM_resultset_record_immediate(ZOOM_resultset s,size_t pos)
1135 {
1136     const char *syntax =
1137         ZOOM_options_get(s->options, "preferredRecordSyntax");
1138     const char *elementSetName =
1139         ZOOM_options_get(s->options, "elementSetName");
1140     const char *schema =
1141         ZOOM_options_get(s->options, "schema");
1142
1143     return ZOOM_record_cache_lookup_i(s, pos, syntax, elementSetName, schema);
1144 }
1145
1146 ZOOM_API(ZOOM_record)
1147     ZOOM_resultset_record(ZOOM_resultset r, size_t pos)
1148 {
1149     ZOOM_record rec = ZOOM_resultset_record_immediate(r, pos);
1150
1151     if (!rec)
1152     {
1153         /*
1154          * MIKE: I think force_sync should always be zero, but I don't
1155          * want to make this change until I get the go-ahead from
1156          * Adam, in case something depends on the old synchronous
1157          * behaviour.
1158          */
1159         int force_sync = 1;
1160         if (getenv("ZOOM_RECORD_NO_FORCE_SYNC")) force_sync = 0;
1161         ZOOM_resultset_retrieve(r, force_sync, pos, 1);
1162         rec = ZOOM_resultset_record_immediate(r, pos);
1163     }
1164     return rec;
1165 }
1166
1167 ZOOM_API(ZOOM_scanset)
1168     ZOOM_connection_scan(ZOOM_connection c, const char *start)
1169 {
1170     ZOOM_scanset s;
1171     ZOOM_query q = ZOOM_query_create();
1172
1173     ZOOM_query_prefix(q, start);
1174
1175     s = ZOOM_connection_scan1(c, q);
1176     ZOOM_query_destroy(q);
1177     return s;
1178
1179 }
1180
1181 ZOOM_API(ZOOM_scanset)
1182     ZOOM_connection_scan1(ZOOM_connection c, ZOOM_query q)
1183 {
1184     ZOOM_scanset scan = 0;
1185     Z_Query *z_query = ZOOM_query_get_Z_Query(q);
1186
1187     if (!z_query)
1188         return 0;
1189     scan = (ZOOM_scanset) xmalloc(sizeof(*scan));
1190     scan->connection = c;
1191     scan->odr = odr_createmem(ODR_DECODE);
1192     scan->options = ZOOM_options_create_with_parent(c->options);
1193     scan->refcount = 1;
1194     scan->scan_response = 0;
1195     scan->srw_scan_response = 0;
1196
1197     scan->query = q;
1198     ZOOM_query_addref(q);
1199     scan->databaseNames = ZOOM_connection_get_databases(c, c->options,
1200                                             &scan->num_databaseNames,
1201                                             scan->odr);
1202
1203     if (1)
1204     {
1205         ZOOM_task task = ZOOM_connection_add_task(c, ZOOM_TASK_SCAN);
1206         task->u.scan.scan = scan;
1207
1208         (scan->refcount)++;
1209         if (!c->async)
1210         {
1211             while (ZOOM_event(1, &c))
1212                 ;
1213         }
1214     }
1215     return scan;
1216 }
1217
1218 ZOOM_API(void)
1219     ZOOM_scanset_destroy(ZOOM_scanset scan)
1220 {
1221     if (!scan)
1222         return;
1223     (scan->refcount)--;
1224     if (scan->refcount == 0)
1225     {
1226         ZOOM_query_destroy(scan->query);
1227
1228         odr_destroy(scan->odr);
1229
1230         ZOOM_options_destroy(scan->options);
1231         xfree(scan);
1232     }
1233 }
1234
1235 static zoom_ret send_package(ZOOM_connection c)
1236 {
1237     ZOOM_Event event;
1238
1239     yaz_log(c->log_details, "%p send_package", c);
1240     if (!c->tasks)
1241         return zoom_complete;
1242     assert (c->tasks->which == ZOOM_TASK_PACKAGE);
1243
1244     event = ZOOM_Event_create(ZOOM_EVENT_SEND_APDU);
1245     ZOOM_connection_put_event(c, event);
1246
1247     c->buf_out = c->tasks->u.package->buf_out;
1248     c->len_out = c->tasks->u.package->len_out;
1249
1250     return ZOOM_send_buf(c);
1251 }
1252
1253 ZOOM_API(size_t)
1254     ZOOM_scanset_size(ZOOM_scanset scan)
1255 {
1256     if (!scan)
1257         return 0;
1258
1259     if (scan->scan_response && scan->scan_response->entries)
1260         return scan->scan_response->entries->num_entries;
1261     else if (scan->srw_scan_response)
1262         return scan->srw_scan_response->num_terms;
1263     return 0;
1264 }
1265
1266 static void ZOOM_scanset_term_x(ZOOM_scanset scan, size_t pos,
1267                                 size_t *occ,
1268                                 const char **value_term, size_t *value_len,
1269                                 const char **disp_term, size_t *disp_len)
1270 {
1271     size_t noent = ZOOM_scanset_size(scan);
1272
1273     *value_term = 0;
1274     *value_len = 0;
1275
1276     *disp_term = 0;
1277     *disp_len = 0;
1278
1279     *occ = 0;
1280     if (pos >= noent)
1281         return;
1282     if (scan->scan_response)
1283     {
1284         Z_ScanResponse *res = scan->scan_response;
1285         if (res->entries->entries[pos]->which == Z_Entry_termInfo)
1286         {
1287             Z_TermInfo *t = res->entries->entries[pos]->u.termInfo;
1288
1289             *value_term = (const char *) t->term->u.general->buf;
1290             *value_len = t->term->u.general->len;
1291             if (t->displayTerm)
1292             {
1293                 *disp_term = t->displayTerm;
1294                 *disp_len = strlen(*disp_term);
1295             }
1296             else if (t->term->which == Z_Term_general)
1297             {
1298                 *disp_term = (const char *) t->term->u.general->buf;
1299                 *disp_len = t->term->u.general->len;
1300             }
1301             *occ = t->globalOccurrences ? *t->globalOccurrences : 0;
1302         }
1303     }
1304     if (scan->srw_scan_response)
1305     {
1306         Z_SRW_scanResponse *res = scan->srw_scan_response;
1307         Z_SRW_scanTerm *t = res->terms + pos;
1308         if (t)
1309         {
1310             *value_term = t->value;
1311             *value_len = strlen(*value_term);
1312
1313             if (t->displayTerm)
1314                 *disp_term = t->displayTerm;
1315             else
1316                 *disp_term = t->value;
1317             *disp_len = strlen(*disp_term);
1318             *occ = t->numberOfRecords ? *t->numberOfRecords : 0;
1319         }
1320     }
1321 }
1322
1323 ZOOM_API(const char *)
1324     ZOOM_scanset_term(ZOOM_scanset scan, size_t pos,
1325                       size_t *occ, size_t *len)
1326 {
1327     const char *value_term = 0;
1328     size_t value_len = 0;
1329     const char *disp_term = 0;
1330     size_t disp_len = 0;
1331
1332     ZOOM_scanset_term_x(scan, pos, occ, &value_term, &value_len,
1333                         &disp_term, &disp_len);
1334
1335     *len = value_len;
1336     return value_term;
1337 }
1338
1339 ZOOM_API(const char *)
1340     ZOOM_scanset_display_term(ZOOM_scanset scan, size_t pos,
1341                               size_t *occ, size_t *len)
1342 {
1343     const char *value_term = 0;
1344     size_t value_len = 0;
1345     const char *disp_term = 0;
1346     size_t disp_len = 0;
1347
1348     ZOOM_scanset_term_x(scan, pos, occ, &value_term, &value_len,
1349                         &disp_term, &disp_len);
1350
1351     *len = disp_len;
1352     return disp_term;
1353 }
1354
1355 ZOOM_API(const char *)
1356     ZOOM_scanset_option_get(ZOOM_scanset scan, const char *key)
1357 {
1358     return ZOOM_options_get(scan->options, key);
1359 }
1360
1361 ZOOM_API(void)
1362     ZOOM_scanset_option_set(ZOOM_scanset scan, const char *key,
1363                             const char *val)
1364 {
1365     ZOOM_options_set(scan->options, key, val);
1366 }
1367
1368
1369 ZOOM_API(ZOOM_package)
1370     ZOOM_connection_package(ZOOM_connection c, ZOOM_options options)
1371 {
1372     ZOOM_package p = (ZOOM_package) xmalloc(sizeof(*p));
1373
1374     p->connection = c;
1375     p->odr_out = odr_createmem(ODR_ENCODE);
1376     p->options = ZOOM_options_create_with_parent2(options, c->options);
1377     p->refcount = 1;
1378     p->buf_out = 0;
1379     p->len_out = 0;
1380     return p;
1381 }
1382
1383 ZOOM_API(void)
1384     ZOOM_package_destroy(ZOOM_package p)
1385 {
1386     if (!p)
1387         return;
1388     (p->refcount)--;
1389     if (p->refcount == 0)
1390     {
1391         odr_destroy(p->odr_out);
1392         xfree(p->buf_out);
1393
1394         ZOOM_options_destroy(p->options);
1395         xfree(p);
1396     }
1397 }
1398
1399 ZOOM_API(const char *)
1400     ZOOM_package_option_get(ZOOM_package p, const char *key)
1401 {
1402     return ZOOM_options_get(p->options, key);
1403 }
1404
1405 ZOOM_API(const char *)
1406     ZOOM_package_option_getl(ZOOM_package p, const char *key, int *lenp)
1407 {
1408     return ZOOM_options_getl(p->options, key, lenp);
1409 }
1410
1411 ZOOM_API(void)
1412     ZOOM_package_option_set(ZOOM_package p, const char *key,
1413                             const char *val)
1414 {
1415     ZOOM_options_set(p->options, key, val);
1416 }
1417
1418 ZOOM_API(void)
1419     ZOOM_package_option_setl(ZOOM_package p, const char *key,
1420                              const char *val, int len)
1421 {
1422     ZOOM_options_setl(p->options, key, val, len);
1423 }
1424
1425 ZOOM_API(int)
1426     ZOOM_connection_exec_task(ZOOM_connection c)
1427 {
1428     ZOOM_task task = c->tasks;
1429     zoom_ret ret = zoom_complete;
1430
1431     if (!task)
1432         return 0;
1433     yaz_log(c->log_details, "%p ZOOM_connection_exec_task type=%d run=%d",
1434             c, task->which, task->running);
1435     if (c->error != ZOOM_ERROR_NONE)
1436     {
1437         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1438                 "removing tasks because of error = %d", c, c->error);
1439         ZOOM_connection_remove_tasks(c);
1440         return 0;
1441     }
1442     if (task->running)
1443     {
1444         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1445                 "task already running", c);
1446         return 0;
1447     }
1448     task->running = 1;
1449     ret = zoom_complete;
1450     if (c->cs || task->which == ZOOM_TASK_CONNECT)
1451     {
1452         switch (task->which)
1453         {
1454         case ZOOM_TASK_SEARCH:
1455             if (c->proto == PROTO_HTTP)
1456                 ret = ZOOM_connection_srw_send_search(c);
1457             else
1458                 ret = ZOOM_connection_Z3950_search(c);
1459             break;
1460         case ZOOM_TASK_CONNECT:
1461             ret = do_connect(c);
1462             break;
1463         case ZOOM_TASK_SCAN:
1464             if (c->proto == PROTO_HTTP)
1465                 ret = ZOOM_connection_srw_send_scan(c);
1466             else
1467                 ret = ZOOM_connection_Z3950_send_scan(c);
1468             break;
1469         case ZOOM_TASK_PACKAGE:
1470             ret = send_package(c);
1471             break;
1472         case ZOOM_TASK_SORT:
1473             c->tasks->u.sort.resultset->r_sort_spec =
1474                 ZOOM_query_get_sortspec(c->tasks->u.sort.q);
1475             ret = send_Z3950_sort(c, c->tasks->u.sort.resultset);
1476             break;
1477         }
1478     }
1479     else
1480     {
1481         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1482                 "remove tasks because no connection exist", c);
1483         ZOOM_connection_remove_tasks(c);
1484     }
1485     if (ret == zoom_complete)
1486     {
1487         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1488                 "task removed (complete)", c);
1489         ZOOM_connection_remove_task(c);
1490         return 0;
1491     }
1492     yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1493             "task pending", c);
1494     return 1;
1495 }
1496
1497 #if YAZ_HAVE_XML2
1498
1499 static zoom_ret send_HTTP_redirect(ZOOM_connection c, const char *uri)
1500 {
1501     Z_GDU *gdu = z_get_HTTP_Request_uri(c->odr_out, uri, 0, c->proxy ? 1 : 0);
1502
1503     gdu->u.HTTP_Request->method = odr_strdup(c->odr_out, "GET");
1504     z_HTTP_header_add(c->odr_out, &gdu->u.HTTP_Request->headers, "Accept",
1505                       "text/xml");
1506     yaz_cookies_request(c->cookies, c->odr_out, gdu->u.HTTP_Request);
1507     if (c->user && c->password)
1508     {
1509         z_HTTP_header_add_basic_auth(c->odr_out, &gdu->u.HTTP_Request->headers,
1510                                      c->user, c->password);
1511     }
1512     return ZOOM_send_GDU(c, gdu);
1513 }
1514
1515 zoom_ret ZOOM_send_GDU(ZOOM_connection c, Z_GDU *gdu)
1516 {
1517     ZOOM_Event event;
1518
1519     int r = z_GDU(c->odr_out, &gdu, 0, 0);
1520     if (!r)
1521         return zoom_complete;
1522     if (c->odr_print)
1523         z_GDU(c->odr_print, &gdu, 0, 0);
1524     if (c->odr_save)
1525         z_GDU(c->odr_save, &gdu, 0, 0);
1526     c->buf_out = odr_getbuf(c->odr_out, &c->len_out, 0);
1527     odr_reset(c->odr_out);
1528
1529     event = ZOOM_Event_create(ZOOM_EVENT_SEND_APDU);
1530     ZOOM_connection_put_event(c, event);
1531
1532     return ZOOM_send_buf(c);
1533 }
1534
1535 #if YAZ_HAVE_XML2
1536 void ZOOM_set_HTTP_error(ZOOM_connection c, int error,
1537                          const char *addinfo, const char *addinfo2)
1538 {
1539     ZOOM_set_dset_error(c, error, "HTTP", addinfo, addinfo2);
1540 }
1541 #endif
1542
1543
1544 static void handle_http(ZOOM_connection c, Z_HTTP_Response *hres)
1545 {
1546     zoom_ret cret = zoom_complete;
1547     int ret = -1;
1548     char *addinfo = 0;
1549     const char *connection_head = z_HTTP_header_lookup(hres->headers,
1550                                                        "Connection");
1551     const char *location;
1552
1553     ZOOM_connection_set_mask(c, 0);
1554     yaz_log(c->log_details, "%p handle_http", c);
1555
1556     yaz_cookies_response(c->cookies, hres);
1557     if ((hres->code == 301 || hres->code == 302) && c->sru_mode == zoom_sru_get
1558         && (location = z_HTTP_header_lookup(hres->headers, "Location")))
1559     {
1560         c->no_redirects++;
1561         if (c->no_redirects > 10)
1562         {
1563             ZOOM_set_HTTP_error(c, hres->code, 0, 0);
1564             c->no_redirects = 0;
1565             ZOOM_connection_close(c);
1566         }
1567         else
1568         {
1569             /* since redirect may change host we just reconnect. A smarter
1570                implementation might check whether it's the same server */
1571
1572             int host_change = 0;
1573             location = yaz_check_location(c->odr_in, c->host_port,
1574                                           location, &host_change);
1575             if (do_connect_host(c, location) == zoom_complete)
1576                 return;  /* connect failed.. */
1577             cs_rcvconnect(c->cs);
1578             send_HTTP_redirect(c, location);
1579             return;
1580         }
1581     }
1582     else
1583     {
1584         ret = ZOOM_handle_sru(c, hres, &cret, &addinfo);
1585         if (ret == 0)
1586         {
1587             if (c->no_redirects) /* end of redirect. change hosts again */
1588                 ZOOM_connection_close(c);
1589         }
1590         c->no_redirects = 0;
1591     }
1592     if (ret)
1593     {
1594         if (hres->code != 200)
1595             ZOOM_set_HTTP_error(c, hres->code, 0, 0);
1596         else
1597         {
1598             yaz_log(YLOG_LOG, "set error... addinfo=%s", addinfo ?
1599                     addinfo : "NULL");
1600             ZOOM_set_error(c, ZOOM_ERROR_DECODE, addinfo);
1601         }
1602         ZOOM_connection_close(c);
1603     }
1604     if (cret == zoom_complete)
1605     {
1606         yaz_log(c->log_details, "removing tasks in handle_http");
1607         ZOOM_connection_remove_task(c);
1608     }
1609     {
1610         int must_close = 0;
1611         if (!strcmp(hres->version, "1.0"))
1612         {
1613             /* HTTP 1.0: only if Keep-Alive we stay alive.. */
1614             if (!connection_head || strcmp(connection_head, "Keep-Alive"))
1615                 must_close = 1;
1616         }
1617         else
1618         {
1619             /* HTTP 1.1: only if no close we stay alive.. */
1620             if (connection_head && !strcmp(connection_head, "close"))
1621                 must_close = 1;
1622         }
1623         if (must_close)
1624         {
1625             ZOOM_connection_close(c);
1626             if (c->tasks)
1627             {
1628                 c->tasks->running = 0;
1629                 ZOOM_connection_insert_task(c, ZOOM_TASK_CONNECT);
1630                 c->reconnect_ok = 0;
1631             }
1632         }
1633         else
1634             c->reconnect_ok = 1; /* if the server closes anyway */
1635     }
1636 }
1637 #endif
1638
1639 static int do_read(ZOOM_connection c)
1640 {
1641     int r, more;
1642     ZOOM_Event event;
1643
1644     event = ZOOM_Event_create(ZOOM_EVENT_RECV_DATA);
1645     ZOOM_connection_put_event(c, event);
1646
1647     r = cs_get(c->cs, &c->buf_in, &c->len_in);
1648     more = cs_more(c->cs);
1649     yaz_log(c->log_details, "%p do_read len=%d more=%d", c, r, more);
1650     if (r == 1)
1651         return 0;
1652     if (r <= 0)
1653     {
1654         if (!ZOOM_test_reconnect(c))
1655         {
1656             ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
1657             ZOOM_connection_close(c);
1658         }
1659     }
1660     else
1661     {
1662         Z_GDU *gdu;
1663         ZOOM_Event event;
1664
1665         odr_reset(c->odr_in);
1666         odr_setbuf(c->odr_in, c->buf_in, r, 0);
1667         event = ZOOM_Event_create(ZOOM_EVENT_RECV_APDU);
1668         ZOOM_connection_put_event(c, event);
1669
1670         if (!z_GDU(c->odr_in, &gdu, 0, 0))
1671         {
1672             int x;
1673             int err = odr_geterrorx(c->odr_in, &x);
1674             char msg[100];
1675             const char *element = odr_getelement(c->odr_in);
1676             yaz_snprintf(msg, sizeof(msg),
1677                     "ODR code %d:%d element=%s offset=%d",
1678                     err, x, element ? element : "<unknown>",
1679                     odr_offset(c->odr_in));
1680             ZOOM_set_error(c, ZOOM_ERROR_DECODE, msg);
1681             if (c->log_api)
1682             {
1683                 FILE *ber_file = yaz_log_file();
1684                 if (ber_file)
1685                     odr_dumpBER(ber_file, c->buf_in, r);
1686             }
1687             ZOOM_connection_close(c);
1688         }
1689         else
1690         {
1691             if (c->odr_print)
1692                 z_GDU(c->odr_print, &gdu, 0, 0);
1693             if (c->odr_save)
1694                 z_GDU(c->odr_save, &gdu, 0, 0);
1695             if (gdu->which == Z_GDU_Z3950)
1696                 ZOOM_handle_Z3950_apdu(c, gdu->u.z3950);
1697             else if (gdu->which == Z_GDU_HTTP_Response)
1698             {
1699 #if YAZ_HAVE_XML2
1700                 handle_http(c, gdu->u.HTTP_Response);
1701 #else
1702                 ZOOM_set_error(c, ZOOM_ERROR_DECODE, 0);
1703                 ZOOM_connection_close(c);
1704 #endif
1705             }
1706         }
1707     }
1708     return 1;
1709 }
1710
1711 static zoom_ret do_write_ex(ZOOM_connection c, char *buf_out, int len_out)
1712 {
1713     int r;
1714     ZOOM_Event event;
1715
1716     event = ZOOM_Event_create(ZOOM_EVENT_SEND_DATA);
1717     ZOOM_connection_put_event(c, event);
1718
1719     yaz_log(c->log_details, "%p do_write_ex len=%d", c, len_out);
1720     if ((r = cs_put(c->cs, buf_out, len_out)) < 0)
1721     {
1722         yaz_log(c->log_details, "%p do_write_ex write failed", c);
1723         if (ZOOM_test_reconnect(c))
1724         {
1725             return zoom_pending;
1726         }
1727         if (c->state == STATE_CONNECTING)
1728             ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
1729         else
1730             ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
1731         ZOOM_connection_close(c);
1732         return zoom_complete;
1733     }
1734     else if (r == 1)
1735     {
1736         int mask = ZOOM_SELECT_EXCEPT;
1737         if (c->cs->io_pending & CS_WANT_WRITE)
1738             mask += ZOOM_SELECT_WRITE;
1739         if (c->cs->io_pending & CS_WANT_READ)
1740             mask += ZOOM_SELECT_READ;
1741         ZOOM_connection_set_mask(c, mask);
1742         yaz_log(c->log_details, "%p do_write_ex write incomplete mask=%d",
1743                 c, c->mask);
1744     }
1745     else
1746     {
1747         ZOOM_connection_set_mask(c, ZOOM_SELECT_READ|ZOOM_SELECT_EXCEPT);
1748         yaz_log(c->log_details, "%p do_write_ex write complete mask=%d",
1749                 c, c->mask);
1750     }
1751     return zoom_pending;
1752 }
1753
1754 zoom_ret ZOOM_send_buf(ZOOM_connection c)
1755 {
1756     return do_write_ex(c, c->buf_out, c->len_out);
1757 }
1758
1759
1760 ZOOM_API(const char *)
1761     ZOOM_connection_option_get(ZOOM_connection c, const char *key)
1762 {
1763     if (!strcmp(key, "APDU"))
1764     {
1765         return c->saveAPDU_wrbuf ? wrbuf_cstr(c->saveAPDU_wrbuf) : "";
1766     }
1767     else
1768         return ZOOM_options_get(c->options, key);
1769 }
1770
1771 ZOOM_API(const char *)
1772     ZOOM_connection_option_getl(ZOOM_connection c, const char *key, int *lenp)
1773 {
1774     if (!strcmp(key, "APDU"))
1775     {
1776         if (c->saveAPDU_wrbuf)
1777         {
1778             *lenp = wrbuf_len(c->saveAPDU_wrbuf);
1779             return wrbuf_cstr(c->saveAPDU_wrbuf);
1780         }
1781         else
1782         {
1783             *lenp = 0;
1784             return "";
1785         }
1786     }
1787     else
1788         return ZOOM_options_getl(c->options, key, lenp);
1789 }
1790
1791 ZOOM_API(void)
1792     ZOOM_connection_option_set(ZOOM_connection c, const char *key,
1793                                const char *val)
1794 {
1795     if (!strcmp(key, "saveAPDU"))
1796     {
1797         if (val && strcmp(val, "0"))
1798         {
1799             if (!c->saveAPDU_wrbuf)
1800                 c->saveAPDU_wrbuf = wrbuf_alloc();
1801             else
1802                 wrbuf_rewind(c->saveAPDU_wrbuf);
1803         }
1804         else
1805         {
1806             wrbuf_destroy(c->saveAPDU_wrbuf);
1807             c->saveAPDU_wrbuf = 0;
1808         }
1809         ZOOM_connection_save_apdu_wrbuf(c, c->saveAPDU_wrbuf);
1810     }
1811     else
1812         ZOOM_options_set(c->options, key, val);
1813 }
1814
1815 ZOOM_API(void)
1816     ZOOM_connection_option_setl(ZOOM_connection c, const char *key,
1817                                 const char *val, int len)
1818 {
1819     ZOOM_options_setl(c->options, key, val, len);
1820 }
1821
1822 ZOOM_API(const char *)
1823     ZOOM_resultset_option_get(ZOOM_resultset r, const char *key)
1824 {
1825     return ZOOM_options_get(r->options, key);
1826 }
1827
1828 ZOOM_API(void)
1829     ZOOM_resultset_option_set(ZOOM_resultset r, const char *key,
1830                               const char *val)
1831 {
1832     ZOOM_options_set(r->options, key, val);
1833 }
1834
1835
1836 ZOOM_API(int)
1837     ZOOM_connection_errcode(ZOOM_connection c)
1838 {
1839     return ZOOM_connection_error(c, 0, 0);
1840 }
1841
1842 ZOOM_API(const char *)
1843     ZOOM_connection_errmsg(ZOOM_connection c)
1844 {
1845     const char *msg;
1846     ZOOM_connection_error(c, &msg, 0);
1847     return msg;
1848 }
1849
1850 ZOOM_API(const char *)
1851     ZOOM_connection_addinfo(ZOOM_connection c)
1852 {
1853     const char *addinfo;
1854     ZOOM_connection_error(c, 0, &addinfo);
1855     return addinfo;
1856 }
1857
1858 ZOOM_API(const char *)
1859     ZOOM_connection_diagset(ZOOM_connection c)
1860 {
1861     const char *diagset;
1862     ZOOM_connection_error_x(c, 0, 0, &diagset);
1863     return diagset;
1864 }
1865
1866 ZOOM_API(const char *)
1867     ZOOM_diag_str(int error)
1868 {
1869     switch (error)
1870     {
1871     case ZOOM_ERROR_NONE:
1872         return "No error";
1873     case ZOOM_ERROR_CONNECT:
1874         return "Connect failed";
1875     case ZOOM_ERROR_MEMORY:
1876         return "Out of memory";
1877     case ZOOM_ERROR_ENCODE:
1878         return "Encoding failed";
1879     case ZOOM_ERROR_DECODE:
1880         return "Decoding failed";
1881     case ZOOM_ERROR_CONNECTION_LOST:
1882         return "Connection lost";
1883     case ZOOM_ERROR_INIT:
1884         return "Init rejected";
1885     case ZOOM_ERROR_INTERNAL:
1886         return "Internal failure";
1887     case ZOOM_ERROR_TIMEOUT:
1888         return "Timeout";
1889     case ZOOM_ERROR_UNSUPPORTED_PROTOCOL:
1890         return "Unsupported protocol";
1891     case ZOOM_ERROR_UNSUPPORTED_QUERY:
1892         return "Unsupported query type";
1893     case ZOOM_ERROR_INVALID_QUERY:
1894         return "Invalid query";
1895     case ZOOM_ERROR_CQL_PARSE:
1896         return "CQL parsing error";
1897     case ZOOM_ERROR_CQL_TRANSFORM:
1898         return "CQL transformation error";
1899     case ZOOM_ERROR_CCL_CONFIG:
1900         return "CCL configuration error";
1901     case ZOOM_ERROR_CCL_PARSE:
1902         return "CCL parsing error";
1903     case ZOOM_ERROR_ES_INVALID_ACTION:
1904         return "Extended Service. invalid action";
1905     case ZOOM_ERROR_ES_INVALID_VERSION:
1906         return "Extended Service. invalid version";
1907     case ZOOM_ERROR_ES_INVALID_SYNTAX:
1908         return "Extended Service. invalid syntax";
1909     case ZOOM_ERROR_MEMCACHED:
1910         return "Memcached";
1911     default:
1912         return diagbib1_str(error);
1913     }
1914 }
1915
1916 ZOOM_API(int)
1917     ZOOM_connection_error_x(ZOOM_connection c, const char **cp,
1918                             const char **addinfo, const char **diagset)
1919 {
1920     int error = c->error;
1921     if (cp)
1922     {
1923         if (!c->diagset || !strcmp(c->diagset, "ZOOM"))
1924             *cp = ZOOM_diag_str(error);
1925         else if (!strcmp(c->diagset, "HTTP"))
1926             *cp = z_HTTP_errmsg(c->error);
1927         else if (!strcmp(c->diagset, "Bib-1"))
1928             *cp = ZOOM_diag_str(error);
1929         else if (!strcmp(c->diagset, "info:srw/diagnostic/1"))
1930             *cp = yaz_diag_srw_str(c->error);
1931         else
1932             *cp = "Unknown error and diagnostic set";
1933     }
1934     if (addinfo)
1935         *addinfo = c->addinfo ? c->addinfo : "";
1936     if (diagset)
1937         *diagset = c->diagset ? c->diagset : "";
1938     return c->error;
1939 }
1940
1941 ZOOM_API(int)
1942     ZOOM_connection_error(ZOOM_connection c, const char **cp,
1943                           const char **addinfo)
1944 {
1945     return ZOOM_connection_error_x(c, cp, addinfo, 0);
1946 }
1947
1948 static void ZOOM_connection_do_io(ZOOM_connection c, int mask)
1949 {
1950     ZOOM_Event event = 0;
1951     int r = cs_look(c->cs);
1952     yaz_log(c->log_details, "%p ZOOM_connection_do_io mask=%d cs_look=%d",
1953             c, mask, r);
1954
1955     if (r == CS_NONE)
1956     {
1957         event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1958         ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
1959         ZOOM_connection_close(c);
1960         ZOOM_connection_put_event(c, event);
1961     }
1962     else if (r == CS_CONNECT)
1963     {
1964         int ret = ret = cs_rcvconnect(c->cs);
1965         yaz_log(c->log_details, "%p ZOOM_connection_do_io "
1966                 "cs_rcvconnect returned %d", c, ret);
1967         if (ret == 1)
1968         {
1969             int mask = ZOOM_SELECT_EXCEPT;
1970             if (c->cs->io_pending & CS_WANT_WRITE)
1971                 mask += ZOOM_SELECT_WRITE;
1972             if (c->cs->io_pending & CS_WANT_READ)
1973                 mask += ZOOM_SELECT_READ;
1974             ZOOM_connection_set_mask(c, mask);
1975             event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1976             ZOOM_connection_put_event(c, event);
1977         }
1978         else if (ret == 0)
1979         {
1980             event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1981             ZOOM_connection_put_event(c, event);
1982             get_cert(c);
1983             if (c->proto == PROTO_Z3950)
1984                 ZOOM_connection_Z3950_send_init(c);
1985             else
1986             {
1987                 /* no init request for SRW .. */
1988                 if (c->tasks->which == ZOOM_TASK_CONNECT)
1989                 {
1990                     ZOOM_connection_remove_task(c);
1991                     ZOOM_connection_set_mask(c, 0);
1992                 }
1993                 ZOOM_connection_exec_task(c);
1994             }
1995             c->state = STATE_ESTABLISHED;
1996         }
1997         else
1998         {
1999             ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
2000             ZOOM_connection_close(c);
2001         }
2002     }
2003     else
2004     {
2005         if (mask & ZOOM_SELECT_EXCEPT)
2006         {
2007             if (!ZOOM_test_reconnect(c))
2008             {
2009                 ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
2010                 ZOOM_connection_close(c);
2011             }
2012             return;
2013         }
2014         if (mask & ZOOM_SELECT_READ)
2015             do_read(c);
2016         if (c->cs && (mask & ZOOM_SELECT_WRITE))
2017             ZOOM_send_buf(c);
2018     }
2019 }
2020
2021 ZOOM_API(int)
2022     ZOOM_connection_last_event(ZOOM_connection cs)
2023 {
2024     if (!cs)
2025         return ZOOM_EVENT_NONE;
2026     return cs->last_event;
2027 }
2028
2029
2030 ZOOM_API(int) ZOOM_connection_fire_event_timeout(ZOOM_connection c)
2031 {
2032     if (c->mask)
2033     {
2034         ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_TIMEOUT);
2035         /* timeout and this connection was waiting */
2036         ZOOM_set_error(c, ZOOM_ERROR_TIMEOUT, 0);
2037         ZOOM_connection_close(c);
2038         ZOOM_connection_put_event(c, event);
2039     }
2040     return 0;
2041 }
2042
2043 ZOOM_API(int)
2044     ZOOM_connection_process(ZOOM_connection c)
2045 {
2046     ZOOM_Event event;
2047     if (!c)
2048         return 0;
2049
2050     event = ZOOM_connection_get_event(c);
2051     if (event)
2052     {
2053         ZOOM_Event_destroy(event);
2054         return 1;
2055     }
2056     ZOOM_connection_exec_task(c);
2057     event = ZOOM_connection_get_event(c);
2058     if (event)
2059     {
2060         ZOOM_Event_destroy(event);
2061         return 1;
2062     }
2063     return 0;
2064 }
2065
2066 ZOOM_API(int)
2067     ZOOM_event_nonblock(int no, ZOOM_connection *cs)
2068 {
2069     int i;
2070
2071     yaz_log(log_details0, "ZOOM_process_event(no=%d,cs=%p)", no, cs);
2072
2073     for (i = 0; i<no; i++)
2074     {
2075         ZOOM_connection c = cs[i];
2076
2077         if (c && ZOOM_connection_process(c))
2078             return i+1;
2079     }
2080     return 0;
2081 }
2082
2083 ZOOM_API(int) ZOOM_connection_fire_event_socket(ZOOM_connection c, int mask)
2084 {
2085     if (c->mask && mask)
2086         ZOOM_connection_do_io(c, mask);
2087     return 0;
2088 }
2089
2090 ZOOM_API(int) ZOOM_connection_get_socket(ZOOM_connection c)
2091 {
2092     if (c->cs)
2093         return cs_fileno(c->cs);
2094     return -1;
2095 }
2096
2097 ZOOM_API(int) ZOOM_connection_set_mask(ZOOM_connection c, int mask)
2098 {
2099     c->mask = mask;
2100     if (!c->cs)
2101         return -1;
2102     return 0;
2103 }
2104
2105 ZOOM_API(int) ZOOM_connection_get_mask(ZOOM_connection c)
2106 {
2107     if (c->cs)
2108         return c->mask;
2109     return 0;
2110 }
2111
2112 ZOOM_API(int) ZOOM_connection_get_timeout(ZOOM_connection c)
2113 {
2114     return ZOOM_options_get_int(c->options, "timeout", 30);
2115 }
2116
2117 ZOOM_API(void) ZOOM_connection_close(ZOOM_connection c)
2118 {
2119     if (c->cs)
2120         cs_close(c->cs);
2121     c->cs = 0;
2122     ZOOM_connection_set_mask(c, 0);
2123     c->state = STATE_IDLE;
2124 }
2125
2126 /*
2127  * Local variables:
2128  * c-basic-offset: 4
2129  * c-file-style: "Stroustrup"
2130  * indent-tabs-mode: nil
2131  * End:
2132  * vim: shiftwidth=4 tabstop=8 expandtab
2133  */
2134