3e5d64742e6dd99728a3c759b40b6532a770ad52
[yaz-moved-to-github.git] / src / zoom-c.c
1 /* This file is part of the YAZ toolkit.
2  * Copyright (C) Index Data
3  * See the file LICENSE for details.
4  */
5 /**
6  * \file zoom-c.c
7  * \brief Implements ZOOM C interface.
8  */
9 #if HAVE_CONFIG_H
10 #include <config.h>
11 #endif
12
13 #include <assert.h>
14 #include <string.h>
15 #include <errno.h>
16 #include "zoom-p.h"
17
18 #include <yaz/yaz-util.h>
19 #include <yaz/xmalloc.h>
20 #include <yaz/otherinfo.h>
21 #include <yaz/log.h>
22 #include <yaz/diagbib1.h>
23 #include <yaz/charneg.h>
24 #include <yaz/query-charset.h>
25 #include <yaz/snprintf.h>
26 #include <yaz/facet.h>
27
28 #include <yaz/shptr.h>
29
30 #if SHPTR
31 YAZ_SHPTR_TYPE(WRBUF)
32 #endif
33
34 static int log_api0 = 0;
35 static int log_details0 = 0;
36
37 static void resultset_destroy(ZOOM_resultset r);
38 static zoom_ret do_write_ex(ZOOM_connection c, char *buf_out, int len_out);
39
40 static void initlog(void)
41 {
42     static int log_level_initialized = 0;
43
44     if (!log_level_initialized)
45     {
46         log_api0 = yaz_log_module_level("zoom");
47         log_details0 = yaz_log_module_level("zoomdetails");
48         log_level_initialized = 1;
49     }
50 }
51
52 static zoom_ret send_HTTP_redirect(ZOOM_connection c, const char *uri);
53
54 void ZOOM_set_dset_error(ZOOM_connection c, int error,
55                          const char *dset,
56                          const char *addinfo, const char *addinfo2)
57 {
58     char *cp;
59
60     xfree(c->addinfo);
61     c->addinfo = 0;
62     c->error = error;
63     if (!c->diagset || strcmp(dset, c->diagset))
64     {
65         xfree(c->diagset);
66         c->diagset = xstrdup(dset);
67         /* remove integer part from SRW diagset .. */
68         if ((cp = strrchr(c->diagset, '/')))
69             *cp = '\0';
70     }
71     if (addinfo && addinfo2)
72     {
73         c->addinfo = (char*) xmalloc(strlen(addinfo) + strlen(addinfo2) + 3);
74         strcpy(c->addinfo, addinfo);
75         strcat(c->addinfo, ": ");
76         strcat(c->addinfo, addinfo2);
77     }
78     else if (addinfo)
79         c->addinfo = xstrdup(addinfo);
80     if (error != ZOOM_ERROR_NONE)
81     {
82         yaz_log(c->log_api, "%p set_dset_error %s %s:%d %s %s",
83                 c, c->host_port ? c->host_port : "<>", dset, error,
84                 addinfo ? addinfo : "",
85                 addinfo2 ? addinfo2 : "");
86     }
87 }
88
89 int ZOOM_uri_to_code(const char *uri)
90 {
91     int code = 0;
92     const char *cp;
93     if ((cp = strrchr(uri, '/')))
94         code = atoi(cp+1);
95     return code;
96 }
97
98 void ZOOM_set_error(ZOOM_connection c, int error, const char *addinfo)
99 {
100     ZOOM_set_dset_error(c, error, "ZOOM", addinfo, 0);
101 }
102
103 static void clear_error(ZOOM_connection c)
104 {
105     /*
106      * If an error is tied to an operation then it's ok to clear: for
107      * example, a diagnostic returned from a search is cleared by a
108      * subsequent search.  However, problems such as Connection Lost
109      * or Init Refused are not cleared, because they are not
110      * recoverable: doing another search doesn't help.
111      */
112
113     ZOOM_connection_remove_events(c);
114     switch (c->error)
115     {
116     case ZOOM_ERROR_CONNECT:
117     case ZOOM_ERROR_MEMORY:
118     case ZOOM_ERROR_DECODE:
119     case ZOOM_ERROR_CONNECTION_LOST:
120     case ZOOM_ERROR_INIT:
121     case ZOOM_ERROR_INTERNAL:
122     case ZOOM_ERROR_UNSUPPORTED_PROTOCOL:
123         break;
124     default:
125         ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
126     }
127 }
128
129 void ZOOM_connection_show_task(ZOOM_task task)
130 {
131     switch(task->which)
132     {
133     case ZOOM_TASK_SEARCH:
134         yaz_log(YLOG_LOG, "search p=%p", task);
135         break;
136     case ZOOM_TASK_CONNECT:
137         yaz_log(YLOG_LOG, "connect p=%p", task);
138         break;
139     case ZOOM_TASK_SCAN:
140         yaz_log(YLOG_LOG, "scan p=%p", task);
141         break;
142     }
143 }
144
145 void ZOOM_connection_show_tasks(ZOOM_connection c)
146 {
147     ZOOM_task task;
148     yaz_log(YLOG_LOG, "connection p=%p tasks", c);
149     for (task = c->tasks; task; task = task->next)
150         ZOOM_connection_show_task(task);
151 }
152
153 ZOOM_task ZOOM_connection_add_task(ZOOM_connection c, int which)
154 {
155     ZOOM_task *taskp = &c->tasks;
156     while (*taskp)
157         taskp = &(*taskp)->next;
158     *taskp = (ZOOM_task) xmalloc(sizeof(**taskp));
159     (*taskp)->running = 0;
160     (*taskp)->which = which;
161     (*taskp)->next = 0;
162     clear_error(c);
163     return *taskp;
164 }
165
166 ZOOM_API(int) ZOOM_connection_is_idle(ZOOM_connection c)
167 {
168     return c->tasks ? 0 : 1;
169 }
170
171 ZOOM_task ZOOM_connection_insert_task(ZOOM_connection c, int which)
172 {
173     ZOOM_task task = (ZOOM_task) xmalloc(sizeof(*task));
174
175     task->next = c->tasks;
176     c->tasks = task;
177
178     task->running = 0;
179     task->which = which;
180     return task;
181 }
182
183 void ZOOM_connection_remove_task(ZOOM_connection c)
184 {
185     ZOOM_task task = c->tasks;
186
187     if (task)
188     {
189         c->tasks = task->next;
190         switch (task->which)
191         {
192         case ZOOM_TASK_SEARCH:
193             resultset_destroy(task->u.search.resultset);
194             xfree(task->u.search.syntax);
195             xfree(task->u.search.elementSetName);
196             xfree(task->u.search.schema);
197             break;
198         case ZOOM_TASK_CONNECT:
199             break;
200         case ZOOM_TASK_SCAN:
201             ZOOM_scanset_destroy(task->u.scan.scan);
202             break;
203         case ZOOM_TASK_PACKAGE:
204             ZOOM_package_destroy(task->u.package);
205             break;
206         case ZOOM_TASK_SORT:
207             resultset_destroy(task->u.sort.resultset);
208             ZOOM_query_destroy(task->u.sort.q);
209             break;
210         default:
211             assert(0);
212         }
213         xfree(task);
214
215         if (!c->tasks)
216         {
217             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_END);
218             ZOOM_connection_put_event(c, event);
219         }
220     }
221 }
222
223 void ZOOM_connection_remove_tasks(ZOOM_connection c)
224 {
225     while (c->tasks)
226         ZOOM_connection_remove_task(c);
227 }
228
229 static void odr_wrbuf_write(ODR o, void *handle, int type,
230                             const char *buf, int len)
231 {
232     WRBUF w = (WRBUF) handle;
233     wrbuf_write(w, buf, len);
234 }
235
236 ZOOM_API(ZOOM_connection)
237     ZOOM_connection_create(ZOOM_options options)
238 {
239     ZOOM_connection c = (ZOOM_connection) xmalloc(sizeof(*c));
240
241     initlog();
242
243     c->log_api = log_api0;
244     c->log_details = log_details0;
245
246     yaz_log(c->log_api, "%p ZOOM_connection_create", c);
247
248     c->proto = PROTO_Z3950;
249     c->cs = 0;
250     ZOOM_connection_set_mask(c, 0);
251     c->reconnect_ok = 0;
252     c->state = STATE_IDLE;
253     c->addinfo = 0;
254     c->diagset = 0;
255     ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
256     c->buf_in = 0;
257     c->len_in = 0;
258     c->buf_out = 0;
259     c->len_out = 0;
260     c->resultsets = 0;
261
262     c->options = ZOOM_options_create_with_parent(options);
263
264     c->host_port = 0;
265     c->proxy = 0;
266     c->tproxy = 0;
267     c->proxy_mode = 0;
268
269     c->charset = c->lang = 0;
270
271     c->cookie_out = 0;
272     c->cookie_in = 0;
273     c->client_IP = 0;
274     c->tasks = 0;
275
276     c->user = 0;
277     c->group = 0;
278     c->password = 0;
279     c->url_authentication = 0;
280
281     c->maximum_record_size = 0;
282     c->preferred_message_size = 0;
283
284     c->odr_in = odr_createmem(ODR_DECODE);
285     c->odr_out = odr_createmem(ODR_ENCODE);
286     c->odr_print = 0;
287     c->odr_save = 0;
288
289     c->async = 0;
290     c->support_named_resultsets = 0;
291     c->last_event = ZOOM_EVENT_NONE;
292
293     c->m_queue_front = 0;
294     c->m_queue_back = 0;
295
296     c->sru_version = 0;
297     c->no_redirects = 0;
298     c->cookies = 0;
299     c->location = 0;
300     c->saveAPDU_wrbuf = 0;
301
302     ZOOM_memcached_init(c);
303     return c;
304 }
305
306 ZOOM_API(void) ZOOM_connection_save_apdu_wrbuf(ZOOM_connection c, WRBUF w)
307 {
308     if (c->odr_save)
309     {
310         odr_destroy(c->odr_save);
311         c->odr_save = 0;
312     }
313     if (w)
314     {
315         c->odr_save = odr_createmem(ODR_PRINT);
316         odr_set_stream(c->odr_save, w, odr_wrbuf_write, 0);
317     }
318 }
319
320 /* set database names. Take local databases (if set); otherwise
321    take databases given in ZURL (if set); otherwise use Default */
322 char **ZOOM_connection_get_databases(ZOOM_connection con, ZOOM_options options,
323                                      int *num, ODR odr)
324 {
325     char **databaseNames;
326     const char *cp = ZOOM_options_get(options, "databaseName");
327
328     if ((!cp || !*cp) && con->host_port)
329         cs_get_host_args(con->host_port, &cp);
330     if (!cp || !*cp)
331         cp = "Default";
332     nmem_strsplit(odr_getmem(odr), "+", cp,  &databaseNames, num);
333     return databaseNames;
334 }
335
336 ZOOM_API(ZOOM_connection)
337     ZOOM_connection_new(const char *host, int portnum)
338 {
339     ZOOM_connection c = ZOOM_connection_create(0);
340
341     ZOOM_connection_connect(c, host, portnum);
342     return c;
343 }
344
345 static zoom_sru_mode get_sru_mode_from_string(const char *s)
346 {
347     if (!s || !*s)
348         return zoom_sru_soap;
349     if (!yaz_matchstr(s, "soap"))
350         return zoom_sru_soap;
351     else if (!yaz_matchstr(s, "get"))
352         return zoom_sru_get;
353     else if (!yaz_matchstr(s, "post"))
354         return zoom_sru_post;
355     else if (!yaz_matchstr(s, "solr"))
356         return zoom_sru_solr;
357     return zoom_sru_error;
358 }
359
360 ZOOM_API(void)
361     ZOOM_connection_connect(ZOOM_connection c,
362                             const char *host, int portnum)
363 {
364     const char *val;
365
366     initlog();
367
368     yaz_log(c->log_api, "%p ZOOM_connection_connect host=%s portnum=%d",
369             c, host ? host : "null", portnum);
370
371     ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
372     ZOOM_connection_remove_tasks(c);
373
374     if (c->cs)
375     {
376         yaz_log(c->log_details, "%p ZOOM_connection_connect reconnect ok", c);
377         c->reconnect_ok = 1;
378         return;
379     }
380     yaz_log(c->log_details, "%p ZOOM_connection_connect connect", c);
381     xfree(c->proxy);
382     c->proxy = 0;
383     val = ZOOM_options_get(c->options, "proxy");
384     if (val && *val)
385     {
386         yaz_log(c->log_details, "%p ZOOM_connection_connect proxy=%s", c, val);
387         c->proxy = xstrdup(val);
388     }
389
390     xfree(c->tproxy);
391     c->tproxy = 0;
392     val = ZOOM_options_get(c->options, "tproxy");
393     if (val && *val)
394     {
395         yaz_log(c->log_details, "%p ZOOM_connection_connect tproxy=%s", c, val);
396         c->tproxy = xstrdup(val);
397     }
398
399     xfree(c->charset);
400     c->charset = 0;
401     val = ZOOM_options_get(c->options, "charset");
402     if (val && *val)
403     {
404         yaz_log(c->log_details, "%p ZOOM_connection_connect charset=%s", c, val);
405         c->charset = xstrdup(val);
406     }
407
408     xfree(c->lang);
409     val = ZOOM_options_get(c->options, "lang");
410     if (val && *val)
411     {
412         yaz_log(c->log_details, "%p ZOOM_connection_connect lang=%s", c, val);
413         c->lang = xstrdup(val);
414     }
415     else
416         c->lang = 0;
417
418     if (host)
419     {
420         char hostn[128];
421         const char *http_lead;
422
423         val = ZOOM_options_get(c->options, "sru");
424         if (val && *val && !strstr(host, "://"))
425             http_lead = "http://";
426         else
427             http_lead = "";
428         c->sru_mode = get_sru_mode_from_string(val);
429
430         xfree(c->host_port);
431         if (portnum)
432         {
433             sprintf(hostn, "%.80s:%d", host, portnum);
434             host = hostn;
435         }
436         c->host_port = xmalloc(strlen(host) + strlen(http_lead) + 1);
437         strcpy(c->host_port, http_lead);
438         strcat(c->host_port, host);
439     }
440
441     {
442         /*
443          * If the "<scheme>:" part of the host string is preceded by one
444          * or more comma-separated <name>=<value> pairs, these are taken
445          * to be options to be set on the connection object.  Among other
446          * applications, this facility can be used to embed authentication
447          * in a host string:
448          *          user=admin,password=secret,tcp:localhost:9999
449          */
450         char *remainder = c->host_port;
451         char *pcolon = strchr(remainder, ':');
452         char *pcomma;
453         char *pequals;
454         while ((pcomma = strchr(remainder, ',')) != 0 &&
455                (pcolon == 0 || pcomma < pcolon))
456         {
457             *pcomma = '\0';
458             if ((pequals = strchr(remainder, '=')) != 0)
459             {
460                 *pequals = '\0';
461                 ZOOM_connection_option_set(c, remainder, pequals+1);
462             }
463             remainder = pcomma+1;
464         }
465
466         if (remainder != c->host_port)
467         {
468             remainder = xstrdup(remainder);
469             xfree(c->host_port);
470             c->host_port = remainder;
471         }
472     }
473
474     xfree(c->sru_version);
475     val = ZOOM_options_get(c->options, "sru_version");
476     c->sru_version = xstrdup(val ? val : "1.2");
477
478     ZOOM_options_set(c->options, "host", c->host_port);
479
480     xfree(c->cookie_out);
481     c->cookie_out = 0;
482     val = ZOOM_options_get(c->options, "cookie");
483     if (val && *val)
484     {
485         yaz_log(c->log_details, "%p ZOOM_connection_connect cookie=%s", c, val);
486         c->cookie_out = xstrdup(val);
487     }
488
489     xfree(c->client_IP);
490     c->client_IP = 0;
491     val = ZOOM_options_get(c->options, "clientIP");
492     if (val && *val)
493     {
494         yaz_log(c->log_details, "%p ZOOM_connection_connect clientIP=%s",
495                 c, val);
496         c->client_IP = xstrdup(val);
497     }
498
499     xfree(c->group);
500     c->group = 0;
501     val = ZOOM_options_get(c->options, "group");
502     if (val && *val)
503         c->group = xstrdup(val);
504
505     xfree(c->user);
506     c->user = 0;
507     val = ZOOM_options_get(c->options, "user");
508     if (val && *val)
509         c->user = xstrdup(val);
510
511     xfree(c->password);
512     c->password = 0;
513     val = ZOOM_options_get(c->options, "password");
514     if (!val)
515         val = ZOOM_options_get(c->options, "pass");
516     if (val && *val)
517         c->password = xstrdup(val);
518
519     val = ZOOM_options_get(c->options, "authenticationMode");
520     if (val && !strcmp(val, "url"))
521         c->url_authentication = 1;
522     else
523         c->url_authentication = 0;
524
525     c->maximum_record_size =
526         ZOOM_options_get_int(c->options, "maximumRecordSize", 64*1024*1024);
527     c->preferred_message_size =
528         ZOOM_options_get_int(c->options, "preferredMessageSize", 64*1024*1024);
529
530     c->async = ZOOM_options_get_bool(c->options, "async", 0);
531
532     yaz_cookies_destroy(c->cookies);
533     c->cookies = yaz_cookies_create();
534
535     if (ZOOM_memcached_configure(c))
536     {
537         ZOOM_connection_remove_tasks(c);
538         return;
539     }
540     if (c->sru_mode == zoom_sru_error)
541     {
542         ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, val);
543         ZOOM_connection_remove_tasks(c);
544         return;
545     }
546
547     if (c->odr_print)
548         odr_destroy(c->odr_print);
549     c->odr_print = 0;
550     val = ZOOM_options_get(c->options, "apdufile");
551     if (val)
552     {
553         c->odr_print = odr_createmem(ODR_PRINT);
554         if (strcmp(val, "-"))
555         {
556             FILE *f = fopen(val, "a");
557             if (!f)
558             {
559                 WRBUF w = wrbuf_alloc();
560                 wrbuf_printf(w, "fopen: %s", val);
561                 ZOOM_set_error(c, ZOOM_ERROR_INTERNAL, wrbuf_cstr(w));
562                 wrbuf_destroy(w);
563                 return;
564             }
565             odr_setprint(c->odr_print, f);
566         }
567     }
568     else if (ZOOM_options_get_bool(c->options, "apdulog", 0))
569     {
570         c->odr_print = odr_createmem(ODR_PRINT);
571         odr_setprint_noclose(c->odr_print, yaz_log_file());
572     }
573
574     yaz_log(c->log_details, "%p ZOOM_connection_connect async=%d", c, c->async);
575     ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
576
577     if (!c->async)
578     {
579         while (ZOOM_event(1, &c))
580             ;
581     }
582 }
583
584 ZOOM_API(void) ZOOM_resultset_release(ZOOM_resultset r)
585 {
586     if (r->connection)
587     {
588         /* remove ourselves from the resultsets in connection */
589         ZOOM_resultset *rp = &r->connection->resultsets;
590         while (1)
591         {
592             assert(*rp);   /* we must be in this list!! */
593             if (*rp == r)
594             {   /* OK, we're here - take us out of it */
595                 *rp = (*rp)->next;
596                 break;
597             }
598             rp = &(*rp)->next;
599         }
600         r->connection = 0;
601     }
602 }
603
604 ZOOM_API(void)
605     ZOOM_connection_destroy(ZOOM_connection c)
606 {
607     ZOOM_resultset r;
608     if (!c)
609         return;
610     yaz_log(c->log_api, "%p ZOOM_connection_destroy", c);
611
612     ZOOM_memcached_destroy(c);
613     if (c->cs)
614         cs_close(c->cs);
615
616     for (r = c->resultsets; r; r = r->next)
617         r->connection = 0;
618
619     xfree(c->buf_in);
620     xfree(c->addinfo);
621     xfree(c->diagset);
622     odr_destroy(c->odr_in);
623     odr_destroy(c->odr_out);
624     if (c->odr_save)
625         odr_destroy(c->odr_save);
626     if (c->odr_print)
627         odr_destroy(c->odr_print);
628     ZOOM_options_destroy(c->options);
629     ZOOM_connection_remove_tasks(c);
630     ZOOM_connection_remove_events(c);
631     xfree(c->host_port);
632     xfree(c->proxy);
633     xfree(c->tproxy);
634     xfree(c->charset);
635     xfree(c->lang);
636     xfree(c->cookie_out);
637     xfree(c->cookie_in);
638     xfree(c->client_IP);
639     xfree(c->user);
640     xfree(c->group);
641     xfree(c->password);
642     xfree(c->sru_version);
643     xfree(c->location);
644     yaz_cookies_destroy(c->cookies);
645     wrbuf_destroy(c->saveAPDU_wrbuf);
646     xfree(c);
647 }
648
649 void ZOOM_resultset_addref(ZOOM_resultset r)
650 {
651     if (r)
652     {
653         yaz_mutex_enter(r->mutex);
654         (r->refcount)++;
655         yaz_log(log_details0, "%p ZOOM_resultset_addref count=%d",
656                 r, r->refcount);
657         yaz_mutex_leave(r->mutex);
658     }
659 }
660
661 static int g_resultsets = 0;
662 static YAZ_MUTEX g_resultset_mutex = 0;
663
664 /* TODO We need to initialize this before running threaded:
665  * call resultset_use(0)  */
666
667 static int resultset_use(int delta) {
668     int resultset_count;
669     if (g_resultset_mutex == 0)
670         yaz_mutex_create(&g_resultset_mutex);
671     yaz_mutex_enter(g_resultset_mutex);
672     g_resultsets += delta;
673     resultset_count = g_resultsets;
674     yaz_mutex_leave(g_resultset_mutex);
675     return resultset_count;
676 }
677
678 int resultsets_count(void) {
679     return resultset_use(0);
680 }
681
682 ZOOM_resultset ZOOM_resultset_create(void)
683 {
684     int i;
685     ZOOM_resultset r = (ZOOM_resultset) xmalloc(sizeof(*r));
686
687     initlog();
688
689     yaz_log(log_details0, "%p ZOOM_resultset_create", r);
690     r->refcount = 1;
691     r->size = 0;
692     r->odr = odr_createmem(ODR_DECODE);
693     r->piggyback = 1;
694     r->setname = 0;
695     r->step = 0;
696     for (i = 0; i<RECORD_HASH_SIZE; i++)
697         r->record_hash[i] = 0;
698     r->r_sort_spec = 0;
699     r->query = 0;
700     r->connection = 0;
701     r->databaseNames = 0;
702     r->num_databaseNames = 0;
703     r->req_facets = 0;
704     r->res_facets = 0;
705     r->num_res_facets = 0;
706     r->facets_names = 0;
707     r->mutex = 0;
708     yaz_mutex_create(&r->mutex);
709 #if SHPTR
710     {
711         WRBUF w = wrbuf_alloc();
712         YAZ_SHPTR_INIT(r->record_wrbuf, w);
713     }
714 #endif
715     resultset_use(1);
716     r->mc_key = 0;
717     r->live_set = 0;
718     return r;
719 }
720
721 ZOOM_API(ZOOM_resultset)
722     ZOOM_connection_search_pqf(ZOOM_connection c, const char *q)
723 {
724     ZOOM_resultset r;
725     ZOOM_query s = ZOOM_query_create();
726
727     ZOOM_query_prefix(s, q);
728
729     r = ZOOM_connection_search(c, s);
730     ZOOM_query_destroy(s);
731     return r;
732 }
733
734 ZOOM_API(ZOOM_resultset)
735     ZOOM_connection_search(ZOOM_connection c, ZOOM_query q)
736 {
737     ZOOM_resultset r = ZOOM_resultset_create();
738     ZOOM_task task;
739     int start, count;
740     const char *syntax, *elementSetName, *schema;
741     yaz_log(c->log_api, "%p ZOOM_connection_search set %p query %p", c, r, q);
742     r->r_sort_spec = ZOOM_query_get_sortspec(q);
743     r->query = q;
744     ZOOM_query_addref(q);
745
746     r->options = ZOOM_options_create_with_parent(c->options);
747
748     r->req_facets = odr_strdup_null(r->odr,
749                                     ZOOM_options_get(r->options, "facets"));
750     start = ZOOM_options_get_int(r->options, "start", 0);
751     count = ZOOM_options_get_int(r->options, "count", 0);
752     {
753         /* If "presentChunk" is defined use that; otherwise "step" */
754         const char *cp = ZOOM_options_get(r->options, "presentChunk");
755         r->step = ZOOM_options_get_int(r->options,
756                                        (cp != 0 ? "presentChunk": "step"), 0);
757     }
758     r->piggyback = ZOOM_options_get_bool(r->options, "piggyback", 1);
759     r->setname = odr_strdup_null(r->odr,
760                                  ZOOM_options_get(r->options, "setname"));
761     r->databaseNames = ZOOM_connection_get_databases(c, c->options,
762                                                      &r->num_databaseNames,
763                                                      r->odr);
764     r->connection = c;
765     r->next = c->resultsets;
766     c->resultsets = r;
767
768     ZOOM_memcached_resultset(r, q);
769
770     if (c->host_port && c->proto == PROTO_HTTP)
771     {
772         if (!c->cs)
773         {
774             yaz_log(c->log_details, "ZOOM_connection_search: no comstack");
775             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
776         }
777         else
778         {
779             yaz_log(c->log_details, "ZOOM_connection_search: reconnect");
780             c->reconnect_ok = 1;
781         }
782     }
783
784     task = ZOOM_connection_add_task(c, ZOOM_TASK_SEARCH);
785     task->u.search.resultset = r;
786     task->u.search.start = start;
787     task->u.search.count = count;
788
789     syntax = ZOOM_options_get(r->options, "preferredRecordSyntax");
790     task->u.search.syntax = syntax ? xstrdup(syntax) : 0;
791     elementSetName = ZOOM_options_get(r->options, "elementSetName");
792     task->u.search.elementSetName = elementSetName ?
793         xstrdup(elementSetName) : 0;
794     schema = ZOOM_options_get(r->options, "schema");
795     task->u.search.schema = schema ? xstrdup(schema) : 0;
796
797     ZOOM_resultset_addref(r);
798
799     if (!c->async)
800     {
801         while (ZOOM_event(1, &c))
802             ;
803     }
804     return r;
805 }
806
807 ZOOM_API(void)
808     ZOOM_resultset_sort(ZOOM_resultset r,
809                          const char *sort_type, const char *sort_spec)
810 {
811     (void) ZOOM_resultset_sort1(r, sort_type, sort_spec);
812 }
813
814 ZOOM_API(int)
815     ZOOM_resultset_sort1(ZOOM_resultset r,
816                          const char *sort_type, const char *sort_spec)
817 {
818     ZOOM_connection c = r->connection;
819     ZOOM_task task;
820     ZOOM_query newq;
821
822     newq = ZOOM_query_create();
823     if (ZOOM_query_sortby(newq, sort_spec) < 0)
824         return -1;
825
826     yaz_log(c->log_api, "%p ZOOM_resultset_sort r=%p sort_type=%s sort_spec=%s",
827             r, r, sort_type, sort_spec);
828     if (!c)
829         return 0;
830
831     if (c->host_port && c->proto == PROTO_HTTP)
832     {
833         if (!c->cs)
834         {
835             yaz_log(c->log_details, "%p ZOOM_resultset_sort: no comstack", r);
836             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
837         }
838         else
839         {
840             yaz_log(c->log_details, "%p ZOOM_resultset_sort: prepare reconnect",
841                     r);
842             c->reconnect_ok = 1;
843         }
844     }
845
846     ZOOM_resultset_cache_reset(r);
847     task = ZOOM_connection_add_task(c, ZOOM_TASK_SORT);
848     task->u.sort.resultset = r;
849     task->u.sort.q = newq;
850
851     ZOOM_resultset_addref(r);
852
853     if (!c->async)
854     {
855         while (ZOOM_event(1, &c))
856             ;
857     }
858
859     return 0;
860 }
861
862 ZOOM_API(void)
863     ZOOM_resultset_destroy(ZOOM_resultset r)
864 {
865     resultset_destroy(r);
866 }
867
868 static void resultset_destroy(ZOOM_resultset r)
869 {
870     if (!r)
871         return;
872     yaz_mutex_enter(r->mutex);
873     (r->refcount)--;
874     yaz_log(log_details0, "%p ZOOM_resultset_destroy r=%p count=%d",
875             r, r, r->refcount);
876     if (r->refcount == 0)
877     {
878         yaz_mutex_leave(r->mutex);
879
880         yaz_log(log_details0, "%p ZOOM_connection resultset_destroy: Deleting resultset (%p) ", r->connection, r);
881         ZOOM_resultset_cache_reset(r);
882         ZOOM_resultset_release(r);
883         ZOOM_query_destroy(r->query);
884         ZOOM_options_destroy(r->options);
885         odr_destroy(r->odr);
886         yaz_mutex_destroy(&r->mutex);
887 #if SHPTR
888         YAZ_SHPTR_DEC(r->record_wrbuf, wrbuf_destroy);
889 #endif
890         wrbuf_destroy(r->mc_key);
891         resultset_use(-1);
892         xfree(r);
893     }
894     else
895         yaz_mutex_leave(r->mutex);
896 }
897
898 ZOOM_API(size_t)
899     ZOOM_resultset_size(ZOOM_resultset r)
900 {
901     return r->size;
902 }
903
904 int ZOOM_test_reconnect(ZOOM_connection c)
905 {
906     ZOOM_Event event;
907
908     if (!c->reconnect_ok)
909         return 0;
910     ZOOM_connection_close(c);
911     c->reconnect_ok = 0;
912     c->tasks->running = 0;
913     ZOOM_connection_insert_task(c, ZOOM_TASK_CONNECT);
914
915     event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
916     ZOOM_connection_put_event(c, event);
917
918     return 1;
919 }
920
921 static void ZOOM_resultset_retrieve(ZOOM_resultset r,
922                                     int force_sync, int start, int count)
923 {
924     ZOOM_task task;
925     ZOOM_connection c;
926     const char *cp;
927     const char *syntax, *elementSetName;
928
929     if (!r)
930         return;
931     yaz_log(log_details0, "%p ZOOM_resultset_retrieve force_sync=%d start=%d"
932             " count=%d", r, force_sync, start, count);
933     c = r->connection;
934     if (!c)
935         return;
936
937     if (c->host_port && c->proto == PROTO_HTTP)
938     {
939         if (!c->cs)
940         {
941             yaz_log(log_details0, "%p ZOOM_resultset_retrieve: no comstack", r);
942             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
943         }
944         else
945         {
946             yaz_log(log_details0, "%p ZOOM_resultset_retrieve: prepare "
947                     "reconnect", r);
948             c->reconnect_ok = 1;
949         }
950     }
951     task = ZOOM_connection_add_task(c, ZOOM_TASK_SEARCH);
952     task->u.search.resultset = r;
953     task->u.search.start = start;
954     task->u.search.count = count;
955
956     syntax = ZOOM_options_get(r->options, "preferredRecordSyntax");
957     task->u.search.syntax = syntax ? xstrdup(syntax) : 0;
958     elementSetName = ZOOM_options_get(r->options, "elementSetName");
959     task->u.search.elementSetName = elementSetName
960         ? xstrdup(elementSetName) : 0;
961
962     cp = ZOOM_options_get(r->options, "schema");
963     task->u.search.schema = cp ? xstrdup(cp) : 0;
964
965     ZOOM_resultset_addref(r);
966
967     if (!r->connection->async || force_sync)
968         while (r->connection && ZOOM_event(1, &r->connection))
969             ;
970 }
971
972 ZOOM_API(void)
973     ZOOM_resultset_records(ZOOM_resultset r, ZOOM_record *recs,
974                            size_t start, size_t count)
975 {
976     int force_present = 0;
977
978     if (!r)
979         return ;
980     yaz_log(log_api0, "%p ZOOM_resultset_records r=%p start=%ld count=%ld",
981             r, r, (long) start, (long) count);
982     if (count && recs)
983         force_present = 1;
984     ZOOM_resultset_retrieve(r, force_present, start, count);
985     if (force_present)
986     {
987         size_t i;
988         for (i = 0; i< count; i++)
989             recs[i] = ZOOM_resultset_record_immediate(r, i+start);
990     }
991 }
992
993 ZOOM_API(size_t)
994     ZOOM_resultset_facets_size(ZOOM_resultset r)
995 {
996     return r->num_res_facets;
997 }
998
999 ZOOM_API(ZOOM_facet_field)
1000     ZOOM_resultset_get_facet_field(ZOOM_resultset r, const char *name)
1001 {
1002     int num = r->num_res_facets;
1003     ZOOM_facet_field *facets = r->res_facets;
1004     int i;
1005     for (i = 0; i < num; i++)
1006         if (!strcmp(facets[i]->facet_name, name))
1007             return facets[i];
1008     return 0;
1009 }
1010
1011 ZOOM_API(ZOOM_facet_field)
1012     ZOOM_resultset_get_facet_field_by_index(ZOOM_resultset r, int idx)
1013 {
1014     int num = r->num_res_facets;
1015     ZOOM_facet_field *facets = r->res_facets;
1016     if (idx >= 0 && idx < num)
1017         return facets[idx];
1018     return 0;
1019 }
1020
1021 ZOOM_API(ZOOM_facet_field *)
1022     ZOOM_resultset_facets(ZOOM_resultset r)
1023 {
1024     return r->res_facets;
1025 }
1026
1027 ZOOM_API(const char**)
1028     ZOOM_resultset_facets_names(ZOOM_resultset r)
1029 {
1030     return (const char **) r->facets_names;
1031 }
1032
1033 ZOOM_API(const char*)
1034     ZOOM_facet_field_name(ZOOM_facet_field field)
1035 {
1036     return field->facet_name;
1037 }
1038
1039 ZOOM_API(size_t)
1040     ZOOM_facet_field_term_count(ZOOM_facet_field field)
1041 {
1042     return field->num_terms;
1043 }
1044
1045 ZOOM_API(const char*)
1046     ZOOM_facet_field_get_term(ZOOM_facet_field field, size_t idx, int *freq)
1047 {
1048     *freq = field->facet_terms[idx].frequency;
1049     return field->facet_terms[idx].term;
1050 }
1051
1052
1053 static void get_cert(ZOOM_connection c)
1054 {
1055     char *cert_buf;
1056     int cert_len;
1057
1058     if (cs_get_peer_certificate_x509(c->cs, &cert_buf, &cert_len))
1059     {
1060         ZOOM_connection_option_setl(c, "sslPeerCert",
1061                                     cert_buf, cert_len);
1062         xfree(cert_buf);
1063     }
1064 }
1065
1066 static zoom_ret do_connect_host(ZOOM_connection c,
1067                                 const char *logical_url);
1068
1069 static zoom_ret do_connect(ZOOM_connection c)
1070 {
1071     return do_connect_host(c, c->host_port);
1072 }
1073
1074 static zoom_ret do_connect_host(ZOOM_connection c, const char *logical_url)
1075 {
1076     void *add;
1077
1078     if (c->cs)
1079         cs_close(c->cs);
1080     c->cs = cs_create_host2(logical_url, CS_FLAGS_DNS_NO_BLOCK, &add,
1081                             c->tproxy ? c->tproxy : c->proxy,
1082                             &c->proxy_mode);
1083     if (!c->proxy)
1084         c->proxy_mode = 0;
1085
1086     if (c->cs && c->cs->protocol == PROTO_HTTP)
1087     {
1088 #if YAZ_HAVE_XML2
1089         c->proto = PROTO_HTTP;
1090 #else
1091         ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, "SRW");
1092         ZOOM_connection_close(c);
1093         return zoom_complete;
1094 #endif
1095     }
1096     if (c->cs)
1097     {
1098         int ret = cs_connect(c->cs, add);
1099         if (ret == 0)
1100         {
1101             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1102             ZOOM_connection_put_event(c, event);
1103             get_cert(c);
1104             if (c->proto == PROTO_Z3950)
1105                 ZOOM_connection_Z3950_send_init(c);
1106             else
1107             {
1108                 /* no init request for SRW .. */
1109                 assert(c->tasks->which == ZOOM_TASK_CONNECT);
1110                 ZOOM_connection_remove_task(c);
1111                 ZOOM_connection_set_mask(c, 0);
1112
1113                 if (c->cs && c->location)
1114                     send_HTTP_redirect(c, c->location);
1115                 else
1116                     ZOOM_connection_exec_task(c);
1117             }
1118             c->state = STATE_ESTABLISHED;
1119             return zoom_pending;
1120         }
1121         else if (ret > 0)
1122         {
1123             int mask = ZOOM_SELECT_EXCEPT;
1124             if (c->cs->io_pending & CS_WANT_WRITE)
1125                 mask += ZOOM_SELECT_WRITE;
1126             if (c->cs->io_pending & CS_WANT_READ)
1127                 mask += ZOOM_SELECT_READ;
1128             ZOOM_connection_set_mask(c, mask);
1129             c->state = STATE_CONNECTING;
1130             return zoom_pending;
1131         }
1132     }
1133     c->state = STATE_IDLE;
1134     ZOOM_set_error(c, ZOOM_ERROR_CONNECT, logical_url);
1135     return zoom_complete;
1136 }
1137
1138 /* returns 1 if PDU was sent OK (still pending )
1139    0 if PDU was not sent OK (nothing to wait for)
1140 */
1141
1142 ZOOM_API(ZOOM_record)
1143     ZOOM_resultset_record_immediate(ZOOM_resultset s,size_t pos)
1144 {
1145     const char *syntax =
1146         ZOOM_options_get(s->options, "preferredRecordSyntax");
1147     const char *elementSetName =
1148         ZOOM_options_get(s->options, "elementSetName");
1149     const char *schema =
1150         ZOOM_options_get(s->options, "schema");
1151
1152     return ZOOM_record_cache_lookup_i(s, pos, syntax, elementSetName, schema);
1153 }
1154
1155 ZOOM_API(ZOOM_record)
1156     ZOOM_resultset_record(ZOOM_resultset r, size_t pos)
1157 {
1158     ZOOM_record rec = ZOOM_resultset_record_immediate(r, pos);
1159
1160     if (!rec)
1161     {
1162         /*
1163          * MIKE: I think force_sync should always be zero, but I don't
1164          * want to make this change until I get the go-ahead from
1165          * Adam, in case something depends on the old synchronous
1166          * behaviour.
1167          */
1168         int force_sync = 1;
1169         if (getenv("ZOOM_RECORD_NO_FORCE_SYNC")) force_sync = 0;
1170         ZOOM_resultset_retrieve(r, force_sync, pos, 1);
1171         rec = ZOOM_resultset_record_immediate(r, pos);
1172     }
1173     return rec;
1174 }
1175
1176 ZOOM_API(ZOOM_scanset)
1177     ZOOM_connection_scan(ZOOM_connection c, const char *start)
1178 {
1179     ZOOM_scanset s;
1180     ZOOM_query q = ZOOM_query_create();
1181
1182     ZOOM_query_prefix(q, start);
1183
1184     s = ZOOM_connection_scan1(c, q);
1185     ZOOM_query_destroy(q);
1186     return s;
1187
1188 }
1189
1190 ZOOM_API(ZOOM_scanset)
1191     ZOOM_connection_scan1(ZOOM_connection c, ZOOM_query q)
1192 {
1193     ZOOM_scanset scan = 0;
1194     Z_Query *z_query = ZOOM_query_get_Z_Query(q);
1195
1196     if (!z_query)
1197         return 0;
1198     scan = (ZOOM_scanset) xmalloc(sizeof(*scan));
1199     scan->connection = c;
1200     scan->odr = odr_createmem(ODR_DECODE);
1201     scan->options = ZOOM_options_create_with_parent(c->options);
1202     scan->refcount = 1;
1203     scan->scan_response = 0;
1204     scan->srw_scan_response = 0;
1205
1206     scan->query = q;
1207     ZOOM_query_addref(q);
1208     scan->databaseNames = ZOOM_connection_get_databases(c, c->options,
1209                                             &scan->num_databaseNames,
1210                                             scan->odr);
1211
1212     if (1)
1213     {
1214         ZOOM_task task = ZOOM_connection_add_task(c, ZOOM_TASK_SCAN);
1215         task->u.scan.scan = scan;
1216
1217         (scan->refcount)++;
1218         if (!c->async)
1219         {
1220             while (ZOOM_event(1, &c))
1221                 ;
1222         }
1223     }
1224     return scan;
1225 }
1226
1227 ZOOM_API(void)
1228     ZOOM_scanset_destroy(ZOOM_scanset scan)
1229 {
1230     if (!scan)
1231         return;
1232     (scan->refcount)--;
1233     if (scan->refcount == 0)
1234     {
1235         ZOOM_query_destroy(scan->query);
1236
1237         odr_destroy(scan->odr);
1238
1239         ZOOM_options_destroy(scan->options);
1240         xfree(scan);
1241     }
1242 }
1243
1244 static zoom_ret send_package(ZOOM_connection c)
1245 {
1246     ZOOM_Event event;
1247
1248     yaz_log(c->log_details, "%p send_package", c);
1249     if (!c->tasks)
1250         return zoom_complete;
1251     assert (c->tasks->which == ZOOM_TASK_PACKAGE);
1252
1253     event = ZOOM_Event_create(ZOOM_EVENT_SEND_APDU);
1254     ZOOM_connection_put_event(c, event);
1255
1256     c->buf_out = c->tasks->u.package->buf_out;
1257     c->len_out = c->tasks->u.package->len_out;
1258
1259     return ZOOM_send_buf(c);
1260 }
1261
1262 ZOOM_API(size_t)
1263     ZOOM_scanset_size(ZOOM_scanset scan)
1264 {
1265     if (!scan)
1266         return 0;
1267
1268     if (scan->scan_response && scan->scan_response->entries)
1269         return scan->scan_response->entries->num_entries;
1270     else if (scan->srw_scan_response)
1271         return scan->srw_scan_response->num_terms;
1272     return 0;
1273 }
1274
1275 static void ZOOM_scanset_term_x(ZOOM_scanset scan, size_t pos,
1276                                 size_t *occ,
1277                                 const char **value_term, size_t *value_len,
1278                                 const char **disp_term, size_t *disp_len)
1279 {
1280     size_t noent = ZOOM_scanset_size(scan);
1281
1282     *value_term = 0;
1283     *value_len = 0;
1284
1285     *disp_term = 0;
1286     *disp_len = 0;
1287
1288     *occ = 0;
1289     if (pos >= noent)
1290         return;
1291     if (scan->scan_response)
1292     {
1293         Z_ScanResponse *res = scan->scan_response;
1294         if (res->entries->entries[pos]->which == Z_Entry_termInfo)
1295         {
1296             Z_TermInfo *t = res->entries->entries[pos]->u.termInfo;
1297
1298             *value_term = (const char *) t->term->u.general->buf;
1299             *value_len = t->term->u.general->len;
1300             if (t->displayTerm)
1301             {
1302                 *disp_term = t->displayTerm;
1303                 *disp_len = strlen(*disp_term);
1304             }
1305             else if (t->term->which == Z_Term_general)
1306             {
1307                 *disp_term = (const char *) t->term->u.general->buf;
1308                 *disp_len = t->term->u.general->len;
1309             }
1310             *occ = t->globalOccurrences ? *t->globalOccurrences : 0;
1311         }
1312     }
1313     if (scan->srw_scan_response)
1314     {
1315         Z_SRW_scanResponse *res = scan->srw_scan_response;
1316         Z_SRW_scanTerm *t = res->terms + pos;
1317         if (t)
1318         {
1319             *value_term = t->value;
1320             *value_len = strlen(*value_term);
1321
1322             if (t->displayTerm)
1323                 *disp_term = t->displayTerm;
1324             else
1325                 *disp_term = t->value;
1326             *disp_len = strlen(*disp_term);
1327             *occ = t->numberOfRecords ? *t->numberOfRecords : 0;
1328         }
1329     }
1330 }
1331
1332 ZOOM_API(const char *)
1333     ZOOM_scanset_term(ZOOM_scanset scan, size_t pos,
1334                       size_t *occ, size_t *len)
1335 {
1336     const char *value_term = 0;
1337     size_t value_len = 0;
1338     const char *disp_term = 0;
1339     size_t disp_len = 0;
1340
1341     ZOOM_scanset_term_x(scan, pos, occ, &value_term, &value_len,
1342                         &disp_term, &disp_len);
1343
1344     *len = value_len;
1345     return value_term;
1346 }
1347
1348 ZOOM_API(const char *)
1349     ZOOM_scanset_display_term(ZOOM_scanset scan, size_t pos,
1350                               size_t *occ, size_t *len)
1351 {
1352     const char *value_term = 0;
1353     size_t value_len = 0;
1354     const char *disp_term = 0;
1355     size_t disp_len = 0;
1356
1357     ZOOM_scanset_term_x(scan, pos, occ, &value_term, &value_len,
1358                         &disp_term, &disp_len);
1359
1360     *len = disp_len;
1361     return disp_term;
1362 }
1363
1364 ZOOM_API(const char *)
1365     ZOOM_scanset_option_get(ZOOM_scanset scan, const char *key)
1366 {
1367     return ZOOM_options_get(scan->options, key);
1368 }
1369
1370 ZOOM_API(void)
1371     ZOOM_scanset_option_set(ZOOM_scanset scan, const char *key,
1372                             const char *val)
1373 {
1374     ZOOM_options_set(scan->options, key, val);
1375 }
1376
1377
1378 ZOOM_API(ZOOM_package)
1379     ZOOM_connection_package(ZOOM_connection c, ZOOM_options options)
1380 {
1381     ZOOM_package p = (ZOOM_package) xmalloc(sizeof(*p));
1382
1383     p->connection = c;
1384     p->odr_out = odr_createmem(ODR_ENCODE);
1385     p->options = ZOOM_options_create_with_parent2(options, c->options);
1386     p->refcount = 1;
1387     p->buf_out = 0;
1388     p->len_out = 0;
1389     return p;
1390 }
1391
1392 ZOOM_API(void)
1393     ZOOM_package_destroy(ZOOM_package p)
1394 {
1395     if (!p)
1396         return;
1397     (p->refcount)--;
1398     if (p->refcount == 0)
1399     {
1400         odr_destroy(p->odr_out);
1401         xfree(p->buf_out);
1402
1403         ZOOM_options_destroy(p->options);
1404         xfree(p);
1405     }
1406 }
1407
1408 ZOOM_API(const char *)
1409     ZOOM_package_option_get(ZOOM_package p, const char *key)
1410 {
1411     return ZOOM_options_get(p->options, key);
1412 }
1413
1414 ZOOM_API(const char *)
1415     ZOOM_package_option_getl(ZOOM_package p, const char *key, int *lenp)
1416 {
1417     return ZOOM_options_getl(p->options, key, lenp);
1418 }
1419
1420 ZOOM_API(void)
1421     ZOOM_package_option_set(ZOOM_package p, const char *key,
1422                             const char *val)
1423 {
1424     ZOOM_options_set(p->options, key, val);
1425 }
1426
1427 ZOOM_API(void)
1428     ZOOM_package_option_setl(ZOOM_package p, const char *key,
1429                              const char *val, int len)
1430 {
1431     ZOOM_options_setl(p->options, key, val, len);
1432 }
1433
1434 ZOOM_API(int)
1435     ZOOM_connection_exec_task(ZOOM_connection c)
1436 {
1437     ZOOM_task task = c->tasks;
1438     zoom_ret ret = zoom_complete;
1439
1440     if (!task)
1441         return 0;
1442     yaz_log(c->log_details, "%p ZOOM_connection_exec_task type=%d run=%d",
1443             c, task->which, task->running);
1444     if (c->error != ZOOM_ERROR_NONE)
1445     {
1446         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1447                 "removing tasks because of error = %d", c, c->error);
1448         ZOOM_connection_remove_tasks(c);
1449         return 0;
1450     }
1451     if (task->running)
1452     {
1453         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1454                 "task already running", c);
1455         return 0;
1456     }
1457     task->running = 1;
1458     ret = zoom_complete;
1459     if (c->cs || task->which == ZOOM_TASK_CONNECT)
1460     {
1461         switch (task->which)
1462         {
1463         case ZOOM_TASK_SEARCH:
1464             if (c->proto == PROTO_HTTP)
1465                 ret = ZOOM_connection_srw_send_search(c);
1466             else
1467                 ret = ZOOM_connection_Z3950_search(c);
1468             break;
1469         case ZOOM_TASK_CONNECT:
1470             ret = do_connect(c);
1471             break;
1472         case ZOOM_TASK_SCAN:
1473             if (c->proto == PROTO_HTTP)
1474                 ret = ZOOM_connection_srw_send_scan(c);
1475             else
1476                 ret = ZOOM_connection_Z3950_send_scan(c);
1477             break;
1478         case ZOOM_TASK_PACKAGE:
1479             ret = send_package(c);
1480             break;
1481         case ZOOM_TASK_SORT:
1482             c->tasks->u.sort.resultset->r_sort_spec =
1483                 ZOOM_query_get_sortspec(c->tasks->u.sort.q);
1484             ret = send_Z3950_sort(c, c->tasks->u.sort.resultset);
1485             break;
1486         }
1487     }
1488     else
1489     {
1490         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1491                 "remove tasks because no connection exist", c);
1492         ZOOM_connection_remove_tasks(c);
1493     }
1494     if (ret == zoom_complete)
1495     {
1496         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1497                 "task removed (complete)", c);
1498         ZOOM_connection_remove_task(c);
1499         return 0;
1500     }
1501     yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1502             "task pending", c);
1503     return 1;
1504 }
1505
1506
1507 static zoom_ret send_HTTP_redirect(ZOOM_connection c, const char *uri)
1508 {
1509     Z_GDU *gdu = z_get_HTTP_Request_uri(c->odr_out, uri, 0, c->proxy ? 1 : 0);
1510
1511     gdu->u.HTTP_Request->method = odr_strdup(c->odr_out, "GET");
1512     z_HTTP_header_add(c->odr_out, &gdu->u.HTTP_Request->headers, "Accept",
1513                       "text/xml");
1514     yaz_cookies_request(c->cookies, c->odr_out, gdu->u.HTTP_Request);
1515     if (c->user && c->password)
1516     {
1517         z_HTTP_header_add_basic_auth(c->odr_out, &gdu->u.HTTP_Request->headers,
1518                                      c->user, c->password);
1519     }
1520     xfree(c->location);
1521     c->location = 0;
1522     return ZOOM_send_GDU(c, gdu);
1523 }
1524
1525
1526 zoom_ret ZOOM_send_GDU(ZOOM_connection c, Z_GDU *gdu)
1527 {
1528     ZOOM_Event event;
1529
1530     int r = z_GDU(c->odr_out, &gdu, 0, 0);
1531     if (!r)
1532         return zoom_complete;
1533     if (c->odr_print)
1534         z_GDU(c->odr_print, &gdu, 0, 0);
1535     if (c->odr_save)
1536         z_GDU(c->odr_save, &gdu, 0, 0);
1537     c->buf_out = odr_getbuf(c->odr_out, &c->len_out, 0);
1538     odr_reset(c->odr_out);
1539
1540     event = ZOOM_Event_create(ZOOM_EVENT_SEND_APDU);
1541     ZOOM_connection_put_event(c, event);
1542
1543     return ZOOM_send_buf(c);
1544 }
1545
1546 void ZOOM_set_HTTP_error(ZOOM_connection c, int error,
1547                          const char *addinfo, const char *addinfo2)
1548 {
1549     ZOOM_set_dset_error(c, error, "HTTP", addinfo, addinfo2);
1550 }
1551
1552 #if YAZ_HAVE_XML2
1553 static void handle_http(ZOOM_connection c, Z_HTTP_Response *hres)
1554 {
1555     zoom_ret cret = zoom_complete;
1556     int ret = -1;
1557     char *addinfo = 0;
1558     const char *connection_head = z_HTTP_header_lookup(hres->headers,
1559                                                        "Connection");
1560     int must_close = 0;
1561     const char *location;
1562
1563     ZOOM_connection_set_mask(c, 0);
1564     yaz_log(c->log_details, "%p handle_http", c);
1565     if (!strcmp(hres->version, "1.0"))
1566     {
1567         /* HTTP 1.0: only if Keep-Alive we stay alive.. */
1568         if (!connection_head || strcmp(connection_head, "Keep-Alive"))
1569              must_close = 1;
1570     }
1571     else
1572     {
1573         /* HTTP 1.1: only if no close we stay alive.. */
1574         if (connection_head && !strcmp(connection_head, "close"))
1575              must_close = 1;
1576     }
1577     yaz_cookies_response(c->cookies, hres);
1578     if ((hres->code == 301 || hres->code == 302) && c->sru_mode == zoom_sru_get
1579         && (location = z_HTTP_header_lookup(hres->headers, "Location")))
1580     {
1581         c->no_redirects++;
1582         if (c->no_redirects > 10)
1583         {
1584             ZOOM_set_HTTP_error(c, hres->code, 0, 0);
1585             c->no_redirects = 0;
1586             ZOOM_connection_close(c);
1587         }
1588         else
1589         {
1590             /* since redirect may change host we just reconnect. A smarter
1591                implementation might check whether it's the same server */
1592
1593             int host_change = 0;
1594             location = yaz_check_location(c->odr_in, c->host_port,
1595                                           location, &host_change);
1596             xfree(c->location);
1597             c->location = xstrdup(location);
1598             do_connect_host(c, location);
1599             return;
1600         }
1601     }
1602     else
1603     {
1604         ret = ZOOM_handle_sru(c, hres, &cret, &addinfo);
1605         if (ret == 0)
1606         {
1607             if (c->no_redirects) /* end of redirect. change hosts again */
1608                 ZOOM_connection_close(c);
1609         }
1610         c->no_redirects = 0;
1611     }
1612     if (ret)
1613     {
1614         if (hres->code != 200)
1615             ZOOM_set_HTTP_error(c, hres->code, 0, 0);
1616         else
1617         {
1618             yaz_log(YLOG_LOG, "set error... addinfo=%s", addinfo ?
1619                     addinfo : "NULL");
1620             ZOOM_set_error(c, ZOOM_ERROR_DECODE, addinfo);
1621         }
1622         ZOOM_connection_close(c);
1623     }
1624     if (cret == zoom_complete)
1625     {
1626         yaz_log(c->log_details, "removing tasks in handle_http");
1627         ZOOM_connection_remove_task(c);
1628     }
1629     if (must_close)
1630     {
1631         ZOOM_connection_close(c);
1632         if (c->tasks)
1633         {
1634             c->tasks->running = 0;
1635             ZOOM_connection_insert_task(c, ZOOM_TASK_CONNECT);
1636             c->reconnect_ok = 0;
1637         }
1638     }
1639     else
1640         c->reconnect_ok = 1; /* if the server closes anyway */
1641 }
1642 #endif
1643
1644 static int do_read(ZOOM_connection c)
1645 {
1646     int r, more;
1647     ZOOM_Event event;
1648
1649     event = ZOOM_Event_create(ZOOM_EVENT_RECV_DATA);
1650     ZOOM_connection_put_event(c, event);
1651
1652     r = cs_get(c->cs, &c->buf_in, &c->len_in);
1653     more = cs_more(c->cs);
1654     yaz_log(c->log_details, "%p do_read len=%d more=%d", c, r, more);
1655     if (r == 1)
1656         return 0;
1657     if (r <= 0)
1658     {
1659         if (!ZOOM_test_reconnect(c))
1660         {
1661             ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
1662             ZOOM_connection_close(c);
1663         }
1664     }
1665     else
1666     {
1667         Z_GDU *gdu;
1668         ZOOM_Event event;
1669
1670         odr_reset(c->odr_in);
1671         odr_setbuf(c->odr_in, c->buf_in, r, 0);
1672         event = ZOOM_Event_create(ZOOM_EVENT_RECV_APDU);
1673         ZOOM_connection_put_event(c, event);
1674
1675         if (!z_GDU(c->odr_in, &gdu, 0, 0))
1676         {
1677             /* even on failures we try to re-connect (HTTP) */
1678             if (!ZOOM_test_reconnect(c))
1679             {
1680                 int x;
1681                 int err = odr_geterrorx(c->odr_in, &x);
1682                 char msg[100];
1683                 const char *element = odr_getelement(c->odr_in);
1684                 yaz_snprintf(msg, sizeof(msg),
1685                              "ODR code %d:%d element=%s offset=%d",
1686                              err, x, element ? element : "<unknown>",
1687                              odr_offset(c->odr_in));
1688                 ZOOM_set_error(c, ZOOM_ERROR_DECODE, msg);
1689                 if (c->log_api)
1690                 {
1691                     FILE *ber_file = yaz_log_file();
1692                     if (ber_file)
1693                         odr_dumpBER(ber_file, c->buf_in, r);
1694                 }
1695                 ZOOM_connection_close(c);
1696             }
1697         }
1698         else
1699         {
1700             if (c->odr_print)
1701                 z_GDU(c->odr_print, &gdu, 0, 0);
1702             if (c->odr_save)
1703                 z_GDU(c->odr_save, &gdu, 0, 0);
1704             if (gdu->which == Z_GDU_Z3950)
1705                 ZOOM_handle_Z3950_apdu(c, gdu->u.z3950);
1706             else if (gdu->which == Z_GDU_HTTP_Response)
1707             {
1708 #if YAZ_HAVE_XML2
1709                 handle_http(c, gdu->u.HTTP_Response);
1710 #else
1711                 ZOOM_set_error(c, ZOOM_ERROR_DECODE, 0);
1712                 ZOOM_connection_close(c);
1713 #endif
1714             }
1715         }
1716     }
1717     return 1;
1718 }
1719
1720 static zoom_ret do_write_ex(ZOOM_connection c, char *buf_out, int len_out)
1721 {
1722     int r;
1723     ZOOM_Event event;
1724
1725     event = ZOOM_Event_create(ZOOM_EVENT_SEND_DATA);
1726     ZOOM_connection_put_event(c, event);
1727
1728     yaz_log(c->log_details, "%p do_write_ex len=%d", c, len_out);
1729     if ((r = cs_put(c->cs, buf_out, len_out)) < 0)
1730     {
1731         yaz_log(c->log_details, "%p do_write_ex write failed", c);
1732         if (ZOOM_test_reconnect(c))
1733         {
1734             return zoom_pending;
1735         }
1736         if (c->state == STATE_CONNECTING)
1737             ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
1738         else
1739             ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
1740         ZOOM_connection_close(c);
1741         return zoom_complete;
1742     }
1743     else if (r == 1)
1744     {
1745         int mask = ZOOM_SELECT_EXCEPT;
1746         if (c->cs->io_pending & CS_WANT_WRITE)
1747             mask += ZOOM_SELECT_WRITE;
1748         if (c->cs->io_pending & CS_WANT_READ)
1749             mask += ZOOM_SELECT_READ;
1750         ZOOM_connection_set_mask(c, mask);
1751         yaz_log(c->log_details, "%p do_write_ex write incomplete mask=%d",
1752                 c, c->mask);
1753     }
1754     else
1755     {
1756         ZOOM_connection_set_mask(c, ZOOM_SELECT_READ|ZOOM_SELECT_EXCEPT);
1757         yaz_log(c->log_details, "%p do_write_ex write complete mask=%d",
1758                 c, c->mask);
1759     }
1760     return zoom_pending;
1761 }
1762
1763 zoom_ret ZOOM_send_buf(ZOOM_connection c)
1764 {
1765     return do_write_ex(c, c->buf_out, c->len_out);
1766 }
1767
1768
1769 ZOOM_API(const char *)
1770     ZOOM_connection_option_get(ZOOM_connection c, const char *key)
1771 {
1772     if (!strcmp(key, "APDU"))
1773     {
1774         return c->saveAPDU_wrbuf ? wrbuf_cstr(c->saveAPDU_wrbuf) : "";
1775     }
1776     else
1777         return ZOOM_options_get(c->options, key);
1778 }
1779
1780 ZOOM_API(const char *)
1781     ZOOM_connection_option_getl(ZOOM_connection c, const char *key, int *lenp)
1782 {
1783     if (!strcmp(key, "APDU"))
1784     {
1785         if (c->saveAPDU_wrbuf)
1786         {
1787             *lenp = wrbuf_len(c->saveAPDU_wrbuf);
1788             return wrbuf_cstr(c->saveAPDU_wrbuf);
1789         }
1790         else
1791         {
1792             *lenp = 0;
1793             return "";
1794         }
1795     }
1796     else
1797         return ZOOM_options_getl(c->options, key, lenp);
1798 }
1799
1800 ZOOM_API(void)
1801     ZOOM_connection_option_set(ZOOM_connection c, const char *key,
1802                                const char *val)
1803 {
1804     if (!strcmp(key, "saveAPDU"))
1805     {
1806         if (val && strcmp(val, "0"))
1807         {
1808             if (!c->saveAPDU_wrbuf)
1809                 c->saveAPDU_wrbuf = wrbuf_alloc();
1810             else
1811                 wrbuf_rewind(c->saveAPDU_wrbuf);
1812         }
1813         else
1814         {
1815             wrbuf_destroy(c->saveAPDU_wrbuf);
1816             c->saveAPDU_wrbuf = 0;
1817         }
1818         ZOOM_connection_save_apdu_wrbuf(c, c->saveAPDU_wrbuf);
1819     }
1820     else
1821         ZOOM_options_set(c->options, key, val);
1822 }
1823
1824 ZOOM_API(void)
1825     ZOOM_connection_option_setl(ZOOM_connection c, const char *key,
1826                                 const char *val, int len)
1827 {
1828     ZOOM_options_setl(c->options, key, val, len);
1829 }
1830
1831 ZOOM_API(const char *)
1832     ZOOM_resultset_option_get(ZOOM_resultset r, const char *key)
1833 {
1834     return ZOOM_options_get(r->options, key);
1835 }
1836
1837 ZOOM_API(void)
1838     ZOOM_resultset_option_set(ZOOM_resultset r, const char *key,
1839                               const char *val)
1840 {
1841     ZOOM_options_set(r->options, key, val);
1842 }
1843
1844
1845 ZOOM_API(int)
1846     ZOOM_connection_errcode(ZOOM_connection c)
1847 {
1848     return ZOOM_connection_error(c, 0, 0);
1849 }
1850
1851 ZOOM_API(const char *)
1852     ZOOM_connection_errmsg(ZOOM_connection c)
1853 {
1854     const char *msg;
1855     ZOOM_connection_error(c, &msg, 0);
1856     return msg;
1857 }
1858
1859 ZOOM_API(const char *)
1860     ZOOM_connection_addinfo(ZOOM_connection c)
1861 {
1862     const char *addinfo;
1863     ZOOM_connection_error(c, 0, &addinfo);
1864     return addinfo;
1865 }
1866
1867 ZOOM_API(const char *)
1868     ZOOM_connection_diagset(ZOOM_connection c)
1869 {
1870     const char *diagset;
1871     ZOOM_connection_error_x(c, 0, 0, &diagset);
1872     return diagset;
1873 }
1874
1875 ZOOM_API(const char *)
1876     ZOOM_diag_str(int error)
1877 {
1878     switch (error)
1879     {
1880     case ZOOM_ERROR_NONE:
1881         return "No error";
1882     case ZOOM_ERROR_CONNECT:
1883         return "Connect failed";
1884     case ZOOM_ERROR_MEMORY:
1885         return "Out of memory";
1886     case ZOOM_ERROR_ENCODE:
1887         return "Encoding failed";
1888     case ZOOM_ERROR_DECODE:
1889         return "Decoding failed";
1890     case ZOOM_ERROR_CONNECTION_LOST:
1891         return "Connection lost";
1892     case ZOOM_ERROR_INIT:
1893         return "Init rejected";
1894     case ZOOM_ERROR_INTERNAL:
1895         return "Internal failure";
1896     case ZOOM_ERROR_TIMEOUT:
1897         return "Timeout";
1898     case ZOOM_ERROR_UNSUPPORTED_PROTOCOL:
1899         return "Unsupported protocol";
1900     case ZOOM_ERROR_UNSUPPORTED_QUERY:
1901         return "Unsupported query type";
1902     case ZOOM_ERROR_INVALID_QUERY:
1903         return "Invalid query";
1904     case ZOOM_ERROR_CQL_PARSE:
1905         return "CQL parsing error";
1906     case ZOOM_ERROR_CQL_TRANSFORM:
1907         return "CQL transformation error";
1908     case ZOOM_ERROR_CCL_CONFIG:
1909         return "CCL configuration error";
1910     case ZOOM_ERROR_CCL_PARSE:
1911         return "CCL parsing error";
1912     case ZOOM_ERROR_ES_INVALID_ACTION:
1913         return "Extended Service. invalid action";
1914     case ZOOM_ERROR_ES_INVALID_VERSION:
1915         return "Extended Service. invalid version";
1916     case ZOOM_ERROR_ES_INVALID_SYNTAX:
1917         return "Extended Service. invalid syntax";
1918     case ZOOM_ERROR_MEMCACHED:
1919         return "Memcached";
1920     default:
1921         return diagbib1_str(error);
1922     }
1923 }
1924
1925 ZOOM_API(int)
1926     ZOOM_connection_error_x(ZOOM_connection c, const char **cp,
1927                             const char **addinfo, const char **diagset)
1928 {
1929     int error = c->error;
1930     if (cp)
1931     {
1932         if (!c->diagset || !strcmp(c->diagset, "ZOOM"))
1933             *cp = ZOOM_diag_str(error);
1934         else if (!strcmp(c->diagset, "HTTP"))
1935             *cp = z_HTTP_errmsg(c->error);
1936         else if (!strcmp(c->diagset, "Bib-1"))
1937             *cp = ZOOM_diag_str(error);
1938         else if (!strcmp(c->diagset, "info:srw/diagnostic/1"))
1939             *cp = yaz_diag_srw_str(c->error);
1940         else
1941             *cp = "Unknown error and diagnostic set";
1942     }
1943     if (addinfo)
1944         *addinfo = c->addinfo ? c->addinfo : "";
1945     if (diagset)
1946         *diagset = c->diagset ? c->diagset : "";
1947     return c->error;
1948 }
1949
1950 ZOOM_API(int)
1951     ZOOM_connection_error(ZOOM_connection c, const char **cp,
1952                           const char **addinfo)
1953 {
1954     return ZOOM_connection_error_x(c, cp, addinfo, 0);
1955 }
1956
1957 static void ZOOM_connection_do_io(ZOOM_connection c, int mask)
1958 {
1959     ZOOM_Event event = 0;
1960     int r = cs_look(c->cs);
1961     yaz_log(c->log_details, "%p ZOOM_connection_do_io mask=%d cs_look=%d",
1962             c, mask, r);
1963
1964     if (r == CS_NONE)
1965     {
1966         event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1967         ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
1968         ZOOM_connection_close(c);
1969         ZOOM_connection_put_event(c, event);
1970     }
1971     else if (r == CS_CONNECT)
1972     {
1973         int ret = ret = cs_rcvconnect(c->cs);
1974         yaz_log(c->log_details, "%p ZOOM_connection_do_io "
1975                 "cs_rcvconnect returned %d", c, ret);
1976         if (ret == 1)
1977         {
1978             int mask = ZOOM_SELECT_EXCEPT;
1979             if (c->cs->io_pending & CS_WANT_WRITE)
1980                 mask += ZOOM_SELECT_WRITE;
1981             if (c->cs->io_pending & CS_WANT_READ)
1982                 mask += ZOOM_SELECT_READ;
1983             ZOOM_connection_set_mask(c, mask);
1984             event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1985             ZOOM_connection_put_event(c, event);
1986         }
1987         else if (ret == 0)
1988         {
1989             event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1990             ZOOM_connection_put_event(c, event);
1991             get_cert(c);
1992             if (c->proto == PROTO_Z3950)
1993                 ZOOM_connection_Z3950_send_init(c);
1994             else
1995             {
1996                 /* no init request for SRW .. */
1997                 if (c->tasks->which == ZOOM_TASK_CONNECT)
1998                 {
1999                     ZOOM_connection_remove_task(c);
2000                     ZOOM_connection_set_mask(c, 0);
2001                 }
2002                 if (c->cs && c->location)
2003                     send_HTTP_redirect(c, c->location);
2004                 else
2005                     ZOOM_connection_exec_task(c);
2006             }
2007             c->state = STATE_ESTABLISHED;
2008         }
2009         else
2010         {
2011             ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
2012             ZOOM_connection_close(c);
2013         }
2014     }
2015     else
2016     {
2017         if (mask & ZOOM_SELECT_EXCEPT)
2018         {
2019             if (!ZOOM_test_reconnect(c))
2020             {
2021                 ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
2022                 ZOOM_connection_close(c);
2023             }
2024             return;
2025         }
2026         if (mask & ZOOM_SELECT_READ)
2027             do_read(c);
2028         if (c->cs && (mask & ZOOM_SELECT_WRITE))
2029             ZOOM_send_buf(c);
2030     }
2031 }
2032
2033 ZOOM_API(int)
2034     ZOOM_connection_last_event(ZOOM_connection cs)
2035 {
2036     if (!cs)
2037         return ZOOM_EVENT_NONE;
2038     return cs->last_event;
2039 }
2040
2041
2042 ZOOM_API(int) ZOOM_connection_fire_event_timeout(ZOOM_connection c)
2043 {
2044     if (c->mask)
2045     {
2046         ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_TIMEOUT);
2047         /* timeout and this connection was waiting */
2048         ZOOM_set_error(c, ZOOM_ERROR_TIMEOUT, 0);
2049         ZOOM_connection_close(c);
2050         ZOOM_connection_put_event(c, event);
2051     }
2052     return 0;
2053 }
2054
2055 ZOOM_API(int)
2056     ZOOM_connection_process(ZOOM_connection c)
2057 {
2058     ZOOM_Event event;
2059     if (!c)
2060         return 0;
2061
2062     event = ZOOM_connection_get_event(c);
2063     if (event)
2064     {
2065         ZOOM_Event_destroy(event);
2066         return 1;
2067     }
2068     ZOOM_connection_exec_task(c);
2069     event = ZOOM_connection_get_event(c);
2070     if (event)
2071     {
2072         ZOOM_Event_destroy(event);
2073         return 1;
2074     }
2075     return 0;
2076 }
2077
2078 ZOOM_API(int)
2079     ZOOM_event_nonblock(int no, ZOOM_connection *cs)
2080 {
2081     int i;
2082
2083     yaz_log(log_details0, "ZOOM_process_event(no=%d,cs=%p)", no, cs);
2084
2085     for (i = 0; i<no; i++)
2086     {
2087         ZOOM_connection c = cs[i];
2088
2089         if (c && ZOOM_connection_process(c))
2090             return i+1;
2091     }
2092     return 0;
2093 }
2094
2095 ZOOM_API(int) ZOOM_connection_fire_event_socket(ZOOM_connection c, int mask)
2096 {
2097     if (c->mask && mask)
2098         ZOOM_connection_do_io(c, mask);
2099     return 0;
2100 }
2101
2102 ZOOM_API(int) ZOOM_connection_get_socket(ZOOM_connection c)
2103 {
2104     if (c->cs)
2105         return cs_fileno(c->cs);
2106     return -1;
2107 }
2108
2109 ZOOM_API(int) ZOOM_connection_set_mask(ZOOM_connection c, int mask)
2110 {
2111     c->mask = mask;
2112     if (!c->cs)
2113         return -1;
2114     return 0;
2115 }
2116
2117 ZOOM_API(int) ZOOM_connection_get_mask(ZOOM_connection c)
2118 {
2119     if (c->cs)
2120         return c->mask;
2121     return 0;
2122 }
2123
2124 ZOOM_API(int) ZOOM_connection_get_timeout(ZOOM_connection c)
2125 {
2126     return ZOOM_options_get_int(c->options, "timeout", 30);
2127 }
2128
2129 ZOOM_API(void) ZOOM_connection_close(ZOOM_connection c)
2130 {
2131     if (c->cs)
2132         cs_close(c->cs);
2133     c->cs = 0;
2134     ZOOM_connection_set_mask(c, 0);
2135     c->state = STATE_IDLE;
2136 }
2137
2138 /*
2139  * Local variables:
2140  * c-basic-offset: 4
2141  * c-file-style: "Stroustrup"
2142  * indent-tabs-mode: nil
2143  * End:
2144  * vim: shiftwidth=4 tabstop=8 expandtab
2145  */
2146