Fix null ptr reference for freeReplyObject YAZ-773
[yaz-moved-to-github.git] / src / zoom-c.c
1 /* This file is part of the YAZ toolkit.
2  * Copyright (C) Index Data
3  * See the file LICENSE for details.
4  */
5 /**
6  * \file zoom-c.c
7  * \brief Implements ZOOM C interface.
8  */
9 #if HAVE_CONFIG_H
10 #include <config.h>
11 #endif
12
13 #include <assert.h>
14 #include <string.h>
15 #include <errno.h>
16 #include "zoom-p.h"
17
18 #include <yaz/yaz-util.h>
19 #include <yaz/xmalloc.h>
20 #include <yaz/otherinfo.h>
21 #include <yaz/log.h>
22 #include <yaz/diagbib1.h>
23 #include <yaz/charneg.h>
24 #include <yaz/query-charset.h>
25 #include <yaz/snprintf.h>
26 #include <yaz/facet.h>
27
28 #include <yaz/shptr.h>
29
30 #if SHPTR
31 YAZ_SHPTR_TYPE(WRBUF)
32 #endif
33
34 static int log_api0 = 0;
35 static int log_details0 = 0;
36
37 static void resultset_destroy(ZOOM_resultset r);
38 static zoom_ret do_write_ex(ZOOM_connection c, char *buf_out, int len_out);
39
40 static void initlog(void)
41 {
42     static int log_level_initialized = 0;
43
44     if (!log_level_initialized)
45     {
46         log_api0 = yaz_log_module_level("zoom");
47         log_details0 = yaz_log_module_level("zoomdetails");
48         log_level_initialized = 1;
49     }
50 }
51
52 void ZOOM_connection_remove_tasks(ZOOM_connection c);
53 static zoom_ret send_HTTP_redirect(ZOOM_connection c, const char *uri);
54
55 void ZOOM_set_dset_error(ZOOM_connection c, int error,
56                          const char *dset,
57                          const char *addinfo, const char *addinfo2)
58 {
59     char *cp;
60
61     xfree(c->addinfo);
62     c->addinfo = 0;
63     c->error = error;
64     if (!c->diagset || strcmp(dset, c->diagset))
65     {
66         xfree(c->diagset);
67         c->diagset = xstrdup(dset);
68         /* remove integer part from SRW diagset .. */
69         if ((cp = strrchr(c->diagset, '/')))
70             *cp = '\0';
71     }
72     if (addinfo && addinfo2)
73     {
74         c->addinfo = (char*) xmalloc(strlen(addinfo) + strlen(addinfo2) + 3);
75         strcpy(c->addinfo, addinfo);
76         strcat(c->addinfo, ": ");
77         strcat(c->addinfo, addinfo2);
78     }
79     else if (addinfo)
80         c->addinfo = xstrdup(addinfo);
81     if (error != ZOOM_ERROR_NONE)
82     {
83         yaz_log(c->log_api, "%p set_dset_error %s %s:%d %s %s",
84                 c, c->host_port ? c->host_port : "<>", dset, error,
85                 addinfo ? addinfo : "",
86                 addinfo2 ? addinfo2 : "");
87         ZOOM_connection_remove_tasks(c);
88     }
89 }
90
91 int ZOOM_uri_to_code(const char *uri)
92 {
93     int code = 0;
94     const char *cp;
95     if ((cp = strrchr(uri, '/')))
96         code = atoi(cp+1);
97     return code;
98 }
99
100 void ZOOM_set_error(ZOOM_connection c, int error, const char *addinfo)
101 {
102     ZOOM_set_dset_error(c, error, "ZOOM", addinfo, 0);
103 }
104
105 static void clear_error(ZOOM_connection c)
106 {
107     /*
108      * If an error is tied to an operation then it's ok to clear: for
109      * example, a diagnostic returned from a search is cleared by a
110      * subsequent search.  However, problems such as Connection Lost
111      * or Init Refused are not cleared, because they are not
112      * recoverable: doing another search doesn't help.
113      */
114
115     ZOOM_connection_remove_events(c);
116     switch (c->error)
117     {
118     case ZOOM_ERROR_CONNECT:
119     case ZOOM_ERROR_MEMORY:
120     case ZOOM_ERROR_DECODE:
121     case ZOOM_ERROR_CONNECTION_LOST:
122     case ZOOM_ERROR_INIT:
123     case ZOOM_ERROR_INTERNAL:
124     case ZOOM_ERROR_UNSUPPORTED_PROTOCOL:
125         break;
126     default:
127         ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
128     }
129 }
130
131 void ZOOM_connection_show_task(ZOOM_task task)
132 {
133     switch(task->which)
134     {
135     case ZOOM_TASK_SEARCH:
136         yaz_log(YLOG_LOG, "search p=%p", task);
137         break;
138     case ZOOM_TASK_CONNECT:
139         yaz_log(YLOG_LOG, "connect p=%p", task);
140         break;
141     case ZOOM_TASK_SCAN:
142         yaz_log(YLOG_LOG, "scan p=%p", task);
143         break;
144     }
145 }
146
147 void ZOOM_connection_show_tasks(ZOOM_connection c)
148 {
149     ZOOM_task task;
150     yaz_log(YLOG_LOG, "connection p=%p tasks", c);
151     for (task = c->tasks; task; task = task->next)
152         ZOOM_connection_show_task(task);
153 }
154
155 ZOOM_task ZOOM_connection_add_task(ZOOM_connection c, int which)
156 {
157     ZOOM_task *taskp = &c->tasks;
158     while (*taskp)
159         taskp = &(*taskp)->next;
160     *taskp = (ZOOM_task) xmalloc(sizeof(**taskp));
161     (*taskp)->running = 0;
162     (*taskp)->which = which;
163     (*taskp)->next = 0;
164     clear_error(c);
165     return *taskp;
166 }
167
168 ZOOM_API(int) ZOOM_connection_is_idle(ZOOM_connection c)
169 {
170     return c->tasks ? 0 : 1;
171 }
172
173 ZOOM_task ZOOM_connection_insert_task(ZOOM_connection c, int which)
174 {
175     ZOOM_task task = (ZOOM_task) xmalloc(sizeof(*task));
176
177     task->next = c->tasks;
178     c->tasks = task;
179
180     task->running = 0;
181     task->which = which;
182     return task;
183 }
184
185 void ZOOM_connection_remove_task(ZOOM_connection c)
186 {
187     ZOOM_task task = c->tasks;
188
189     if (task)
190     {
191         c->tasks = task->next;
192         switch (task->which)
193         {
194         case ZOOM_TASK_SEARCH:
195             resultset_destroy(task->u.search.resultset);
196             xfree(task->u.search.syntax);
197             xfree(task->u.search.elementSetName);
198             xfree(task->u.search.schema);
199             break;
200         case ZOOM_TASK_CONNECT:
201             break;
202         case ZOOM_TASK_SCAN:
203             ZOOM_scanset_destroy(task->u.scan.scan);
204             break;
205         case ZOOM_TASK_PACKAGE:
206             ZOOM_package_destroy(task->u.package);
207             break;
208         case ZOOM_TASK_SORT:
209             resultset_destroy(task->u.sort.resultset);
210             ZOOM_query_destroy(task->u.sort.q);
211             break;
212         default:
213             assert(0);
214         }
215         xfree(task);
216
217         if (!c->tasks)
218         {
219             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_END);
220             ZOOM_connection_put_event(c, event);
221         }
222     }
223 }
224
225 void ZOOM_connection_remove_tasks(ZOOM_connection c)
226 {
227     while (c->tasks)
228         ZOOM_connection_remove_task(c);
229 }
230
231 static void odr_wrbuf_write(ODR o, void *handle, int type,
232                             const char *buf, int len)
233 {
234     WRBUF w = (WRBUF) handle;
235     wrbuf_write(w, buf, len);
236 }
237
238 ZOOM_API(ZOOM_connection)
239     ZOOM_connection_create(ZOOM_options options)
240 {
241     ZOOM_connection c = (ZOOM_connection) xmalloc(sizeof(*c));
242
243     initlog();
244
245     c->log_api = log_api0;
246     c->log_details = log_details0;
247
248     yaz_log(c->log_api, "%p ZOOM_connection_create", c);
249
250     c->proto = PROTO_Z3950;
251     c->cs = 0;
252     ZOOM_connection_set_mask(c, 0);
253     c->reconnect_ok = 0;
254     c->state = STATE_IDLE;
255     c->addinfo = 0;
256     c->diagset = 0;
257     ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
258     c->buf_in = 0;
259     c->len_in = 0;
260     c->buf_out = 0;
261     c->len_out = 0;
262     c->resultsets = 0;
263
264     c->options = ZOOM_options_create_with_parent(options);
265
266     c->host_port = 0;
267     c->proxy = 0;
268     c->tproxy = 0;
269
270     c->charset = c->lang = 0;
271
272     c->cookie_out = 0;
273     c->cookie_in = 0;
274     c->client_IP = 0;
275     c->tasks = 0;
276
277     c->user = 0;
278     c->group = 0;
279     c->password = 0;
280     c->url_authentication = 0;
281
282     c->maximum_record_size = 0;
283     c->preferred_message_size = 0;
284
285     c->odr_in = odr_createmem(ODR_DECODE);
286     c->odr_out = odr_createmem(ODR_ENCODE);
287     c->odr_print = 0;
288     c->odr_save = 0;
289
290     c->async = 0;
291     c->support_named_resultsets = 0;
292     c->last_event = ZOOM_EVENT_NONE;
293
294     c->m_queue_front = 0;
295     c->m_queue_back = 0;
296
297     c->sru_version = 0;
298     c->no_redirects = 0;
299     c->cookies = 0;
300     c->location = 0;
301     c->saveAPDU_wrbuf = 0;
302
303     ZOOM_memcached_init(c);
304     return c;
305 }
306
307 ZOOM_API(void) ZOOM_connection_save_apdu_wrbuf(ZOOM_connection c, WRBUF w)
308 {
309     if (c->odr_save)
310     {
311         odr_destroy(c->odr_save);
312         c->odr_save = 0;
313     }
314     if (w)
315     {
316         c->odr_save = odr_createmem(ODR_PRINT);
317         odr_set_stream(c->odr_save, w, odr_wrbuf_write, 0);
318     }
319 }
320
321 /* set database names. Take local databases (if set); otherwise
322    take databases given in ZURL (if set); otherwise use Default */
323 char **ZOOM_connection_get_databases(ZOOM_connection con, ZOOM_options options,
324                                      int *num, ODR odr)
325 {
326     char **databaseNames;
327     const char *cp = ZOOM_options_get(options, "databaseName");
328
329     if ((!cp || !*cp) && con->host_port)
330         cs_get_host_args(con->host_port, &cp);
331     if (!cp || !*cp)
332         cp = "Default";
333     nmem_strsplit(odr_getmem(odr), "+", cp,  &databaseNames, num);
334     return databaseNames;
335 }
336
337 ZOOM_API(ZOOM_connection)
338     ZOOM_connection_new(const char *host, int portnum)
339 {
340     ZOOM_connection c = ZOOM_connection_create(0);
341
342     ZOOM_connection_connect(c, host, portnum);
343     return c;
344 }
345
346 static zoom_sru_mode get_sru_mode_from_string(const char *s)
347 {
348     if (!s || !*s)
349         return zoom_sru_soap;
350     if (!yaz_matchstr(s, "soap"))
351         return zoom_sru_soap;
352     else if (!yaz_matchstr(s, "get"))
353         return zoom_sru_get;
354     else if (!yaz_matchstr(s, "post"))
355         return zoom_sru_post;
356     else if (!yaz_matchstr(s, "solr"))
357         return zoom_sru_solr;
358     return zoom_sru_error;
359 }
360
361 ZOOM_API(void)
362     ZOOM_connection_connect(ZOOM_connection c,
363                             const char *host, int portnum)
364 {
365     const char *val;
366
367     initlog();
368
369     yaz_log(c->log_api, "%p ZOOM_connection_connect host=%s portnum=%d",
370             c, host ? host : "null", portnum);
371
372     ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
373     ZOOM_connection_remove_tasks(c);
374
375     if (c->cs)
376     {
377         yaz_log(c->log_details, "%p ZOOM_connection_connect reconnect ok", c);
378         c->reconnect_ok = 1;
379         return;
380     }
381     yaz_log(c->log_details, "%p ZOOM_connection_connect connect", c);
382     xfree(c->proxy);
383     c->proxy = 0;
384     val = ZOOM_options_get(c->options, "proxy");
385     if (val && *val)
386     {
387         yaz_log(c->log_details, "%p ZOOM_connection_connect proxy=%s", c, val);
388         c->proxy = xstrdup(val);
389     }
390
391     xfree(c->tproxy);
392     c->tproxy = 0;
393     val = ZOOM_options_get(c->options, "tproxy");
394     if (val && *val)
395     {
396         yaz_log(c->log_details, "%p ZOOM_connection_connect tproxy=%s", c, val);
397         c->tproxy = xstrdup(val);
398     }
399
400     xfree(c->charset);
401     c->charset = 0;
402     val = ZOOM_options_get(c->options, "charset");
403     if (val && *val)
404     {
405         yaz_log(c->log_details, "%p ZOOM_connection_connect charset=%s", c, val);
406         c->charset = xstrdup(val);
407     }
408
409     xfree(c->lang);
410     val = ZOOM_options_get(c->options, "lang");
411     if (val && *val)
412     {
413         yaz_log(c->log_details, "%p ZOOM_connection_connect lang=%s", c, val);
414         c->lang = xstrdup(val);
415     }
416     else
417         c->lang = 0;
418
419     if (host)
420     {
421         char hostn[128];
422         const char *http_lead;
423
424         val = ZOOM_options_get(c->options, "sru");
425         if (val && *val && !strstr(host, "://"))
426             http_lead = "http://";
427         else
428             http_lead = "";
429         c->sru_mode = get_sru_mode_from_string(val);
430
431         xfree(c->host_port);
432         if (portnum)
433         {
434             sprintf(hostn, "%.80s:%d", host, portnum);
435             host = hostn;
436         }
437         c->host_port = xmalloc(strlen(host) + strlen(http_lead) + 1);
438         strcpy(c->host_port, http_lead);
439         strcat(c->host_port, host);
440     }
441
442     {
443         /*
444          * If the "<scheme>:" part of the host string is preceded by one
445          * or more comma-separated <name>=<value> pairs, these are taken
446          * to be options to be set on the connection object.  Among other
447          * applications, this facility can be used to embed authentication
448          * in a host string:
449          *          user=admin,password=secret,tcp:localhost:9999
450          */
451         char *remainder = c->host_port;
452         char *pcolon = strchr(remainder, ':');
453         char *pcomma;
454         char *pequals;
455         while ((pcomma = strchr(remainder, ',')) != 0 &&
456                (pcolon == 0 || pcomma < pcolon))
457         {
458             *pcomma = '\0';
459             if ((pequals = strchr(remainder, '=')) != 0)
460             {
461                 *pequals = '\0';
462                 ZOOM_connection_option_set(c, remainder, pequals+1);
463             }
464             remainder = pcomma+1;
465         }
466
467         if (remainder != c->host_port)
468         {
469             remainder = xstrdup(remainder);
470             xfree(c->host_port);
471             c->host_port = remainder;
472         }
473     }
474
475     xfree(c->sru_version);
476     val = ZOOM_options_get(c->options, "sru_version");
477     c->sru_version = xstrdup(val ? val : "1.2");
478
479     ZOOM_options_set(c->options, "host", c->host_port);
480
481     xfree(c->cookie_out);
482     c->cookie_out = 0;
483     val = ZOOM_options_get(c->options, "cookie");
484     if (val && *val)
485     {
486         yaz_log(c->log_details, "%p ZOOM_connection_connect cookie=%s", c, val);
487         c->cookie_out = xstrdup(val);
488     }
489
490     xfree(c->client_IP);
491     c->client_IP = 0;
492     val = ZOOM_options_get(c->options, "clientIP");
493     if (val && *val)
494     {
495         yaz_log(c->log_details, "%p ZOOM_connection_connect clientIP=%s",
496                 c, val);
497         c->client_IP = xstrdup(val);
498     }
499
500     xfree(c->group);
501     c->group = 0;
502     val = ZOOM_options_get(c->options, "group");
503     if (val && *val)
504         c->group = xstrdup(val);
505
506     xfree(c->user);
507     c->user = 0;
508     val = ZOOM_options_get(c->options, "user");
509     if (val && *val)
510         c->user = xstrdup(val);
511
512     xfree(c->password);
513     c->password = 0;
514     val = ZOOM_options_get(c->options, "password");
515     if (!val)
516         val = ZOOM_options_get(c->options, "pass");
517     if (val && *val)
518         c->password = xstrdup(val);
519
520     val = ZOOM_options_get(c->options, "authenticationMode");
521     if (val && !strcmp(val, "url"))
522         c->url_authentication = 1;
523     else
524         c->url_authentication = 0;
525
526     c->maximum_record_size =
527         ZOOM_options_get_int(c->options, "maximumRecordSize", 64*1024*1024);
528     c->preferred_message_size =
529         ZOOM_options_get_int(c->options, "preferredMessageSize", 64*1024*1024);
530
531     c->async = ZOOM_options_get_bool(c->options, "async", 0);
532
533     yaz_cookies_destroy(c->cookies);
534     c->cookies = yaz_cookies_create();
535
536     if (ZOOM_memcached_configure(c))
537     {
538         ZOOM_connection_remove_tasks(c);
539         return;
540     }
541     if (c->sru_mode == zoom_sru_error)
542     {
543         ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, val);
544         ZOOM_connection_remove_tasks(c);
545         return;
546     }
547
548     if (c->odr_print)
549         odr_destroy(c->odr_print);
550     c->odr_print = 0;
551     val = ZOOM_options_get(c->options, "apdufile");
552     if (val)
553     {
554         c->odr_print = odr_createmem(ODR_PRINT);
555         if (strcmp(val, "-"))
556         {
557             FILE *f = fopen(val, "a");
558             if (!f)
559             {
560                 WRBUF w = wrbuf_alloc();
561                 wrbuf_printf(w, "fopen: %s", val);
562                 ZOOM_set_error(c, ZOOM_ERROR_INTERNAL, wrbuf_cstr(w));
563                 wrbuf_destroy(w);
564                 return;
565             }
566             odr_setprint(c->odr_print, f);
567         }
568     }
569     else if (ZOOM_options_get_bool(c->options, "apdulog", 0))
570     {
571         c->odr_print = odr_createmem(ODR_PRINT);
572         odr_setprint_noclose(c->odr_print, yaz_log_file());
573     }
574
575     yaz_log(c->log_details, "%p ZOOM_connection_connect async=%d", c, c->async);
576     ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
577
578     if (!c->async)
579     {
580         while (ZOOM_event(1, &c))
581             ;
582     }
583 }
584
585 ZOOM_API(void) ZOOM_resultset_release(ZOOM_resultset r)
586 {
587     if (r->connection)
588     {
589         /* remove ourselves from the resultsets in connection */
590         ZOOM_resultset *rp = &r->connection->resultsets;
591         while (1)
592         {
593             assert(*rp);   /* we must be in this list!! */
594             if (*rp == r)
595             {   /* OK, we're here - take us out of it */
596                 *rp = (*rp)->next;
597                 break;
598             }
599             rp = &(*rp)->next;
600         }
601         r->connection = 0;
602     }
603 }
604
605 ZOOM_API(void)
606     ZOOM_connection_destroy(ZOOM_connection c)
607 {
608     ZOOM_resultset r;
609     if (!c)
610         return;
611     yaz_log(c->log_api, "%p ZOOM_connection_destroy", c);
612
613     ZOOM_memcached_destroy(c);
614     if (c->cs)
615         cs_close(c->cs);
616
617     for (r = c->resultsets; r; r = r->next)
618         r->connection = 0;
619
620     xfree(c->buf_in);
621     xfree(c->addinfo);
622     xfree(c->diagset);
623     odr_destroy(c->odr_in);
624     odr_destroy(c->odr_out);
625     if (c->odr_save)
626         odr_destroy(c->odr_save);
627     if (c->odr_print)
628         odr_destroy(c->odr_print);
629     ZOOM_options_destroy(c->options);
630     ZOOM_connection_remove_tasks(c);
631     ZOOM_connection_remove_events(c);
632     xfree(c->host_port);
633     xfree(c->proxy);
634     xfree(c->tproxy);
635     xfree(c->charset);
636     xfree(c->lang);
637     xfree(c->cookie_out);
638     xfree(c->cookie_in);
639     xfree(c->client_IP);
640     xfree(c->user);
641     xfree(c->group);
642     xfree(c->password);
643     xfree(c->sru_version);
644     xfree(c->location);
645     yaz_cookies_destroy(c->cookies);
646     wrbuf_destroy(c->saveAPDU_wrbuf);
647     xfree(c);
648 }
649
650 void ZOOM_resultset_addref(ZOOM_resultset r)
651 {
652     if (r)
653     {
654         yaz_mutex_enter(r->mutex);
655         (r->refcount)++;
656         yaz_log(log_details0, "%p ZOOM_resultset_addref count=%d",
657                 r, r->refcount);
658         yaz_mutex_leave(r->mutex);
659     }
660 }
661
662 static int g_resultsets = 0;
663 static YAZ_MUTEX g_resultset_mutex = 0;
664
665 /* TODO We need to initialize this before running threaded:
666  * call resultset_use(0)  */
667
668 static int resultset_use(int delta) {
669     int resultset_count;
670     if (g_resultset_mutex == 0)
671         yaz_mutex_create(&g_resultset_mutex);
672     yaz_mutex_enter(g_resultset_mutex);
673     g_resultsets += delta;
674     resultset_count = g_resultsets;
675     yaz_mutex_leave(g_resultset_mutex);
676     return resultset_count;
677 }
678
679 int resultsets_count(void) {
680     return resultset_use(0);
681 }
682
683 ZOOM_resultset ZOOM_resultset_create(void)
684 {
685     int i;
686     ZOOM_resultset r = (ZOOM_resultset) xmalloc(sizeof(*r));
687
688     initlog();
689
690     yaz_log(log_details0, "%p ZOOM_resultset_create", r);
691     r->refcount = 1;
692     r->size = 0;
693     r->odr = odr_createmem(ODR_DECODE);
694     r->piggyback = 1;
695     r->setname = 0;
696     r->step = 0;
697     for (i = 0; i<RECORD_HASH_SIZE; i++)
698         r->record_hash[i] = 0;
699     r->r_sort_spec = 0;
700     r->query = 0;
701     r->connection = 0;
702     r->databaseNames = 0;
703     r->num_databaseNames = 0;
704     r->req_facets = 0;
705     r->res_facets = 0;
706     r->num_res_facets = 0;
707     r->facets_names = 0;
708     r->mutex = 0;
709     yaz_mutex_create(&r->mutex);
710 #if SHPTR
711     {
712         WRBUF w = wrbuf_alloc();
713         YAZ_SHPTR_INIT(r->record_wrbuf, w);
714     }
715 #endif
716     resultset_use(1);
717     r->mc_key = 0;
718     r->live_set = 0;
719     return r;
720 }
721
722 ZOOM_API(ZOOM_resultset)
723     ZOOM_connection_search_pqf(ZOOM_connection c, const char *q)
724 {
725     ZOOM_resultset r;
726     ZOOM_query s = ZOOM_query_create();
727
728     ZOOM_query_prefix(s, q);
729
730     r = ZOOM_connection_search(c, s);
731     ZOOM_query_destroy(s);
732     return r;
733 }
734
735 ZOOM_API(ZOOM_resultset)
736     ZOOM_connection_search(ZOOM_connection c, ZOOM_query q)
737 {
738     ZOOM_resultset r = ZOOM_resultset_create();
739     ZOOM_task task;
740     int start, count;
741     const char *syntax, *elementSetName, *schema;
742     yaz_log(c->log_api, "%p ZOOM_connection_search set %p query %p", c, r, q);
743     r->r_sort_spec = ZOOM_query_get_sortspec(q);
744     r->query = q;
745     ZOOM_query_addref(q);
746
747     r->options = ZOOM_options_create_with_parent(c->options);
748
749     r->req_facets = odr_strdup_null(r->odr,
750                                     ZOOM_options_get(r->options, "facets"));
751     start = ZOOM_options_get_int(r->options, "start", 0);
752     count = ZOOM_options_get_int(r->options, "count", 0);
753     {
754         /* If "presentChunk" is defined use that; otherwise "step" */
755         const char *cp = ZOOM_options_get(r->options, "presentChunk");
756         r->step = ZOOM_options_get_int(r->options,
757                                        (cp != 0 ? "presentChunk": "step"), 0);
758     }
759     r->piggyback = ZOOM_options_get_bool(r->options, "piggyback", 1);
760     r->setname = odr_strdup_null(r->odr,
761                                  ZOOM_options_get(r->options, "setname"));
762     r->databaseNames = ZOOM_connection_get_databases(c, c->options,
763                                                      &r->num_databaseNames,
764                                                      r->odr);
765     r->connection = c;
766     r->next = c->resultsets;
767     c->resultsets = r;
768
769     ZOOM_memcached_resultset(r, q);
770
771     if (c->host_port && c->proto == PROTO_HTTP)
772     {
773         if (!c->cs)
774         {
775             yaz_log(c->log_details, "ZOOM_connection_search: no comstack");
776             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
777         }
778         else
779         {
780             yaz_log(c->log_details, "ZOOM_connection_search: reconnect");
781             c->reconnect_ok = 1;
782         }
783     }
784
785     task = ZOOM_connection_add_task(c, ZOOM_TASK_SEARCH);
786     task->u.search.resultset = r;
787     task->u.search.start = start;
788     task->u.search.count = count;
789
790     syntax = ZOOM_options_get(r->options, "preferredRecordSyntax");
791     task->u.search.syntax = syntax ? xstrdup(syntax) : 0;
792     elementSetName = ZOOM_options_get(r->options, "elementSetName");
793     task->u.search.elementSetName = elementSetName ?
794         xstrdup(elementSetName) : 0;
795     schema = ZOOM_options_get(r->options, "schema");
796     task->u.search.schema = schema ? xstrdup(schema) : 0;
797
798     ZOOM_resultset_addref(r);
799
800     if (!c->async)
801     {
802         while (ZOOM_event(1, &c))
803             ;
804     }
805     return r;
806 }
807
808 ZOOM_API(void)
809     ZOOM_resultset_sort(ZOOM_resultset r,
810                          const char *sort_type, const char *sort_spec)
811 {
812     (void) ZOOM_resultset_sort1(r, sort_type, sort_spec);
813 }
814
815 ZOOM_API(int)
816     ZOOM_resultset_sort1(ZOOM_resultset r,
817                          const char *sort_type, const char *sort_spec)
818 {
819     ZOOM_connection c = r->connection;
820     ZOOM_task task;
821     ZOOM_query newq;
822
823     newq = ZOOM_query_create();
824     if (ZOOM_query_sortby(newq, sort_spec) < 0)
825         return -1;
826
827     yaz_log(c->log_api, "%p ZOOM_resultset_sort r=%p sort_type=%s sort_spec=%s",
828             r, r, sort_type, sort_spec);
829     if (!c)
830         return 0;
831
832     if (c->host_port && c->proto == PROTO_HTTP)
833     {
834         if (!c->cs)
835         {
836             yaz_log(c->log_details, "%p ZOOM_resultset_sort: no comstack", r);
837             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
838         }
839         else
840         {
841             yaz_log(c->log_details, "%p ZOOM_resultset_sort: prepare reconnect",
842                     r);
843             c->reconnect_ok = 1;
844         }
845     }
846
847     ZOOM_resultset_cache_reset(r);
848     task = ZOOM_connection_add_task(c, ZOOM_TASK_SORT);
849     task->u.sort.resultset = r;
850     task->u.sort.q = newq;
851
852     ZOOM_resultset_addref(r);
853
854     if (!c->async)
855     {
856         while (ZOOM_event(1, &c))
857             ;
858     }
859
860     return 0;
861 }
862
863 ZOOM_API(void)
864     ZOOM_resultset_destroy(ZOOM_resultset r)
865 {
866     resultset_destroy(r);
867 }
868
869 static void resultset_destroy(ZOOM_resultset r)
870 {
871     if (!r)
872         return;
873     yaz_mutex_enter(r->mutex);
874     (r->refcount)--;
875     yaz_log(log_details0, "%p ZOOM_resultset_destroy r=%p count=%d",
876             r, r, r->refcount);
877     if (r->refcount == 0)
878     {
879         yaz_mutex_leave(r->mutex);
880
881         yaz_log(log_details0, "%p ZOOM_connection resultset_destroy: Deleting resultset (%p) ", r->connection, r);
882         ZOOM_resultset_cache_reset(r);
883         ZOOM_resultset_release(r);
884         ZOOM_query_destroy(r->query);
885         ZOOM_options_destroy(r->options);
886         odr_destroy(r->odr);
887         yaz_mutex_destroy(&r->mutex);
888 #if SHPTR
889         YAZ_SHPTR_DEC(r->record_wrbuf, wrbuf_destroy);
890 #endif
891         wrbuf_destroy(r->mc_key);
892         resultset_use(-1);
893         xfree(r);
894     }
895     else
896         yaz_mutex_leave(r->mutex);
897 }
898
899 ZOOM_API(size_t)
900     ZOOM_resultset_size(ZOOM_resultset r)
901 {
902     return r->size;
903 }
904
905 int ZOOM_test_reconnect(ZOOM_connection c)
906 {
907     ZOOM_Event event;
908
909     if (!c->reconnect_ok)
910         return 0;
911     ZOOM_connection_close(c);
912     c->reconnect_ok = 0;
913     c->tasks->running = 0;
914     ZOOM_connection_insert_task(c, ZOOM_TASK_CONNECT);
915
916     event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
917     ZOOM_connection_put_event(c, event);
918
919     return 1;
920 }
921
922 static void ZOOM_resultset_retrieve(ZOOM_resultset r,
923                                     int force_sync, int start, int count)
924 {
925     ZOOM_task task;
926     ZOOM_connection c;
927     const char *cp;
928     const char *syntax, *elementSetName;
929
930     if (!r)
931         return;
932     yaz_log(log_details0, "%p ZOOM_resultset_retrieve force_sync=%d start=%d"
933             " count=%d", r, force_sync, start, count);
934     c = r->connection;
935     if (!c)
936         return;
937
938     if (c->host_port && c->proto == PROTO_HTTP)
939     {
940         if (!c->cs)
941         {
942             yaz_log(log_details0, "%p ZOOM_resultset_retrieve: no comstack", r);
943             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
944         }
945         else
946         {
947             yaz_log(log_details0, "%p ZOOM_resultset_retrieve: prepare "
948                     "reconnect", r);
949             c->reconnect_ok = 1;
950         }
951     }
952     task = ZOOM_connection_add_task(c, ZOOM_TASK_SEARCH);
953     task->u.search.resultset = r;
954     task->u.search.start = start;
955     task->u.search.count = count;
956
957     syntax = ZOOM_options_get(r->options, "preferredRecordSyntax");
958     task->u.search.syntax = syntax ? xstrdup(syntax) : 0;
959     elementSetName = ZOOM_options_get(r->options, "elementSetName");
960     task->u.search.elementSetName = elementSetName
961         ? xstrdup(elementSetName) : 0;
962
963     cp = ZOOM_options_get(r->options, "schema");
964     task->u.search.schema = cp ? xstrdup(cp) : 0;
965
966     ZOOM_resultset_addref(r);
967
968     if (!r->connection->async || force_sync)
969         while (r->connection && ZOOM_event(1, &r->connection))
970             ;
971 }
972
973 ZOOM_API(void)
974     ZOOM_resultset_records(ZOOM_resultset r, ZOOM_record *recs,
975                            size_t start, size_t count)
976 {
977     int force_present = 0;
978
979     if (!r)
980         return ;
981     yaz_log(log_api0, "%p ZOOM_resultset_records r=%p start=%ld count=%ld",
982             r, r, (long) start, (long) count);
983     if (count && recs)
984         force_present = 1;
985     ZOOM_resultset_retrieve(r, force_present, start, count);
986     if (force_present)
987     {
988         size_t i;
989         for (i = 0; i< count; i++)
990             recs[i] = ZOOM_resultset_record_immediate(r, i+start);
991     }
992 }
993
994 ZOOM_API(size_t)
995     ZOOM_resultset_facets_size(ZOOM_resultset r)
996 {
997     return r->num_res_facets;
998 }
999
1000 ZOOM_API(ZOOM_facet_field)
1001     ZOOM_resultset_get_facet_field(ZOOM_resultset r, const char *name)
1002 {
1003     int num = r->num_res_facets;
1004     ZOOM_facet_field *facets = r->res_facets;
1005     int i;
1006     for (i = 0; i < num; i++)
1007         if (!strcmp(facets[i]->facet_name, name))
1008             return facets[i];
1009     return 0;
1010 }
1011
1012 ZOOM_API(ZOOM_facet_field)
1013     ZOOM_resultset_get_facet_field_by_index(ZOOM_resultset r, int idx)
1014 {
1015     int num = r->num_res_facets;
1016     ZOOM_facet_field *facets = r->res_facets;
1017     if (idx >= 0 && idx < num)
1018         return facets[idx];
1019     return 0;
1020 }
1021
1022 ZOOM_API(ZOOM_facet_field *)
1023     ZOOM_resultset_facets(ZOOM_resultset r)
1024 {
1025     return r->res_facets;
1026 }
1027
1028 ZOOM_API(const char**)
1029     ZOOM_resultset_facets_names(ZOOM_resultset r)
1030 {
1031     return (const char **) r->facets_names;
1032 }
1033
1034 ZOOM_API(const char*)
1035     ZOOM_facet_field_name(ZOOM_facet_field field)
1036 {
1037     return field->facet_name;
1038 }
1039
1040 ZOOM_API(size_t)
1041     ZOOM_facet_field_term_count(ZOOM_facet_field field)
1042 {
1043     return field->num_terms;
1044 }
1045
1046 ZOOM_API(const char*)
1047     ZOOM_facet_field_get_term(ZOOM_facet_field field, size_t idx, int *freq)
1048 {
1049     *freq = field->facet_terms[idx].frequency;
1050     return field->facet_terms[idx].term;
1051 }
1052
1053
1054 static void get_cert(ZOOM_connection c)
1055 {
1056     char *cert_buf;
1057     int cert_len;
1058
1059     if (cs_get_peer_certificate_x509(c->cs, &cert_buf, &cert_len))
1060     {
1061         ZOOM_connection_option_setl(c, "sslPeerCert",
1062                                     cert_buf, cert_len);
1063         xfree(cert_buf);
1064     }
1065 }
1066
1067 static zoom_ret do_connect_host(ZOOM_connection c,
1068                                 const char *logical_url);
1069
1070 static zoom_ret do_connect(ZOOM_connection c)
1071 {
1072     return do_connect_host(c, c->host_port);
1073 }
1074
1075 static zoom_ret do_connect_host(ZOOM_connection c, const char *logical_url)
1076 {
1077     void *add;
1078
1079     if (c->cs)
1080         cs_close(c->cs);
1081     c->cs = cs_create_host_proxy(logical_url, CS_FLAGS_DNS_NO_BLOCK, &add,
1082                                  c->tproxy ? c->tproxy : c->proxy);
1083
1084     if (c->cs && c->cs->protocol == PROTO_HTTP)
1085     {
1086 #if YAZ_HAVE_XML2
1087         c->proto = PROTO_HTTP;
1088 #else
1089         ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, "SRW");
1090         ZOOM_connection_close(c);
1091         return zoom_complete;
1092 #endif
1093     }
1094     if (c->cs)
1095     {
1096         int ret = cs_connect(c->cs, add);
1097         if (ret == 0)
1098         {
1099             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1100             ZOOM_connection_put_event(c, event);
1101             get_cert(c);
1102             if (c->proto == PROTO_Z3950)
1103                 ZOOM_connection_Z3950_send_init(c);
1104             else
1105             {
1106                 /* no init request for SRW .. */
1107                 assert(c->tasks->which == ZOOM_TASK_CONNECT);
1108                 ZOOM_connection_remove_task(c);
1109                 ZOOM_connection_set_mask(c, 0);
1110
1111                 if (c->cs && c->location)
1112                     send_HTTP_redirect(c, c->location);
1113                 else
1114                     ZOOM_connection_exec_task(c);
1115             }
1116             c->state = STATE_ESTABLISHED;
1117             return zoom_pending;
1118         }
1119         else if (ret > 0)
1120         {
1121             int mask = ZOOM_SELECT_EXCEPT;
1122             if (c->cs->io_pending & CS_WANT_WRITE)
1123                 mask += ZOOM_SELECT_WRITE;
1124             if (c->cs->io_pending & CS_WANT_READ)
1125                 mask += ZOOM_SELECT_READ;
1126             ZOOM_connection_set_mask(c, mask);
1127             c->state = STATE_CONNECTING;
1128             return zoom_pending;
1129         }
1130     }
1131     c->state = STATE_IDLE;
1132     ZOOM_set_error(c, ZOOM_ERROR_CONNECT, logical_url);
1133     return zoom_complete;
1134 }
1135
1136 /* returns 1 if PDU was sent OK (still pending )
1137    0 if PDU was not sent OK (nothing to wait for)
1138 */
1139
1140 ZOOM_API(ZOOM_record)
1141     ZOOM_resultset_record_immediate(ZOOM_resultset s,size_t pos)
1142 {
1143     const char *syntax =
1144         ZOOM_options_get(s->options, "preferredRecordSyntax");
1145     const char *elementSetName =
1146         ZOOM_options_get(s->options, "elementSetName");
1147     const char *schema =
1148         ZOOM_options_get(s->options, "schema");
1149
1150     return ZOOM_record_cache_lookup_i(s, pos, syntax, elementSetName, schema);
1151 }
1152
1153 ZOOM_API(ZOOM_record)
1154     ZOOM_resultset_record(ZOOM_resultset r, size_t pos)
1155 {
1156     ZOOM_record rec = ZOOM_resultset_record_immediate(r, pos);
1157
1158     if (!rec)
1159     {
1160         /*
1161          * MIKE: I think force_sync should always be zero, but I don't
1162          * want to make this change until I get the go-ahead from
1163          * Adam, in case something depends on the old synchronous
1164          * behaviour.
1165          */
1166         int force_sync = 1;
1167         if (getenv("ZOOM_RECORD_NO_FORCE_SYNC")) force_sync = 0;
1168         ZOOM_resultset_retrieve(r, force_sync, pos, 1);
1169         rec = ZOOM_resultset_record_immediate(r, pos);
1170     }
1171     return rec;
1172 }
1173
1174 ZOOM_API(ZOOM_scanset)
1175     ZOOM_connection_scan(ZOOM_connection c, const char *start)
1176 {
1177     ZOOM_scanset s;
1178     ZOOM_query q = ZOOM_query_create();
1179
1180     ZOOM_query_prefix(q, start);
1181
1182     s = ZOOM_connection_scan1(c, q);
1183     ZOOM_query_destroy(q);
1184     return s;
1185
1186 }
1187
1188 ZOOM_API(ZOOM_scanset)
1189     ZOOM_connection_scan1(ZOOM_connection c, ZOOM_query q)
1190 {
1191     ZOOM_scanset scan = 0;
1192     Z_Query *z_query = ZOOM_query_get_Z_Query(q);
1193
1194     if (!z_query)
1195         return 0;
1196     scan = (ZOOM_scanset) xmalloc(sizeof(*scan));
1197     scan->connection = c;
1198     scan->odr = odr_createmem(ODR_DECODE);
1199     scan->options = ZOOM_options_create_with_parent(c->options);
1200     scan->refcount = 1;
1201     scan->scan_response = 0;
1202     scan->srw_scan_response = 0;
1203
1204     scan->query = q;
1205     ZOOM_query_addref(q);
1206     scan->databaseNames = ZOOM_connection_get_databases(c, c->options,
1207                                             &scan->num_databaseNames,
1208                                             scan->odr);
1209
1210     if (1)
1211     {
1212         ZOOM_task task = ZOOM_connection_add_task(c, ZOOM_TASK_SCAN);
1213         task->u.scan.scan = scan;
1214
1215         (scan->refcount)++;
1216         if (!c->async)
1217         {
1218             while (ZOOM_event(1, &c))
1219                 ;
1220         }
1221     }
1222     return scan;
1223 }
1224
1225 ZOOM_API(void)
1226     ZOOM_scanset_destroy(ZOOM_scanset scan)
1227 {
1228     if (!scan)
1229         return;
1230     (scan->refcount)--;
1231     if (scan->refcount == 0)
1232     {
1233         ZOOM_query_destroy(scan->query);
1234
1235         odr_destroy(scan->odr);
1236
1237         ZOOM_options_destroy(scan->options);
1238         xfree(scan);
1239     }
1240 }
1241
1242 static zoom_ret send_package(ZOOM_connection c)
1243 {
1244     ZOOM_Event event;
1245
1246     yaz_log(c->log_details, "%p send_package", c);
1247     if (!c->tasks)
1248         return zoom_complete;
1249     assert (c->tasks->which == ZOOM_TASK_PACKAGE);
1250
1251     event = ZOOM_Event_create(ZOOM_EVENT_SEND_APDU);
1252     ZOOM_connection_put_event(c, event);
1253
1254     c->buf_out = c->tasks->u.package->buf_out;
1255     c->len_out = c->tasks->u.package->len_out;
1256
1257     return ZOOM_send_buf(c);
1258 }
1259
1260 ZOOM_API(size_t)
1261     ZOOM_scanset_size(ZOOM_scanset scan)
1262 {
1263     if (!scan)
1264         return 0;
1265
1266     if (scan->scan_response && scan->scan_response->entries)
1267         return scan->scan_response->entries->num_entries;
1268     else if (scan->srw_scan_response)
1269         return scan->srw_scan_response->num_terms;
1270     return 0;
1271 }
1272
1273 static void ZOOM_scanset_term_x(ZOOM_scanset scan, size_t pos,
1274                                 size_t *occ,
1275                                 const char **value_term, size_t *value_len,
1276                                 const char **disp_term, size_t *disp_len)
1277 {
1278     size_t noent = ZOOM_scanset_size(scan);
1279
1280     *value_term = 0;
1281     *value_len = 0;
1282
1283     *disp_term = 0;
1284     *disp_len = 0;
1285
1286     *occ = 0;
1287     if (pos >= noent)
1288         return;
1289     if (scan->scan_response)
1290     {
1291         Z_ScanResponse *res = scan->scan_response;
1292         if (res->entries->entries[pos]->which == Z_Entry_termInfo)
1293         {
1294             Z_TermInfo *t = res->entries->entries[pos]->u.termInfo;
1295
1296             *value_term = (const char *) t->term->u.general->buf;
1297             *value_len = t->term->u.general->len;
1298             if (t->displayTerm)
1299             {
1300                 *disp_term = t->displayTerm;
1301                 *disp_len = strlen(*disp_term);
1302             }
1303             else if (t->term->which == Z_Term_general)
1304             {
1305                 *disp_term = (const char *) t->term->u.general->buf;
1306                 *disp_len = t->term->u.general->len;
1307             }
1308             *occ = t->globalOccurrences ? *t->globalOccurrences : 0;
1309         }
1310     }
1311     if (scan->srw_scan_response)
1312     {
1313         Z_SRW_scanResponse *res = scan->srw_scan_response;
1314         Z_SRW_scanTerm *t = res->terms + pos;
1315         if (t)
1316         {
1317             *value_term = t->value;
1318             *value_len = strlen(*value_term);
1319
1320             if (t->displayTerm)
1321                 *disp_term = t->displayTerm;
1322             else
1323                 *disp_term = t->value;
1324             *disp_len = strlen(*disp_term);
1325             *occ = t->numberOfRecords ? *t->numberOfRecords : 0;
1326         }
1327     }
1328 }
1329
1330 ZOOM_API(const char *)
1331     ZOOM_scanset_term(ZOOM_scanset scan, size_t pos,
1332                       size_t *occ, size_t *len)
1333 {
1334     const char *value_term = 0;
1335     size_t value_len = 0;
1336     const char *disp_term = 0;
1337     size_t disp_len = 0;
1338
1339     ZOOM_scanset_term_x(scan, pos, occ, &value_term, &value_len,
1340                         &disp_term, &disp_len);
1341
1342     *len = value_len;
1343     return value_term;
1344 }
1345
1346 ZOOM_API(const char *)
1347     ZOOM_scanset_display_term(ZOOM_scanset scan, size_t pos,
1348                               size_t *occ, size_t *len)
1349 {
1350     const char *value_term = 0;
1351     size_t value_len = 0;
1352     const char *disp_term = 0;
1353     size_t disp_len = 0;
1354
1355     ZOOM_scanset_term_x(scan, pos, occ, &value_term, &value_len,
1356                         &disp_term, &disp_len);
1357
1358     *len = disp_len;
1359     return disp_term;
1360 }
1361
1362 ZOOM_API(const char *)
1363     ZOOM_scanset_option_get(ZOOM_scanset scan, const char *key)
1364 {
1365     return ZOOM_options_get(scan->options, key);
1366 }
1367
1368 ZOOM_API(void)
1369     ZOOM_scanset_option_set(ZOOM_scanset scan, const char *key,
1370                             const char *val)
1371 {
1372     ZOOM_options_set(scan->options, key, val);
1373 }
1374
1375
1376 ZOOM_API(ZOOM_package)
1377     ZOOM_connection_package(ZOOM_connection c, ZOOM_options options)
1378 {
1379     ZOOM_package p = (ZOOM_package) xmalloc(sizeof(*p));
1380
1381     p->connection = c;
1382     p->odr_out = odr_createmem(ODR_ENCODE);
1383     p->options = ZOOM_options_create_with_parent2(options, c->options);
1384     p->refcount = 1;
1385     p->buf_out = 0;
1386     p->len_out = 0;
1387     return p;
1388 }
1389
1390 ZOOM_API(void)
1391     ZOOM_package_destroy(ZOOM_package p)
1392 {
1393     if (!p)
1394         return;
1395     (p->refcount)--;
1396     if (p->refcount == 0)
1397     {
1398         odr_destroy(p->odr_out);
1399         xfree(p->buf_out);
1400
1401         ZOOM_options_destroy(p->options);
1402         xfree(p);
1403     }
1404 }
1405
1406 ZOOM_API(const char *)
1407     ZOOM_package_option_get(ZOOM_package p, const char *key)
1408 {
1409     return ZOOM_options_get(p->options, key);
1410 }
1411
1412 ZOOM_API(const char *)
1413     ZOOM_package_option_getl(ZOOM_package p, const char *key, int *lenp)
1414 {
1415     return ZOOM_options_getl(p->options, key, lenp);
1416 }
1417
1418 ZOOM_API(void)
1419     ZOOM_package_option_set(ZOOM_package p, const char *key,
1420                             const char *val)
1421 {
1422     ZOOM_options_set(p->options, key, val);
1423 }
1424
1425 ZOOM_API(void)
1426     ZOOM_package_option_setl(ZOOM_package p, const char *key,
1427                              const char *val, int len)
1428 {
1429     ZOOM_options_setl(p->options, key, val, len);
1430 }
1431
1432 ZOOM_API(int)
1433     ZOOM_connection_exec_task(ZOOM_connection c)
1434 {
1435     ZOOM_task task = c->tasks;
1436     zoom_ret ret = zoom_complete;
1437
1438     if (!task)
1439         return 0;
1440     yaz_log(c->log_details, "%p ZOOM_connection_exec_task type=%d run=%d",
1441             c, task->which, task->running);
1442     if (c->error != ZOOM_ERROR_NONE)
1443     {
1444         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1445                 "removing tasks because of error = %d", c, c->error);
1446         ZOOM_connection_remove_tasks(c);
1447         return 0;
1448     }
1449     if (task->running)
1450     {
1451         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1452                 "task already running", c);
1453         return 0;
1454     }
1455     task->running = 1;
1456     ret = zoom_complete;
1457     if (c->cs || task->which == ZOOM_TASK_CONNECT)
1458     {
1459         switch (task->which)
1460         {
1461         case ZOOM_TASK_SEARCH:
1462             if (c->proto == PROTO_HTTP)
1463                 ret = ZOOM_connection_srw_send_search(c);
1464             else
1465                 ret = ZOOM_connection_Z3950_search(c);
1466             break;
1467         case ZOOM_TASK_CONNECT:
1468             ret = do_connect(c);
1469             break;
1470         case ZOOM_TASK_SCAN:
1471             if (c->proto == PROTO_HTTP)
1472                 ret = ZOOM_connection_srw_send_scan(c);
1473             else
1474                 ret = ZOOM_connection_Z3950_send_scan(c);
1475             break;
1476         case ZOOM_TASK_PACKAGE:
1477             ret = send_package(c);
1478             break;
1479         case ZOOM_TASK_SORT:
1480             c->tasks->u.sort.resultset->r_sort_spec =
1481                 ZOOM_query_get_sortspec(c->tasks->u.sort.q);
1482             ret = send_Z3950_sort(c, c->tasks->u.sort.resultset);
1483             break;
1484         }
1485     }
1486     else
1487     {
1488         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1489                 "remove tasks because no connection exist", c);
1490         ZOOM_connection_remove_tasks(c);
1491     }
1492     if (ret == zoom_complete)
1493     {
1494         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1495                 "task removed (complete)", c);
1496         ZOOM_connection_remove_task(c);
1497         return 0;
1498     }
1499     yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1500             "task pending", c);
1501     return 1;
1502 }
1503
1504
1505 static zoom_ret send_HTTP_redirect(ZOOM_connection c, const char *uri)
1506 {
1507     Z_GDU *gdu = z_get_HTTP_Request_uri(c->odr_out, uri, 0, c->proxy ? 1 : 0);
1508
1509     gdu->u.HTTP_Request->method = odr_strdup(c->odr_out, "GET");
1510     z_HTTP_header_add(c->odr_out, &gdu->u.HTTP_Request->headers, "Accept",
1511                       "text/xml");
1512     yaz_cookies_request(c->cookies, c->odr_out, gdu->u.HTTP_Request);
1513     if (c->user && c->password)
1514     {
1515         z_HTTP_header_add_basic_auth(c->odr_out, &gdu->u.HTTP_Request->headers,
1516                                      c->user, c->password);
1517     }
1518     xfree(c->location);
1519     c->location = 0;
1520     return ZOOM_send_GDU(c, gdu);
1521 }
1522
1523
1524 zoom_ret ZOOM_send_GDU(ZOOM_connection c, Z_GDU *gdu)
1525 {
1526     ZOOM_Event event;
1527
1528     int r = z_GDU(c->odr_out, &gdu, 0, 0);
1529     if (!r)
1530         return zoom_complete;
1531     if (c->odr_print)
1532         z_GDU(c->odr_print, &gdu, 0, 0);
1533     if (c->odr_save)
1534         z_GDU(c->odr_save, &gdu, 0, 0);
1535     c->buf_out = odr_getbuf(c->odr_out, &c->len_out, 0);
1536     odr_reset(c->odr_out);
1537
1538     event = ZOOM_Event_create(ZOOM_EVENT_SEND_APDU);
1539     ZOOM_connection_put_event(c, event);
1540
1541     return ZOOM_send_buf(c);
1542 }
1543
1544 void ZOOM_set_HTTP_error(ZOOM_connection c, int error,
1545                          const char *addinfo, const char *addinfo2)
1546 {
1547     ZOOM_set_dset_error(c, error, "HTTP", addinfo, addinfo2);
1548 }
1549
1550 #if YAZ_HAVE_XML2
1551 static void handle_http(ZOOM_connection c, Z_HTTP_Response *hres)
1552 {
1553     zoom_ret cret = zoom_complete;
1554     int ret = -1;
1555     char *addinfo = 0;
1556     const char *connection_head = z_HTTP_header_lookup(hres->headers,
1557                                                        "Connection");
1558     const char *location;
1559
1560     ZOOM_connection_set_mask(c, 0);
1561     yaz_log(c->log_details, "%p handle_http", c);
1562
1563     yaz_cookies_response(c->cookies, hres);
1564     if ((hres->code == 301 || hres->code == 302) && c->sru_mode == zoom_sru_get
1565         && (location = z_HTTP_header_lookup(hres->headers, "Location")))
1566     {
1567         c->no_redirects++;
1568         if (c->no_redirects > 10)
1569         {
1570             ZOOM_set_HTTP_error(c, hres->code, 0, 0);
1571             c->no_redirects = 0;
1572             ZOOM_connection_close(c);
1573         }
1574         else
1575         {
1576             /* since redirect may change host we just reconnect. A smarter
1577                implementation might check whether it's the same server */
1578
1579             int host_change = 0;
1580             location = yaz_check_location(c->odr_in, c->host_port,
1581                                           location, &host_change);
1582             xfree(c->location);
1583             c->location = xstrdup(location);
1584             do_connect_host(c, location);
1585             return;
1586         }
1587     }
1588     else
1589     {
1590         ret = ZOOM_handle_sru(c, hres, &cret, &addinfo);
1591         if (ret == 0)
1592         {
1593             if (c->no_redirects) /* end of redirect. change hosts again */
1594                 ZOOM_connection_close(c);
1595         }
1596         c->no_redirects = 0;
1597     }
1598     if (ret)
1599     {
1600         if (hres->code != 200)
1601             ZOOM_set_HTTP_error(c, hres->code, 0, 0);
1602         else
1603         {
1604             yaz_log(YLOG_LOG, "set error... addinfo=%s", addinfo ?
1605                     addinfo : "NULL");
1606             ZOOM_set_error(c, ZOOM_ERROR_DECODE, addinfo);
1607         }
1608         ZOOM_connection_close(c);
1609     }
1610     if (cret == zoom_complete)
1611     {
1612         yaz_log(c->log_details, "removing tasks in handle_http");
1613         ZOOM_connection_remove_task(c);
1614     }
1615     {
1616         int must_close = 0;
1617         if (!strcmp(hres->version, "1.0"))
1618         {
1619             /* HTTP 1.0: only if Keep-Alive we stay alive.. */
1620             if (!connection_head || strcmp(connection_head, "Keep-Alive"))
1621                 must_close = 1;
1622         }
1623         else
1624         {
1625             /* HTTP 1.1: only if no close we stay alive.. */
1626             if (connection_head && !strcmp(connection_head, "close"))
1627                 must_close = 1;
1628         }
1629         if (must_close)
1630         {
1631             ZOOM_connection_close(c);
1632             if (c->tasks)
1633             {
1634                 c->tasks->running = 0;
1635                 ZOOM_connection_insert_task(c, ZOOM_TASK_CONNECT);
1636                 c->reconnect_ok = 0;
1637             }
1638         }
1639         else
1640             c->reconnect_ok = 1; /* if the server closes anyway */
1641     }
1642 }
1643 #endif
1644
1645 static int do_read(ZOOM_connection c)
1646 {
1647     int r, more;
1648     ZOOM_Event event;
1649
1650     event = ZOOM_Event_create(ZOOM_EVENT_RECV_DATA);
1651     ZOOM_connection_put_event(c, event);
1652
1653     r = cs_get(c->cs, &c->buf_in, &c->len_in);
1654     more = cs_more(c->cs);
1655     yaz_log(c->log_details, "%p do_read len=%d more=%d", c, r, more);
1656     if (r == 1)
1657         return 0;
1658     if (r <= 0)
1659     {
1660         if (!ZOOM_test_reconnect(c))
1661         {
1662             ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
1663             ZOOM_connection_close(c);
1664         }
1665     }
1666     else
1667     {
1668         Z_GDU *gdu;
1669         ZOOM_Event event;
1670
1671         odr_reset(c->odr_in);
1672         odr_setbuf(c->odr_in, c->buf_in, r, 0);
1673         event = ZOOM_Event_create(ZOOM_EVENT_RECV_APDU);
1674         ZOOM_connection_put_event(c, event);
1675
1676         if (!z_GDU(c->odr_in, &gdu, 0, 0))
1677         {
1678             int x;
1679             int err = odr_geterrorx(c->odr_in, &x);
1680             char msg[100];
1681             const char *element = odr_getelement(c->odr_in);
1682             yaz_snprintf(msg, sizeof(msg),
1683                     "ODR code %d:%d element=%s offset=%d",
1684                     err, x, element ? element : "<unknown>",
1685                     odr_offset(c->odr_in));
1686             ZOOM_set_error(c, ZOOM_ERROR_DECODE, msg);
1687             if (c->log_api)
1688             {
1689                 FILE *ber_file = yaz_log_file();
1690                 if (ber_file)
1691                     odr_dumpBER(ber_file, c->buf_in, r);
1692             }
1693             ZOOM_connection_close(c);
1694         }
1695         else
1696         {
1697             if (c->odr_print)
1698                 z_GDU(c->odr_print, &gdu, 0, 0);
1699             if (c->odr_save)
1700                 z_GDU(c->odr_save, &gdu, 0, 0);
1701             if (gdu->which == Z_GDU_Z3950)
1702                 ZOOM_handle_Z3950_apdu(c, gdu->u.z3950);
1703             else if (gdu->which == Z_GDU_HTTP_Response)
1704             {
1705 #if YAZ_HAVE_XML2
1706                 handle_http(c, gdu->u.HTTP_Response);
1707 #else
1708                 ZOOM_set_error(c, ZOOM_ERROR_DECODE, 0);
1709                 ZOOM_connection_close(c);
1710 #endif
1711             }
1712         }
1713     }
1714     return 1;
1715 }
1716
1717 static zoom_ret do_write_ex(ZOOM_connection c, char *buf_out, int len_out)
1718 {
1719     int r;
1720     ZOOM_Event event;
1721
1722     event = ZOOM_Event_create(ZOOM_EVENT_SEND_DATA);
1723     ZOOM_connection_put_event(c, event);
1724
1725     yaz_log(c->log_details, "%p do_write_ex len=%d", c, len_out);
1726     if ((r = cs_put(c->cs, buf_out, len_out)) < 0)
1727     {
1728         yaz_log(c->log_details, "%p do_write_ex write failed", c);
1729         if (ZOOM_test_reconnect(c))
1730         {
1731             return zoom_pending;
1732         }
1733         if (c->state == STATE_CONNECTING)
1734             ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
1735         else
1736             ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
1737         ZOOM_connection_close(c);
1738         return zoom_complete;
1739     }
1740     else if (r == 1)
1741     {
1742         int mask = ZOOM_SELECT_EXCEPT;
1743         if (c->cs->io_pending & CS_WANT_WRITE)
1744             mask += ZOOM_SELECT_WRITE;
1745         if (c->cs->io_pending & CS_WANT_READ)
1746             mask += ZOOM_SELECT_READ;
1747         ZOOM_connection_set_mask(c, mask);
1748         yaz_log(c->log_details, "%p do_write_ex write incomplete mask=%d",
1749                 c, c->mask);
1750     }
1751     else
1752     {
1753         ZOOM_connection_set_mask(c, ZOOM_SELECT_READ|ZOOM_SELECT_EXCEPT);
1754         yaz_log(c->log_details, "%p do_write_ex write complete mask=%d",
1755                 c, c->mask);
1756     }
1757     return zoom_pending;
1758 }
1759
1760 zoom_ret ZOOM_send_buf(ZOOM_connection c)
1761 {
1762     return do_write_ex(c, c->buf_out, c->len_out);
1763 }
1764
1765
1766 ZOOM_API(const char *)
1767     ZOOM_connection_option_get(ZOOM_connection c, const char *key)
1768 {
1769     if (!strcmp(key, "APDU"))
1770     {
1771         return c->saveAPDU_wrbuf ? wrbuf_cstr(c->saveAPDU_wrbuf) : "";
1772     }
1773     else
1774         return ZOOM_options_get(c->options, key);
1775 }
1776
1777 ZOOM_API(const char *)
1778     ZOOM_connection_option_getl(ZOOM_connection c, const char *key, int *lenp)
1779 {
1780     if (!strcmp(key, "APDU"))
1781     {
1782         if (c->saveAPDU_wrbuf)
1783         {
1784             *lenp = wrbuf_len(c->saveAPDU_wrbuf);
1785             return wrbuf_cstr(c->saveAPDU_wrbuf);
1786         }
1787         else
1788         {
1789             *lenp = 0;
1790             return "";
1791         }
1792     }
1793     else
1794         return ZOOM_options_getl(c->options, key, lenp);
1795 }
1796
1797 ZOOM_API(void)
1798     ZOOM_connection_option_set(ZOOM_connection c, const char *key,
1799                                const char *val)
1800 {
1801     if (!strcmp(key, "saveAPDU"))
1802     {
1803         if (val && strcmp(val, "0"))
1804         {
1805             if (!c->saveAPDU_wrbuf)
1806                 c->saveAPDU_wrbuf = wrbuf_alloc();
1807             else
1808                 wrbuf_rewind(c->saveAPDU_wrbuf);
1809         }
1810         else
1811         {
1812             wrbuf_destroy(c->saveAPDU_wrbuf);
1813             c->saveAPDU_wrbuf = 0;
1814         }
1815         ZOOM_connection_save_apdu_wrbuf(c, c->saveAPDU_wrbuf);
1816     }
1817     else
1818         ZOOM_options_set(c->options, key, val);
1819 }
1820
1821 ZOOM_API(void)
1822     ZOOM_connection_option_setl(ZOOM_connection c, const char *key,
1823                                 const char *val, int len)
1824 {
1825     ZOOM_options_setl(c->options, key, val, len);
1826 }
1827
1828 ZOOM_API(const char *)
1829     ZOOM_resultset_option_get(ZOOM_resultset r, const char *key)
1830 {
1831     return ZOOM_options_get(r->options, key);
1832 }
1833
1834 ZOOM_API(void)
1835     ZOOM_resultset_option_set(ZOOM_resultset r, const char *key,
1836                               const char *val)
1837 {
1838     ZOOM_options_set(r->options, key, val);
1839 }
1840
1841
1842 ZOOM_API(int)
1843     ZOOM_connection_errcode(ZOOM_connection c)
1844 {
1845     return ZOOM_connection_error(c, 0, 0);
1846 }
1847
1848 ZOOM_API(const char *)
1849     ZOOM_connection_errmsg(ZOOM_connection c)
1850 {
1851     const char *msg;
1852     ZOOM_connection_error(c, &msg, 0);
1853     return msg;
1854 }
1855
1856 ZOOM_API(const char *)
1857     ZOOM_connection_addinfo(ZOOM_connection c)
1858 {
1859     const char *addinfo;
1860     ZOOM_connection_error(c, 0, &addinfo);
1861     return addinfo;
1862 }
1863
1864 ZOOM_API(const char *)
1865     ZOOM_connection_diagset(ZOOM_connection c)
1866 {
1867     const char *diagset;
1868     ZOOM_connection_error_x(c, 0, 0, &diagset);
1869     return diagset;
1870 }
1871
1872 ZOOM_API(const char *)
1873     ZOOM_diag_str(int error)
1874 {
1875     switch (error)
1876     {
1877     case ZOOM_ERROR_NONE:
1878         return "No error";
1879     case ZOOM_ERROR_CONNECT:
1880         return "Connect failed";
1881     case ZOOM_ERROR_MEMORY:
1882         return "Out of memory";
1883     case ZOOM_ERROR_ENCODE:
1884         return "Encoding failed";
1885     case ZOOM_ERROR_DECODE:
1886         return "Decoding failed";
1887     case ZOOM_ERROR_CONNECTION_LOST:
1888         return "Connection lost";
1889     case ZOOM_ERROR_INIT:
1890         return "Init rejected";
1891     case ZOOM_ERROR_INTERNAL:
1892         return "Internal failure";
1893     case ZOOM_ERROR_TIMEOUT:
1894         return "Timeout";
1895     case ZOOM_ERROR_UNSUPPORTED_PROTOCOL:
1896         return "Unsupported protocol";
1897     case ZOOM_ERROR_UNSUPPORTED_QUERY:
1898         return "Unsupported query type";
1899     case ZOOM_ERROR_INVALID_QUERY:
1900         return "Invalid query";
1901     case ZOOM_ERROR_CQL_PARSE:
1902         return "CQL parsing error";
1903     case ZOOM_ERROR_CQL_TRANSFORM:
1904         return "CQL transformation error";
1905     case ZOOM_ERROR_CCL_CONFIG:
1906         return "CCL configuration error";
1907     case ZOOM_ERROR_CCL_PARSE:
1908         return "CCL parsing error";
1909     case ZOOM_ERROR_ES_INVALID_ACTION:
1910         return "Extended Service. invalid action";
1911     case ZOOM_ERROR_ES_INVALID_VERSION:
1912         return "Extended Service. invalid version";
1913     case ZOOM_ERROR_ES_INVALID_SYNTAX:
1914         return "Extended Service. invalid syntax";
1915     case ZOOM_ERROR_MEMCACHED:
1916         return "Memcached";
1917     default:
1918         return diagbib1_str(error);
1919     }
1920 }
1921
1922 ZOOM_API(int)
1923     ZOOM_connection_error_x(ZOOM_connection c, const char **cp,
1924                             const char **addinfo, const char **diagset)
1925 {
1926     int error = c->error;
1927     if (cp)
1928     {
1929         if (!c->diagset || !strcmp(c->diagset, "ZOOM"))
1930             *cp = ZOOM_diag_str(error);
1931         else if (!strcmp(c->diagset, "HTTP"))
1932             *cp = z_HTTP_errmsg(c->error);
1933         else if (!strcmp(c->diagset, "Bib-1"))
1934             *cp = ZOOM_diag_str(error);
1935         else if (!strcmp(c->diagset, "info:srw/diagnostic/1"))
1936             *cp = yaz_diag_srw_str(c->error);
1937         else
1938             *cp = "Unknown error and diagnostic set";
1939     }
1940     if (addinfo)
1941         *addinfo = c->addinfo ? c->addinfo : "";
1942     if (diagset)
1943         *diagset = c->diagset ? c->diagset : "";
1944     return c->error;
1945 }
1946
1947 ZOOM_API(int)
1948     ZOOM_connection_error(ZOOM_connection c, const char **cp,
1949                           const char **addinfo)
1950 {
1951     return ZOOM_connection_error_x(c, cp, addinfo, 0);
1952 }
1953
1954 static void ZOOM_connection_do_io(ZOOM_connection c, int mask)
1955 {
1956     ZOOM_Event event = 0;
1957     int r = cs_look(c->cs);
1958     yaz_log(c->log_details, "%p ZOOM_connection_do_io mask=%d cs_look=%d",
1959             c, mask, r);
1960
1961     if (r == CS_NONE)
1962     {
1963         event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1964         ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
1965         ZOOM_connection_close(c);
1966         ZOOM_connection_put_event(c, event);
1967     }
1968     else if (r == CS_CONNECT)
1969     {
1970         int ret = ret = cs_rcvconnect(c->cs);
1971         yaz_log(c->log_details, "%p ZOOM_connection_do_io "
1972                 "cs_rcvconnect returned %d", c, ret);
1973         if (ret == 1)
1974         {
1975             int mask = ZOOM_SELECT_EXCEPT;
1976             if (c->cs->io_pending & CS_WANT_WRITE)
1977                 mask += ZOOM_SELECT_WRITE;
1978             if (c->cs->io_pending & CS_WANT_READ)
1979                 mask += ZOOM_SELECT_READ;
1980             ZOOM_connection_set_mask(c, mask);
1981             event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1982             ZOOM_connection_put_event(c, event);
1983         }
1984         else if (ret == 0)
1985         {
1986             event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1987             ZOOM_connection_put_event(c, event);
1988             get_cert(c);
1989             if (c->proto == PROTO_Z3950)
1990                 ZOOM_connection_Z3950_send_init(c);
1991             else
1992             {
1993                 /* no init request for SRW .. */
1994                 if (c->tasks->which == ZOOM_TASK_CONNECT)
1995                 {
1996                     ZOOM_connection_remove_task(c);
1997                     ZOOM_connection_set_mask(c, 0);
1998                 }
1999                 if (c->cs && c->location)
2000                     send_HTTP_redirect(c, c->location);
2001                 else
2002                     ZOOM_connection_exec_task(c);
2003             }
2004             c->state = STATE_ESTABLISHED;
2005         }
2006         else
2007         {
2008             ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
2009             ZOOM_connection_close(c);
2010         }
2011     }
2012     else
2013     {
2014         if (mask & ZOOM_SELECT_EXCEPT)
2015         {
2016             if (!ZOOM_test_reconnect(c))
2017             {
2018                 ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
2019                 ZOOM_connection_close(c);
2020             }
2021             return;
2022         }
2023         if (mask & ZOOM_SELECT_READ)
2024             do_read(c);
2025         if (c->cs && (mask & ZOOM_SELECT_WRITE))
2026             ZOOM_send_buf(c);
2027     }
2028 }
2029
2030 ZOOM_API(int)
2031     ZOOM_connection_last_event(ZOOM_connection cs)
2032 {
2033     if (!cs)
2034         return ZOOM_EVENT_NONE;
2035     return cs->last_event;
2036 }
2037
2038
2039 ZOOM_API(int) ZOOM_connection_fire_event_timeout(ZOOM_connection c)
2040 {
2041     if (c->mask)
2042     {
2043         ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_TIMEOUT);
2044         /* timeout and this connection was waiting */
2045         ZOOM_set_error(c, ZOOM_ERROR_TIMEOUT, 0);
2046         ZOOM_connection_close(c);
2047         ZOOM_connection_put_event(c, event);
2048     }
2049     return 0;
2050 }
2051
2052 ZOOM_API(int)
2053     ZOOM_connection_process(ZOOM_connection c)
2054 {
2055     ZOOM_Event event;
2056     if (!c)
2057         return 0;
2058
2059     event = ZOOM_connection_get_event(c);
2060     if (event)
2061     {
2062         ZOOM_Event_destroy(event);
2063         return 1;
2064     }
2065     ZOOM_connection_exec_task(c);
2066     event = ZOOM_connection_get_event(c);
2067     if (event)
2068     {
2069         ZOOM_Event_destroy(event);
2070         return 1;
2071     }
2072     return 0;
2073 }
2074
2075 ZOOM_API(int)
2076     ZOOM_event_nonblock(int no, ZOOM_connection *cs)
2077 {
2078     int i;
2079
2080     yaz_log(log_details0, "ZOOM_process_event(no=%d,cs=%p)", no, cs);
2081
2082     for (i = 0; i<no; i++)
2083     {
2084         ZOOM_connection c = cs[i];
2085
2086         if (c && ZOOM_connection_process(c))
2087             return i+1;
2088     }
2089     return 0;
2090 }
2091
2092 ZOOM_API(int) ZOOM_connection_fire_event_socket(ZOOM_connection c, int mask)
2093 {
2094     if (c->mask && mask)
2095         ZOOM_connection_do_io(c, mask);
2096     return 0;
2097 }
2098
2099 ZOOM_API(int) ZOOM_connection_get_socket(ZOOM_connection c)
2100 {
2101     if (c->cs)
2102         return cs_fileno(c->cs);
2103     return -1;
2104 }
2105
2106 ZOOM_API(int) ZOOM_connection_set_mask(ZOOM_connection c, int mask)
2107 {
2108     c->mask = mask;
2109     if (!c->cs)
2110         return -1;
2111     return 0;
2112 }
2113
2114 ZOOM_API(int) ZOOM_connection_get_mask(ZOOM_connection c)
2115 {
2116     if (c->cs)
2117         return c->mask;
2118     return 0;
2119 }
2120
2121 ZOOM_API(int) ZOOM_connection_get_timeout(ZOOM_connection c)
2122 {
2123     return ZOOM_options_get_int(c->options, "timeout", 30);
2124 }
2125
2126 ZOOM_API(void) ZOOM_connection_close(ZOOM_connection c)
2127 {
2128     if (c->cs)
2129         cs_close(c->cs);
2130     c->cs = 0;
2131     ZOOM_connection_set_mask(c, 0);
2132     c->state = STATE_IDLE;
2133 }
2134
2135 /*
2136  * Local variables:
2137  * c-basic-offset: 4
2138  * c-file-style: "Stroustrup"
2139  * indent-tabs-mode: nil
2140  * End:
2141  * vim: shiftwidth=4 tabstop=8 expandtab
2142  */
2143