ZOOM: handle trailing slash for Z39.50 urls
[yaz-moved-to-github.git] / src / zoom-c.c
1 /* This file is part of the YAZ toolkit.
2  * Copyright (C) 1995-2012 Index Data
3  * See the file LICENSE for details.
4  */
5 /**
6  * \file zoom-c.c
7  * \brief Implements ZOOM C interface.
8  */
9 #if HAVE_CONFIG_H
10 #include <config.h>
11 #endif
12
13 #include <assert.h>
14 #include <string.h>
15 #include <errno.h>
16 #include "zoom-p.h"
17
18 #include <yaz/yaz-util.h>
19 #include <yaz/xmalloc.h>
20 #include <yaz/otherinfo.h>
21 #include <yaz/log.h>
22 #include <yaz/diagbib1.h>
23 #include <yaz/charneg.h>
24 #include <yaz/query-charset.h>
25 #include <yaz/snprintf.h>
26 #include <yaz/facet.h>
27
28 #include <yaz/shptr.h>
29
30 #if SHPTR
31 YAZ_SHPTR_TYPE(WRBUF)
32 #endif
33
34 static int log_api0 = 0;
35 static int log_details0 = 0;
36
37 static void resultset_destroy(ZOOM_resultset r);
38 static zoom_ret do_write_ex(ZOOM_connection c, char *buf_out, int len_out);
39
40 static void initlog(void)
41 {
42     static int log_level_initialized = 0;
43
44     if (!log_level_initialized)
45     {
46         log_api0 = yaz_log_module_level("zoom");
47         log_details0 = yaz_log_module_level("zoomdetails");
48         log_level_initialized = 1;
49     }
50 }
51
52 void ZOOM_connection_remove_tasks(ZOOM_connection c);
53
54 void ZOOM_set_dset_error(ZOOM_connection c, int error,
55                          const char *dset,
56                          const char *addinfo, const char *addinfo2)
57 {
58     char *cp;
59
60     xfree(c->addinfo);
61     c->addinfo = 0;
62     c->error = error;
63     if (!c->diagset || strcmp(dset, c->diagset))
64     {
65         xfree(c->diagset);
66         c->diagset = xstrdup(dset);
67         /* remove integer part from SRW diagset .. */
68         if ((cp = strrchr(c->diagset, '/')))
69             *cp = '\0';
70     }
71     if (addinfo && addinfo2)
72     {
73         c->addinfo = (char*) xmalloc(strlen(addinfo) + strlen(addinfo2) + 3);
74         strcpy(c->addinfo, addinfo);
75         strcat(c->addinfo, ": ");
76         strcat(c->addinfo, addinfo2);
77     }
78     else if (addinfo)
79         c->addinfo = xstrdup(addinfo);
80     if (error != ZOOM_ERROR_NONE)
81     {
82         yaz_log(c->log_api, "%p set_dset_error %s %s:%d %s %s",
83                 c, c->host_port ? c->host_port : "<>", dset, error,
84                 addinfo ? addinfo : "",
85                 addinfo2 ? addinfo2 : "");
86         ZOOM_connection_remove_tasks(c);
87     }
88 }
89
90 int ZOOM_uri_to_code(const char *uri)
91 {
92     int code = 0;       
93     const char *cp;
94     if ((cp = strrchr(uri, '/')))
95         code = atoi(cp+1);
96     return code;
97 }
98
99
100
101 void ZOOM_set_error(ZOOM_connection c, int error, const char *addinfo)
102 {
103     ZOOM_set_dset_error(c, error, "ZOOM", addinfo, 0);
104 }
105
106 static void clear_error(ZOOM_connection c)
107 {
108     /*
109      * If an error is tied to an operation then it's ok to clear: for
110      * example, a diagnostic returned from a search is cleared by a
111      * subsequent search.  However, problems such as Connection Lost
112      * or Init Refused are not cleared, because they are not
113      * recoverable: doing another search doesn't help.
114      */
115
116     ZOOM_connection_remove_events(c);
117     switch (c->error)
118     {
119     case ZOOM_ERROR_CONNECT:
120     case ZOOM_ERROR_MEMORY:
121     case ZOOM_ERROR_DECODE:
122     case ZOOM_ERROR_CONNECTION_LOST:
123     case ZOOM_ERROR_INIT:
124     case ZOOM_ERROR_INTERNAL:
125     case ZOOM_ERROR_UNSUPPORTED_PROTOCOL:
126         break;
127     default:
128         ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
129     }
130 }
131
132 void ZOOM_connection_show_task(ZOOM_task task)
133 {
134     switch(task->which)
135     {
136     case ZOOM_TASK_SEARCH:
137         yaz_log(YLOG_LOG, "search p=%p", task);
138         break;
139     case ZOOM_TASK_RETRIEVE:
140         yaz_log(YLOG_LOG, "retrieve p=%p", task);
141         break;
142     case ZOOM_TASK_CONNECT:
143         yaz_log(YLOG_LOG, "connect p=%p", task);
144         break;
145     case ZOOM_TASK_SCAN:
146         yaz_log(YLOG_LOG, "scan p=%p", task);
147         break;
148     }
149 }
150
151 void ZOOM_connection_show_tasks(ZOOM_connection c)
152 {
153     ZOOM_task task;
154     yaz_log(YLOG_LOG, "connection p=%p tasks", c);
155     for (task = c->tasks; task; task = task->next)
156         ZOOM_connection_show_task(task);
157 }
158
159 ZOOM_task ZOOM_connection_add_task(ZOOM_connection c, int which)
160 {
161     ZOOM_task *taskp = &c->tasks;
162     while (*taskp)
163         taskp = &(*taskp)->next;
164     *taskp = (ZOOM_task) xmalloc(sizeof(**taskp));
165     (*taskp)->running = 0;
166     (*taskp)->which = which;
167     (*taskp)->next = 0;
168     clear_error(c);
169     return *taskp;
170 }
171
172 ZOOM_API(int) ZOOM_connection_is_idle(ZOOM_connection c)
173 {
174     return c->tasks ? 0 : 1;
175 }
176
177 ZOOM_task ZOOM_connection_insert_task(ZOOM_connection c, int which)
178 {
179     ZOOM_task task = (ZOOM_task) xmalloc(sizeof(*task));
180
181     task->next = c->tasks;
182     c->tasks = task;
183
184     task->running = 0;
185     task->which = which;
186     return task;
187 }
188
189 void ZOOM_connection_remove_task(ZOOM_connection c)
190 {
191     ZOOM_task task = c->tasks;
192
193     if (task)
194     {
195         c->tasks = task->next;
196         switch (task->which)
197         {
198         case ZOOM_TASK_SEARCH:
199             resultset_destroy(task->u.search.resultset);
200             xfree(task->u.search.syntax);
201             xfree(task->u.search.elementSetName);
202             break;
203         case ZOOM_TASK_RETRIEVE:
204             resultset_destroy(task->u.retrieve.resultset);
205             xfree(task->u.retrieve.syntax);
206             xfree(task->u.retrieve.elementSetName);
207             break;
208         case ZOOM_TASK_CONNECT:
209             break;
210         case ZOOM_TASK_SCAN:
211             ZOOM_scanset_destroy(task->u.scan.scan);
212             break;
213         case ZOOM_TASK_PACKAGE:
214             ZOOM_package_destroy(task->u.package);
215             break;
216         case ZOOM_TASK_SORT:
217             resultset_destroy(task->u.sort.resultset);
218             ZOOM_query_destroy(task->u.sort.q);
219             break;
220         default:
221             assert(0);
222         }
223         xfree(task);
224
225         if (!c->tasks)
226         {
227             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_END);
228             ZOOM_connection_put_event(c, event);
229         }
230     }
231 }
232
233 void ZOOM_connection_remove_tasks(ZOOM_connection c)
234 {
235     while (c->tasks)
236         ZOOM_connection_remove_task(c);
237 }
238
239 static void odr_wrbuf_write(ODR o, void *handle, int type,
240                             const char *buf, int len)
241 {
242     WRBUF w = (WRBUF) handle;
243     wrbuf_write(w, buf, len);
244 }
245
246 ZOOM_API(ZOOM_connection)
247     ZOOM_connection_create(ZOOM_options options)
248 {
249     ZOOM_connection c = (ZOOM_connection) xmalloc(sizeof(*c));
250
251     initlog();
252
253     c->log_api = log_api0;
254     c->log_details = log_details0;
255
256     yaz_log(c->log_api, "%p ZOOM_connection_create", c);
257
258     c->proto = PROTO_Z3950;
259     c->cs = 0;
260     ZOOM_connection_set_mask(c, 0);
261     c->reconnect_ok = 0;
262     c->state = STATE_IDLE;
263     c->addinfo = 0;
264     c->diagset = 0;
265     ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
266     c->buf_in = 0;
267     c->len_in = 0;
268     c->buf_out = 0;
269     c->len_out = 0;
270     c->resultsets = 0;
271
272     c->options = ZOOM_options_create_with_parent(options);
273
274     c->host_port = 0;
275     c->proxy = 0;
276     c->tproxy = 0;
277     
278     c->charset = c->lang = 0;
279
280     c->cookie_out = 0;
281     c->cookie_in = 0;
282     c->client_IP = 0;
283     c->tasks = 0;
284
285     c->user = 0;
286     c->group = 0;
287     c->password = 0;
288
289     c->maximum_record_size = 0;
290     c->preferred_message_size = 0;
291
292     c->odr_in = odr_createmem(ODR_DECODE);
293     c->odr_out = odr_createmem(ODR_ENCODE);
294     c->odr_print = 0;
295     c->odr_save = 0;
296
297     c->async = 0;
298     c->support_named_resultsets = 0;
299     c->last_event = ZOOM_EVENT_NONE;
300
301     c->m_queue_front = 0;
302     c->m_queue_back = 0;
303
304     c->sru_version = 0;
305     c->no_redirects = 0;
306     return c;
307 }
308
309 ZOOM_API(void) ZOOM_connection_save_apdu_wrbuf(ZOOM_connection c, WRBUF w)
310 {
311     if (c->odr_save)
312     {
313         odr_destroy(c->odr_save);
314         c->odr_save = 0;
315     }
316     if (w)
317     {
318         c->odr_save = odr_createmem(ODR_PRINT);
319         odr_set_stream(c->odr_save, w, odr_wrbuf_write, 0);
320     }
321 }
322
323 /* set database names. Take local databases (if set); otherwise
324    take databases given in ZURL (if set); otherwise use Default */
325 char **ZOOM_connection_get_databases(ZOOM_connection con, ZOOM_options options,
326                                      int *num, ODR odr)
327 {
328     char **databaseNames;
329     const char *cp = ZOOM_options_get(options, "databaseName");
330     
331     if ((!cp || !*cp) && con->host_port)
332     {
333         if (strncmp(con->host_port, "unix:", 5) == 0)
334             cp = strchr(con->host_port+5, ':');
335         else
336             cp = strchr(con->host_port, '/');
337         if (cp)
338             cp++;
339     }
340     if (!cp || !*cp)
341         cp = "Default";
342     nmem_strsplit(odr_getmem(odr), "+", cp,  &databaseNames, num);
343     return databaseNames;
344 }
345
346 ZOOM_API(ZOOM_connection)
347     ZOOM_connection_new(const char *host, int portnum)
348 {
349     ZOOM_connection c = ZOOM_connection_create(0);
350
351     ZOOM_connection_connect(c, host, portnum);
352     return c;
353 }
354
355 static zoom_sru_mode get_sru_mode_from_string(const char *s)
356 {
357     if (!s || !*s)
358         return zoom_sru_soap;
359     if (!yaz_matchstr(s, "soap"))
360         return zoom_sru_soap;
361     else if (!yaz_matchstr(s, "get"))
362         return zoom_sru_get;
363     else if (!yaz_matchstr(s, "post"))
364         return zoom_sru_post;
365     else if (!yaz_matchstr(s, "solr"))
366         return zoom_sru_solr;
367     return zoom_sru_error;
368 }
369
370 ZOOM_API(void)
371     ZOOM_connection_connect(ZOOM_connection c,
372                             const char *host, int portnum)
373 {
374     const char *val;
375
376     initlog();
377
378     yaz_log(c->log_api, "%p ZOOM_connection_connect host=%s portnum=%d",
379             c, host ? host : "null", portnum);
380
381     ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
382     ZOOM_connection_remove_tasks(c);
383
384     if (c->odr_print)
385     {
386         odr_setprint(c->odr_print, 0); /* prevent destroy from fclose'ing */
387         odr_destroy(c->odr_print);
388     }
389     if (ZOOM_options_get_bool(c->options, "apdulog", 0))
390     {
391         c->odr_print = odr_createmem(ODR_PRINT);
392         odr_setprint(c->odr_print, yaz_log_file());
393     }
394     else
395         c->odr_print = 0;
396
397     if (c->cs)
398     {
399         yaz_log(c->log_details, "%p ZOOM_connection_connect reconnect ok", c);
400         c->reconnect_ok = 1;
401         return;
402     }
403     yaz_log(c->log_details, "%p ZOOM_connection_connect connect", c);
404     xfree(c->proxy);
405     c->proxy = 0;
406     val = ZOOM_options_get(c->options, "proxy");
407     if (val && *val)
408     {
409         yaz_log(c->log_details, "%p ZOOM_connection_connect proxy=%s", c, val);
410         c->proxy = xstrdup(val);
411     }
412
413     xfree(c->tproxy);
414     c->tproxy = 0;
415     val = ZOOM_options_get(c->options, "tproxy");
416     if (val && *val)
417     {
418         yaz_log(c->log_details, "%p ZOOM_connection_connect tproxy=%s", c, val);
419         c->tproxy = xstrdup(val);
420     }
421
422
423     xfree(c->charset);
424     c->charset = 0;
425     val = ZOOM_options_get(c->options, "charset");
426     if (val && *val)
427     {
428         yaz_log(c->log_details, "%p ZOOM_connection_connect charset=%s", c, val);
429         c->charset = xstrdup(val);
430     }
431
432     xfree(c->lang);
433     val = ZOOM_options_get(c->options, "lang");
434     if (val && *val)
435     {
436         yaz_log(c->log_details, "%p ZOOM_connection_connect lang=%s", c, val);
437         c->lang = xstrdup(val);
438     }
439     else
440         c->lang = 0;
441
442     if (host)
443     {
444         xfree(c->host_port);
445         if (portnum)
446         {
447             char hostn[128];
448             sprintf(hostn, "%.80s:%d", host, portnum);
449             c->host_port = xstrdup(hostn);
450         }
451         else
452             c->host_port = xstrdup(host);
453     }        
454
455     {
456         /*
457          * If the "<scheme>:" part of the host string is preceded by one
458          * or more comma-separated <name>=<value> pairs, these are taken
459          * to be options to be set on the connection object.  Among other
460          * applications, this facility can be used to embed authentication
461          * in a host string:
462          *          user=admin,password=secret,tcp:localhost:9999
463          */
464         char *remainder = c->host_port;
465         char *pcolon = strchr(remainder, ':');
466         char *pcomma;
467         char *pequals;
468         while ((pcomma = strchr(remainder, ',')) != 0 &&
469                (pcolon == 0 || pcomma < pcolon)) {
470             *pcomma = '\0';
471             if ((pequals = strchr(remainder, '=')) != 0) {
472                 *pequals = '\0';
473                 /*printf("# setting '%s'='%s'\n", remainder, pequals+1);*/
474                 ZOOM_connection_option_set(c, remainder, pequals+1);
475             }
476             remainder = pcomma+1;
477         }
478
479         if (remainder != c->host_port) {
480             xfree(c->host_port);
481             c->host_port = xstrdup(remainder);
482             /*printf("# reset hp='%s'\n", remainder);*/
483         }
484     }
485
486     val = ZOOM_options_get(c->options, "sru");
487     c->sru_mode = get_sru_mode_from_string(val);
488
489     xfree(c->sru_version);
490     val = ZOOM_options_get(c->options, "sru_version");
491     c->sru_version = xstrdup(val ? val : "1.2");
492
493     ZOOM_options_set(c->options, "host", c->host_port);
494
495     xfree(c->cookie_out);
496     c->cookie_out = 0;
497     val = ZOOM_options_get(c->options, "cookie");
498     if (val && *val)
499     { 
500         yaz_log(c->log_details, "%p ZOOM_connection_connect cookie=%s", c, val);
501         c->cookie_out = xstrdup(val);
502     }
503
504     xfree(c->client_IP);
505     c->client_IP = 0;
506     val = ZOOM_options_get(c->options, "clientIP");
507     if (val && *val)
508     {
509         yaz_log(c->log_details, "%p ZOOM_connection_connect clientIP=%s",
510                 c, val);
511         c->client_IP = xstrdup(val);
512     }
513
514     xfree(c->group);
515     c->group = 0;
516     val = ZOOM_options_get(c->options, "group");
517     if (val && *val)
518         c->group = xstrdup(val);
519
520     xfree(c->user);
521     c->user = 0;
522     val = ZOOM_options_get(c->options, "user");
523     if (val && *val)
524         c->user = xstrdup(val);
525
526     xfree(c->password);
527     c->password = 0;
528     val = ZOOM_options_get(c->options, "password");
529     if (!val)
530         val = ZOOM_options_get(c->options, "pass");
531
532     if (val && *val)
533         c->password = xstrdup(val);
534     
535     c->maximum_record_size =
536         ZOOM_options_get_int(c->options, "maximumRecordSize", 1024*1024);
537     c->preferred_message_size =
538         ZOOM_options_get_int(c->options, "preferredMessageSize", 1024*1024);
539
540     c->async = ZOOM_options_get_bool(c->options, "async", 0);
541
542     if (c->sru_mode == zoom_sru_error)
543     {
544         ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, val);
545         ZOOM_connection_remove_tasks(c);
546         return;
547     }
548
549     yaz_log(c->log_details, "%p ZOOM_connection_connect async=%d", c, c->async);
550     ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
551
552     if (!c->async)
553     {
554         while (ZOOM_event(1, &c))
555             ;
556     }
557 }
558
559 ZOOM_API(void) ZOOM_resultset_release(ZOOM_resultset r)
560 {
561 #if ZOOM_RESULT_LISTS
562 #else
563     if (r->connection)
564     {
565         /* remove ourselves from the resultsets in connection */
566         ZOOM_resultset *rp = &r->connection->resultsets;
567         while (1)
568         {
569             assert(*rp);   /* we must be in this list!! */
570             if (*rp == r)
571             {   /* OK, we're here - take us out of it */
572                 *rp = (*rp)->next;
573                 break;
574             }
575             rp = &(*rp)->next;
576         }
577         r->connection = 0;
578     }
579 #endif
580 }
581
582 ZOOM_API(void)
583     ZOOM_connection_destroy(ZOOM_connection c)
584 {
585 #if ZOOM_RESULT_LISTS
586     ZOOM_resultsets list;
587 #else
588     ZOOM_resultset r;
589 #endif
590     if (!c)
591         return;
592     yaz_log(c->log_api, "%p ZOOM_connection_destroy", c);
593     if (c->cs)
594         cs_close(c->cs);
595
596 #if ZOOM_RESULT_LISTS
597     /* Remove the connection's usage of resultsets */
598     list = c->resultsets;
599     while (list) {
600         ZOOM_resultsets removed = list;
601         ZOOM_resultset_destroy(list->resultset);
602         list = list->next;
603         xfree(removed);
604     }
605 #else
606     for (r = c->resultsets; r; r = r->next)
607         r->connection = 0;
608 #endif
609
610     xfree(c->buf_in);
611     xfree(c->addinfo);
612     xfree(c->diagset);
613     odr_destroy(c->odr_in);
614     odr_destroy(c->odr_out);
615     if (c->odr_save)
616         odr_destroy(c->odr_save);
617     if (c->odr_print)
618     {
619         odr_setprint(c->odr_print, 0); /* prevent destroy from fclose'ing */
620         odr_destroy(c->odr_print);
621     }
622     ZOOM_options_destroy(c->options);
623     ZOOM_connection_remove_tasks(c);
624     ZOOM_connection_remove_events(c);
625     xfree(c->host_port);
626     xfree(c->proxy);
627     xfree(c->tproxy);
628     xfree(c->charset);
629     xfree(c->lang);
630     xfree(c->cookie_out);
631     xfree(c->cookie_in);
632     xfree(c->client_IP);
633     xfree(c->user);
634     xfree(c->group);
635     xfree(c->password);
636     xfree(c->sru_version);
637     xfree(c);
638 }
639
640 void ZOOM_resultset_addref(ZOOM_resultset r)
641 {
642     if (r)
643     {
644         yaz_mutex_enter(r->mutex);
645         (r->refcount)++;
646         yaz_log(log_details0, "%p ZOOM_resultset_addref count=%d",
647                 r, r->refcount);
648         yaz_mutex_leave(r->mutex);
649     }
650 }
651
652 static int g_resultsets = 0;
653 static YAZ_MUTEX g_resultset_mutex = 0;
654
655 /* TODO We need to initialize this before running threaded:
656  * call resultset_use(0)  */
657
658 static int resultset_use(int delta) {
659     int resultset_count;
660     if (g_resultset_mutex == 0)
661         yaz_mutex_create(&g_resultset_mutex);
662     yaz_mutex_enter(g_resultset_mutex);
663     g_resultsets += delta;
664     resultset_count = g_resultsets;
665     yaz_mutex_leave(g_resultset_mutex);
666     return resultset_count;
667 }
668
669 int resultsets_count(void) {
670     return resultset_use(0);
671 }
672
673 ZOOM_resultset ZOOM_resultset_create(void)
674 {
675     int i;
676     ZOOM_resultset r = (ZOOM_resultset) xmalloc(sizeof(*r));
677
678     initlog();
679
680     yaz_log(log_details0, "%p ZOOM_resultset_create", r);
681     r->refcount = 1;
682     r->size = 0;
683     r->odr = odr_createmem(ODR_ENCODE);
684     r->piggyback = 1;
685     r->setname = 0;
686     r->schema = 0;
687     r->step = 0;
688     for (i = 0; i<RECORD_HASH_SIZE; i++)
689         r->record_hash[i] = 0;
690     r->r_sort_spec = 0;
691     r->query = 0;
692     r->connection = 0;
693     r->databaseNames = 0;
694     r->num_databaseNames = 0;
695     r->facets = 0;
696     r->num_facets = 0;
697     r->facets_names = 0;
698     r->mutex = 0;
699     yaz_mutex_create(&r->mutex);
700 #if SHPTR
701     {
702         WRBUF w = wrbuf_alloc();
703         YAZ_SHPTR_INIT(r->record_wrbuf, w);
704     }
705 #endif
706     resultset_use(1);
707     return r;
708 }
709
710 ZOOM_API(ZOOM_resultset)
711     ZOOM_connection_search_pqf(ZOOM_connection c, const char *q)
712 {
713     ZOOM_resultset r;
714     ZOOM_query s = ZOOM_query_create();
715
716     ZOOM_query_prefix(s, q);
717
718     r = ZOOM_connection_search(c, s);
719     ZOOM_query_destroy(s);
720     return r;
721 }
722
723 ZOOM_API(ZOOM_resultset)
724     ZOOM_connection_search(ZOOM_connection c, ZOOM_query q)
725 {
726     ZOOM_resultset r = ZOOM_resultset_create();
727     ZOOM_task task;
728     const char *cp;
729     int start, count;
730     const char *syntax, *elementSetName;
731 #if ZOOM_RESULT_LISTS
732     ZOOM_resultsets set;
733 #endif
734
735     yaz_log(c->log_api, "%p ZOOM_connection_search set %p query %p", c, r, q);
736     r->r_sort_spec = ZOOM_query_get_sortspec(q);
737     r->query = q;
738
739     r->options = ZOOM_options_create_with_parent(c->options);
740     
741     start = ZOOM_options_get_int(r->options, "start", 0);
742     count = ZOOM_options_get_int(r->options, "count", 0);
743     {
744         /* If "presentChunk" is defined use that; otherwise "step" */
745         const char *cp = ZOOM_options_get(r->options, "presentChunk");
746         r->step = ZOOM_options_get_int(r->options,
747                                        (cp != 0 ? "presentChunk": "step"), 0);
748     }
749     r->piggyback = ZOOM_options_get_bool(r->options, "piggyback", 1);
750     cp = ZOOM_options_get(r->options, "setname");
751     if (cp)
752         r->setname = xstrdup(cp);
753     cp = ZOOM_options_get(r->options, "schema");
754     if (cp)
755         r->schema = xstrdup(cp);
756
757     r->databaseNames = ZOOM_connection_get_databases(c, c->options, &r->num_databaseNames,
758                                          r->odr);
759     
760     r->connection = c;
761
762 #if ZOOM_RESULT_LISTS
763     yaz_log(log_details, "%p ZOOM_connection_search: Adding new resultset (%p) to resultsets (%p) ", c, r, c->resultsets);
764     set = xmalloc(sizeof(*set));
765     ZOOM_resultset_addref(r);
766     set->resultset = r;
767     set->next = c->resultsets;
768     c->resultsets = set;
769 #else
770     r->next = c->resultsets;
771     c->resultsets = r;
772 #endif
773     if (c->host_port && c->proto == PROTO_HTTP)
774     {
775         if (!c->cs)
776         {
777             yaz_log(c->log_details, "ZOOM_connection_search: no comstack");
778             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
779         }
780         else
781         {
782             yaz_log(c->log_details, "ZOOM_connection_search: reconnect");
783             c->reconnect_ok = 1;
784         }
785     }
786
787     task = ZOOM_connection_add_task(c, ZOOM_TASK_SEARCH);
788     task->u.search.resultset = r;
789     task->u.search.start = start;
790     task->u.search.count = count;
791     task->u.search.recv_search_fired = 0;
792
793     syntax = ZOOM_options_get(r->options, "preferredRecordSyntax"); 
794     task->u.search.syntax = syntax ? xstrdup(syntax) : 0;
795     elementSetName = ZOOM_options_get(r->options, "elementSetName");
796     task->u.search.elementSetName = elementSetName 
797         ? xstrdup(elementSetName) : 0;
798    
799     ZOOM_resultset_addref(r);
800
801     ZOOM_query_addref(q);
802
803     if (!c->async)
804     {
805         while (ZOOM_event(1, &c))
806             ;
807     }
808     return r;
809 }
810
811 ZOOM_API(void)
812     ZOOM_resultset_sort(ZOOM_resultset r,
813                          const char *sort_type, const char *sort_spec)
814 {
815     (void) ZOOM_resultset_sort1(r, sort_type, sort_spec);
816 }
817
818 ZOOM_API(int)
819     ZOOM_resultset_sort1(ZOOM_resultset r,
820                          const char *sort_type, const char *sort_spec)
821 {
822     ZOOM_connection c = r->connection;
823     ZOOM_task task;
824     ZOOM_query newq;
825
826     newq = ZOOM_query_create();
827     if (ZOOM_query_sortby(newq, sort_spec) < 0)
828         return -1;
829
830     yaz_log(c->log_api, "%p ZOOM_resultset_sort r=%p sort_type=%s sort_spec=%s",
831             r, r, sort_type, sort_spec);
832     if (!c)
833         return 0;
834
835     if (c->host_port && c->proto == PROTO_HTTP)
836     {
837         if (!c->cs)
838         {
839             yaz_log(c->log_details, "%p ZOOM_resultset_sort: no comstack", r);
840             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
841         }
842         else
843         {
844             yaz_log(c->log_details, "%p ZOOM_resultset_sort: prepare reconnect",
845                     r);
846             c->reconnect_ok = 1;
847         }
848     }
849     
850     ZOOM_resultset_cache_reset(r);
851     task = ZOOM_connection_add_task(c, ZOOM_TASK_SORT);
852     task->u.sort.resultset = r;
853     task->u.sort.q = newq;
854
855     ZOOM_resultset_addref(r);  
856
857     if (!c->async)
858     {
859         while (ZOOM_event(1, &c))
860             ;
861     }
862
863     return 0;
864 }
865
866 ZOOM_API(void)
867     ZOOM_resultset_destroy(ZOOM_resultset r)
868 {
869     resultset_destroy(r);
870 }
871
872 static void resultset_destroy(ZOOM_resultset r)
873 {
874     if (!r)
875         return;
876     yaz_mutex_enter(r->mutex);
877     (r->refcount)--;
878     yaz_log(log_details0, "%p ZOOM_resultset_destroy r=%p count=%d",
879             r, r, r->refcount);
880     if (r->refcount == 0)
881     {
882         yaz_mutex_leave(r->mutex);
883
884         yaz_log(log_details0, "%p ZOOM_connection resultset_destroy: Deleting resultset (%p) ", r->connection, r);
885         ZOOM_resultset_cache_reset(r);
886         ZOOM_resultset_release(r);
887         ZOOM_query_destroy(r->query);
888         ZOOM_options_destroy(r->options);
889         odr_destroy(r->odr);
890         xfree(r->setname);
891         xfree(r->schema);
892         yaz_mutex_destroy(&r->mutex);
893 #if SHPTR
894         YAZ_SHPTR_DEC(r->record_wrbuf, wrbuf_destroy);
895 #endif
896         resultset_use(-1);
897         xfree(r);
898     }
899     else
900         yaz_mutex_leave(r->mutex);
901 }
902
903 ZOOM_API(size_t)
904     ZOOM_resultset_size(ZOOM_resultset r)
905 {
906     return r->size;
907 }
908
909 int ZOOM_test_reconnect(ZOOM_connection c)
910 {
911     ZOOM_Event event;
912
913     if (!c->reconnect_ok)
914         return 0;
915     ZOOM_connection_close(c);
916     c->reconnect_ok = 0;
917     c->tasks->running = 0;
918     ZOOM_connection_insert_task(c, ZOOM_TASK_CONNECT);
919
920     event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
921     ZOOM_connection_put_event(c, event);
922
923     return 1;
924 }
925
926 static void ZOOM_resultset_retrieve(ZOOM_resultset r,
927                                     int force_sync, int start, int count)
928 {
929     ZOOM_task task;
930     ZOOM_connection c;
931     const char *cp;
932     const char *syntax, *elementSetName;
933
934     if (!r)
935         return;
936     yaz_log(log_details0, "%p ZOOM_resultset_retrieve force_sync=%d start=%d"
937             " count=%d", r, force_sync, start, count);
938     c = r->connection;
939     if (!c)
940         return;
941
942     if (c->host_port && c->proto == PROTO_HTTP)
943     {
944         if (!c->cs)
945         {
946             yaz_log(log_details0, "%p ZOOM_resultset_retrieve: no comstack", r);
947             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
948         }
949         else
950         {
951             yaz_log(log_details0, "%p ZOOM_resultset_retrieve: prepare "
952                     "reconnect", r);
953             c->reconnect_ok = 1;
954         }
955     }
956     task = ZOOM_connection_add_task(c, ZOOM_TASK_RETRIEVE);
957     task->u.retrieve.resultset = r;
958     task->u.retrieve.start = start;
959     task->u.retrieve.count = count;
960
961     syntax = ZOOM_options_get(r->options, "preferredRecordSyntax"); 
962     task->u.retrieve.syntax = syntax ? xstrdup(syntax) : 0;
963     elementSetName = ZOOM_options_get(r->options, "elementSetName");
964     task->u.retrieve.elementSetName = elementSetName 
965         ? xstrdup(elementSetName) : 0;
966
967     cp = ZOOM_options_get(r->options, "schema");
968     if (cp)
969     {
970         if (!r->schema || strcmp(r->schema, cp))
971         {
972             xfree(r->schema);
973             r->schema = xstrdup(cp);
974         }
975     }
976
977     ZOOM_resultset_addref(r);
978
979     if (!r->connection->async || force_sync)
980         while (r->connection && ZOOM_event(1, &r->connection))
981             ;
982 }
983
984 ZOOM_API(void)
985     ZOOM_resultset_records(ZOOM_resultset r, ZOOM_record *recs,
986                            size_t start, size_t count)
987 {
988     int force_present = 0;
989
990     if (!r)
991         return ;
992     yaz_log(log_api0, "%p ZOOM_resultset_records r=%p start=%ld count=%ld",
993             r, r, (long) start, (long) count);
994     if (count && recs)
995         force_present = 1;
996     ZOOM_resultset_retrieve(r, force_present, start, count);
997     if (force_present)
998     {
999         size_t i;
1000         for (i = 0; i< count; i++)
1001             recs[i] = ZOOM_resultset_record_immediate(r, i+start);
1002     }
1003 }
1004
1005 ZOOM_API(size_t)
1006     ZOOM_resultset_facets_size(ZOOM_resultset r) {
1007     return r->num_facets;
1008 }
1009
1010 ZOOM_API(ZOOM_facet_field)
1011     ZOOM_resultset_get_facet_field(ZOOM_resultset r, const char *name) {
1012     int num = r->num_facets;
1013     ZOOM_facet_field *facets = r->facets;
1014     int index;
1015     for (index = 0; index < num; index++) {
1016         if (!strcmp(facets[index]->facet_name, name)) {
1017             return facets[index];
1018         }
1019     }
1020     return 0;
1021 }
1022
1023
1024 ZOOM_API(ZOOM_facet_field *)
1025     ZOOM_resultset_facets(ZOOM_resultset r)
1026 {
1027     return r->facets;
1028 }
1029
1030 ZOOM_API(const char**)
1031     ZOOM_resultset_facet_names(ZOOM_resultset r)
1032 {
1033     return (const char **) r->facets_names;
1034 }
1035
1036 ZOOM_API(const char*)
1037     ZOOM_facet_field_name(ZOOM_facet_field field)
1038 {
1039     return field->facet_name;
1040 }
1041
1042 ZOOM_API(size_t)
1043     ZOOM_facet_field_term_count(ZOOM_facet_field field)
1044 {
1045     return field->num_terms;
1046 }
1047
1048 ZOOM_API(const char*)
1049     ZOOM_facet_field_get_term(ZOOM_facet_field field, size_t idx, int *freq) {
1050     *freq = field->facet_terms[idx].frequency;
1051     return field->facet_terms[idx].term;
1052 }
1053
1054
1055 static void get_cert(ZOOM_connection c)
1056 {
1057     char *cert_buf;
1058     int cert_len;
1059     
1060     if (cs_get_peer_certificate_x509(c->cs, &cert_buf, &cert_len))
1061     {
1062         ZOOM_connection_option_setl(c, "sslPeerCert",
1063                                     cert_buf, cert_len);
1064         xfree(cert_buf);
1065     }
1066 }
1067
1068 static zoom_ret do_connect_host(ZOOM_connection c,
1069                                 const char *logical_url);
1070
1071 static zoom_ret do_connect(ZOOM_connection c)
1072 {
1073     return do_connect_host(c, c->host_port);
1074 }
1075
1076 static zoom_ret do_connect_host(ZOOM_connection c, const char *logical_url)
1077 {
1078     void *add;
1079
1080     if (c->cs)
1081         cs_close(c->cs);
1082     c->cs = cs_create_host_proxy(logical_url, 0, &add,
1083                                  c->tproxy ? c->tproxy : c->proxy);
1084     
1085     if (c->cs && c->cs->protocol == PROTO_HTTP)
1086     {
1087 #if YAZ_HAVE_XML2
1088         c->proto = PROTO_HTTP;
1089 #else
1090         ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, "SRW");
1091         ZOOM_connection_close(c);
1092         return zoom_complete;
1093 #endif
1094     }
1095     if (c->cs)
1096     {
1097         int ret = cs_connect(c->cs, add);
1098         if (ret == 0)
1099         {
1100             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1101             ZOOM_connection_put_event(c, event);
1102             get_cert(c);
1103             if (c->proto == PROTO_Z3950)
1104                 ZOOM_connection_Z3950_send_init(c);
1105             else
1106             {
1107                 /* no init request for SRW .. */
1108                 assert(c->tasks->which == ZOOM_TASK_CONNECT);
1109                 ZOOM_connection_remove_task(c);
1110                 ZOOM_connection_set_mask(c, 0);
1111                 ZOOM_connection_exec_task(c);
1112             }
1113             c->state = STATE_ESTABLISHED;
1114             return zoom_pending;
1115         }
1116         else if (ret > 0)
1117         {
1118             int mask = ZOOM_SELECT_EXCEPT;
1119             if (c->cs->io_pending & CS_WANT_WRITE)
1120                 mask += ZOOM_SELECT_WRITE;
1121             if (c->cs->io_pending & CS_WANT_READ)
1122                 mask += ZOOM_SELECT_READ;
1123             ZOOM_connection_set_mask(c, mask);
1124             c->state = STATE_CONNECTING; 
1125             return zoom_pending;
1126         }
1127     }
1128     c->state = STATE_IDLE;
1129     ZOOM_set_error(c, ZOOM_ERROR_CONNECT, logical_url);
1130     return zoom_complete;
1131 }
1132
1133 /* returns 1 if PDU was sent OK (still pending )
1134    0 if PDU was not sent OK (nothing to wait for) 
1135 */
1136
1137 ZOOM_API(ZOOM_record)
1138     ZOOM_resultset_record_immediate(ZOOM_resultset s,size_t pos)
1139 {
1140     const char *syntax =
1141         ZOOM_options_get(s->options, "preferredRecordSyntax"); 
1142     const char *elementSetName =
1143         ZOOM_options_get(s->options, "elementSetName");
1144
1145     return ZOOM_record_cache_lookup(s, pos, syntax, elementSetName);
1146 }
1147
1148 ZOOM_API(ZOOM_record)
1149     ZOOM_resultset_record(ZOOM_resultset r, size_t pos)
1150 {
1151     ZOOM_record rec = ZOOM_resultset_record_immediate(r, pos);
1152
1153     if (!rec)
1154     {
1155         /*
1156          * MIKE: I think force_sync should always be zero, but I don't
1157          * want to make this change until I get the go-ahead from
1158          * Adam, in case something depends on the old synchronous
1159          * behaviour.
1160          */
1161         int force_sync = 1;
1162         if (getenv("ZOOM_RECORD_NO_FORCE_SYNC")) force_sync = 0;
1163         ZOOM_resultset_retrieve(r, force_sync, pos, 1);
1164         rec = ZOOM_resultset_record_immediate(r, pos);
1165     }
1166     return rec;
1167 }
1168
1169 ZOOM_API(ZOOM_scanset)
1170     ZOOM_connection_scan(ZOOM_connection c, const char *start)
1171 {
1172     ZOOM_scanset s;
1173     ZOOM_query q = ZOOM_query_create();
1174
1175     ZOOM_query_prefix(q, start);
1176
1177     s = ZOOM_connection_scan1(c, q);
1178     ZOOM_query_destroy(q);
1179     return s;
1180
1181 }
1182
1183 ZOOM_API(ZOOM_scanset)
1184     ZOOM_connection_scan1(ZOOM_connection c, ZOOM_query q)
1185 {
1186     ZOOM_scanset scan = 0;
1187     Z_Query *z_query = ZOOM_query_get_Z_Query(q);
1188
1189     if (!z_query)
1190         return 0;
1191     scan = (ZOOM_scanset) xmalloc(sizeof(*scan));
1192     scan->connection = c;
1193     scan->odr = odr_createmem(ODR_DECODE);
1194     scan->options = ZOOM_options_create_with_parent(c->options);
1195     scan->refcount = 1;
1196     scan->scan_response = 0;
1197     scan->srw_scan_response = 0;
1198
1199     scan->query = q;
1200     ZOOM_query_addref(q);
1201     scan->databaseNames = ZOOM_connection_get_databases(c, c->options,
1202                                             &scan->num_databaseNames,
1203                                             scan->odr);
1204
1205     if (1)
1206     {
1207         ZOOM_task task = ZOOM_connection_add_task(c, ZOOM_TASK_SCAN);
1208         task->u.scan.scan = scan;
1209         
1210         (scan->refcount)++;
1211         if (!c->async)
1212         {
1213             while (ZOOM_event(1, &c))
1214                 ;
1215         }
1216     }
1217     return scan;
1218 }
1219
1220 ZOOM_API(void)
1221     ZOOM_scanset_destroy(ZOOM_scanset scan)
1222 {
1223     if (!scan)
1224         return;
1225     (scan->refcount)--;
1226     if (scan->refcount == 0)
1227     {
1228         ZOOM_query_destroy(scan->query);
1229
1230         odr_destroy(scan->odr);
1231         
1232         ZOOM_options_destroy(scan->options);
1233         xfree(scan);
1234     }
1235 }
1236
1237 static zoom_ret send_package(ZOOM_connection c)
1238 {
1239     ZOOM_Event event;
1240
1241     yaz_log(c->log_details, "%p send_package", c);
1242     if (!c->tasks)
1243         return zoom_complete;
1244     assert (c->tasks->which == ZOOM_TASK_PACKAGE);
1245     
1246     event = ZOOM_Event_create(ZOOM_EVENT_SEND_APDU);
1247     ZOOM_connection_put_event(c, event);
1248     
1249     c->buf_out = c->tasks->u.package->buf_out;
1250     c->len_out = c->tasks->u.package->len_out;
1251
1252     return ZOOM_send_buf(c);
1253 }
1254
1255 ZOOM_API(size_t)
1256     ZOOM_scanset_size(ZOOM_scanset scan)
1257 {
1258     if (!scan)
1259         return 0;
1260
1261     if (scan->scan_response && scan->scan_response->entries)
1262         return scan->scan_response->entries->num_entries;
1263     else if (scan->srw_scan_response)
1264         return scan->srw_scan_response->num_terms;
1265     return 0;
1266 }
1267
1268 static void ZOOM_scanset_term_x(ZOOM_scanset scan, size_t pos,
1269                                 size_t *occ,
1270                                 const char **value_term, size_t *value_len,
1271                                 const char **disp_term, size_t *disp_len)
1272 {
1273     size_t noent = ZOOM_scanset_size(scan);
1274     
1275     *value_term = 0;
1276     *value_len = 0;
1277
1278     *disp_term = 0;
1279     *disp_len = 0;
1280
1281     *occ = 0;
1282     if (pos >= noent)
1283         return;
1284     if (scan->scan_response)
1285     {
1286         Z_ScanResponse *res = scan->scan_response;
1287         if (res->entries->entries[pos]->which == Z_Entry_termInfo)
1288         {
1289             Z_TermInfo *t = res->entries->entries[pos]->u.termInfo;
1290             
1291             *value_term = (const char *) t->term->u.general->buf;
1292             *value_len = t->term->u.general->len;
1293             if (t->displayTerm)
1294             {
1295                 *disp_term = t->displayTerm;
1296                 *disp_len = strlen(*disp_term);
1297             }
1298             else if (t->term->which == Z_Term_general)
1299             {
1300                 *disp_term = (const char *) t->term->u.general->buf;
1301                 *disp_len = t->term->u.general->len;
1302             }
1303             *occ = t->globalOccurrences ? *t->globalOccurrences : 0;
1304         }
1305     }
1306     if (scan->srw_scan_response)
1307     {
1308         Z_SRW_scanResponse *res = scan->srw_scan_response;
1309         Z_SRW_scanTerm *t = res->terms + pos;
1310         if (t)
1311         {
1312             *value_term = t->value;
1313             *value_len = strlen(*value_term);
1314
1315             if (t->displayTerm)
1316                 *disp_term = t->displayTerm;
1317             else
1318                 *disp_term = t->value;
1319             *disp_len = strlen(*disp_term);
1320             *occ = t->numberOfRecords ? *t->numberOfRecords : 0;
1321         }
1322     }
1323 }
1324
1325 ZOOM_API(const char *)
1326     ZOOM_scanset_term(ZOOM_scanset scan, size_t pos,
1327                       size_t *occ, size_t *len)
1328 {
1329     const char *value_term = 0;
1330     size_t value_len = 0;
1331     const char *disp_term = 0;
1332     size_t disp_len = 0;
1333
1334     ZOOM_scanset_term_x(scan, pos, occ, &value_term, &value_len,
1335                         &disp_term, &disp_len);
1336     
1337     *len = value_len;
1338     return value_term;
1339 }
1340
1341 ZOOM_API(const char *)
1342     ZOOM_scanset_display_term(ZOOM_scanset scan, size_t pos,
1343                               size_t *occ, size_t *len)
1344 {
1345     const char *value_term = 0;
1346     size_t value_len = 0;
1347     const char *disp_term = 0;
1348     size_t disp_len = 0;
1349
1350     ZOOM_scanset_term_x(scan, pos, occ, &value_term, &value_len,
1351                         &disp_term, &disp_len);
1352     
1353     *len = disp_len;
1354     return disp_term;
1355 }
1356
1357 ZOOM_API(const char *)
1358     ZOOM_scanset_option_get(ZOOM_scanset scan, const char *key)
1359 {
1360     return ZOOM_options_get(scan->options, key);
1361 }
1362
1363 ZOOM_API(void)
1364     ZOOM_scanset_option_set(ZOOM_scanset scan, const char *key,
1365                             const char *val)
1366 {
1367     ZOOM_options_set(scan->options, key, val);
1368 }
1369
1370
1371 ZOOM_API(ZOOM_package)
1372     ZOOM_connection_package(ZOOM_connection c, ZOOM_options options)
1373 {
1374     ZOOM_package p = (ZOOM_package) xmalloc(sizeof(*p));
1375
1376     p->connection = c;
1377     p->odr_out = odr_createmem(ODR_ENCODE);
1378     p->options = ZOOM_options_create_with_parent2(options, c->options);
1379     p->refcount = 1;
1380     p->buf_out = 0;
1381     p->len_out = 0;
1382     return p;
1383 }
1384
1385 ZOOM_API(void)
1386     ZOOM_package_destroy(ZOOM_package p)
1387 {
1388     if (!p)
1389         return;
1390     (p->refcount)--;
1391     if (p->refcount == 0)
1392     {
1393         odr_destroy(p->odr_out);
1394         xfree(p->buf_out);
1395         
1396         ZOOM_options_destroy(p->options);
1397         xfree(p);
1398     }
1399 }
1400
1401 ZOOM_API(const char *)
1402     ZOOM_package_option_get(ZOOM_package p, const char *key)
1403 {
1404     return ZOOM_options_get(p->options, key);
1405 }
1406
1407 ZOOM_API(const char *)
1408     ZOOM_package_option_getl(ZOOM_package p, const char *key, int *lenp)
1409 {
1410     return ZOOM_options_getl(p->options, key, lenp);
1411 }
1412
1413 ZOOM_API(void)
1414     ZOOM_package_option_set(ZOOM_package p, const char *key,
1415                             const char *val)
1416 {
1417     ZOOM_options_set(p->options, key, val);
1418 }
1419
1420 ZOOM_API(void)
1421     ZOOM_package_option_setl(ZOOM_package p, const char *key,
1422                              const char *val, int len)
1423 {
1424     ZOOM_options_setl(p->options, key, val, len);
1425 }
1426
1427 ZOOM_API(int)
1428     ZOOM_connection_exec_task(ZOOM_connection c)
1429 {
1430     ZOOM_task task = c->tasks;
1431     zoom_ret ret = zoom_complete;
1432
1433     if (!task)
1434         return 0;
1435     yaz_log(c->log_details, "%p ZOOM_connection_exec_task type=%d run=%d",
1436             c, task->which, task->running);
1437     if (c->error != ZOOM_ERROR_NONE)
1438     {
1439         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1440                 "removing tasks because of error = %d", c, c->error);
1441         ZOOM_connection_remove_tasks(c);
1442         return 0;
1443     }
1444     if (task->running)
1445     {
1446         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1447                 "task already running", c);
1448         return 0;
1449     }
1450     task->running = 1;
1451     ret = zoom_complete;
1452     if (c->cs || task->which == ZOOM_TASK_CONNECT)
1453     {
1454         switch (task->which)
1455         {
1456         case ZOOM_TASK_SEARCH:
1457             if (c->proto == PROTO_HTTP)
1458                 ret = ZOOM_connection_srw_send_search(c);
1459             else
1460                 ret = ZOOM_connection_Z3950_send_search(c);
1461             break;
1462         case ZOOM_TASK_RETRIEVE:
1463             if (c->proto == PROTO_HTTP)
1464                 ret = ZOOM_connection_srw_send_search(c);
1465             else
1466                 ret = send_Z3950_present(c);
1467             break;
1468         case ZOOM_TASK_CONNECT:
1469             ret = do_connect(c);
1470             break;
1471         case ZOOM_TASK_SCAN:
1472             if (c->proto == PROTO_HTTP)
1473                 ret = ZOOM_connection_srw_send_scan(c);
1474             else
1475                 ret = ZOOM_connection_Z3950_send_scan(c);
1476             break;
1477         case ZOOM_TASK_PACKAGE:
1478             ret = send_package(c);
1479             break;
1480         case ZOOM_TASK_SORT:
1481             c->tasks->u.sort.resultset->r_sort_spec = 
1482                 ZOOM_query_get_sortspec(c->tasks->u.sort.q);
1483             ret = send_Z3950_sort(c, c->tasks->u.sort.resultset);
1484             break;
1485         }
1486     }
1487     else
1488     {
1489         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1490                 "remove tasks because no connection exist", c);
1491         ZOOM_connection_remove_tasks(c);
1492     }
1493     if (ret == zoom_complete)
1494     {
1495         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1496                 "task removed (complete)", c);
1497         ZOOM_connection_remove_task(c);
1498         return 0;
1499     }
1500     yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1501             "task pending", c);
1502     return 1;
1503 }
1504
1505 #if YAZ_HAVE_XML2
1506
1507 static zoom_ret send_HTTP_redirect(ZOOM_connection c, const char *uri,
1508                                   Z_HTTP_Response *cookie_hres)
1509 {
1510     struct Z_HTTP_Header *h;
1511     char *combined_cookies = 0;
1512     int combined_cookies_len = 0;
1513     Z_GDU *gdu = z_get_HTTP_Request_uri(c->odr_out, uri, 0, c->proxy ? 1 : 0);
1514
1515     gdu->u.HTTP_Request->method = odr_strdup(c->odr_out, "GET");
1516     z_HTTP_header_add(c->odr_out, &gdu->u.HTTP_Request->headers, "Accept",
1517                       "text/xml");
1518
1519     for (h = cookie_hres->headers; h; h = h->next)
1520     {
1521         if (!strcmp(h->name, "Set-Cookie"))
1522         {
1523             char *cp;
1524
1525             if (!(cp = strchr(h->value, ';')))
1526                 cp = h->value + strlen(h->value);
1527             if (cp - h->value >= 1) {
1528                 combined_cookies = xrealloc(combined_cookies, combined_cookies_len + cp - h->value + 3);
1529                 memcpy(combined_cookies+combined_cookies_len, h->value, cp - h->value);
1530                 combined_cookies[combined_cookies_len + cp - h->value] = '\0';
1531                 strcat(combined_cookies,"; ");
1532                 combined_cookies_len = strlen(combined_cookies);
1533             }
1534         }
1535     }
1536
1537     if (combined_cookies_len)
1538     {
1539         z_HTTP_header_add(c->odr_out, &gdu->u.HTTP_Request->headers,
1540                           "Cookie", combined_cookies);
1541         xfree(combined_cookies);
1542     }
1543
1544     if (c->user && c->password)
1545     {
1546         z_HTTP_header_add_basic_auth(c->odr_out, &gdu->u.HTTP_Request->headers,
1547                                      c->user, c->password);
1548     }
1549     if (!z_GDU(c->odr_out, &gdu, 0, 0))
1550         return zoom_complete;
1551     if (c->odr_print)
1552         z_GDU(c->odr_print, &gdu, 0, 0);
1553     if (c->odr_save)
1554         z_GDU(c->odr_save, &gdu, 0, 0);
1555     c->buf_out = odr_getbuf(c->odr_out, &c->len_out, 0);
1556
1557     odr_reset(c->odr_out);
1558     return ZOOM_send_buf(c);
1559 }
1560
1561 #if YAZ_HAVE_XML2
1562 void ZOOM_set_HTTP_error(ZOOM_connection c, int error,
1563                          const char *addinfo, const char *addinfo2)
1564 {
1565     ZOOM_set_dset_error(c, error, "HTTP", addinfo, addinfo2);
1566 }
1567 #endif
1568
1569
1570 static void handle_http(ZOOM_connection c, Z_HTTP_Response *hres)
1571 {
1572     zoom_ret cret = zoom_complete;
1573     int ret = -1;
1574     char *addinfo = 0;
1575     const char *connection_head = z_HTTP_header_lookup(hres->headers,
1576                                                        "Connection");
1577     const char *location;
1578
1579     ZOOM_connection_set_mask(c, 0);
1580     yaz_log(c->log_details, "%p handle_http", c);
1581
1582     if ((hres->code == 301 || hres->code == 302) && c->sru_mode == zoom_sru_get
1583         && (location = z_HTTP_header_lookup(hres->headers, "Location")))
1584     {
1585         c->no_redirects++;
1586         if (c->no_redirects > 10)
1587         {
1588             ZOOM_set_HTTP_error(c, hres->code, 0, 0);
1589             c->no_redirects = 0;
1590             ZOOM_connection_close(c);
1591         }
1592         else
1593         {
1594             /* since redirect may change host we just reconnect. A smarter
1595                implementation might check whether it's the same server */
1596             do_connect_host(c, location);
1597             send_HTTP_redirect(c, location, hres);
1598             /* we're OK for now. Operation is not really complete */
1599             return;
1600         }
1601     }
1602     else 
1603     {  
1604         ret = ZOOM_handle_sru(c, hres, &cret, &addinfo);
1605         if (ret == 0)
1606         {
1607             if (c->no_redirects) /* end of redirect. change hosts again */
1608                 ZOOM_connection_close(c);
1609         }
1610         c->no_redirects = 0;
1611     }
1612     if (ret)
1613     {
1614         if (hres->code != 200)
1615             ZOOM_set_HTTP_error(c, hres->code, 0, 0);
1616         else
1617         {
1618             yaz_log(YLOG_LOG, "set error... addinfo=%s", addinfo ?
1619                     addinfo : "NULL");
1620             ZOOM_set_error(c, ZOOM_ERROR_DECODE, addinfo);
1621         }
1622         ZOOM_connection_close(c);
1623     }
1624     if (cret == zoom_complete)
1625     {
1626         yaz_log(c->log_details, "removing tasks in handle_http");
1627         ZOOM_connection_remove_task(c);
1628     }
1629     {
1630         int must_close = 0;
1631         if (!strcmp(hres->version, "1.0"))
1632         {
1633             /* HTTP 1.0: only if Keep-Alive we stay alive.. */
1634             if (!connection_head || strcmp(connection_head, "Keep-Alive"))
1635                 must_close = 1;
1636         }
1637         else
1638         {
1639             /* HTTP 1.1: only if no close we stay alive.. */
1640             if (connection_head && !strcmp(connection_head, "close"))
1641                 must_close = 1;
1642         }
1643         if (must_close)
1644         {
1645             ZOOM_connection_close(c);
1646             if (c->tasks)
1647             {
1648                 c->tasks->running = 0;
1649                 ZOOM_connection_insert_task(c, ZOOM_TASK_CONNECT);
1650                 c->reconnect_ok = 0;
1651             }
1652         }
1653         else
1654             c->reconnect_ok = 1; /* if the server closes anyway */
1655     }
1656 }
1657 #endif
1658
1659 static int do_read(ZOOM_connection c)
1660 {
1661     int r, more;
1662     ZOOM_Event event;
1663     
1664     event = ZOOM_Event_create(ZOOM_EVENT_RECV_DATA);
1665     ZOOM_connection_put_event(c, event);
1666     
1667     r = cs_get(c->cs, &c->buf_in, &c->len_in);
1668     more = cs_more(c->cs);
1669     yaz_log(c->log_details, "%p do_read len=%d more=%d", c, r, more);
1670     if (r == 1)
1671         return 0;
1672     if (r <= 0)
1673     {
1674         if (!ZOOM_test_reconnect(c))
1675         {
1676             ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
1677             ZOOM_connection_close(c);
1678         }
1679     }
1680     else
1681     {
1682         Z_GDU *gdu;
1683         ZOOM_Event event;
1684
1685         odr_reset(c->odr_in);
1686         odr_setbuf(c->odr_in, c->buf_in, r, 0);
1687         event = ZOOM_Event_create(ZOOM_EVENT_RECV_APDU);
1688         ZOOM_connection_put_event(c, event);
1689
1690         if (!z_GDU(c->odr_in, &gdu, 0, 0))
1691         {
1692             int x;
1693             int err = odr_geterrorx(c->odr_in, &x);
1694             char msg[100];
1695             const char *element = odr_getelement(c->odr_in);
1696             yaz_snprintf(msg, sizeof(msg),
1697                     "ODR code %d:%d element=%s offset=%d",
1698                     err, x, element ? element : "<unknown>",
1699                     odr_offset(c->odr_in));
1700             ZOOM_set_error(c, ZOOM_ERROR_DECODE, msg);
1701             if (c->log_api)
1702             {
1703                 FILE *ber_file = yaz_log_file();
1704                 if (ber_file)
1705                     odr_dumpBER(ber_file, c->buf_in, r);
1706             }
1707             ZOOM_connection_close(c);
1708         }
1709         else
1710         {
1711             if (c->odr_print)
1712                 z_GDU(c->odr_print, &gdu, 0, 0);
1713             if (c->odr_save)
1714                 z_GDU(c->odr_save, &gdu, 0, 0);
1715             if (gdu->which == Z_GDU_Z3950)
1716                 ZOOM_handle_Z3950_apdu(c, gdu->u.z3950);
1717             else if (gdu->which == Z_GDU_HTTP_Response)
1718             {
1719 #if YAZ_HAVE_XML2
1720                 handle_http(c, gdu->u.HTTP_Response);
1721 #else
1722                 ZOOM_set_error(c, ZOOM_ERROR_DECODE, 0);
1723                 ZOOM_connection_close(c);
1724 #endif
1725             }
1726         }
1727     }
1728     return 1;
1729 }
1730
1731 static zoom_ret do_write_ex(ZOOM_connection c, char *buf_out, int len_out)
1732 {
1733     int r;
1734     ZOOM_Event event;
1735     
1736     event = ZOOM_Event_create(ZOOM_EVENT_SEND_DATA);
1737     ZOOM_connection_put_event(c, event);
1738
1739     yaz_log(c->log_details, "%p do_write_ex len=%d", c, len_out);
1740     if ((r = cs_put(c->cs, buf_out, len_out)) < 0)
1741     {
1742         yaz_log(c->log_details, "%p do_write_ex write failed", c);
1743         if (ZOOM_test_reconnect(c))
1744         {
1745             return zoom_pending;
1746         }
1747         if (c->state == STATE_CONNECTING)
1748             ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
1749         else
1750             ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
1751         ZOOM_connection_close(c);
1752         return zoom_complete;
1753     }
1754     else if (r == 1)
1755     {    
1756         int mask = ZOOM_SELECT_EXCEPT;
1757         if (c->cs->io_pending & CS_WANT_WRITE)
1758             mask += ZOOM_SELECT_WRITE;
1759         if (c->cs->io_pending & CS_WANT_READ)
1760             mask += ZOOM_SELECT_READ;
1761         ZOOM_connection_set_mask(c, mask);
1762         yaz_log(c->log_details, "%p do_write_ex write incomplete mask=%d",
1763                 c, c->mask);
1764     }
1765     else
1766     {
1767         ZOOM_connection_set_mask(c, ZOOM_SELECT_READ|ZOOM_SELECT_EXCEPT);
1768         yaz_log(c->log_details, "%p do_write_ex write complete mask=%d",
1769                 c, c->mask);
1770     }
1771     return zoom_pending;
1772 }
1773
1774 zoom_ret ZOOM_send_buf(ZOOM_connection c)
1775 {
1776     return do_write_ex(c, c->buf_out, c->len_out);
1777 }
1778
1779
1780 ZOOM_API(const char *)
1781     ZOOM_connection_option_get(ZOOM_connection c, const char *key)
1782 {
1783     return ZOOM_options_get(c->options, key);
1784 }
1785
1786 ZOOM_API(const char *)
1787     ZOOM_connection_option_getl(ZOOM_connection c, const char *key, int *lenp)
1788 {
1789     return ZOOM_options_getl(c->options, key, lenp);
1790 }
1791
1792 ZOOM_API(void)
1793     ZOOM_connection_option_set(ZOOM_connection c, const char *key,
1794                                const char *val)
1795 {
1796     ZOOM_options_set(c->options, key, val);
1797 }
1798
1799 ZOOM_API(void)
1800     ZOOM_connection_option_setl(ZOOM_connection c, const char *key,
1801                                 const char *val, int len)
1802 {
1803     ZOOM_options_setl(c->options, key, val, len);
1804 }
1805
1806 ZOOM_API(const char *)
1807     ZOOM_resultset_option_get(ZOOM_resultset r, const char *key)
1808 {
1809     return ZOOM_options_get(r->options, key);
1810 }
1811
1812 ZOOM_API(void)
1813     ZOOM_resultset_option_set(ZOOM_resultset r, const char *key,
1814                               const char *val)
1815 {
1816     ZOOM_options_set(r->options, key, val);
1817 }
1818
1819
1820 ZOOM_API(int)
1821     ZOOM_connection_errcode(ZOOM_connection c)
1822 {
1823     return ZOOM_connection_error(c, 0, 0);
1824 }
1825
1826 ZOOM_API(const char *)
1827     ZOOM_connection_errmsg(ZOOM_connection c)
1828 {
1829     const char *msg;
1830     ZOOM_connection_error(c, &msg, 0);
1831     return msg;
1832 }
1833
1834 ZOOM_API(const char *)
1835     ZOOM_connection_addinfo(ZOOM_connection c)
1836 {
1837     const char *addinfo;
1838     ZOOM_connection_error(c, 0, &addinfo);
1839     return addinfo;
1840 }
1841
1842 ZOOM_API(const char *)
1843     ZOOM_connection_diagset(ZOOM_connection c)
1844 {
1845     const char *diagset;
1846     ZOOM_connection_error_x(c, 0, 0, &diagset);
1847     return diagset;
1848 }
1849
1850 ZOOM_API(const char *)
1851     ZOOM_diag_str(int error)
1852 {
1853     switch (error)
1854     {
1855     case ZOOM_ERROR_NONE:
1856         return "No error";
1857     case ZOOM_ERROR_CONNECT:
1858         return "Connect failed";
1859     case ZOOM_ERROR_MEMORY:
1860         return "Out of memory";
1861     case ZOOM_ERROR_ENCODE:
1862         return "Encoding failed";
1863     case ZOOM_ERROR_DECODE:
1864         return "Decoding failed";
1865     case ZOOM_ERROR_CONNECTION_LOST:
1866         return "Connection lost";
1867     case ZOOM_ERROR_INIT:
1868         return "Init rejected";
1869     case ZOOM_ERROR_INTERNAL:
1870         return "Internal failure";
1871     case ZOOM_ERROR_TIMEOUT:
1872         return "Timeout";
1873     case ZOOM_ERROR_UNSUPPORTED_PROTOCOL:
1874         return "Unsupported protocol";
1875     case ZOOM_ERROR_UNSUPPORTED_QUERY:
1876         return "Unsupported query type";
1877     case ZOOM_ERROR_INVALID_QUERY:
1878         return "Invalid query";
1879     case ZOOM_ERROR_CQL_PARSE:
1880         return "CQL parsing error";
1881     case ZOOM_ERROR_CQL_TRANSFORM:
1882         return "CQL transformation error";
1883     case ZOOM_ERROR_CCL_CONFIG:
1884         return "CCL configuration error";
1885     case ZOOM_ERROR_CCL_PARSE:
1886         return "CCL parsing error";
1887     case ZOOM_ERROR_ES_INVALID_ACTION:
1888         return "Extended Service. invalid action";
1889     case ZOOM_ERROR_ES_INVALID_VERSION:
1890         return "Extended Service. invalid version";
1891     case ZOOM_ERROR_ES_INVALID_SYNTAX:
1892         return "Extended Service. invalid syntax";
1893     default:
1894         return diagbib1_str(error);
1895     }
1896 }
1897
1898 ZOOM_API(int)
1899     ZOOM_connection_error_x(ZOOM_connection c, const char **cp,
1900                             const char **addinfo, const char **diagset)
1901 {
1902     int error = c->error;
1903     if (cp)
1904     {
1905         if (!c->diagset || !strcmp(c->diagset, "ZOOM"))
1906             *cp = ZOOM_diag_str(error);
1907         else if (!strcmp(c->diagset, "HTTP"))
1908             *cp = z_HTTP_errmsg(c->error);
1909         else if (!strcmp(c->diagset, "Bib-1"))
1910             *cp = ZOOM_diag_str(error);
1911         else if (!strcmp(c->diagset, "info:srw/diagnostic/1"))
1912             *cp = yaz_diag_srw_str(c->error);
1913         else
1914             *cp = "Unknown error and diagnostic set";
1915     }
1916     if (addinfo)
1917         *addinfo = c->addinfo ? c->addinfo : "";
1918     if (diagset)
1919         *diagset = c->diagset ? c->diagset : "";
1920     return c->error;
1921 }
1922
1923 ZOOM_API(int)
1924     ZOOM_connection_error(ZOOM_connection c, const char **cp,
1925                           const char **addinfo)
1926 {
1927     return ZOOM_connection_error_x(c, cp, addinfo, 0);
1928 }
1929
1930 static void ZOOM_connection_do_io(ZOOM_connection c, int mask)
1931 {
1932     ZOOM_Event event = 0;
1933     int r = cs_look(c->cs);
1934     yaz_log(c->log_details, "%p ZOOM_connection_do_io mask=%d cs_look=%d",
1935             c, mask, r);
1936     
1937     if (r == CS_NONE)
1938     {
1939         event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1940         ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
1941         ZOOM_connection_close(c);
1942         ZOOM_connection_put_event(c, event);
1943     }
1944     else if (r == CS_CONNECT)
1945     {
1946         int ret = ret = cs_rcvconnect(c->cs);
1947         yaz_log(c->log_details, "%p ZOOM_connection_do_io "
1948                 "cs_rcvconnect returned %d", c, ret);
1949         if (ret == 1)
1950         {
1951             int mask = ZOOM_SELECT_EXCEPT;
1952             if (c->cs->io_pending & CS_WANT_WRITE)
1953                 mask += ZOOM_SELECT_WRITE;
1954             if (c->cs->io_pending & CS_WANT_READ)
1955                 mask += ZOOM_SELECT_READ;
1956             ZOOM_connection_set_mask(c, mask);
1957             event = ZOOM_Event_create(ZOOM_EVENT_NONE);
1958             ZOOM_connection_put_event(c, event);
1959         }
1960         else if (ret == 0)
1961         {
1962             event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1963             ZOOM_connection_put_event(c, event);
1964             get_cert(c);
1965             if (c->proto == PROTO_Z3950)
1966                 ZOOM_connection_Z3950_send_init(c);
1967             else
1968             {
1969                 /* no init request for SRW .. */
1970                 assert(c->tasks->which == ZOOM_TASK_CONNECT);
1971                 ZOOM_connection_remove_task(c);
1972                 ZOOM_connection_set_mask(c, 0);
1973                 ZOOM_connection_exec_task(c);
1974             }
1975             c->state = STATE_ESTABLISHED;
1976         }
1977         else
1978         {
1979             ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
1980             ZOOM_connection_close(c);
1981         }
1982     }
1983     else
1984     {
1985         if (mask & ZOOM_SELECT_EXCEPT)
1986         {
1987             if (!ZOOM_test_reconnect(c))
1988             {
1989                 ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
1990                 ZOOM_connection_close(c);
1991             }
1992             return;
1993         }
1994         if (mask & ZOOM_SELECT_READ)
1995             do_read(c);
1996         if (c->cs && (mask & ZOOM_SELECT_WRITE))
1997             ZOOM_send_buf(c);
1998     }
1999 }
2000
2001 ZOOM_API(int)
2002     ZOOM_connection_last_event(ZOOM_connection cs)
2003 {
2004     if (!cs)
2005         return ZOOM_EVENT_NONE;
2006     return cs->last_event;
2007 }
2008
2009
2010 ZOOM_API(int) ZOOM_connection_fire_event_timeout(ZOOM_connection c)
2011 {
2012     if (c->mask)
2013     {
2014         ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_TIMEOUT);
2015         /* timeout and this connection was waiting */
2016         ZOOM_set_error(c, ZOOM_ERROR_TIMEOUT, 0);
2017         ZOOM_connection_close(c);
2018         ZOOM_connection_put_event(c, event);
2019     }
2020     return 0;
2021 }
2022
2023 ZOOM_API(int)
2024     ZOOM_connection_process(ZOOM_connection c)
2025 {
2026     ZOOM_Event event;
2027     if (!c)
2028         return 0;
2029
2030     event = ZOOM_connection_get_event(c);
2031     if (event)
2032     {
2033         ZOOM_Event_destroy(event);
2034         return 1;
2035     }
2036     ZOOM_connection_exec_task(c);
2037     event = ZOOM_connection_get_event(c);
2038     if (event)
2039     {
2040         ZOOM_Event_destroy(event);
2041         return 1;
2042     }
2043     return 0;
2044 }
2045
2046 ZOOM_API(int)
2047     ZOOM_event_nonblock(int no, ZOOM_connection *cs)
2048 {
2049     int i;
2050
2051     yaz_log(log_details0, "ZOOM_process_event(no=%d,cs=%p)", no, cs);
2052     
2053     for (i = 0; i<no; i++)
2054     {
2055         ZOOM_connection c = cs[i];
2056
2057         if (c && ZOOM_connection_process(c))
2058             return i+1;
2059     }
2060     return 0;
2061 }
2062
2063 ZOOM_API(int) ZOOM_connection_fire_event_socket(ZOOM_connection c, int mask)
2064 {
2065     if (c->mask && mask)
2066         ZOOM_connection_do_io(c, mask);
2067     return 0;
2068 }
2069
2070 ZOOM_API(int) ZOOM_connection_get_socket(ZOOM_connection c)
2071 {
2072     if (c->cs)
2073         return cs_fileno(c->cs);
2074     return -1;
2075 }
2076
2077 ZOOM_API(int) ZOOM_connection_set_mask(ZOOM_connection c, int mask)
2078 {
2079     c->mask = mask;
2080     if (!c->cs)
2081         return -1; 
2082     return 0;
2083 }
2084
2085 ZOOM_API(int) ZOOM_connection_get_mask(ZOOM_connection c)
2086 {
2087     if (c->cs)
2088         return c->mask;
2089     return 0;
2090 }
2091
2092 ZOOM_API(int) ZOOM_connection_get_timeout(ZOOM_connection c)
2093 {
2094     return ZOOM_options_get_int(c->options, "timeout", 30);
2095 }
2096
2097 ZOOM_API(void) ZOOM_connection_close(ZOOM_connection c)
2098 {
2099     if (c->cs)
2100         cs_close(c->cs);
2101     c->cs = 0;
2102     ZOOM_connection_set_mask(c, 0);
2103     c->state = STATE_IDLE;
2104 }
2105
2106 /*
2107  * Local variables:
2108  * c-basic-offset: 4
2109  * c-file-style: "Stroustrup"
2110  * indent-tabs-mode: nil
2111  * End:
2112  * vim: shiftwidth=4 tabstop=8 expandtab
2113  */
2114