Prefix URL with http:// if sru setting hints SRU/Solr mode YAZ-731
[yaz-moved-to-github.git] / src / zoom-c.c
1 /* This file is part of the YAZ toolkit.
2  * Copyright (C) Index Data
3  * See the file LICENSE for details.
4  */
5 /**
6  * \file zoom-c.c
7  * \brief Implements ZOOM C interface.
8  */
9 #if HAVE_CONFIG_H
10 #include <config.h>
11 #endif
12
13 #include <assert.h>
14 #include <string.h>
15 #include <errno.h>
16 #include "zoom-p.h"
17
18 #include <yaz/yaz-util.h>
19 #include <yaz/xmalloc.h>
20 #include <yaz/otherinfo.h>
21 #include <yaz/log.h>
22 #include <yaz/diagbib1.h>
23 #include <yaz/charneg.h>
24 #include <yaz/query-charset.h>
25 #include <yaz/snprintf.h>
26 #include <yaz/facet.h>
27
28 #include <yaz/shptr.h>
29
30 #if SHPTR
31 YAZ_SHPTR_TYPE(WRBUF)
32 #endif
33
34 static int log_api0 = 0;
35 static int log_details0 = 0;
36
37 static void resultset_destroy(ZOOM_resultset r);
38 static zoom_ret do_write_ex(ZOOM_connection c, char *buf_out, int len_out);
39
40 static void initlog(void)
41 {
42     static int log_level_initialized = 0;
43
44     if (!log_level_initialized)
45     {
46         log_api0 = yaz_log_module_level("zoom");
47         log_details0 = yaz_log_module_level("zoomdetails");
48         log_level_initialized = 1;
49     }
50 }
51
52 void ZOOM_connection_remove_tasks(ZOOM_connection c);
53
54 void ZOOM_set_dset_error(ZOOM_connection c, int error,
55                          const char *dset,
56                          const char *addinfo, const char *addinfo2)
57 {
58     char *cp;
59
60     xfree(c->addinfo);
61     c->addinfo = 0;
62     c->error = error;
63     if (!c->diagset || strcmp(dset, c->diagset))
64     {
65         xfree(c->diagset);
66         c->diagset = xstrdup(dset);
67         /* remove integer part from SRW diagset .. */
68         if ((cp = strrchr(c->diagset, '/')))
69             *cp = '\0';
70     }
71     if (addinfo && addinfo2)
72     {
73         c->addinfo = (char*) xmalloc(strlen(addinfo) + strlen(addinfo2) + 3);
74         strcpy(c->addinfo, addinfo);
75         strcat(c->addinfo, ": ");
76         strcat(c->addinfo, addinfo2);
77     }
78     else if (addinfo)
79         c->addinfo = xstrdup(addinfo);
80     if (error != ZOOM_ERROR_NONE)
81     {
82         yaz_log(c->log_api, "%p set_dset_error %s %s:%d %s %s",
83                 c, c->host_port ? c->host_port : "<>", dset, error,
84                 addinfo ? addinfo : "",
85                 addinfo2 ? addinfo2 : "");
86         ZOOM_connection_remove_tasks(c);
87     }
88 }
89
90 int ZOOM_uri_to_code(const char *uri)
91 {
92     int code = 0;
93     const char *cp;
94     if ((cp = strrchr(uri, '/')))
95         code = atoi(cp+1);
96     return code;
97 }
98
99
100
101 void ZOOM_set_error(ZOOM_connection c, int error, const char *addinfo)
102 {
103     ZOOM_set_dset_error(c, error, "ZOOM", addinfo, 0);
104 }
105
106 static void clear_error(ZOOM_connection c)
107 {
108     /*
109      * If an error is tied to an operation then it's ok to clear: for
110      * example, a diagnostic returned from a search is cleared by a
111      * subsequent search.  However, problems such as Connection Lost
112      * or Init Refused are not cleared, because they are not
113      * recoverable: doing another search doesn't help.
114      */
115
116     ZOOM_connection_remove_events(c);
117     switch (c->error)
118     {
119     case ZOOM_ERROR_CONNECT:
120     case ZOOM_ERROR_MEMORY:
121     case ZOOM_ERROR_DECODE:
122     case ZOOM_ERROR_CONNECTION_LOST:
123     case ZOOM_ERROR_INIT:
124     case ZOOM_ERROR_INTERNAL:
125     case ZOOM_ERROR_UNSUPPORTED_PROTOCOL:
126         break;
127     default:
128         ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
129     }
130 }
131
132 void ZOOM_connection_show_task(ZOOM_task task)
133 {
134     switch(task->which)
135     {
136     case ZOOM_TASK_SEARCH:
137         yaz_log(YLOG_LOG, "search p=%p", task);
138         break;
139     case ZOOM_TASK_RETRIEVE:
140         yaz_log(YLOG_LOG, "retrieve p=%p", task);
141         break;
142     case ZOOM_TASK_CONNECT:
143         yaz_log(YLOG_LOG, "connect p=%p", task);
144         break;
145     case ZOOM_TASK_SCAN:
146         yaz_log(YLOG_LOG, "scan p=%p", task);
147         break;
148     }
149 }
150
151 void ZOOM_connection_show_tasks(ZOOM_connection c)
152 {
153     ZOOM_task task;
154     yaz_log(YLOG_LOG, "connection p=%p tasks", c);
155     for (task = c->tasks; task; task = task->next)
156         ZOOM_connection_show_task(task);
157 }
158
159 ZOOM_task ZOOM_connection_add_task(ZOOM_connection c, int which)
160 {
161     ZOOM_task *taskp = &c->tasks;
162     while (*taskp)
163         taskp = &(*taskp)->next;
164     *taskp = (ZOOM_task) xmalloc(sizeof(**taskp));
165     (*taskp)->running = 0;
166     (*taskp)->which = which;
167     (*taskp)->next = 0;
168     clear_error(c);
169     return *taskp;
170 }
171
172 ZOOM_API(int) ZOOM_connection_is_idle(ZOOM_connection c)
173 {
174     return c->tasks ? 0 : 1;
175 }
176
177 ZOOM_task ZOOM_connection_insert_task(ZOOM_connection c, int which)
178 {
179     ZOOM_task task = (ZOOM_task) xmalloc(sizeof(*task));
180
181     task->next = c->tasks;
182     c->tasks = task;
183
184     task->running = 0;
185     task->which = which;
186     return task;
187 }
188
189 void ZOOM_connection_remove_task(ZOOM_connection c)
190 {
191     ZOOM_task task = c->tasks;
192
193     if (task)
194     {
195         c->tasks = task->next;
196         switch (task->which)
197         {
198         case ZOOM_TASK_SEARCH:
199             resultset_destroy(task->u.search.resultset);
200             xfree(task->u.search.syntax);
201             xfree(task->u.search.elementSetName);
202             xfree(task->u.search.schema);
203             break;
204         case ZOOM_TASK_RETRIEVE:
205             resultset_destroy(task->u.retrieve.resultset);
206             xfree(task->u.retrieve.syntax);
207             xfree(task->u.retrieve.elementSetName);
208             xfree(task->u.retrieve.schema);
209             break;
210         case ZOOM_TASK_CONNECT:
211             break;
212         case ZOOM_TASK_SCAN:
213             ZOOM_scanset_destroy(task->u.scan.scan);
214             break;
215         case ZOOM_TASK_PACKAGE:
216             ZOOM_package_destroy(task->u.package);
217             break;
218         case ZOOM_TASK_SORT:
219             resultset_destroy(task->u.sort.resultset);
220             ZOOM_query_destroy(task->u.sort.q);
221             break;
222         default:
223             assert(0);
224         }
225         xfree(task);
226
227         if (!c->tasks)
228         {
229             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_END);
230             ZOOM_connection_put_event(c, event);
231         }
232     }
233 }
234
235 void ZOOM_connection_remove_tasks(ZOOM_connection c)
236 {
237     while (c->tasks)
238         ZOOM_connection_remove_task(c);
239 }
240
241 static void odr_wrbuf_write(ODR o, void *handle, int type,
242                             const char *buf, int len)
243 {
244     WRBUF w = (WRBUF) handle;
245     wrbuf_write(w, buf, len);
246 }
247
248 ZOOM_API(ZOOM_connection)
249     ZOOM_connection_create(ZOOM_options options)
250 {
251     ZOOM_connection c = (ZOOM_connection) xmalloc(sizeof(*c));
252
253     initlog();
254
255     c->log_api = log_api0;
256     c->log_details = log_details0;
257
258     yaz_log(c->log_api, "%p ZOOM_connection_create", c);
259
260     c->proto = PROTO_Z3950;
261     c->cs = 0;
262     ZOOM_connection_set_mask(c, 0);
263     c->reconnect_ok = 0;
264     c->state = STATE_IDLE;
265     c->addinfo = 0;
266     c->diagset = 0;
267     ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
268     c->buf_in = 0;
269     c->len_in = 0;
270     c->buf_out = 0;
271     c->len_out = 0;
272     c->resultsets = 0;
273
274     c->options = ZOOM_options_create_with_parent(options);
275
276     c->host_port = 0;
277     c->proxy = 0;
278     c->tproxy = 0;
279
280     c->charset = c->lang = 0;
281
282     c->cookie_out = 0;
283     c->cookie_in = 0;
284     c->client_IP = 0;
285     c->tasks = 0;
286
287     c->user = 0;
288     c->group = 0;
289     c->password = 0;
290     c->url_authentication = 0;
291
292     c->maximum_record_size = 0;
293     c->preferred_message_size = 0;
294
295     c->odr_in = odr_createmem(ODR_DECODE);
296     c->odr_out = odr_createmem(ODR_ENCODE);
297     c->odr_print = 0;
298     c->odr_save = 0;
299
300     c->async = 0;
301     c->support_named_resultsets = 0;
302     c->last_event = ZOOM_EVENT_NONE;
303
304     c->m_queue_front = 0;
305     c->m_queue_back = 0;
306
307     c->sru_version = 0;
308     c->no_redirects = 0;
309     c->cookies = 0;
310     c->saveAPDU_wrbuf = 0;
311     return c;
312 }
313
314 ZOOM_API(void) ZOOM_connection_save_apdu_wrbuf(ZOOM_connection c, WRBUF w)
315 {
316     if (c->odr_save)
317     {
318         odr_destroy(c->odr_save);
319         c->odr_save = 0;
320     }
321     if (w)
322     {
323         c->odr_save = odr_createmem(ODR_PRINT);
324         odr_set_stream(c->odr_save, w, odr_wrbuf_write, 0);
325     }
326 }
327
328 /* set database names. Take local databases (if set); otherwise
329    take databases given in ZURL (if set); otherwise use Default */
330 char **ZOOM_connection_get_databases(ZOOM_connection con, ZOOM_options options,
331                                      int *num, ODR odr)
332 {
333     char **databaseNames;
334     const char *cp = ZOOM_options_get(options, "databaseName");
335
336     if ((!cp || !*cp) && con->host_port)
337         cs_get_host_args(con->host_port, &cp);
338     if (!cp || !*cp)
339         cp = "Default";
340     nmem_strsplit(odr_getmem(odr), "+", cp,  &databaseNames, num);
341     return databaseNames;
342 }
343
344 ZOOM_API(ZOOM_connection)
345     ZOOM_connection_new(const char *host, int portnum)
346 {
347     ZOOM_connection c = ZOOM_connection_create(0);
348
349     ZOOM_connection_connect(c, host, portnum);
350     return c;
351 }
352
353 static zoom_sru_mode get_sru_mode_from_string(const char *s)
354 {
355     if (!s || !*s)
356         return zoom_sru_soap;
357     if (!yaz_matchstr(s, "soap"))
358         return zoom_sru_soap;
359     else if (!yaz_matchstr(s, "get"))
360         return zoom_sru_get;
361     else if (!yaz_matchstr(s, "post"))
362         return zoom_sru_post;
363     else if (!yaz_matchstr(s, "solr"))
364         return zoom_sru_solr;
365     return zoom_sru_error;
366 }
367
368 ZOOM_API(void)
369     ZOOM_connection_connect(ZOOM_connection c,
370                             const char *host, int portnum)
371 {
372     const char *val;
373     const char *http_lead;
374
375     initlog();
376
377     yaz_log(c->log_api, "%p ZOOM_connection_connect host=%s portnum=%d",
378             c, host ? host : "null", portnum);
379
380     ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
381     ZOOM_connection_remove_tasks(c);
382
383     if (c->odr_print)
384     {
385         odr_setprint(c->odr_print, 0); /* prevent destroy from fclose'ing */
386         odr_destroy(c->odr_print);
387     }
388     if (ZOOM_options_get_bool(c->options, "apdulog", 0))
389     {
390         c->odr_print = odr_createmem(ODR_PRINT);
391         odr_setprint(c->odr_print, yaz_log_file());
392     }
393     else
394         c->odr_print = 0;
395
396     if (c->cs)
397     {
398         yaz_log(c->log_details, "%p ZOOM_connection_connect reconnect ok", c);
399         c->reconnect_ok = 1;
400         return;
401     }
402     yaz_log(c->log_details, "%p ZOOM_connection_connect connect", c);
403     xfree(c->proxy);
404     c->proxy = 0;
405     val = ZOOM_options_get(c->options, "proxy");
406     if (val && *val)
407     {
408         yaz_log(c->log_details, "%p ZOOM_connection_connect proxy=%s", c, val);
409         c->proxy = xstrdup(val);
410     }
411
412     xfree(c->tproxy);
413     c->tproxy = 0;
414     val = ZOOM_options_get(c->options, "tproxy");
415     if (val && *val)
416     {
417         yaz_log(c->log_details, "%p ZOOM_connection_connect tproxy=%s", c, val);
418         c->tproxy = xstrdup(val);
419     }
420
421     xfree(c->charset);
422     c->charset = 0;
423     val = ZOOM_options_get(c->options, "charset");
424     if (val && *val)
425     {
426         yaz_log(c->log_details, "%p ZOOM_connection_connect charset=%s", c, val);
427         c->charset = xstrdup(val);
428     }
429
430     xfree(c->lang);
431     val = ZOOM_options_get(c->options, "lang");
432     if (val && *val)
433     {
434         yaz_log(c->log_details, "%p ZOOM_connection_connect lang=%s", c, val);
435         c->lang = xstrdup(val);
436     }
437     else
438         c->lang = 0;
439
440     val = ZOOM_options_get(c->options, "sru");
441     if (val && *val && !strstr(host, "://"))
442         http_lead = "http://";
443     else
444         http_lead = "";
445     c->sru_mode = get_sru_mode_from_string(val);
446
447     if (host)
448     {
449         char hostn[128];
450         xfree(c->host_port);
451         if (portnum)
452         {
453             sprintf(hostn, "%.80s:%d", host, portnum);
454             host = hostn;
455         }
456         c->host_port = xmalloc(strlen(host) + strlen(http_lead) + 1);
457         strcpy(c->host_port, http_lead);
458         strcat(c->host_port, host);
459     }
460
461     {
462         /*
463          * If the "<scheme>:" part of the host string is preceded by one
464          * or more comma-separated <name>=<value> pairs, these are taken
465          * to be options to be set on the connection object.  Among other
466          * applications, this facility can be used to embed authentication
467          * in a host string:
468          *          user=admin,password=secret,tcp:localhost:9999
469          */
470         char *remainder = c->host_port;
471         char *pcolon = strchr(remainder, ':');
472         char *pcomma;
473         char *pequals;
474         while ((pcomma = strchr(remainder, ',')) != 0 &&
475                (pcolon == 0 || pcomma < pcolon))
476         {
477             *pcomma = '\0';
478             if ((pequals = strchr(remainder, '=')) != 0)
479             {
480                 *pequals = '\0';
481                 ZOOM_connection_option_set(c, remainder, pequals+1);
482             }
483             remainder = pcomma+1;
484         }
485
486         if (remainder != c->host_port)
487         {
488             remainder = xstrdup(remainder);
489             xfree(c->host_port);
490             c->host_port = remainder;
491         }
492     }
493
494     xfree(c->sru_version);
495     val = ZOOM_options_get(c->options, "sru_version");
496     c->sru_version = xstrdup(val ? val : "1.2");
497
498     ZOOM_options_set(c->options, "host", c->host_port);
499
500     xfree(c->cookie_out);
501     c->cookie_out = 0;
502     val = ZOOM_options_get(c->options, "cookie");
503     if (val && *val)
504     {
505         yaz_log(c->log_details, "%p ZOOM_connection_connect cookie=%s", c, val);
506         c->cookie_out = xstrdup(val);
507     }
508
509     xfree(c->client_IP);
510     c->client_IP = 0;
511     val = ZOOM_options_get(c->options, "clientIP");
512     if (val && *val)
513     {
514         yaz_log(c->log_details, "%p ZOOM_connection_connect clientIP=%s",
515                 c, val);
516         c->client_IP = xstrdup(val);
517     }
518
519     xfree(c->group);
520     c->group = 0;
521     val = ZOOM_options_get(c->options, "group");
522     if (val && *val)
523         c->group = xstrdup(val);
524
525     xfree(c->user);
526     c->user = 0;
527     val = ZOOM_options_get(c->options, "user");
528     if (val && *val)
529         c->user = xstrdup(val);
530
531     xfree(c->password);
532     c->password = 0;
533     val = ZOOM_options_get(c->options, "password");
534     if (!val)
535         val = ZOOM_options_get(c->options, "pass");
536     if (val && *val)
537         c->password = xstrdup(val);
538
539     val = ZOOM_options_get(c->options, "authenticationMode");
540     if (val && !strcmp(val, "url"))
541         c->url_authentication = 1;
542     else
543         c->url_authentication = 0;
544
545     c->maximum_record_size =
546         ZOOM_options_get_int(c->options, "maximumRecordSize", 64*1024*1024);
547     c->preferred_message_size =
548         ZOOM_options_get_int(c->options, "preferredMessageSize", 64*1024*1024);
549
550     c->async = ZOOM_options_get_bool(c->options, "async", 0);
551
552     yaz_cookies_destroy(c->cookies);
553     c->cookies = yaz_cookies_create();
554
555     if (c->sru_mode == zoom_sru_error)
556     {
557         ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, val);
558         ZOOM_connection_remove_tasks(c);
559         return;
560     }
561
562     yaz_log(c->log_details, "%p ZOOM_connection_connect async=%d", c, c->async);
563     ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
564
565     if (!c->async)
566     {
567         while (ZOOM_event(1, &c))
568             ;
569     }
570 }
571
572 ZOOM_API(void) ZOOM_resultset_release(ZOOM_resultset r)
573 {
574     if (r->connection)
575     {
576         /* remove ourselves from the resultsets in connection */
577         ZOOM_resultset *rp = &r->connection->resultsets;
578         while (1)
579         {
580             assert(*rp);   /* we must be in this list!! */
581             if (*rp == r)
582             {   /* OK, we're here - take us out of it */
583                 *rp = (*rp)->next;
584                 break;
585             }
586             rp = &(*rp)->next;
587         }
588         r->connection = 0;
589     }
590 }
591
592 ZOOM_API(void)
593     ZOOM_connection_destroy(ZOOM_connection c)
594 {
595     ZOOM_resultset r;
596     if (!c)
597         return;
598     yaz_log(c->log_api, "%p ZOOM_connection_destroy", c);
599     if (c->cs)
600         cs_close(c->cs);
601
602     for (r = c->resultsets; r; r = r->next)
603         r->connection = 0;
604
605     xfree(c->buf_in);
606     xfree(c->addinfo);
607     xfree(c->diagset);
608     odr_destroy(c->odr_in);
609     odr_destroy(c->odr_out);
610     if (c->odr_save)
611         odr_destroy(c->odr_save);
612     if (c->odr_print)
613     {
614         odr_setprint(c->odr_print, 0); /* prevent destroy from fclose'ing */
615         odr_destroy(c->odr_print);
616     }
617     ZOOM_options_destroy(c->options);
618     ZOOM_connection_remove_tasks(c);
619     ZOOM_connection_remove_events(c);
620     xfree(c->host_port);
621     xfree(c->proxy);
622     xfree(c->tproxy);
623     xfree(c->charset);
624     xfree(c->lang);
625     xfree(c->cookie_out);
626     xfree(c->cookie_in);
627     xfree(c->client_IP);
628     xfree(c->user);
629     xfree(c->group);
630     xfree(c->password);
631     xfree(c->sru_version);
632     yaz_cookies_destroy(c->cookies);
633     wrbuf_destroy(c->saveAPDU_wrbuf);
634     xfree(c);
635 }
636
637 void ZOOM_resultset_addref(ZOOM_resultset r)
638 {
639     if (r)
640     {
641         yaz_mutex_enter(r->mutex);
642         (r->refcount)++;
643         yaz_log(log_details0, "%p ZOOM_resultset_addref count=%d",
644                 r, r->refcount);
645         yaz_mutex_leave(r->mutex);
646     }
647 }
648
649 static int g_resultsets = 0;
650 static YAZ_MUTEX g_resultset_mutex = 0;
651
652 /* TODO We need to initialize this before running threaded:
653  * call resultset_use(0)  */
654
655 static int resultset_use(int delta) {
656     int resultset_count;
657     if (g_resultset_mutex == 0)
658         yaz_mutex_create(&g_resultset_mutex);
659     yaz_mutex_enter(g_resultset_mutex);
660     g_resultsets += delta;
661     resultset_count = g_resultsets;
662     yaz_mutex_leave(g_resultset_mutex);
663     return resultset_count;
664 }
665
666 int resultsets_count(void) {
667     return resultset_use(0);
668 }
669
670 ZOOM_resultset ZOOM_resultset_create(void)
671 {
672     int i;
673     ZOOM_resultset r = (ZOOM_resultset) xmalloc(sizeof(*r));
674
675     initlog();
676
677     yaz_log(log_details0, "%p ZOOM_resultset_create", r);
678     r->refcount = 1;
679     r->size = 0;
680     r->odr = odr_createmem(ODR_ENCODE);
681     r->piggyback = 1;
682     r->setname = 0;
683     r->step = 0;
684     for (i = 0; i<RECORD_HASH_SIZE; i++)
685         r->record_hash[i] = 0;
686     r->r_sort_spec = 0;
687     r->query = 0;
688     r->connection = 0;
689     r->databaseNames = 0;
690     r->num_databaseNames = 0;
691     r->facets = 0;
692     r->num_facets = 0;
693     r->facets_names = 0;
694     r->mutex = 0;
695     yaz_mutex_create(&r->mutex);
696 #if SHPTR
697     {
698         WRBUF w = wrbuf_alloc();
699         YAZ_SHPTR_INIT(r->record_wrbuf, w);
700     }
701 #endif
702     resultset_use(1);
703     return r;
704 }
705
706 ZOOM_API(ZOOM_resultset)
707     ZOOM_connection_search_pqf(ZOOM_connection c, const char *q)
708 {
709     ZOOM_resultset r;
710     ZOOM_query s = ZOOM_query_create();
711
712     ZOOM_query_prefix(s, q);
713
714     r = ZOOM_connection_search(c, s);
715     ZOOM_query_destroy(s);
716     return r;
717 }
718
719 ZOOM_API(ZOOM_resultset)
720     ZOOM_connection_search(ZOOM_connection c, ZOOM_query q)
721 {
722     ZOOM_resultset r = ZOOM_resultset_create();
723     ZOOM_task task;
724     const char *cp;
725     int start, count;
726     const char *syntax, *elementSetName, *schema;
727
728     yaz_log(c->log_api, "%p ZOOM_connection_search set %p query %p", c, r, q);
729     r->r_sort_spec = ZOOM_query_get_sortspec(q);
730     r->query = q;
731
732     r->options = ZOOM_options_create_with_parent(c->options);
733
734     start = ZOOM_options_get_int(r->options, "start", 0);
735     count = ZOOM_options_get_int(r->options, "count", 0);
736     {
737         /* If "presentChunk" is defined use that; otherwise "step" */
738         const char *cp = ZOOM_options_get(r->options, "presentChunk");
739         r->step = ZOOM_options_get_int(r->options,
740                                        (cp != 0 ? "presentChunk": "step"), 0);
741     }
742     r->piggyback = ZOOM_options_get_bool(r->options, "piggyback", 1);
743     cp = ZOOM_options_get(r->options, "setname");
744     if (cp)
745         r->setname = xstrdup(cp);
746
747     r->databaseNames = ZOOM_connection_get_databases(c, c->options, &r->num_databaseNames,
748                                          r->odr);
749
750     r->connection = c;
751     r->next = c->resultsets;
752     c->resultsets = r;
753     if (c->host_port && c->proto == PROTO_HTTP)
754     {
755         if (!c->cs)
756         {
757             yaz_log(c->log_details, "ZOOM_connection_search: no comstack");
758             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
759         }
760         else
761         {
762             yaz_log(c->log_details, "ZOOM_connection_search: reconnect");
763             c->reconnect_ok = 1;
764         }
765     }
766
767     task = ZOOM_connection_add_task(c, ZOOM_TASK_SEARCH);
768     task->u.search.resultset = r;
769     task->u.search.start = start;
770     task->u.search.count = count;
771     task->u.search.recv_search_fired = 0;
772
773     syntax = ZOOM_options_get(r->options, "preferredRecordSyntax");
774     task->u.search.syntax = syntax ? xstrdup(syntax) : 0;
775     elementSetName = ZOOM_options_get(r->options, "elementSetName");
776     task->u.search.elementSetName = elementSetName ?
777         xstrdup(elementSetName) : 0;
778     schema = ZOOM_options_get(r->options, "schema");
779     task->u.search.schema = schema ? xstrdup(schema) : 0;
780
781     ZOOM_resultset_addref(r);
782
783     ZOOM_query_addref(q);
784
785     if (!c->async)
786     {
787         while (ZOOM_event(1, &c))
788             ;
789     }
790     return r;
791 }
792
793 ZOOM_API(void)
794     ZOOM_resultset_sort(ZOOM_resultset r,
795                          const char *sort_type, const char *sort_spec)
796 {
797     (void) ZOOM_resultset_sort1(r, sort_type, sort_spec);
798 }
799
800 ZOOM_API(int)
801     ZOOM_resultset_sort1(ZOOM_resultset r,
802                          const char *sort_type, const char *sort_spec)
803 {
804     ZOOM_connection c = r->connection;
805     ZOOM_task task;
806     ZOOM_query newq;
807
808     newq = ZOOM_query_create();
809     if (ZOOM_query_sortby(newq, sort_spec) < 0)
810         return -1;
811
812     yaz_log(c->log_api, "%p ZOOM_resultset_sort r=%p sort_type=%s sort_spec=%s",
813             r, r, sort_type, sort_spec);
814     if (!c)
815         return 0;
816
817     if (c->host_port && c->proto == PROTO_HTTP)
818     {
819         if (!c->cs)
820         {
821             yaz_log(c->log_details, "%p ZOOM_resultset_sort: no comstack", r);
822             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
823         }
824         else
825         {
826             yaz_log(c->log_details, "%p ZOOM_resultset_sort: prepare reconnect",
827                     r);
828             c->reconnect_ok = 1;
829         }
830     }
831
832     ZOOM_resultset_cache_reset(r);
833     task = ZOOM_connection_add_task(c, ZOOM_TASK_SORT);
834     task->u.sort.resultset = r;
835     task->u.sort.q = newq;
836
837     ZOOM_resultset_addref(r);
838
839     if (!c->async)
840     {
841         while (ZOOM_event(1, &c))
842             ;
843     }
844
845     return 0;
846 }
847
848 ZOOM_API(void)
849     ZOOM_resultset_destroy(ZOOM_resultset r)
850 {
851     resultset_destroy(r);
852 }
853
854 static void resultset_destroy(ZOOM_resultset r)
855 {
856     if (!r)
857         return;
858     yaz_mutex_enter(r->mutex);
859     (r->refcount)--;
860     yaz_log(log_details0, "%p ZOOM_resultset_destroy r=%p count=%d",
861             r, r, r->refcount);
862     if (r->refcount == 0)
863     {
864         yaz_mutex_leave(r->mutex);
865
866         yaz_log(log_details0, "%p ZOOM_connection resultset_destroy: Deleting resultset (%p) ", r->connection, r);
867         ZOOM_resultset_cache_reset(r);
868         ZOOM_resultset_release(r);
869         ZOOM_query_destroy(r->query);
870         ZOOM_options_destroy(r->options);
871         odr_destroy(r->odr);
872         xfree(r->setname);
873         yaz_mutex_destroy(&r->mutex);
874 #if SHPTR
875         YAZ_SHPTR_DEC(r->record_wrbuf, wrbuf_destroy);
876 #endif
877         resultset_use(-1);
878         xfree(r);
879     }
880     else
881         yaz_mutex_leave(r->mutex);
882 }
883
884 ZOOM_API(size_t)
885     ZOOM_resultset_size(ZOOM_resultset r)
886 {
887     return r->size;
888 }
889
890 int ZOOM_test_reconnect(ZOOM_connection c)
891 {
892     ZOOM_Event event;
893
894     if (!c->reconnect_ok)
895         return 0;
896     ZOOM_connection_close(c);
897     c->reconnect_ok = 0;
898     c->tasks->running = 0;
899     ZOOM_connection_insert_task(c, ZOOM_TASK_CONNECT);
900
901     event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
902     ZOOM_connection_put_event(c, event);
903
904     return 1;
905 }
906
907 static void ZOOM_resultset_retrieve(ZOOM_resultset r,
908                                     int force_sync, int start, int count)
909 {
910     ZOOM_task task;
911     ZOOM_connection c;
912     const char *cp;
913     const char *syntax, *elementSetName;
914
915     if (!r)
916         return;
917     yaz_log(log_details0, "%p ZOOM_resultset_retrieve force_sync=%d start=%d"
918             " count=%d", r, force_sync, start, count);
919     c = r->connection;
920     if (!c)
921         return;
922
923     if (c->host_port && c->proto == PROTO_HTTP)
924     {
925         if (!c->cs)
926         {
927             yaz_log(log_details0, "%p ZOOM_resultset_retrieve: no comstack", r);
928             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
929         }
930         else
931         {
932             yaz_log(log_details0, "%p ZOOM_resultset_retrieve: prepare "
933                     "reconnect", r);
934             c->reconnect_ok = 1;
935         }
936     }
937     task = ZOOM_connection_add_task(c, ZOOM_TASK_RETRIEVE);
938     task->u.retrieve.resultset = r;
939     task->u.retrieve.start = start;
940     task->u.retrieve.count = count;
941
942     syntax = ZOOM_options_get(r->options, "preferredRecordSyntax");
943     task->u.retrieve.syntax = syntax ? xstrdup(syntax) : 0;
944     elementSetName = ZOOM_options_get(r->options, "elementSetName");
945     task->u.retrieve.elementSetName = elementSetName
946         ? xstrdup(elementSetName) : 0;
947
948     cp = ZOOM_options_get(r->options, "schema");
949     task->u.retrieve.schema = cp ? xstrdup(cp) : 0;
950
951     ZOOM_resultset_addref(r);
952
953     if (!r->connection->async || force_sync)
954         while (r->connection && ZOOM_event(1, &r->connection))
955             ;
956 }
957
958 ZOOM_API(void)
959     ZOOM_resultset_records(ZOOM_resultset r, ZOOM_record *recs,
960                            size_t start, size_t count)
961 {
962     int force_present = 0;
963
964     if (!r)
965         return ;
966     yaz_log(log_api0, "%p ZOOM_resultset_records r=%p start=%ld count=%ld",
967             r, r, (long) start, (long) count);
968     if (count && recs)
969         force_present = 1;
970     ZOOM_resultset_retrieve(r, force_present, start, count);
971     if (force_present)
972     {
973         size_t i;
974         for (i = 0; i< count; i++)
975             recs[i] = ZOOM_resultset_record_immediate(r, i+start);
976     }
977 }
978
979 ZOOM_API(size_t)
980     ZOOM_resultset_facets_size(ZOOM_resultset r) {
981     return r->num_facets;
982 }
983
984 ZOOM_API(ZOOM_facet_field)
985     ZOOM_resultset_get_facet_field(ZOOM_resultset r, const char *name) {
986     int num = r->num_facets;
987     ZOOM_facet_field *facets = r->facets;
988     int index;
989     for (index = 0; index < num; index++) {
990         if (!strcmp(facets[index]->facet_name, name)) {
991             return facets[index];
992         }
993     }
994     return 0;
995 }
996
997 ZOOM_API(ZOOM_facet_field)
998     ZOOM_resultset_get_facet_field_by_index(ZOOM_resultset r, int index) {
999     int num = r->num_facets;
1000     ZOOM_facet_field *facets = r->facets;
1001     if (index >= 0 && index < num) {
1002         return facets[index];
1003     }
1004     return 0;
1005 }
1006
1007 ZOOM_API(ZOOM_facet_field *)
1008     ZOOM_resultset_facets(ZOOM_resultset r)
1009 {
1010     return r->facets;
1011 }
1012
1013 ZOOM_API(const char**)
1014     ZOOM_resultset_facets_names(ZOOM_resultset r)
1015 {
1016     return (const char **) r->facets_names;
1017 }
1018
1019 ZOOM_API(const char*)
1020     ZOOM_facet_field_name(ZOOM_facet_field field)
1021 {
1022     return field->facet_name;
1023 }
1024
1025 ZOOM_API(size_t)
1026     ZOOM_facet_field_term_count(ZOOM_facet_field field)
1027 {
1028     return field->num_terms;
1029 }
1030
1031 ZOOM_API(const char*)
1032     ZOOM_facet_field_get_term(ZOOM_facet_field field, size_t idx, int *freq) {
1033     *freq = field->facet_terms[idx].frequency;
1034     return field->facet_terms[idx].term;
1035 }
1036
1037
1038 static void get_cert(ZOOM_connection c)
1039 {
1040     char *cert_buf;
1041     int cert_len;
1042
1043     if (cs_get_peer_certificate_x509(c->cs, &cert_buf, &cert_len))
1044     {
1045         ZOOM_connection_option_setl(c, "sslPeerCert",
1046                                     cert_buf, cert_len);
1047         xfree(cert_buf);
1048     }
1049 }
1050
1051 static zoom_ret do_connect_host(ZOOM_connection c,
1052                                 const char *logical_url);
1053
1054 static zoom_ret do_connect(ZOOM_connection c)
1055 {
1056     return do_connect_host(c, c->host_port);
1057 }
1058
1059 static zoom_ret do_connect_host(ZOOM_connection c, const char *logical_url)
1060 {
1061     void *add;
1062
1063     if (c->cs)
1064         cs_close(c->cs);
1065     c->cs = cs_create_host_proxy(logical_url, 0, &add,
1066                                  c->tproxy ? c->tproxy : c->proxy);
1067
1068     if (c->cs && c->cs->protocol == PROTO_HTTP)
1069     {
1070 #if YAZ_HAVE_XML2
1071         c->proto = PROTO_HTTP;
1072 #else
1073         ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, "SRW");
1074         ZOOM_connection_close(c);
1075         return zoom_complete;
1076 #endif
1077     }
1078     if (c->cs)
1079     {
1080         int ret = cs_connect(c->cs, add);
1081         if (ret == 0)
1082         {
1083             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1084             ZOOM_connection_put_event(c, event);
1085             get_cert(c);
1086             if (c->proto == PROTO_Z3950)
1087                 ZOOM_connection_Z3950_send_init(c);
1088             else
1089             {
1090                 /* no init request for SRW .. */
1091                 assert(c->tasks->which == ZOOM_TASK_CONNECT);
1092                 ZOOM_connection_remove_task(c);
1093                 ZOOM_connection_set_mask(c, 0);
1094                 ZOOM_connection_exec_task(c);
1095             }
1096             c->state = STATE_ESTABLISHED;
1097             return zoom_pending;
1098         }
1099         else if (ret > 0)
1100         {
1101             int mask = ZOOM_SELECT_EXCEPT;
1102             if (c->cs->io_pending & CS_WANT_WRITE)
1103                 mask += ZOOM_SELECT_WRITE;
1104             if (c->cs->io_pending & CS_WANT_READ)
1105                 mask += ZOOM_SELECT_READ;
1106             ZOOM_connection_set_mask(c, mask);
1107             c->state = STATE_CONNECTING;
1108             return zoom_pending;
1109         }
1110     }
1111     c->state = STATE_IDLE;
1112     ZOOM_set_error(c, ZOOM_ERROR_CONNECT, logical_url);
1113     return zoom_complete;
1114 }
1115
1116 /* returns 1 if PDU was sent OK (still pending )
1117    0 if PDU was not sent OK (nothing to wait for)
1118 */
1119
1120 ZOOM_API(ZOOM_record)
1121     ZOOM_resultset_record_immediate(ZOOM_resultset s,size_t pos)
1122 {
1123     const char *syntax =
1124         ZOOM_options_get(s->options, "preferredRecordSyntax");
1125     const char *elementSetName =
1126         ZOOM_options_get(s->options, "elementSetName");
1127     const char *schema =
1128         ZOOM_options_get(s->options, "schema");
1129
1130     return ZOOM_record_cache_lookup(s, pos, syntax, elementSetName, schema);
1131 }
1132
1133 ZOOM_API(ZOOM_record)
1134     ZOOM_resultset_record(ZOOM_resultset r, size_t pos)
1135 {
1136     ZOOM_record rec = ZOOM_resultset_record_immediate(r, pos);
1137
1138     if (!rec)
1139     {
1140         /*
1141          * MIKE: I think force_sync should always be zero, but I don't
1142          * want to make this change until I get the go-ahead from
1143          * Adam, in case something depends on the old synchronous
1144          * behaviour.
1145          */
1146         int force_sync = 1;
1147         if (getenv("ZOOM_RECORD_NO_FORCE_SYNC")) force_sync = 0;
1148         ZOOM_resultset_retrieve(r, force_sync, pos, 1);
1149         rec = ZOOM_resultset_record_immediate(r, pos);
1150     }
1151     return rec;
1152 }
1153
1154 ZOOM_API(ZOOM_scanset)
1155     ZOOM_connection_scan(ZOOM_connection c, const char *start)
1156 {
1157     ZOOM_scanset s;
1158     ZOOM_query q = ZOOM_query_create();
1159
1160     ZOOM_query_prefix(q, start);
1161
1162     s = ZOOM_connection_scan1(c, q);
1163     ZOOM_query_destroy(q);
1164     return s;
1165
1166 }
1167
1168 ZOOM_API(ZOOM_scanset)
1169     ZOOM_connection_scan1(ZOOM_connection c, ZOOM_query q)
1170 {
1171     ZOOM_scanset scan = 0;
1172     Z_Query *z_query = ZOOM_query_get_Z_Query(q);
1173
1174     if (!z_query)
1175         return 0;
1176     scan = (ZOOM_scanset) xmalloc(sizeof(*scan));
1177     scan->connection = c;
1178     scan->odr = odr_createmem(ODR_DECODE);
1179     scan->options = ZOOM_options_create_with_parent(c->options);
1180     scan->refcount = 1;
1181     scan->scan_response = 0;
1182     scan->srw_scan_response = 0;
1183
1184     scan->query = q;
1185     ZOOM_query_addref(q);
1186     scan->databaseNames = ZOOM_connection_get_databases(c, c->options,
1187                                             &scan->num_databaseNames,
1188                                             scan->odr);
1189
1190     if (1)
1191     {
1192         ZOOM_task task = ZOOM_connection_add_task(c, ZOOM_TASK_SCAN);
1193         task->u.scan.scan = scan;
1194
1195         (scan->refcount)++;
1196         if (!c->async)
1197         {
1198             while (ZOOM_event(1, &c))
1199                 ;
1200         }
1201     }
1202     return scan;
1203 }
1204
1205 ZOOM_API(void)
1206     ZOOM_scanset_destroy(ZOOM_scanset scan)
1207 {
1208     if (!scan)
1209         return;
1210     (scan->refcount)--;
1211     if (scan->refcount == 0)
1212     {
1213         ZOOM_query_destroy(scan->query);
1214
1215         odr_destroy(scan->odr);
1216
1217         ZOOM_options_destroy(scan->options);
1218         xfree(scan);
1219     }
1220 }
1221
1222 static zoom_ret send_package(ZOOM_connection c)
1223 {
1224     ZOOM_Event event;
1225
1226     yaz_log(c->log_details, "%p send_package", c);
1227     if (!c->tasks)
1228         return zoom_complete;
1229     assert (c->tasks->which == ZOOM_TASK_PACKAGE);
1230
1231     event = ZOOM_Event_create(ZOOM_EVENT_SEND_APDU);
1232     ZOOM_connection_put_event(c, event);
1233
1234     c->buf_out = c->tasks->u.package->buf_out;
1235     c->len_out = c->tasks->u.package->len_out;
1236
1237     return ZOOM_send_buf(c);
1238 }
1239
1240 ZOOM_API(size_t)
1241     ZOOM_scanset_size(ZOOM_scanset scan)
1242 {
1243     if (!scan)
1244         return 0;
1245
1246     if (scan->scan_response && scan->scan_response->entries)
1247         return scan->scan_response->entries->num_entries;
1248     else if (scan->srw_scan_response)
1249         return scan->srw_scan_response->num_terms;
1250     return 0;
1251 }
1252
1253 static void ZOOM_scanset_term_x(ZOOM_scanset scan, size_t pos,
1254                                 size_t *occ,
1255                                 const char **value_term, size_t *value_len,
1256                                 const char **disp_term, size_t *disp_len)
1257 {
1258     size_t noent = ZOOM_scanset_size(scan);
1259
1260     *value_term = 0;
1261     *value_len = 0;
1262
1263     *disp_term = 0;
1264     *disp_len = 0;
1265
1266     *occ = 0;
1267     if (pos >= noent)
1268         return;
1269     if (scan->scan_response)
1270     {
1271         Z_ScanResponse *res = scan->scan_response;
1272         if (res->entries->entries[pos]->which == Z_Entry_termInfo)
1273         {
1274             Z_TermInfo *t = res->entries->entries[pos]->u.termInfo;
1275
1276             *value_term = (const char *) t->term->u.general->buf;
1277             *value_len = t->term->u.general->len;
1278             if (t->displayTerm)
1279             {
1280                 *disp_term = t->displayTerm;
1281                 *disp_len = strlen(*disp_term);
1282             }
1283             else if (t->term->which == Z_Term_general)
1284             {
1285                 *disp_term = (const char *) t->term->u.general->buf;
1286                 *disp_len = t->term->u.general->len;
1287             }
1288             *occ = t->globalOccurrences ? *t->globalOccurrences : 0;
1289         }
1290     }
1291     if (scan->srw_scan_response)
1292     {
1293         Z_SRW_scanResponse *res = scan->srw_scan_response;
1294         Z_SRW_scanTerm *t = res->terms + pos;
1295         if (t)
1296         {
1297             *value_term = t->value;
1298             *value_len = strlen(*value_term);
1299
1300             if (t->displayTerm)
1301                 *disp_term = t->displayTerm;
1302             else
1303                 *disp_term = t->value;
1304             *disp_len = strlen(*disp_term);
1305             *occ = t->numberOfRecords ? *t->numberOfRecords : 0;
1306         }
1307     }
1308 }
1309
1310 ZOOM_API(const char *)
1311     ZOOM_scanset_term(ZOOM_scanset scan, size_t pos,
1312                       size_t *occ, size_t *len)
1313 {
1314     const char *value_term = 0;
1315     size_t value_len = 0;
1316     const char *disp_term = 0;
1317     size_t disp_len = 0;
1318
1319     ZOOM_scanset_term_x(scan, pos, occ, &value_term, &value_len,
1320                         &disp_term, &disp_len);
1321
1322     *len = value_len;
1323     return value_term;
1324 }
1325
1326 ZOOM_API(const char *)
1327     ZOOM_scanset_display_term(ZOOM_scanset scan, size_t pos,
1328                               size_t *occ, size_t *len)
1329 {
1330     const char *value_term = 0;
1331     size_t value_len = 0;
1332     const char *disp_term = 0;
1333     size_t disp_len = 0;
1334
1335     ZOOM_scanset_term_x(scan, pos, occ, &value_term, &value_len,
1336                         &disp_term, &disp_len);
1337
1338     *len = disp_len;
1339     return disp_term;
1340 }
1341
1342 ZOOM_API(const char *)
1343     ZOOM_scanset_option_get(ZOOM_scanset scan, const char *key)
1344 {
1345     return ZOOM_options_get(scan->options, key);
1346 }
1347
1348 ZOOM_API(void)
1349     ZOOM_scanset_option_set(ZOOM_scanset scan, const char *key,
1350                             const char *val)
1351 {
1352     ZOOM_options_set(scan->options, key, val);
1353 }
1354
1355
1356 ZOOM_API(ZOOM_package)
1357     ZOOM_connection_package(ZOOM_connection c, ZOOM_options options)
1358 {
1359     ZOOM_package p = (ZOOM_package) xmalloc(sizeof(*p));
1360
1361     p->connection = c;
1362     p->odr_out = odr_createmem(ODR_ENCODE);
1363     p->options = ZOOM_options_create_with_parent2(options, c->options);
1364     p->refcount = 1;
1365     p->buf_out = 0;
1366     p->len_out = 0;
1367     return p;
1368 }
1369
1370 ZOOM_API(void)
1371     ZOOM_package_destroy(ZOOM_package p)
1372 {
1373     if (!p)
1374         return;
1375     (p->refcount)--;
1376     if (p->refcount == 0)
1377     {
1378         odr_destroy(p->odr_out);
1379         xfree(p->buf_out);
1380
1381         ZOOM_options_destroy(p->options);
1382         xfree(p);
1383     }
1384 }
1385
1386 ZOOM_API(const char *)
1387     ZOOM_package_option_get(ZOOM_package p, const char *key)
1388 {
1389     return ZOOM_options_get(p->options, key);
1390 }
1391
1392 ZOOM_API(const char *)
1393     ZOOM_package_option_getl(ZOOM_package p, const char *key, int *lenp)
1394 {
1395     return ZOOM_options_getl(p->options, key, lenp);
1396 }
1397
1398 ZOOM_API(void)
1399     ZOOM_package_option_set(ZOOM_package p, const char *key,
1400                             const char *val)
1401 {
1402     ZOOM_options_set(p->options, key, val);
1403 }
1404
1405 ZOOM_API(void)
1406     ZOOM_package_option_setl(ZOOM_package p, const char *key,
1407                              const char *val, int len)
1408 {
1409     ZOOM_options_setl(p->options, key, val, len);
1410 }
1411
1412 ZOOM_API(int)
1413     ZOOM_connection_exec_task(ZOOM_connection c)
1414 {
1415     ZOOM_task task = c->tasks;
1416     zoom_ret ret = zoom_complete;
1417
1418     if (!task)
1419         return 0;
1420     yaz_log(c->log_details, "%p ZOOM_connection_exec_task type=%d run=%d",
1421             c, task->which, task->running);
1422     if (c->error != ZOOM_ERROR_NONE)
1423     {
1424         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1425                 "removing tasks because of error = %d", c, c->error);
1426         ZOOM_connection_remove_tasks(c);
1427         return 0;
1428     }
1429     if (task->running)
1430     {
1431         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1432                 "task already running", c);
1433         return 0;
1434     }
1435     task->running = 1;
1436     ret = zoom_complete;
1437     if (c->cs || task->which == ZOOM_TASK_CONNECT)
1438     {
1439         switch (task->which)
1440         {
1441         case ZOOM_TASK_SEARCH:
1442             if (c->proto == PROTO_HTTP)
1443                 ret = ZOOM_connection_srw_send_search(c);
1444             else
1445                 ret = ZOOM_connection_Z3950_send_search(c);
1446             break;
1447         case ZOOM_TASK_RETRIEVE:
1448             if (c->proto == PROTO_HTTP)
1449                 ret = ZOOM_connection_srw_send_search(c);
1450             else
1451                 ret = send_Z3950_present(c);
1452             break;
1453         case ZOOM_TASK_CONNECT:
1454             ret = do_connect(c);
1455             break;
1456         case ZOOM_TASK_SCAN:
1457             if (c->proto == PROTO_HTTP)
1458                 ret = ZOOM_connection_srw_send_scan(c);
1459             else
1460                 ret = ZOOM_connection_Z3950_send_scan(c);
1461             break;
1462         case ZOOM_TASK_PACKAGE:
1463             ret = send_package(c);
1464             break;
1465         case ZOOM_TASK_SORT:
1466             c->tasks->u.sort.resultset->r_sort_spec =
1467                 ZOOM_query_get_sortspec(c->tasks->u.sort.q);
1468             ret = send_Z3950_sort(c, c->tasks->u.sort.resultset);
1469             break;
1470         }
1471     }
1472     else
1473     {
1474         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1475                 "remove tasks because no connection exist", c);
1476         ZOOM_connection_remove_tasks(c);
1477     }
1478     if (ret == zoom_complete)
1479     {
1480         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1481                 "task removed (complete)", c);
1482         ZOOM_connection_remove_task(c);
1483         return 0;
1484     }
1485     yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1486             "task pending", c);
1487     return 1;
1488 }
1489
1490 #if YAZ_HAVE_XML2
1491
1492 static zoom_ret send_HTTP_redirect(ZOOM_connection c, const char *uri)
1493 {
1494     Z_GDU *gdu = z_get_HTTP_Request_uri(c->odr_out, uri, 0, c->proxy ? 1 : 0);
1495
1496     gdu->u.HTTP_Request->method = odr_strdup(c->odr_out, "GET");
1497     z_HTTP_header_add(c->odr_out, &gdu->u.HTTP_Request->headers, "Accept",
1498                       "text/xml");
1499     yaz_cookies_request(c->cookies, c->odr_out, gdu->u.HTTP_Request);
1500     if (c->user && c->password)
1501     {
1502         z_HTTP_header_add_basic_auth(c->odr_out, &gdu->u.HTTP_Request->headers,
1503                                      c->user, c->password);
1504     }
1505     return ZOOM_send_GDU(c, gdu);
1506 }
1507
1508 zoom_ret ZOOM_send_GDU(ZOOM_connection c, Z_GDU *gdu)
1509 {
1510     ZOOM_Event event;
1511
1512     int r = z_GDU(c->odr_out, &gdu, 0, 0);
1513     if (!r)
1514         return zoom_complete;
1515     if (c->odr_print)
1516         z_GDU(c->odr_print, &gdu, 0, 0);
1517     if (c->odr_save)
1518         z_GDU(c->odr_save, &gdu, 0, 0);
1519     c->buf_out = odr_getbuf(c->odr_out, &c->len_out, 0);
1520     odr_reset(c->odr_out);
1521
1522     event = ZOOM_Event_create(ZOOM_EVENT_SEND_APDU);
1523     ZOOM_connection_put_event(c, event);
1524
1525     return ZOOM_send_buf(c);
1526 }
1527
1528 #if YAZ_HAVE_XML2
1529 void ZOOM_set_HTTP_error(ZOOM_connection c, int error,
1530                          const char *addinfo, const char *addinfo2)
1531 {
1532     ZOOM_set_dset_error(c, error, "HTTP", addinfo, addinfo2);
1533 }
1534 #endif
1535
1536
1537 static void handle_http(ZOOM_connection c, Z_HTTP_Response *hres)
1538 {
1539     zoom_ret cret = zoom_complete;
1540     int ret = -1;
1541     char *addinfo = 0;
1542     const char *connection_head = z_HTTP_header_lookup(hres->headers,
1543                                                        "Connection");
1544     const char *location;
1545
1546     ZOOM_connection_set_mask(c, 0);
1547     yaz_log(c->log_details, "%p handle_http", c);
1548
1549     yaz_cookies_response(c->cookies, hres);
1550     if ((hres->code == 301 || hres->code == 302) && c->sru_mode == zoom_sru_get
1551         && (location = z_HTTP_header_lookup(hres->headers, "Location")))
1552     {
1553         c->no_redirects++;
1554         if (c->no_redirects > 10)
1555         {
1556             ZOOM_set_HTTP_error(c, hres->code, 0, 0);
1557             c->no_redirects = 0;
1558             ZOOM_connection_close(c);
1559         }
1560         else
1561         {
1562             /* since redirect may change host we just reconnect. A smarter
1563                implementation might check whether it's the same server */
1564
1565             int host_change = 0;
1566             location = yaz_check_location(c->odr_in, c->host_port,
1567                                           location, &host_change);
1568             if (do_connect_host(c, location) == zoom_complete)
1569                 return;  /* connect failed.. */
1570             send_HTTP_redirect(c, location);
1571             return;
1572         }
1573     }
1574     else
1575     {
1576         ret = ZOOM_handle_sru(c, hres, &cret, &addinfo);
1577         if (ret == 0)
1578         {
1579             if (c->no_redirects) /* end of redirect. change hosts again */
1580                 ZOOM_connection_close(c);
1581         }
1582         c->no_redirects = 0;
1583     }
1584     if (ret)
1585     {
1586         if (hres->code != 200)
1587             ZOOM_set_HTTP_error(c, hres->code, 0, 0);
1588         else
1589         {
1590             yaz_log(YLOG_LOG, "set error... addinfo=%s", addinfo ?
1591                     addinfo : "NULL");
1592             ZOOM_set_error(c, ZOOM_ERROR_DECODE, addinfo);
1593         }
1594         ZOOM_connection_close(c);
1595     }
1596     if (cret == zoom_complete)
1597     {
1598         yaz_log(c->log_details, "removing tasks in handle_http");
1599         ZOOM_connection_remove_task(c);
1600     }
1601     {
1602         int must_close = 0;
1603         if (!strcmp(hres->version, "1.0"))
1604         {
1605             /* HTTP 1.0: only if Keep-Alive we stay alive.. */
1606             if (!connection_head || strcmp(connection_head, "Keep-Alive"))
1607                 must_close = 1;
1608         }
1609         else
1610         {
1611             /* HTTP 1.1: only if no close we stay alive.. */
1612             if (connection_head && !strcmp(connection_head, "close"))
1613                 must_close = 1;
1614         }
1615         if (must_close)
1616         {
1617             ZOOM_connection_close(c);
1618             if (c->tasks)
1619             {
1620                 c->tasks->running = 0;
1621                 ZOOM_connection_insert_task(c, ZOOM_TASK_CONNECT);
1622                 c->reconnect_ok = 0;
1623             }
1624         }
1625         else
1626             c->reconnect_ok = 1; /* if the server closes anyway */
1627     }
1628 }
1629 #endif
1630
1631 static int do_read(ZOOM_connection c)
1632 {
1633     int r, more;
1634     ZOOM_Event event;
1635
1636     event = ZOOM_Event_create(ZOOM_EVENT_RECV_DATA);
1637     ZOOM_connection_put_event(c, event);
1638
1639     r = cs_get(c->cs, &c->buf_in, &c->len_in);
1640     more = cs_more(c->cs);
1641     yaz_log(c->log_details, "%p do_read len=%d more=%d", c, r, more);
1642     if (r == 1)
1643         return 0;
1644     if (r <= 0)
1645     {
1646         if (!ZOOM_test_reconnect(c))
1647         {
1648             ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
1649             ZOOM_connection_close(c);
1650         }
1651     }
1652     else
1653     {
1654         Z_GDU *gdu;
1655         ZOOM_Event event;
1656
1657         odr_reset(c->odr_in);
1658         odr_setbuf(c->odr_in, c->buf_in, r, 0);
1659         event = ZOOM_Event_create(ZOOM_EVENT_RECV_APDU);
1660         ZOOM_connection_put_event(c, event);
1661
1662         if (!z_GDU(c->odr_in, &gdu, 0, 0))
1663         {
1664             int x;
1665             int err = odr_geterrorx(c->odr_in, &x);
1666             char msg[100];
1667             const char *element = odr_getelement(c->odr_in);
1668             yaz_snprintf(msg, sizeof(msg),
1669                     "ODR code %d:%d element=%s offset=%d",
1670                     err, x, element ? element : "<unknown>",
1671                     odr_offset(c->odr_in));
1672             ZOOM_set_error(c, ZOOM_ERROR_DECODE, msg);
1673             if (c->log_api)
1674             {
1675                 FILE *ber_file = yaz_log_file();
1676                 if (ber_file)
1677                     odr_dumpBER(ber_file, c->buf_in, r);
1678             }
1679             ZOOM_connection_close(c);
1680         }
1681         else
1682         {
1683             if (c->odr_print)
1684                 z_GDU(c->odr_print, &gdu, 0, 0);
1685             if (c->odr_save)
1686                 z_GDU(c->odr_save, &gdu, 0, 0);
1687             if (gdu->which == Z_GDU_Z3950)
1688                 ZOOM_handle_Z3950_apdu(c, gdu->u.z3950);
1689             else if (gdu->which == Z_GDU_HTTP_Response)
1690             {
1691 #if YAZ_HAVE_XML2
1692                 handle_http(c, gdu->u.HTTP_Response);
1693 #else
1694                 ZOOM_set_error(c, ZOOM_ERROR_DECODE, 0);
1695                 ZOOM_connection_close(c);
1696 #endif
1697             }
1698         }
1699     }
1700     return 1;
1701 }
1702
1703 static zoom_ret do_write_ex(ZOOM_connection c, char *buf_out, int len_out)
1704 {
1705     int r;
1706     ZOOM_Event event;
1707
1708     event = ZOOM_Event_create(ZOOM_EVENT_SEND_DATA);
1709     ZOOM_connection_put_event(c, event);
1710
1711     yaz_log(c->log_details, "%p do_write_ex len=%d", c, len_out);
1712     if ((r = cs_put(c->cs, buf_out, len_out)) < 0)
1713     {
1714         yaz_log(c->log_details, "%p do_write_ex write failed", c);
1715         if (ZOOM_test_reconnect(c))
1716         {
1717             return zoom_pending;
1718         }
1719         if (c->state == STATE_CONNECTING)
1720             ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
1721         else
1722             ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
1723         ZOOM_connection_close(c);
1724         return zoom_complete;
1725     }
1726     else if (r == 1)
1727     {
1728         int mask = ZOOM_SELECT_EXCEPT;
1729         if (c->cs->io_pending & CS_WANT_WRITE)
1730             mask += ZOOM_SELECT_WRITE;
1731         if (c->cs->io_pending & CS_WANT_READ)
1732             mask += ZOOM_SELECT_READ;
1733         ZOOM_connection_set_mask(c, mask);
1734         yaz_log(c->log_details, "%p do_write_ex write incomplete mask=%d",
1735                 c, c->mask);
1736     }
1737     else
1738     {
1739         ZOOM_connection_set_mask(c, ZOOM_SELECT_READ|ZOOM_SELECT_EXCEPT);
1740         yaz_log(c->log_details, "%p do_write_ex write complete mask=%d",
1741                 c, c->mask);
1742     }
1743     return zoom_pending;
1744 }
1745
1746 zoom_ret ZOOM_send_buf(ZOOM_connection c)
1747 {
1748     return do_write_ex(c, c->buf_out, c->len_out);
1749 }
1750
1751
1752 ZOOM_API(const char *)
1753     ZOOM_connection_option_get(ZOOM_connection c, const char *key)
1754 {
1755     if (!strcmp(key, "APDU"))
1756     {
1757         return c->saveAPDU_wrbuf ? wrbuf_cstr(c->saveAPDU_wrbuf) : "";
1758     }
1759     else
1760         return ZOOM_options_get(c->options, key);
1761 }
1762
1763 ZOOM_API(const char *)
1764     ZOOM_connection_option_getl(ZOOM_connection c, const char *key, int *lenp)
1765 {
1766     if (!strcmp(key, "APDU"))
1767     {
1768         if (c->saveAPDU_wrbuf)
1769         {
1770             *lenp = wrbuf_len(c->saveAPDU_wrbuf);
1771             return wrbuf_cstr(c->saveAPDU_wrbuf);
1772         }
1773         else
1774         {
1775             *lenp = 0;
1776             return "";
1777         }
1778     }
1779     else
1780         return ZOOM_options_getl(c->options, key, lenp);
1781 }
1782
1783 ZOOM_API(void)
1784     ZOOM_connection_option_set(ZOOM_connection c, const char *key,
1785                                const char *val)
1786 {
1787     if (!strcmp(key, "saveAPDU"))
1788     {
1789         if (val && strcmp(val, "0"))
1790         {
1791             if (!c->saveAPDU_wrbuf)
1792                 c->saveAPDU_wrbuf = wrbuf_alloc();
1793             else
1794                 wrbuf_rewind(c->saveAPDU_wrbuf);
1795         }
1796         else
1797         {
1798             wrbuf_destroy(c->saveAPDU_wrbuf);
1799             c->saveAPDU_wrbuf = 0;
1800         }
1801         ZOOM_connection_save_apdu_wrbuf(c, c->saveAPDU_wrbuf);
1802     }
1803     else
1804         ZOOM_options_set(c->options, key, val);
1805 }
1806
1807 ZOOM_API(void)
1808     ZOOM_connection_option_setl(ZOOM_connection c, const char *key,
1809                                 const char *val, int len)
1810 {
1811     ZOOM_options_setl(c->options, key, val, len);
1812 }
1813
1814 ZOOM_API(const char *)
1815     ZOOM_resultset_option_get(ZOOM_resultset r, const char *key)
1816 {
1817     return ZOOM_options_get(r->options, key);
1818 }
1819
1820 ZOOM_API(void)
1821     ZOOM_resultset_option_set(ZOOM_resultset r, const char *key,
1822                               const char *val)
1823 {
1824     ZOOM_options_set(r->options, key, val);
1825 }
1826
1827
1828 ZOOM_API(int)
1829     ZOOM_connection_errcode(ZOOM_connection c)
1830 {
1831     return ZOOM_connection_error(c, 0, 0);
1832 }
1833
1834 ZOOM_API(const char *)
1835     ZOOM_connection_errmsg(ZOOM_connection c)
1836 {
1837     const char *msg;
1838     ZOOM_connection_error(c, &msg, 0);
1839     return msg;
1840 }
1841
1842 ZOOM_API(const char *)
1843     ZOOM_connection_addinfo(ZOOM_connection c)
1844 {
1845     const char *addinfo;
1846     ZOOM_connection_error(c, 0, &addinfo);
1847     return addinfo;
1848 }
1849
1850 ZOOM_API(const char *)
1851     ZOOM_connection_diagset(ZOOM_connection c)
1852 {
1853     const char *diagset;
1854     ZOOM_connection_error_x(c, 0, 0, &diagset);
1855     return diagset;
1856 }
1857
1858 ZOOM_API(const char *)
1859     ZOOM_diag_str(int error)
1860 {
1861     switch (error)
1862     {
1863     case ZOOM_ERROR_NONE:
1864         return "No error";
1865     case ZOOM_ERROR_CONNECT:
1866         return "Connect failed";
1867     case ZOOM_ERROR_MEMORY:
1868         return "Out of memory";
1869     case ZOOM_ERROR_ENCODE:
1870         return "Encoding failed";
1871     case ZOOM_ERROR_DECODE:
1872         return "Decoding failed";
1873     case ZOOM_ERROR_CONNECTION_LOST:
1874         return "Connection lost";
1875     case ZOOM_ERROR_INIT:
1876         return "Init rejected";
1877     case ZOOM_ERROR_INTERNAL:
1878         return "Internal failure";
1879     case ZOOM_ERROR_TIMEOUT:
1880         return "Timeout";
1881     case ZOOM_ERROR_UNSUPPORTED_PROTOCOL:
1882         return "Unsupported protocol";
1883     case ZOOM_ERROR_UNSUPPORTED_QUERY:
1884         return "Unsupported query type";
1885     case ZOOM_ERROR_INVALID_QUERY:
1886         return "Invalid query";
1887     case ZOOM_ERROR_CQL_PARSE:
1888         return "CQL parsing error";
1889     case ZOOM_ERROR_CQL_TRANSFORM:
1890         return "CQL transformation error";
1891     case ZOOM_ERROR_CCL_CONFIG:
1892         return "CCL configuration error";
1893     case ZOOM_ERROR_CCL_PARSE:
1894         return "CCL parsing error";
1895     case ZOOM_ERROR_ES_INVALID_ACTION:
1896         return "Extended Service. invalid action";
1897     case ZOOM_ERROR_ES_INVALID_VERSION:
1898         return "Extended Service. invalid version";
1899     case ZOOM_ERROR_ES_INVALID_SYNTAX:
1900         return "Extended Service. invalid syntax";
1901     default:
1902         return diagbib1_str(error);
1903     }
1904 }
1905
1906 ZOOM_API(int)
1907     ZOOM_connection_error_x(ZOOM_connection c, const char **cp,
1908                             const char **addinfo, const char **diagset)
1909 {
1910     int error = c->error;
1911     if (cp)
1912     {
1913         if (!c->diagset || !strcmp(c->diagset, "ZOOM"))
1914             *cp = ZOOM_diag_str(error);
1915         else if (!strcmp(c->diagset, "HTTP"))
1916             *cp = z_HTTP_errmsg(c->error);
1917         else if (!strcmp(c->diagset, "Bib-1"))
1918             *cp = ZOOM_diag_str(error);
1919         else if (!strcmp(c->diagset, "info:srw/diagnostic/1"))
1920             *cp = yaz_diag_srw_str(c->error);
1921         else
1922             *cp = "Unknown error and diagnostic set";
1923     }
1924     if (addinfo)
1925         *addinfo = c->addinfo ? c->addinfo : "";
1926     if (diagset)
1927         *diagset = c->diagset ? c->diagset : "";
1928     return c->error;
1929 }
1930
1931 ZOOM_API(int)
1932     ZOOM_connection_error(ZOOM_connection c, const char **cp,
1933                           const char **addinfo)
1934 {
1935     return ZOOM_connection_error_x(c, cp, addinfo, 0);
1936 }
1937
1938 static void ZOOM_connection_do_io(ZOOM_connection c, int mask)
1939 {
1940     ZOOM_Event event = 0;
1941     int r = cs_look(c->cs);
1942     yaz_log(c->log_details, "%p ZOOM_connection_do_io mask=%d cs_look=%d",
1943             c, mask, r);
1944
1945     if (r == CS_NONE)
1946     {
1947         event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1948         ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
1949         ZOOM_connection_close(c);
1950         ZOOM_connection_put_event(c, event);
1951     }
1952     else if (r == CS_CONNECT)
1953     {
1954         int ret = ret = cs_rcvconnect(c->cs);
1955         yaz_log(c->log_details, "%p ZOOM_connection_do_io "
1956                 "cs_rcvconnect returned %d", c, ret);
1957         if (ret == 1)
1958         {
1959             int mask = ZOOM_SELECT_EXCEPT;
1960             if (c->cs->io_pending & CS_WANT_WRITE)
1961                 mask += ZOOM_SELECT_WRITE;
1962             if (c->cs->io_pending & CS_WANT_READ)
1963                 mask += ZOOM_SELECT_READ;
1964             ZOOM_connection_set_mask(c, mask);
1965             event = ZOOM_Event_create(ZOOM_EVENT_NONE);
1966             ZOOM_connection_put_event(c, event);
1967         }
1968         else if (ret == 0)
1969         {
1970             event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1971             ZOOM_connection_put_event(c, event);
1972             get_cert(c);
1973             if (c->proto == PROTO_Z3950)
1974                 ZOOM_connection_Z3950_send_init(c);
1975             else
1976             {
1977                 /* no init request for SRW .. */
1978                 assert(c->tasks->which == ZOOM_TASK_CONNECT);
1979                 ZOOM_connection_remove_task(c);
1980                 ZOOM_connection_set_mask(c, 0);
1981                 ZOOM_connection_exec_task(c);
1982             }
1983             c->state = STATE_ESTABLISHED;
1984         }
1985         else
1986         {
1987             ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
1988             ZOOM_connection_close(c);
1989         }
1990     }
1991     else
1992     {
1993         if (mask & ZOOM_SELECT_EXCEPT)
1994         {
1995             if (!ZOOM_test_reconnect(c))
1996             {
1997                 ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
1998                 ZOOM_connection_close(c);
1999             }
2000             return;
2001         }
2002         if (mask & ZOOM_SELECT_READ)
2003             do_read(c);
2004         if (c->cs && (mask & ZOOM_SELECT_WRITE))
2005             ZOOM_send_buf(c);
2006     }
2007 }
2008
2009 ZOOM_API(int)
2010     ZOOM_connection_last_event(ZOOM_connection cs)
2011 {
2012     if (!cs)
2013         return ZOOM_EVENT_NONE;
2014     return cs->last_event;
2015 }
2016
2017
2018 ZOOM_API(int) ZOOM_connection_fire_event_timeout(ZOOM_connection c)
2019 {
2020     if (c->mask)
2021     {
2022         ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_TIMEOUT);
2023         /* timeout and this connection was waiting */
2024         ZOOM_set_error(c, ZOOM_ERROR_TIMEOUT, 0);
2025         ZOOM_connection_close(c);
2026         ZOOM_connection_put_event(c, event);
2027     }
2028     return 0;
2029 }
2030
2031 ZOOM_API(int)
2032     ZOOM_connection_process(ZOOM_connection c)
2033 {
2034     ZOOM_Event event;
2035     if (!c)
2036         return 0;
2037
2038     event = ZOOM_connection_get_event(c);
2039     if (event)
2040     {
2041         ZOOM_Event_destroy(event);
2042         return 1;
2043     }
2044     ZOOM_connection_exec_task(c);
2045     event = ZOOM_connection_get_event(c);
2046     if (event)
2047     {
2048         ZOOM_Event_destroy(event);
2049         return 1;
2050     }
2051     return 0;
2052 }
2053
2054 ZOOM_API(int)
2055     ZOOM_event_nonblock(int no, ZOOM_connection *cs)
2056 {
2057     int i;
2058
2059     yaz_log(log_details0, "ZOOM_process_event(no=%d,cs=%p)", no, cs);
2060
2061     for (i = 0; i<no; i++)
2062     {
2063         ZOOM_connection c = cs[i];
2064
2065         if (c && ZOOM_connection_process(c))
2066             return i+1;
2067     }
2068     return 0;
2069 }
2070
2071 ZOOM_API(int) ZOOM_connection_fire_event_socket(ZOOM_connection c, int mask)
2072 {
2073     if (c->mask && mask)
2074         ZOOM_connection_do_io(c, mask);
2075     return 0;
2076 }
2077
2078 ZOOM_API(int) ZOOM_connection_get_socket(ZOOM_connection c)
2079 {
2080     if (c->cs)
2081         return cs_fileno(c->cs);
2082     return -1;
2083 }
2084
2085 ZOOM_API(int) ZOOM_connection_set_mask(ZOOM_connection c, int mask)
2086 {
2087     c->mask = mask;
2088     if (!c->cs)
2089         return -1;
2090     return 0;
2091 }
2092
2093 ZOOM_API(int) ZOOM_connection_get_mask(ZOOM_connection c)
2094 {
2095     if (c->cs)
2096         return c->mask;
2097     return 0;
2098 }
2099
2100 ZOOM_API(int) ZOOM_connection_get_timeout(ZOOM_connection c)
2101 {
2102     return ZOOM_options_get_int(c->options, "timeout", 30);
2103 }
2104
2105 ZOOM_API(void) ZOOM_connection_close(ZOOM_connection c)
2106 {
2107     if (c->cs)
2108         cs_close(c->cs);
2109     c->cs = 0;
2110     ZOOM_connection_set_mask(c, 0);
2111     c->state = STATE_IDLE;
2112 }
2113
2114 /*
2115  * Local variables:
2116  * c-basic-offset: 4
2117  * c-file-style: "Stroustrup"
2118  * indent-tabs-mode: nil
2119  * End:
2120  * vim: shiftwidth=4 tabstop=8 expandtab
2121  */
2122