005e7241c8ff6c8f3370d9232f8a8d4c3af0b278
[yaz-moved-to-github.git] / src / zoom-c.c
1 /* This file is part of the YAZ toolkit.
2  * Copyright (C) 1995-2011 Index Data
3  * See the file LICENSE for details.
4  */
5 /**
6  * \file zoom-c.c
7  * \brief Implements ZOOM C interface.
8  */
9 #if HAVE_CONFIG_H
10 #include <config.h>
11 #endif
12
13 #include <assert.h>
14 #include <string.h>
15 #include <errno.h>
16 #include "zoom-p.h"
17
18 #include <yaz/yaz-util.h>
19 #include <yaz/xmalloc.h>
20 #include <yaz/otherinfo.h>
21 #include <yaz/log.h>
22 #include <yaz/diagbib1.h>
23 #include <yaz/charneg.h>
24 #include <yaz/query-charset.h>
25 #include <yaz/snprintf.h>
26 #include <yaz/facet.h>
27
28 #include <yaz/shptr.h>
29
30 #if SHPTR
31 YAZ_SHPTR_TYPE(WRBUF)
32 #endif
33
34 static int log_api0 = 0;
35 static int log_details0 = 0;
36
37 static void resultset_destroy(ZOOM_resultset r);
38 static zoom_ret do_write_ex(ZOOM_connection c, char *buf_out, int len_out);
39
40 static void initlog(void)
41 {
42     static int log_level_initialized = 0;
43
44     if (!log_level_initialized)
45     {
46         log_api0 = yaz_log_module_level("zoom");
47         log_details0 = yaz_log_module_level("zoomdetails");
48         log_level_initialized = 1;
49     }
50 }
51
52 void ZOOM_connection_remove_tasks(ZOOM_connection c);
53
54 void ZOOM_set_dset_error(ZOOM_connection c, int error,
55                          const char *dset,
56                          const char *addinfo, const char *addinfo2)
57 {
58     char *cp;
59
60     xfree(c->addinfo);
61     c->addinfo = 0;
62     c->error = error;
63     if (!c->diagset || strcmp(dset, c->diagset))
64     {
65         xfree(c->diagset);
66         c->diagset = xstrdup(dset);
67         /* remove integer part from SRW diagset .. */
68         if ((cp = strrchr(c->diagset, '/')))
69             *cp = '\0';
70     }
71     if (addinfo && addinfo2)
72     {
73         c->addinfo = (char*) xmalloc(strlen(addinfo) + strlen(addinfo2) + 2);
74         strcpy(c->addinfo, addinfo);
75         strcat(c->addinfo, addinfo2);
76     }
77     else if (addinfo)
78         c->addinfo = xstrdup(addinfo);
79     if (error != ZOOM_ERROR_NONE)
80     {
81         yaz_log(c->log_api, "%p set_dset_error %s %s:%d %s %s",
82                 c, c->host_port ? c->host_port : "<>", dset, error,
83                 addinfo ? addinfo : "",
84                 addinfo2 ? addinfo2 : "");
85         ZOOM_connection_remove_tasks(c);
86     }
87 }
88
89 int ZOOM_uri_to_code(const char *uri)
90 {
91     int code = 0;       
92     const char *cp;
93     if ((cp = strrchr(uri, '/')))
94         code = atoi(cp+1);
95     return code;
96 }
97
98
99
100 void ZOOM_set_error(ZOOM_connection c, int error, const char *addinfo)
101 {
102     ZOOM_set_dset_error(c, error, "ZOOM", addinfo, 0);
103 }
104
105 static void clear_error(ZOOM_connection c)
106 {
107     /*
108      * If an error is tied to an operation then it's ok to clear: for
109      * example, a diagnostic returned from a search is cleared by a
110      * subsequent search.  However, problems such as Connection Lost
111      * or Init Refused are not cleared, because they are not
112      * recoverable: doing another search doesn't help.
113      */
114
115     ZOOM_connection_remove_events(c);
116     switch (c->error)
117     {
118     case ZOOM_ERROR_CONNECT:
119     case ZOOM_ERROR_MEMORY:
120     case ZOOM_ERROR_DECODE:
121     case ZOOM_ERROR_CONNECTION_LOST:
122     case ZOOM_ERROR_INIT:
123     case ZOOM_ERROR_INTERNAL:
124     case ZOOM_ERROR_UNSUPPORTED_PROTOCOL:
125         break;
126     default:
127         ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
128     }
129 }
130
131 void ZOOM_connection_show_task(ZOOM_task task)
132 {
133     switch(task->which)
134     {
135     case ZOOM_TASK_SEARCH:
136         yaz_log(YLOG_LOG, "search p=%p", task);
137         break;
138     case ZOOM_TASK_RETRIEVE:
139         yaz_log(YLOG_LOG, "retrieve p=%p", task);
140         break;
141     case ZOOM_TASK_CONNECT:
142         yaz_log(YLOG_LOG, "connect p=%p", task);
143         break;
144     case ZOOM_TASK_SCAN:
145         yaz_log(YLOG_LOG, "scan p=%p", task);
146         break;
147     }
148 }
149
150 void ZOOM_connection_show_tasks(ZOOM_connection c)
151 {
152     ZOOM_task task;
153     yaz_log(YLOG_LOG, "connection p=%p tasks", c);
154     for (task = c->tasks; task; task = task->next)
155         ZOOM_connection_show_task(task);
156 }
157
158 ZOOM_task ZOOM_connection_add_task(ZOOM_connection c, int which)
159 {
160     ZOOM_task *taskp = &c->tasks;
161     while (*taskp)
162         taskp = &(*taskp)->next;
163     *taskp = (ZOOM_task) xmalloc(sizeof(**taskp));
164     (*taskp)->running = 0;
165     (*taskp)->which = which;
166     (*taskp)->next = 0;
167     clear_error(c);
168     return *taskp;
169 }
170
171 ZOOM_API(int) ZOOM_connection_is_idle(ZOOM_connection c)
172 {
173     return c->tasks ? 0 : 1;
174 }
175
176 ZOOM_task ZOOM_connection_insert_task(ZOOM_connection c, int which)
177 {
178     ZOOM_task task = (ZOOM_task) xmalloc(sizeof(*task));
179
180     task->next = c->tasks;
181     c->tasks = task;
182
183     task->running = 0;
184     task->which = which;
185     return task;
186 }
187
188 void ZOOM_connection_remove_task(ZOOM_connection c)
189 {
190     ZOOM_task task = c->tasks;
191
192     if (task)
193     {
194         c->tasks = task->next;
195         switch (task->which)
196         {
197         case ZOOM_TASK_SEARCH:
198             resultset_destroy(task->u.search.resultset);
199             xfree(task->u.search.syntax);
200             xfree(task->u.search.elementSetName);
201             break;
202         case ZOOM_TASK_RETRIEVE:
203             resultset_destroy(task->u.retrieve.resultset);
204             xfree(task->u.retrieve.syntax);
205             xfree(task->u.retrieve.elementSetName);
206             break;
207         case ZOOM_TASK_CONNECT:
208             break;
209         case ZOOM_TASK_SCAN:
210             ZOOM_scanset_destroy(task->u.scan.scan);
211             break;
212         case ZOOM_TASK_PACKAGE:
213             ZOOM_package_destroy(task->u.package);
214             break;
215         case ZOOM_TASK_SORT:
216             resultset_destroy(task->u.sort.resultset);
217             ZOOM_query_destroy(task->u.sort.q);
218             break;
219         default:
220             assert(0);
221         }
222         xfree(task);
223
224         if (!c->tasks)
225         {
226             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_END);
227             ZOOM_connection_put_event(c, event);
228         }
229     }
230 }
231
232 void ZOOM_connection_remove_tasks(ZOOM_connection c)
233 {
234     while (c->tasks)
235         ZOOM_connection_remove_task(c);
236 }
237
238
239 ZOOM_API(ZOOM_connection)
240     ZOOM_connection_create(ZOOM_options options)
241 {
242     ZOOM_connection c = (ZOOM_connection) xmalloc(sizeof(*c));
243
244     initlog();
245
246     c->log_api = log_api0;
247     c->log_details = log_details0;
248
249     yaz_log(c->log_api, "%p ZOOM_connection_create", c);
250
251     c->proto = PROTO_Z3950;
252     c->cs = 0;
253     ZOOM_connection_set_mask(c, 0);
254     c->reconnect_ok = 0;
255     c->state = STATE_IDLE;
256     c->addinfo = 0;
257     c->diagset = 0;
258     ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
259     c->buf_in = 0;
260     c->len_in = 0;
261     c->buf_out = 0;
262     c->len_out = 0;
263     c->resultsets = 0;
264
265     c->options = ZOOM_options_create_with_parent(options);
266
267     c->host_port = 0;
268     c->path = 0;
269     c->proxy = 0;
270     
271     c->charset = c->lang = 0;
272
273     c->cookie_out = 0;
274     c->cookie_in = 0;
275     c->client_IP = 0;
276     c->tasks = 0;
277
278     c->user = 0;
279     c->group = 0;
280     c->password = 0;
281
282     c->maximum_record_size = 0;
283     c->preferred_message_size = 0;
284
285     c->odr_in = odr_createmem(ODR_DECODE);
286     c->odr_out = odr_createmem(ODR_ENCODE);
287     c->odr_print = 0;
288
289     c->async = 0;
290     c->support_named_resultsets = 0;
291     c->last_event = ZOOM_EVENT_NONE;
292
293     c->m_queue_front = 0;
294     c->m_queue_back = 0;
295
296     c->sru_version = 0;
297     c->no_redirects = 0;
298     return c;
299 }
300
301
302 /* set database names. Take local databases (if set); otherwise
303    take databases given in ZURL (if set); otherwise use Default */
304 char **ZOOM_connection_get_databases(ZOOM_connection con, ZOOM_options options,
305                                      int *num, ODR odr)
306 {
307     char **databaseNames;
308     const char *cp = ZOOM_options_get(options, "databaseName");
309     
310     if ((!cp || !*cp) && con->host_port)
311     {
312         if (strncmp(con->host_port, "unix:", 5) == 0)
313             cp = strchr(con->host_port+5, ':');
314         else
315             cp = strchr(con->host_port, '/');
316         if (cp)
317             cp++;
318     }
319     if (!cp)
320         cp = "Default";
321     nmem_strsplit(odr_getmem(odr), "+", cp,  &databaseNames, num);
322     return databaseNames;
323 }
324
325 ZOOM_API(ZOOM_connection)
326     ZOOM_connection_new(const char *host, int portnum)
327 {
328     ZOOM_connection c = ZOOM_connection_create(0);
329
330     ZOOM_connection_connect(c, host, portnum);
331     return c;
332 }
333
334 static zoom_sru_mode get_sru_mode_from_string(const char *s)
335 {
336     if (!s || !*s)
337         return zoom_sru_soap;
338     if (!yaz_matchstr(s, "soap"))
339         return zoom_sru_soap;
340     else if (!yaz_matchstr(s, "get"))
341         return zoom_sru_get;
342     else if (!yaz_matchstr(s, "post"))
343         return zoom_sru_post;
344     else if (!yaz_matchstr(s, "solr"))
345         return zoom_sru_solr;
346     return zoom_sru_error;
347 }
348
349 ZOOM_API(void)
350     ZOOM_connection_connect(ZOOM_connection c,
351                             const char *host, int portnum)
352 {
353     const char *val;
354     ZOOM_task task;
355
356     initlog();
357
358     yaz_log(c->log_api, "%p ZOOM_connection_connect host=%s portnum=%d",
359             c, host ? host : "null", portnum);
360
361     ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
362     ZOOM_connection_remove_tasks(c);
363
364     if (c->odr_print)
365     {
366         odr_setprint(c->odr_print, 0); /* prevent destroy from fclose'ing */
367         odr_destroy(c->odr_print);
368     }
369     if (ZOOM_options_get_bool(c->options, "apdulog", 0))
370     {
371         c->odr_print = odr_createmem(ODR_PRINT);
372         odr_setprint(c->odr_print, yaz_log_file());
373     }
374     else
375         c->odr_print = 0;
376
377     if (c->cs)
378     {
379         yaz_log(c->log_details, "%p ZOOM_connection_connect reconnect ok", c);
380         c->reconnect_ok = 1;
381         return;
382     }
383     yaz_log(c->log_details, "%p ZOOM_connection_connect connect", c);
384     xfree(c->proxy);
385     c->proxy = 0;
386     val = ZOOM_options_get(c->options, "proxy");
387     if (val && *val)
388     {
389         yaz_log(c->log_details, "%p ZOOM_connection_connect proxy=%s", c, val);
390         c->proxy = xstrdup(val);
391     }
392
393     xfree(c->charset);
394     c->charset = 0;
395     val = ZOOM_options_get(c->options, "charset");
396     if (val && *val)
397     {
398         yaz_log(c->log_details, "%p ZOOM_connection_connect charset=%s", c, val);
399         c->charset = xstrdup(val);
400     }
401
402     xfree(c->lang);
403     val = ZOOM_options_get(c->options, "lang");
404     if (val && *val)
405     {
406         yaz_log(c->log_details, "%p ZOOM_connection_connect lang=%s", c, val);
407         c->lang = xstrdup(val);
408     }
409     else
410         c->lang = 0;
411
412     if (host)
413     {
414         xfree(c->host_port);
415         if (portnum)
416         {
417             char hostn[128];
418             sprintf(hostn, "%.80s:%d", host, portnum);
419             c->host_port = xstrdup(hostn);
420         }
421         else
422             c->host_port = xstrdup(host);
423     }        
424
425     {
426         /*
427          * If the "<scheme>:" part of the host string is preceded by one
428          * or more comma-separated <name>=<value> pairs, these are taken
429          * to be options to be set on the connection object.  Among other
430          * applications, this facility can be used to embed authentication
431          * in a host string:
432          *          user=admin,password=secret,tcp:localhost:9999
433          */
434         char *remainder = c->host_port;
435         char *pcolon = strchr(remainder, ':');
436         char *pcomma;
437         char *pequals;
438         while ((pcomma = strchr(remainder, ',')) != 0 &&
439                (pcolon == 0 || pcomma < pcolon)) {
440             *pcomma = '\0';
441             if ((pequals = strchr(remainder, '=')) != 0) {
442                 *pequals = '\0';
443                 /*printf("# setting '%s'='%s'\n", remainder, pequals+1);*/
444                 ZOOM_connection_option_set(c, remainder, pequals+1);
445             }
446             remainder = pcomma+1;
447         }
448
449         if (remainder != c->host_port) {
450             xfree(c->host_port);
451             c->host_port = xstrdup(remainder);
452             /*printf("# reset hp='%s'\n", remainder);*/
453         }
454     }
455
456     val = ZOOM_options_get(c->options, "sru");
457     c->sru_mode = get_sru_mode_from_string(val);
458
459     xfree(c->sru_version);
460     val = ZOOM_options_get(c->options, "sru_version");
461     c->sru_version = xstrdup(val ? val : "1.2");
462
463     ZOOM_options_set(c->options, "host", c->host_port);
464
465     xfree(c->cookie_out);
466     c->cookie_out = 0;
467     val = ZOOM_options_get(c->options, "cookie");
468     if (val && *val)
469     { 
470         yaz_log(c->log_details, "%p ZOOM_connection_connect cookie=%s", c, val);
471         c->cookie_out = xstrdup(val);
472     }
473
474     xfree(c->client_IP);
475     c->client_IP = 0;
476     val = ZOOM_options_get(c->options, "clientIP");
477     if (val && *val)
478     {
479         yaz_log(c->log_details, "%p ZOOM_connection_connect clientIP=%s",
480                 c, val);
481         c->client_IP = xstrdup(val);
482     }
483
484     xfree(c->group);
485     c->group = 0;
486     val = ZOOM_options_get(c->options, "group");
487     if (val && *val)
488         c->group = xstrdup(val);
489
490     xfree(c->user);
491     c->user = 0;
492     val = ZOOM_options_get(c->options, "user");
493     if (val && *val)
494         c->user = xstrdup(val);
495
496     xfree(c->password);
497     c->password = 0;
498     val = ZOOM_options_get(c->options, "password");
499     if (!val)
500         val = ZOOM_options_get(c->options, "pass");
501
502     if (val && *val)
503         c->password = xstrdup(val);
504     
505     c->maximum_record_size =
506         ZOOM_options_get_int(c->options, "maximumRecordSize", 1024*1024);
507     c->preferred_message_size =
508         ZOOM_options_get_int(c->options, "preferredMessageSize", 1024*1024);
509
510     c->async = ZOOM_options_get_bool(c->options, "async", 0);
511     yaz_log(c->log_details, "%p ZOOM_connection_connect async=%d", c, c->async);
512  
513     task = ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
514
515     if (!c->async)
516     {
517         while (ZOOM_event(1, &c))
518             ;
519     }
520 }
521
522 ZOOM_API(void) ZOOM_resultset_release(ZOOM_resultset r)
523 {
524 #if ZOOM_RESULT_LISTS
525 #else
526     if (r->connection)
527     {
528         /* remove ourselves from the resultsets in connection */
529         ZOOM_resultset *rp = &r->connection->resultsets;
530         while (1)
531         {
532             assert(*rp);   /* we must be in this list!! */
533             if (*rp == r)
534             {   /* OK, we're here - take us out of it */
535                 *rp = (*rp)->next;
536                 break;
537             }
538             rp = &(*rp)->next;
539         }
540         r->connection = 0;
541     }
542 #endif
543 }
544
545 ZOOM_API(void)
546     ZOOM_connection_destroy(ZOOM_connection c)
547 {
548 #if ZOOM_RESULT_LISTS
549     ZOOM_resultsets list;
550 #else
551     ZOOM_resultset r;
552 #endif
553     if (!c)
554         return;
555     yaz_log(c->log_api, "%p ZOOM_connection_destroy", c);
556     if (c->cs)
557         cs_close(c->cs);
558
559 #if ZOOM_RESULT_LISTS
560     // Remove the connection's usage of resultsets
561     list = c->resultsets;
562     while (list) {
563         ZOOM_resultsets removed = list;
564         ZOOM_resultset_destroy(list->resultset);
565         list = list->next;
566         xfree(removed);
567     }
568 #else
569     for (r = c->resultsets; r; r = r->next)
570         r->connection = 0;
571 #endif
572
573     xfree(c->buf_in);
574     xfree(c->addinfo);
575     xfree(c->diagset);
576     odr_destroy(c->odr_in);
577     odr_destroy(c->odr_out);
578     if (c->odr_print)
579     {
580         odr_setprint(c->odr_print, 0); /* prevent destroy from fclose'ing */
581         odr_destroy(c->odr_print);
582     }
583     ZOOM_options_destroy(c->options);
584     ZOOM_connection_remove_tasks(c);
585     ZOOM_connection_remove_events(c);
586     xfree(c->host_port);
587     xfree(c->path);
588     xfree(c->proxy);
589     xfree(c->charset);
590     xfree(c->lang);
591     xfree(c->cookie_out);
592     xfree(c->cookie_in);
593     xfree(c->client_IP);
594     xfree(c->user);
595     xfree(c->group);
596     xfree(c->password);
597     xfree(c->sru_version);
598     xfree(c);
599 }
600
601 void ZOOM_resultset_addref(ZOOM_resultset r)
602 {
603     if (r)
604     {
605         yaz_mutex_enter(r->mutex);
606         (r->refcount)++;
607         yaz_log(log_details0, "%p ZOOM_resultset_addref count=%d",
608                 r, r->refcount);
609         yaz_mutex_leave(r->mutex);
610     }
611 }
612
613 static int g_resultsets = 0;
614 static YAZ_MUTEX g_resultset_mutex = 0;
615
616 /* TODO We need to initialize this before running threaded:
617  * call resultset_use(0)  */
618
619 static int resultset_use(int delta) {
620     int resultset_count;
621     if (g_resultset_mutex == 0)
622         yaz_mutex_create(&g_resultset_mutex);
623     yaz_mutex_enter(g_resultset_mutex);
624     g_resultsets += delta;
625     resultset_count = g_resultsets;
626     yaz_mutex_leave(g_resultset_mutex);
627     return resultset_count;
628 }
629
630 int resultsets_count(void) {
631     return resultset_use(0);
632 }
633
634 ZOOM_resultset ZOOM_resultset_create(void)
635 {
636     int i;
637     ZOOM_resultset r = (ZOOM_resultset) xmalloc(sizeof(*r));
638
639     initlog();
640
641     yaz_log(log_details0, "%p ZOOM_resultset_create", r);
642     r->refcount = 1;
643     r->size = 0;
644     r->odr = odr_createmem(ODR_ENCODE);
645     r->piggyback = 1;
646     r->setname = 0;
647     r->schema = 0;
648     r->step = 0;
649     for (i = 0; i<RECORD_HASH_SIZE; i++)
650         r->record_hash[i] = 0;
651     r->r_sort_spec = 0;
652     r->query = 0;
653     r->connection = 0;
654     r->databaseNames = 0;
655     r->num_databaseNames = 0;
656     r->facets = 0;
657     r->num_facets = 0;
658     r->facets_names = 0;
659     r->mutex = 0;
660     yaz_mutex_create(&r->mutex);
661 #if SHPTR
662     {
663         WRBUF w = wrbuf_alloc();
664         YAZ_SHPTR_INIT(r->record_wrbuf, w);
665     }
666 #endif
667     resultset_use(1);
668     return r;
669 }
670
671 ZOOM_API(ZOOM_resultset)
672     ZOOM_connection_search_pqf(ZOOM_connection c, const char *q)
673 {
674     ZOOM_resultset r;
675     ZOOM_query s = ZOOM_query_create();
676
677     ZOOM_query_prefix(s, q);
678
679     r = ZOOM_connection_search(c, s);
680     ZOOM_query_destroy(s);
681     return r;
682 }
683
684 ZOOM_API(ZOOM_resultset)
685     ZOOM_connection_search(ZOOM_connection c, ZOOM_query q)
686 {
687     ZOOM_resultset r = ZOOM_resultset_create();
688     ZOOM_task task;
689     const char *cp;
690     int start, count;
691     const char *syntax, *elementSetName;
692 #if ZOOM_RESULT_LISTS
693     ZOOM_resultsets set;
694 #endif
695
696     yaz_log(c->log_api, "%p ZOOM_connection_search set %p query %p", c, r, q);
697     r->r_sort_spec = ZOOM_query_get_sortspec(q);
698     r->query = q;
699
700     r->options = ZOOM_options_create_with_parent(c->options);
701     
702     start = ZOOM_options_get_int(r->options, "start", 0);
703     count = ZOOM_options_get_int(r->options, "count", 0);
704     {
705         /* If "presentChunk" is defined use that; otherwise "step" */
706         const char *cp = ZOOM_options_get(r->options, "presentChunk");
707         r->step = ZOOM_options_get_int(r->options,
708                                        (cp != 0 ? "presentChunk": "step"), 0);
709     }
710     r->piggyback = ZOOM_options_get_bool(r->options, "piggyback", 1);
711     cp = ZOOM_options_get(r->options, "setname");
712     if (cp)
713         r->setname = xstrdup(cp);
714     cp = ZOOM_options_get(r->options, "schema");
715     if (cp)
716         r->schema = xstrdup(cp);
717
718     r->databaseNames = ZOOM_connection_get_databases(c, c->options, &r->num_databaseNames,
719                                          r->odr);
720     
721     r->connection = c;
722
723 #if ZOOM_RESULT_LISTS
724     yaz_log(log_details, "%p ZOOM_connection_search: Adding new resultset (%p) to resultsets (%p) ", c, r, c->resultsets);
725     set = xmalloc(sizeof(*set));
726     ZOOM_resultset_addref(r);
727     set->resultset = r;
728     set->next = c->resultsets;
729     c->resultsets = set;
730 #else
731     r->next = c->resultsets;
732     c->resultsets = r;
733 #endif
734     if (c->host_port && c->proto == PROTO_HTTP)
735     {
736         if (!c->cs)
737         {
738             yaz_log(c->log_details, "ZOOM_connection_search: no comstack");
739             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
740         }
741         else
742         {
743             yaz_log(c->log_details, "ZOOM_connection_search: reconnect");
744             c->reconnect_ok = 1;
745         }
746     }
747
748     task = ZOOM_connection_add_task(c, ZOOM_TASK_SEARCH);
749     task->u.search.resultset = r;
750     task->u.search.start = start;
751     task->u.search.count = count;
752     task->u.search.recv_search_fired = 0;
753
754     syntax = ZOOM_options_get(r->options, "preferredRecordSyntax"); 
755     task->u.search.syntax = syntax ? xstrdup(syntax) : 0;
756     elementSetName = ZOOM_options_get(r->options, "elementSetName");
757     task->u.search.elementSetName = elementSetName 
758         ? xstrdup(elementSetName) : 0;
759    
760     ZOOM_resultset_addref(r);
761
762     ZOOM_query_addref(q);
763
764     if (!c->async)
765     {
766         while (ZOOM_event(1, &c))
767             ;
768     }
769     return r;
770 }
771
772 ZOOM_API(void)
773     ZOOM_resultset_sort(ZOOM_resultset r,
774                          const char *sort_type, const char *sort_spec)
775 {
776     (void) ZOOM_resultset_sort1(r, sort_type, sort_spec);
777 }
778
779 ZOOM_API(int)
780     ZOOM_resultset_sort1(ZOOM_resultset r,
781                          const char *sort_type, const char *sort_spec)
782 {
783     ZOOM_connection c = r->connection;
784     ZOOM_task task;
785     ZOOM_query newq;
786
787     newq = ZOOM_query_create();
788     if (ZOOM_query_sortby(newq, sort_spec) < 0)
789         return -1;
790
791     yaz_log(c->log_api, "%p ZOOM_resultset_sort r=%p sort_type=%s sort_spec=%s",
792             r, r, sort_type, sort_spec);
793     if (!c)
794         return 0;
795
796     if (c->host_port && c->proto == PROTO_HTTP)
797     {
798         if (!c->cs)
799         {
800             yaz_log(c->log_details, "%p ZOOM_resultset_sort: no comstack", r);
801             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
802         }
803         else
804         {
805             yaz_log(c->log_details, "%p ZOOM_resultset_sort: prepare reconnect",
806                     r);
807             c->reconnect_ok = 1;
808         }
809     }
810     
811     ZOOM_resultset_cache_reset(r);
812     task = ZOOM_connection_add_task(c, ZOOM_TASK_SORT);
813     task->u.sort.resultset = r;
814     task->u.sort.q = newq;
815
816     ZOOM_resultset_addref(r);  
817
818     if (!c->async)
819     {
820         while (ZOOM_event(1, &c))
821             ;
822     }
823
824     return 0;
825 }
826
827 ZOOM_API(void)
828     ZOOM_resultset_destroy(ZOOM_resultset r)
829 {
830     resultset_destroy(r);
831 }
832
833 static void resultset_destroy(ZOOM_resultset r)
834 {
835     if (!r)
836         return;
837     yaz_mutex_enter(r->mutex);
838     (r->refcount)--;
839     yaz_log(log_details0, "%p ZOOM_resultset_destroy r=%p count=%d",
840             r, r, r->refcount);
841     if (r->refcount == 0)
842     {
843         yaz_mutex_leave(r->mutex);
844
845         yaz_log(log_details0, "%p ZOOM_connection resultset_destroy: Deleting resultset (%p) ", r->connection, r);
846         ZOOM_resultset_cache_reset(r);
847         ZOOM_resultset_release(r);
848         ZOOM_query_destroy(r->query);
849         ZOOM_options_destroy(r->options);
850         odr_destroy(r->odr);
851         xfree(r->setname);
852         xfree(r->schema);
853         yaz_mutex_destroy(&r->mutex);
854 #if SHPTR
855         YAZ_SHPTR_DEC(r->record_wrbuf, wrbuf_destroy);
856 #endif
857         resultset_use(-1);
858         xfree(r);
859     }
860     else
861         yaz_mutex_leave(r->mutex);
862 }
863
864 ZOOM_API(size_t)
865     ZOOM_resultset_size(ZOOM_resultset r)
866 {
867     return r->size;
868 }
869
870 int ZOOM_test_reconnect(ZOOM_connection c)
871 {
872     ZOOM_Event event;
873
874     if (!c->reconnect_ok)
875         return 0;
876     ZOOM_connection_close(c);
877     c->reconnect_ok = 0;
878     c->tasks->running = 0;
879     ZOOM_connection_insert_task(c, ZOOM_TASK_CONNECT);
880
881     event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
882     ZOOM_connection_put_event(c, event);
883
884     return 1;
885 }
886
887 static void ZOOM_resultset_retrieve(ZOOM_resultset r,
888                                     int force_sync, int start, int count)
889 {
890     ZOOM_task task;
891     ZOOM_connection c;
892     const char *cp;
893     const char *syntax, *elementSetName;
894
895     if (!r)
896         return;
897     yaz_log(log_details0, "%p ZOOM_resultset_retrieve force_sync=%d start=%d"
898             " count=%d", r, force_sync, start, count);
899     c = r->connection;
900     if (!c)
901         return;
902
903     if (c->host_port && c->proto == PROTO_HTTP)
904     {
905         if (!c->cs)
906         {
907             yaz_log(log_details0, "%p ZOOM_resultset_retrieve: no comstack", r);
908             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
909         }
910         else
911         {
912             yaz_log(log_details0, "%p ZOOM_resultset_retrieve: prepare "
913                     "reconnect", r);
914             c->reconnect_ok = 1;
915         }
916     }
917     task = ZOOM_connection_add_task(c, ZOOM_TASK_RETRIEVE);
918     task->u.retrieve.resultset = r;
919     task->u.retrieve.start = start;
920     task->u.retrieve.count = count;
921
922     syntax = ZOOM_options_get(r->options, "preferredRecordSyntax"); 
923     task->u.retrieve.syntax = syntax ? xstrdup(syntax) : 0;
924     elementSetName = ZOOM_options_get(r->options, "elementSetName");
925     task->u.retrieve.elementSetName = elementSetName 
926         ? xstrdup(elementSetName) : 0;
927
928     cp = ZOOM_options_get(r->options, "schema");
929     if (cp)
930     {
931         if (!r->schema || strcmp(r->schema, cp))
932         {
933             xfree(r->schema);
934             r->schema = xstrdup(cp);
935         }
936     }
937
938     ZOOM_resultset_addref(r);
939
940     if (!r->connection->async || force_sync)
941         while (r->connection && ZOOM_event(1, &r->connection))
942             ;
943 }
944
945 ZOOM_API(void)
946     ZOOM_resultset_records(ZOOM_resultset r, ZOOM_record *recs,
947                            size_t start, size_t count)
948 {
949     int force_present = 0;
950
951     if (!r)
952         return ;
953     yaz_log(log_api0, "%p ZOOM_resultset_records r=%p start=%ld count=%ld",
954             r, r, (long) start, (long) count);
955     if (count && recs)
956         force_present = 1;
957     ZOOM_resultset_retrieve(r, force_present, start, count);
958     if (force_present)
959     {
960         size_t i;
961         for (i = 0; i< count; i++)
962             recs[i] = ZOOM_resultset_record_immediate(r, i+start);
963     }
964 }
965
966 ZOOM_API(size_t)
967     ZOOM_resultset_facets_size(ZOOM_resultset r) {
968     return r->num_facets;
969 }
970
971 ZOOM_API(ZOOM_facet_field)
972     ZOOM_resultset_get_facet_field(ZOOM_resultset r, const char *name) {
973     int num = r->num_facets;
974     ZOOM_facet_field *facets = r->facets;
975     int index;
976     for (index = 0; index < num; index++) {
977         if (!strcmp(facets[index]->facet_name, name)) {
978             return facets[index];
979         }
980     }
981     return 0;
982 }
983
984
985 ZOOM_API(ZOOM_facet_field *)
986     ZOOM_resultset_facets(ZOOM_resultset r)
987 {
988     return r->facets;
989 }
990
991 ZOOM_API(const char**)
992     ZOOM_resultset_facet_names(ZOOM_resultset r)
993 {
994     return (const char **) r->facets_names;
995 }
996
997 ZOOM_API(const char*)
998     ZOOM_facet_field_name(ZOOM_facet_field field)
999 {
1000     return field->facet_name;
1001 }
1002
1003 ZOOM_API(size_t)
1004     ZOOM_facet_field_term_count(ZOOM_facet_field field)
1005 {
1006     return field->num_terms;
1007 }
1008
1009 ZOOM_API(const char*)
1010     ZOOM_facet_field_get_term(ZOOM_facet_field field, size_t idx, int *freq) {
1011     *freq = field->facet_terms[idx].frequency;
1012     return field->facet_terms[idx].term;
1013 }
1014
1015
1016 static void get_cert(ZOOM_connection c)
1017 {
1018     char *cert_buf;
1019     int cert_len;
1020     
1021     if (cs_get_peer_certificate_x509(c->cs, &cert_buf, &cert_len))
1022     {
1023         ZOOM_connection_option_setl(c, "sslPeerCert",
1024                                     cert_buf, cert_len);
1025         xfree(cert_buf);
1026     }
1027 }
1028
1029 static zoom_ret do_connect_host(ZOOM_connection c,
1030                                 const char *effective_host,
1031                                 const char *logical_url);
1032
1033 static zoom_ret do_connect(ZOOM_connection c)
1034 {
1035     const char *effective_host;
1036
1037     if (c->proxy)
1038         effective_host = c->proxy;
1039     else
1040         effective_host = c->host_port;
1041     return do_connect_host(c, effective_host, c->host_port);
1042 }
1043
1044 static zoom_ret do_connect_host(ZOOM_connection c, const char *effective_host,
1045     const char *logical_url)
1046 {
1047     void *add;
1048
1049     yaz_log(c->log_details, "%p do_connect effective_host=%s", c, effective_host);
1050
1051     if (c->cs)
1052         cs_close(c->cs);
1053     c->cs = cs_create_host(effective_host, 0, &add);
1054
1055     if (c->cs && c->cs->protocol == PROTO_HTTP)
1056     {
1057 #if YAZ_HAVE_XML2
1058         if (logical_url)
1059         {
1060             const char *db = 0;
1061             
1062             c->proto = PROTO_HTTP;
1063             cs_get_host_args(logical_url, &db);
1064             xfree(c->path);
1065             
1066             c->path = xmalloc(strlen(db) * 3 + 2);
1067             yaz_encode_sru_dbpath_buf(c->path, db);
1068         }
1069 #else
1070         ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, "SRW");
1071         ZOOM_connection_close(c);
1072         return zoom_complete;
1073 #endif
1074     }
1075     if (c->cs)
1076     {
1077         int ret = cs_connect(c->cs, add);
1078         if (ret == 0)
1079         {
1080             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1081             ZOOM_connection_put_event(c, event);
1082             get_cert(c);
1083             if (c->proto == PROTO_Z3950)
1084                 ZOOM_connection_Z3950_send_init(c);
1085             else
1086             {
1087                 /* no init request for SRW .. */
1088                 assert(c->tasks->which == ZOOM_TASK_CONNECT);
1089                 ZOOM_connection_remove_task(c);
1090                 ZOOM_connection_set_mask(c, 0);
1091                 ZOOM_connection_exec_task(c);
1092             }
1093             c->state = STATE_ESTABLISHED;
1094             return zoom_pending;
1095         }
1096         else if (ret > 0)
1097         {
1098             int mask = ZOOM_SELECT_EXCEPT;
1099             if (c->cs->io_pending & CS_WANT_WRITE)
1100                 mask += ZOOM_SELECT_WRITE;
1101             if (c->cs->io_pending & CS_WANT_READ)
1102                 mask += ZOOM_SELECT_READ;
1103             ZOOM_connection_set_mask(c, mask);
1104             c->state = STATE_CONNECTING; 
1105             return zoom_pending;
1106         }
1107     }
1108     c->state = STATE_IDLE;
1109     ZOOM_set_error(c, ZOOM_ERROR_CONNECT, logical_url);
1110     return zoom_complete;
1111 }
1112
1113 /* returns 1 if PDU was sent OK (still pending )
1114    0 if PDU was not sent OK (nothing to wait for) 
1115 */
1116
1117 ZOOM_API(ZOOM_record)
1118     ZOOM_resultset_record_immediate(ZOOM_resultset s,size_t pos)
1119 {
1120     const char *syntax =
1121         ZOOM_options_get(s->options, "preferredRecordSyntax"); 
1122     const char *elementSetName =
1123         ZOOM_options_get(s->options, "elementSetName");
1124
1125     return ZOOM_record_cache_lookup(s, pos, syntax, elementSetName);
1126 }
1127
1128 ZOOM_API(ZOOM_record)
1129     ZOOM_resultset_record(ZOOM_resultset r, size_t pos)
1130 {
1131     ZOOM_record rec = ZOOM_resultset_record_immediate(r, pos);
1132
1133     if (!rec)
1134     {
1135         /*
1136          * MIKE: I think force_sync should always be zero, but I don't
1137          * want to make this change until I get the go-ahead from
1138          * Adam, in case something depends on the old synchronous
1139          * behaviour.
1140          */
1141         int force_sync = 1;
1142         if (getenv("ZOOM_RECORD_NO_FORCE_SYNC")) force_sync = 0;
1143         ZOOM_resultset_retrieve(r, force_sync, pos, 1);
1144         rec = ZOOM_resultset_record_immediate(r, pos);
1145     }
1146     return rec;
1147 }
1148
1149 ZOOM_API(ZOOM_scanset)
1150     ZOOM_connection_scan(ZOOM_connection c, const char *start)
1151 {
1152     ZOOM_scanset s;
1153     ZOOM_query q = ZOOM_query_create();
1154
1155     ZOOM_query_prefix(q, start);
1156
1157     s = ZOOM_connection_scan1(c, q);
1158     ZOOM_query_destroy(q);
1159     return s;
1160
1161 }
1162
1163 ZOOM_API(ZOOM_scanset)
1164     ZOOM_connection_scan1(ZOOM_connection c, ZOOM_query q)
1165 {
1166     ZOOM_scanset scan = 0;
1167     Z_Query *z_query = ZOOM_query_get_Z_Query(q);
1168
1169     if (!z_query)
1170         return 0;
1171     scan = (ZOOM_scanset) xmalloc(sizeof(*scan));
1172     scan->connection = c;
1173     scan->odr = odr_createmem(ODR_DECODE);
1174     scan->options = ZOOM_options_create_with_parent(c->options);
1175     scan->refcount = 1;
1176     scan->scan_response = 0;
1177     scan->srw_scan_response = 0;
1178
1179     scan->query = q;
1180     ZOOM_query_addref(q);
1181     scan->databaseNames = ZOOM_connection_get_databases(c, c->options,
1182                                             &scan->num_databaseNames,
1183                                             scan->odr);
1184
1185     if (1)
1186     {
1187         ZOOM_task task = ZOOM_connection_add_task(c, ZOOM_TASK_SCAN);
1188         task->u.scan.scan = scan;
1189         
1190         (scan->refcount)++;
1191         if (!c->async)
1192         {
1193             while (ZOOM_event(1, &c))
1194                 ;
1195         }
1196     }
1197     return scan;
1198 }
1199
1200 ZOOM_API(void)
1201     ZOOM_scanset_destroy(ZOOM_scanset scan)
1202 {
1203     if (!scan)
1204         return;
1205     (scan->refcount)--;
1206     if (scan->refcount == 0)
1207     {
1208         ZOOM_query_destroy(scan->query);
1209
1210         odr_destroy(scan->odr);
1211         
1212         ZOOM_options_destroy(scan->options);
1213         xfree(scan);
1214     }
1215 }
1216
1217 static zoom_ret send_package(ZOOM_connection c)
1218 {
1219     ZOOM_Event event;
1220
1221     yaz_log(c->log_details, "%p send_package", c);
1222     if (!c->tasks)
1223         return zoom_complete;
1224     assert (c->tasks->which == ZOOM_TASK_PACKAGE);
1225     
1226     event = ZOOM_Event_create(ZOOM_EVENT_SEND_APDU);
1227     ZOOM_connection_put_event(c, event);
1228     
1229     c->buf_out = c->tasks->u.package->buf_out;
1230     c->len_out = c->tasks->u.package->len_out;
1231
1232     return ZOOM_send_buf(c);
1233 }
1234
1235 ZOOM_API(size_t)
1236     ZOOM_scanset_size(ZOOM_scanset scan)
1237 {
1238     if (!scan)
1239         return 0;
1240
1241     if (scan->scan_response && scan->scan_response->entries)
1242         return scan->scan_response->entries->num_entries;
1243     else if (scan->srw_scan_response)
1244         return scan->srw_scan_response->num_terms;
1245     return 0;
1246 }
1247
1248 static void ZOOM_scanset_term_x(ZOOM_scanset scan, size_t pos,
1249                                 size_t *occ,
1250                                 const char **value_term, size_t *value_len,
1251                                 const char **disp_term, size_t *disp_len)
1252 {
1253     size_t noent = ZOOM_scanset_size(scan);
1254     
1255     *value_term = 0;
1256     *value_len = 0;
1257
1258     *disp_term = 0;
1259     *disp_len = 0;
1260
1261     *occ = 0;
1262     if (pos >= noent)
1263         return;
1264     if (scan->scan_response)
1265     {
1266         Z_ScanResponse *res = scan->scan_response;
1267         if (res->entries->entries[pos]->which == Z_Entry_termInfo)
1268         {
1269             Z_TermInfo *t = res->entries->entries[pos]->u.termInfo;
1270             
1271             *value_term = (const char *) t->term->u.general->buf;
1272             *value_len = t->term->u.general->len;
1273             if (t->displayTerm)
1274             {
1275                 *disp_term = t->displayTerm;
1276                 *disp_len = strlen(*disp_term);
1277             }
1278             else if (t->term->which == Z_Term_general)
1279             {
1280                 *disp_term = (const char *) t->term->u.general->buf;
1281                 *disp_len = t->term->u.general->len;
1282             }
1283             *occ = t->globalOccurrences ? *t->globalOccurrences : 0;
1284         }
1285     }
1286     if (scan->srw_scan_response)
1287     {
1288         Z_SRW_scanResponse *res = scan->srw_scan_response;
1289         Z_SRW_scanTerm *t = res->terms + pos;
1290         if (t)
1291         {
1292             *value_term = t->value;
1293             *value_len = strlen(*value_term);
1294
1295             if (t->displayTerm)
1296                 *disp_term = t->displayTerm;
1297             else
1298                 *disp_term = t->value;
1299             *disp_len = strlen(*disp_term);
1300             *occ = t->numberOfRecords ? *t->numberOfRecords : 0;
1301         }
1302     }
1303 }
1304
1305 ZOOM_API(const char *)
1306     ZOOM_scanset_term(ZOOM_scanset scan, size_t pos,
1307                       size_t *occ, size_t *len)
1308 {
1309     const char *value_term = 0;
1310     size_t value_len = 0;
1311     const char *disp_term = 0;
1312     size_t disp_len = 0;
1313
1314     ZOOM_scanset_term_x(scan, pos, occ, &value_term, &value_len,
1315                         &disp_term, &disp_len);
1316     
1317     *len = value_len;
1318     return value_term;
1319 }
1320
1321 ZOOM_API(const char *)
1322     ZOOM_scanset_display_term(ZOOM_scanset scan, size_t pos,
1323                               size_t *occ, size_t *len)
1324 {
1325     const char *value_term = 0;
1326     size_t value_len = 0;
1327     const char *disp_term = 0;
1328     size_t disp_len = 0;
1329
1330     ZOOM_scanset_term_x(scan, pos, occ, &value_term, &value_len,
1331                         &disp_term, &disp_len);
1332     
1333     *len = disp_len;
1334     return disp_term;
1335 }
1336
1337 ZOOM_API(const char *)
1338     ZOOM_scanset_option_get(ZOOM_scanset scan, const char *key)
1339 {
1340     return ZOOM_options_get(scan->options, key);
1341 }
1342
1343 ZOOM_API(void)
1344     ZOOM_scanset_option_set(ZOOM_scanset scan, const char *key,
1345                             const char *val)
1346 {
1347     ZOOM_options_set(scan->options, key, val);
1348 }
1349
1350
1351 ZOOM_API(ZOOM_package)
1352     ZOOM_connection_package(ZOOM_connection c, ZOOM_options options)
1353 {
1354     ZOOM_package p = (ZOOM_package) xmalloc(sizeof(*p));
1355
1356     p->connection = c;
1357     p->odr_out = odr_createmem(ODR_ENCODE);
1358     p->options = ZOOM_options_create_with_parent2(options, c->options);
1359     p->refcount = 1;
1360     p->buf_out = 0;
1361     p->len_out = 0;
1362     return p;
1363 }
1364
1365 ZOOM_API(void)
1366     ZOOM_package_destroy(ZOOM_package p)
1367 {
1368     if (!p)
1369         return;
1370     (p->refcount)--;
1371     if (p->refcount == 0)
1372     {
1373         odr_destroy(p->odr_out);
1374         xfree(p->buf_out);
1375         
1376         ZOOM_options_destroy(p->options);
1377         xfree(p);
1378     }
1379 }
1380
1381 ZOOM_API(const char *)
1382     ZOOM_package_option_get(ZOOM_package p, const char *key)
1383 {
1384     return ZOOM_options_get(p->options, key);
1385 }
1386
1387 ZOOM_API(const char *)
1388     ZOOM_package_option_getl(ZOOM_package p, const char *key, int *lenp)
1389 {
1390     return ZOOM_options_getl(p->options, key, lenp);
1391 }
1392
1393 ZOOM_API(void)
1394     ZOOM_package_option_set(ZOOM_package p, const char *key,
1395                             const char *val)
1396 {
1397     ZOOM_options_set(p->options, key, val);
1398 }
1399
1400 ZOOM_API(void)
1401     ZOOM_package_option_setl(ZOOM_package p, const char *key,
1402                              const char *val, int len)
1403 {
1404     ZOOM_options_setl(p->options, key, val, len);
1405 }
1406
1407 ZOOM_API(int)
1408     ZOOM_connection_exec_task(ZOOM_connection c)
1409 {
1410     ZOOM_task task = c->tasks;
1411     zoom_ret ret = zoom_complete;
1412
1413     if (!task)
1414         return 0;
1415     yaz_log(c->log_details, "%p ZOOM_connection_exec_task type=%d run=%d",
1416             c, task->which, task->running);
1417     if (c->error != ZOOM_ERROR_NONE)
1418     {
1419         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1420                 "removing tasks because of error = %d", c, c->error);
1421         ZOOM_connection_remove_tasks(c);
1422         return 0;
1423     }
1424     if (task->running)
1425     {
1426         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1427                 "task already running", c);
1428         return 0;
1429     }
1430     task->running = 1;
1431     ret = zoom_complete;
1432     if (c->cs || task->which == ZOOM_TASK_CONNECT)
1433     {
1434         switch (task->which)
1435         {
1436         case ZOOM_TASK_SEARCH:
1437             if (c->proto == PROTO_HTTP)
1438                 ret = ZOOM_connection_srw_send_search(c);
1439             else
1440                 ret = ZOOM_connection_Z3950_send_search(c);
1441             break;
1442         case ZOOM_TASK_RETRIEVE:
1443             if (c->proto == PROTO_HTTP)
1444                 ret = ZOOM_connection_srw_send_search(c);
1445             else
1446                 ret = send_Z3950_present(c);
1447             break;
1448         case ZOOM_TASK_CONNECT:
1449             ret = do_connect(c);
1450             break;
1451         case ZOOM_TASK_SCAN:
1452             if (c->proto == PROTO_HTTP)
1453                 ret = ZOOM_connection_srw_send_scan(c);
1454             else
1455                 ret = ZOOM_connection_Z3950_send_scan(c);
1456             break;
1457         case ZOOM_TASK_PACKAGE:
1458             ret = send_package(c);
1459             break;
1460         case ZOOM_TASK_SORT:
1461             c->tasks->u.sort.resultset->r_sort_spec = 
1462                 ZOOM_query_get_sortspec(c->tasks->u.sort.q);
1463             ret = send_Z3950_sort(c, c->tasks->u.sort.resultset);
1464             break;
1465         }
1466     }
1467     else
1468     {
1469         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1470                 "remove tasks because no connection exist", c);
1471         ZOOM_connection_remove_tasks(c);
1472     }
1473     if (ret == zoom_complete)
1474     {
1475         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1476                 "task removed (complete)", c);
1477         ZOOM_connection_remove_task(c);
1478         return 0;
1479     }
1480     yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1481             "task pending", c);
1482     return 1;
1483 }
1484
1485 #if YAZ_HAVE_XML2
1486 static Z_GDU *get_HTTP_Request_url(ODR odr, const char *url)
1487 {
1488     Z_GDU *p = z_get_HTTP_Request(odr);
1489     const char *host = url;
1490     const char *cp0 = strstr(host, "://");
1491     const char *cp1 = 0;
1492     if (cp0)
1493         cp0 = cp0+3;
1494     else
1495         cp0 = host;
1496     
1497     cp1 = strchr(cp0, '/');
1498     if (!cp1)
1499         cp1 = cp0 + strlen(cp0);
1500     
1501     if (cp0 && cp1)
1502     {
1503         char *h = (char*) odr_malloc(odr, cp1 - cp0 + 1);
1504         memcpy (h, cp0, cp1 - cp0);
1505         h[cp1-cp0] = '\0';
1506         z_HTTP_header_add(odr, &p->u.HTTP_Request->headers, "Host", h);
1507     }
1508     p->u.HTTP_Request->path = odr_strdup(odr, *cp1 ? cp1 : "/");
1509     return p;
1510 }
1511
1512 static zoom_ret send_HTTP_redirect(ZOOM_connection c, const char *uri,
1513                                   Z_HTTP_Response *cookie_hres)
1514 {
1515     struct Z_HTTP_Header *h;
1516     Z_GDU *gdu = get_HTTP_Request_url(c->odr_out, uri);
1517     char *combined_cookies;
1518     int combined_cookies_len = 0;
1519
1520     gdu->u.HTTP_Request->method = odr_strdup(c->odr_out, "GET");
1521     z_HTTP_header_add(c->odr_out, &gdu->u.HTTP_Request->headers, "Accept",
1522                       "text/xml");
1523
1524     for (h = cookie_hres->headers; h; h = h->next)
1525     {
1526         if (!strcmp(h->name, "Set-Cookie"))
1527         {
1528             char *cp;
1529
1530             if (!(cp = strchr(h->value, ';')))
1531                 cp = h->value + strlen(h->value);
1532             if (cp - h->value >= 1) {
1533                 combined_cookies = xrealloc(combined_cookies, combined_cookies_len + cp - h->value + 3);
1534                 memcpy(combined_cookies+combined_cookies_len, h->value, cp - h->value);
1535                 combined_cookies[combined_cookies_len + cp - h->value] = '\0';
1536                 strcat(combined_cookies,"; ");
1537                 combined_cookies_len = strlen(combined_cookies);
1538             }
1539         }
1540     }
1541
1542     if (combined_cookies_len)
1543     {
1544         z_HTTP_header_add(c->odr_out, &gdu->u.HTTP_Request->headers,
1545                           "Cookie", combined_cookies);
1546         xfree(combined_cookies);
1547     }
1548
1549     if (c->user && c->password)
1550     {
1551         z_HTTP_header_add_basic_auth(c->odr_out, &gdu->u.HTTP_Request->headers,
1552                                      c->user, c->password);
1553     }
1554     if (!z_GDU(c->odr_out, &gdu, 0, 0))
1555         return zoom_complete;
1556     if (c->odr_print)
1557         z_GDU(c->odr_print, &gdu, 0, 0);
1558     c->buf_out = odr_getbuf(c->odr_out, &c->len_out, 0);
1559
1560     odr_reset(c->odr_out);
1561     return ZOOM_send_buf(c);
1562 }
1563
1564 #if YAZ_HAVE_XML2
1565 void ZOOM_set_HTTP_error(ZOOM_connection c, int error,
1566                          const char *addinfo, const char *addinfo2)
1567 {
1568     ZOOM_set_dset_error(c, error, "HTTP", addinfo, addinfo2);
1569 }
1570 #endif
1571
1572
1573 static void handle_http(ZOOM_connection c, Z_HTTP_Response *hres)
1574 {
1575     zoom_ret cret = zoom_complete;
1576     int ret = -1;
1577     const char *addinfo = 0;
1578     const char *connection_head = z_HTTP_header_lookup(hres->headers,
1579                                                        "Connection");
1580     const char *location;
1581
1582     ZOOM_connection_set_mask(c, 0);
1583     yaz_log(c->log_details, "%p handle_http", c);
1584     
1585     if ((hres->code == 301 || hres->code == 302) && c->sru_mode == zoom_sru_get
1586         && (location = z_HTTP_header_lookup(hres->headers, "Location")))
1587     {
1588         c->no_redirects++;
1589         if (c->no_redirects > 10)
1590         {
1591             ZOOM_set_HTTP_error(c, hres->code, 0, 0);
1592             c->no_redirects = 0;
1593             ZOOM_connection_close(c);
1594         }
1595         else
1596         {
1597             /* since redirect may change host we just reconnect. A smarter
1598                implementation might check whether it's the same server */
1599             do_connect_host(c, location, 0);
1600             send_HTTP_redirect(c, location, hres);
1601             /* we're OK for now. Operation is not really complete */
1602             ret = 0;
1603             cret = zoom_pending;
1604         }
1605     }
1606     else 
1607     {  
1608         ret = ZOOM_handle_sru(c, hres, &cret);
1609         if (ret == 0)
1610         {
1611             if (c->no_redirects) /* end of redirect. change hosts again */
1612                 ZOOM_connection_close(c);
1613         }
1614         c->no_redirects = 0;
1615     }
1616     if (ret)
1617     {
1618         if (hres->code != 200)
1619             ZOOM_set_HTTP_error(c, hres->code, 0, 0);
1620         else
1621             ZOOM_set_error(c, ZOOM_ERROR_DECODE, addinfo);
1622         ZOOM_connection_close(c);
1623     }
1624     if (cret == zoom_complete)
1625     {
1626         yaz_log(c->log_details, "removing tasks in handle_http");
1627         ZOOM_connection_remove_task(c);
1628     }
1629     {
1630         int must_close = 0;
1631         if (!strcmp(hres->version, "1.0"))
1632         {
1633             /* HTTP 1.0: only if Keep-Alive we stay alive.. */
1634             if (!connection_head || strcmp(connection_head, "Keep-Alive"))
1635                 must_close = 1;
1636         }
1637         else
1638         {
1639             /* HTTP 1.1: only if no close we stay alive.. */
1640             if (connection_head && !strcmp(connection_head, "close"))
1641                 must_close = 1;
1642         }
1643         if (must_close)
1644         {
1645             ZOOM_connection_close(c);
1646             if (c->tasks)
1647             {
1648                 c->tasks->running = 0;
1649                 ZOOM_connection_insert_task(c, ZOOM_TASK_CONNECT);
1650                 c->reconnect_ok = 0;
1651             }
1652         }
1653     }
1654 }
1655 #endif
1656
1657 static int do_read(ZOOM_connection c)
1658 {
1659     int r, more;
1660     ZOOM_Event event;
1661     
1662     event = ZOOM_Event_create(ZOOM_EVENT_RECV_DATA);
1663     ZOOM_connection_put_event(c, event);
1664     
1665     r = cs_get(c->cs, &c->buf_in, &c->len_in);
1666     more = cs_more(c->cs);
1667     yaz_log(c->log_details, "%p do_read len=%d more=%d", c, r, more);
1668     if (r == 1)
1669         return 0;
1670     if (r <= 0)
1671     {
1672         if (!ZOOM_test_reconnect(c))
1673         {
1674             ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
1675             ZOOM_connection_close(c);
1676         }
1677     }
1678     else
1679     {
1680         Z_GDU *gdu;
1681         ZOOM_Event event;
1682
1683         odr_reset(c->odr_in);
1684         odr_setbuf(c->odr_in, c->buf_in, r, 0);
1685         event = ZOOM_Event_create(ZOOM_EVENT_RECV_APDU);
1686         ZOOM_connection_put_event(c, event);
1687
1688         if (!z_GDU(c->odr_in, &gdu, 0, 0))
1689         {
1690             int x;
1691             int err = odr_geterrorx(c->odr_in, &x);
1692             char msg[100];
1693             const char *element = odr_getelement(c->odr_in);
1694             yaz_snprintf(msg, sizeof(msg),
1695                     "ODR code %d:%d element=%s offset=%d",
1696                     err, x, element ? element : "<unknown>",
1697                     odr_offset(c->odr_in));
1698             ZOOM_set_error(c, ZOOM_ERROR_DECODE, msg);
1699             if (c->log_api)
1700             {
1701                 FILE *ber_file = yaz_log_file();
1702                 if (ber_file)
1703                     odr_dumpBER(ber_file, c->buf_in, r);
1704             }
1705             ZOOM_connection_close(c);
1706         }
1707         else
1708         {
1709             if (c->odr_print)
1710                 z_GDU(c->odr_print, &gdu, 0, 0);
1711             if (gdu->which == Z_GDU_Z3950)
1712                 ZOOM_handle_Z3950_apdu(c, gdu->u.z3950);
1713             else if (gdu->which == Z_GDU_HTTP_Response)
1714             {
1715 #if YAZ_HAVE_XML2
1716                 handle_http(c, gdu->u.HTTP_Response);
1717 #else
1718                 ZOOM_set_error(c, ZOOM_ERROR_DECODE, 0);
1719                 ZOOM_connection_close(c);
1720 #endif
1721             }
1722         }
1723         c->reconnect_ok = 0;
1724     }
1725     return 1;
1726 }
1727
1728 static zoom_ret do_write_ex(ZOOM_connection c, char *buf_out, int len_out)
1729 {
1730     int r;
1731     ZOOM_Event event;
1732     
1733     event = ZOOM_Event_create(ZOOM_EVENT_SEND_DATA);
1734     ZOOM_connection_put_event(c, event);
1735
1736     yaz_log(c->log_details, "%p do_write_ex len=%d", c, len_out);
1737     if ((r = cs_put(c->cs, buf_out, len_out)) < 0)
1738     {
1739         yaz_log(c->log_details, "%p do_write_ex write failed", c);
1740         if (ZOOM_test_reconnect(c))
1741         {
1742             return zoom_pending;
1743         }
1744         if (c->state == STATE_CONNECTING)
1745             ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
1746         else
1747             ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
1748         ZOOM_connection_close(c);
1749         return zoom_complete;
1750     }
1751     else if (r == 1)
1752     {    
1753         int mask = ZOOM_SELECT_EXCEPT;
1754         if (c->cs->io_pending & CS_WANT_WRITE)
1755             mask += ZOOM_SELECT_WRITE;
1756         if (c->cs->io_pending & CS_WANT_READ)
1757             mask += ZOOM_SELECT_READ;
1758         ZOOM_connection_set_mask(c, mask);
1759         yaz_log(c->log_details, "%p do_write_ex write incomplete mask=%d",
1760                 c, c->mask);
1761     }
1762     else
1763     {
1764         ZOOM_connection_set_mask(c, ZOOM_SELECT_READ|ZOOM_SELECT_EXCEPT);
1765         yaz_log(c->log_details, "%p do_write_ex write complete mask=%d",
1766                 c, c->mask);
1767     }
1768     return zoom_pending;
1769 }
1770
1771 zoom_ret ZOOM_send_buf(ZOOM_connection c)
1772 {
1773     return do_write_ex(c, c->buf_out, c->len_out);
1774 }
1775
1776
1777 ZOOM_API(const char *)
1778     ZOOM_connection_option_get(ZOOM_connection c, const char *key)
1779 {
1780     return ZOOM_options_get(c->options, key);
1781 }
1782
1783 ZOOM_API(const char *)
1784     ZOOM_connection_option_getl(ZOOM_connection c, const char *key, int *lenp)
1785 {
1786     return ZOOM_options_getl(c->options, key, lenp);
1787 }
1788
1789 ZOOM_API(void)
1790     ZOOM_connection_option_set(ZOOM_connection c, const char *key,
1791                                const char *val)
1792 {
1793     ZOOM_options_set(c->options, key, val);
1794 }
1795
1796 ZOOM_API(void)
1797     ZOOM_connection_option_setl(ZOOM_connection c, const char *key,
1798                                 const char *val, int len)
1799 {
1800     ZOOM_options_setl(c->options, key, val, len);
1801 }
1802
1803 ZOOM_API(const char *)
1804     ZOOM_resultset_option_get(ZOOM_resultset r, const char *key)
1805 {
1806     return ZOOM_options_get(r->options, key);
1807 }
1808
1809 ZOOM_API(void)
1810     ZOOM_resultset_option_set(ZOOM_resultset r, const char *key,
1811                               const char *val)
1812 {
1813     ZOOM_options_set(r->options, key, val);
1814 }
1815
1816
1817 ZOOM_API(int)
1818     ZOOM_connection_errcode(ZOOM_connection c)
1819 {
1820     return ZOOM_connection_error(c, 0, 0);
1821 }
1822
1823 ZOOM_API(const char *)
1824     ZOOM_connection_errmsg(ZOOM_connection c)
1825 {
1826     const char *msg;
1827     ZOOM_connection_error(c, &msg, 0);
1828     return msg;
1829 }
1830
1831 ZOOM_API(const char *)
1832     ZOOM_connection_addinfo(ZOOM_connection c)
1833 {
1834     const char *addinfo;
1835     ZOOM_connection_error(c, 0, &addinfo);
1836     return addinfo;
1837 }
1838
1839 ZOOM_API(const char *)
1840     ZOOM_connection_diagset(ZOOM_connection c)
1841 {
1842     const char *diagset;
1843     ZOOM_connection_error_x(c, 0, 0, &diagset);
1844     return diagset;
1845 }
1846
1847 ZOOM_API(const char *)
1848     ZOOM_diag_str(int error)
1849 {
1850     switch (error)
1851     {
1852     case ZOOM_ERROR_NONE:
1853         return "No error";
1854     case ZOOM_ERROR_CONNECT:
1855         return "Connect failed";
1856     case ZOOM_ERROR_MEMORY:
1857         return "Out of memory";
1858     case ZOOM_ERROR_ENCODE:
1859         return "Encoding failed";
1860     case ZOOM_ERROR_DECODE:
1861         return "Decoding failed";
1862     case ZOOM_ERROR_CONNECTION_LOST:
1863         return "Connection lost";
1864     case ZOOM_ERROR_INIT:
1865         return "Init rejected";
1866     case ZOOM_ERROR_INTERNAL:
1867         return "Internal failure";
1868     case ZOOM_ERROR_TIMEOUT:
1869         return "Timeout";
1870     case ZOOM_ERROR_UNSUPPORTED_PROTOCOL:
1871         return "Unsupported protocol";
1872     case ZOOM_ERROR_UNSUPPORTED_QUERY:
1873         return "Unsupported query type";
1874     case ZOOM_ERROR_INVALID_QUERY:
1875         return "Invalid query";
1876     case ZOOM_ERROR_CQL_PARSE:
1877         return "CQL parsing error";
1878     case ZOOM_ERROR_CQL_TRANSFORM:
1879         return "CQL transformation error";
1880     case ZOOM_ERROR_CCL_CONFIG:
1881         return "CCL configuration error";
1882     case ZOOM_ERROR_CCL_PARSE:
1883         return "CCL parsing error";
1884     default:
1885         return diagbib1_str(error);
1886     }
1887 }
1888
1889 ZOOM_API(int)
1890     ZOOM_connection_error_x(ZOOM_connection c, const char **cp,
1891                             const char **addinfo, const char **diagset)
1892 {
1893     int error = c->error;
1894     if (cp)
1895     {
1896         if (!c->diagset || !strcmp(c->diagset, "ZOOM"))
1897             *cp = ZOOM_diag_str(error);
1898         else if (!strcmp(c->diagset, "HTTP"))
1899             *cp = z_HTTP_errmsg(c->error);
1900         else if (!strcmp(c->diagset, "Bib-1"))
1901             *cp = ZOOM_diag_str(error);
1902         else if (!strcmp(c->diagset, "info:srw/diagnostic/1"))
1903             *cp = yaz_diag_srw_str(c->error);
1904         else
1905             *cp = "Unknown error and diagnostic set";
1906     }
1907     if (addinfo)
1908         *addinfo = c->addinfo ? c->addinfo : "";
1909     if (diagset)
1910         *diagset = c->diagset ? c->diagset : "";
1911     return c->error;
1912 }
1913
1914 ZOOM_API(int)
1915     ZOOM_connection_error(ZOOM_connection c, const char **cp,
1916                           const char **addinfo)
1917 {
1918     return ZOOM_connection_error_x(c, cp, addinfo, 0);
1919 }
1920
1921 static void ZOOM_connection_do_io(ZOOM_connection c, int mask)
1922 {
1923     ZOOM_Event event = 0;
1924     int r = cs_look(c->cs);
1925     yaz_log(c->log_details, "%p ZOOM_connection_do_io mask=%d cs_look=%d",
1926             c, mask, r);
1927     
1928     if (r == CS_NONE)
1929     {
1930         event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1931         ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
1932         ZOOM_connection_close(c);
1933         ZOOM_connection_put_event(c, event);
1934     }
1935     else if (r == CS_CONNECT)
1936     {
1937         int ret = ret = cs_rcvconnect(c->cs);
1938         yaz_log(c->log_details, "%p ZOOM_connection_do_io "
1939                 "cs_rcvconnect returned %d", c, ret);
1940         if (ret == 1)
1941         {
1942             int mask = ZOOM_SELECT_EXCEPT;
1943             if (c->cs->io_pending & CS_WANT_WRITE)
1944                 mask += ZOOM_SELECT_WRITE;
1945             if (c->cs->io_pending & CS_WANT_READ)
1946                 mask += ZOOM_SELECT_READ;
1947             ZOOM_connection_set_mask(c, mask);
1948             event = ZOOM_Event_create(ZOOM_EVENT_NONE);
1949             ZOOM_connection_put_event(c, event);
1950         }
1951         else if (ret == 0)
1952         {
1953             event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1954             ZOOM_connection_put_event(c, event);
1955             get_cert(c);
1956             if (c->proto == PROTO_Z3950)
1957                 ZOOM_connection_Z3950_send_init(c);
1958             else
1959             {
1960                 /* no init request for SRW .. */
1961                 assert(c->tasks->which == ZOOM_TASK_CONNECT);
1962                 ZOOM_connection_remove_task(c);
1963                 ZOOM_connection_set_mask(c, 0);
1964                 ZOOM_connection_exec_task(c);
1965             }
1966             c->state = STATE_ESTABLISHED;
1967         }
1968         else
1969         {
1970             ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
1971             ZOOM_connection_close(c);
1972         }
1973     }
1974     else
1975     {
1976         if (mask & ZOOM_SELECT_EXCEPT)
1977         {
1978             if (!ZOOM_test_reconnect(c))
1979             {
1980                 ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
1981                 ZOOM_connection_close(c);
1982             }
1983             return;
1984         }
1985         if (mask & ZOOM_SELECT_READ)
1986             do_read(c);
1987         if (c->cs && (mask & ZOOM_SELECT_WRITE))
1988             ZOOM_send_buf(c);
1989     }
1990 }
1991
1992 ZOOM_API(int)
1993     ZOOM_connection_last_event(ZOOM_connection cs)
1994 {
1995     if (!cs)
1996         return ZOOM_EVENT_NONE;
1997     return cs->last_event;
1998 }
1999
2000
2001 ZOOM_API(int) ZOOM_connection_fire_event_timeout(ZOOM_connection c)
2002 {
2003     if (c->mask)
2004     {
2005         ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_TIMEOUT);
2006         /* timeout and this connection was waiting */
2007         ZOOM_set_error(c, ZOOM_ERROR_TIMEOUT, 0);
2008         ZOOM_connection_close(c);
2009         ZOOM_connection_put_event(c, event);
2010     }
2011     return 0;
2012 }
2013
2014 ZOOM_API(int)
2015     ZOOM_connection_process(ZOOM_connection c)
2016 {
2017     ZOOM_Event event;
2018     if (!c)
2019         return 0;
2020
2021     event = ZOOM_connection_get_event(c);
2022     if (event)
2023     {
2024         ZOOM_Event_destroy(event);
2025         return 1;
2026     }
2027     ZOOM_connection_exec_task(c);
2028     event = ZOOM_connection_get_event(c);
2029     if (event)
2030     {
2031         ZOOM_Event_destroy(event);
2032         return 1;
2033     }
2034     return 0;
2035 }
2036
2037 ZOOM_API(int)
2038     ZOOM_event_nonblock(int no, ZOOM_connection *cs)
2039 {
2040     int i;
2041
2042     yaz_log(log_details0, "ZOOM_process_event(no=%d,cs=%p)", no, cs);
2043     
2044     for (i = 0; i<no; i++)
2045     {
2046         ZOOM_connection c = cs[i];
2047
2048         if (c && ZOOM_connection_process(c))
2049             return i+1;
2050     }
2051     return 0;
2052 }
2053
2054 ZOOM_API(int) ZOOM_connection_fire_event_socket(ZOOM_connection c, int mask)
2055 {
2056     if (c->mask && mask)
2057         ZOOM_connection_do_io(c, mask);
2058     return 0;
2059 }
2060
2061 ZOOM_API(int) ZOOM_connection_get_socket(ZOOM_connection c)
2062 {
2063     if (c->cs)
2064         return cs_fileno(c->cs);
2065     return -1;
2066 }
2067
2068 ZOOM_API(int) ZOOM_connection_set_mask(ZOOM_connection c, int mask)
2069 {
2070     c->mask = mask;
2071     if (!c->cs)
2072         return -1; 
2073     return 0;
2074 }
2075
2076 ZOOM_API(int) ZOOM_connection_get_mask(ZOOM_connection c)
2077 {
2078     if (c->cs)
2079         return c->mask;
2080     return 0;
2081 }
2082
2083 ZOOM_API(int) ZOOM_connection_get_timeout(ZOOM_connection c)
2084 {
2085     return ZOOM_options_get_int(c->options, "timeout", 30);
2086 }
2087
2088 ZOOM_API(void) ZOOM_connection_close(ZOOM_connection c)
2089 {
2090     if (c->cs)
2091         cs_close(c->cs);
2092     c->cs = 0;
2093     ZOOM_connection_set_mask(c, 0);
2094     c->state = STATE_IDLE;
2095 }
2096
2097 /*
2098  * Local variables:
2099  * c-basic-offset: 4
2100  * c-file-style: "Stroustrup"
2101  * indent-tabs-mode: nil
2102  * End:
2103  * vim: shiftwidth=4 tabstop=8 expandtab
2104  */
2105