Refactor ZOOM SRU stuff to separate source
[yaz-moved-to-github.git] / src / zoom-c.c
1 /* This file is part of the YAZ toolkit.
2  * Copyright (C) 1995-2010 Index Data
3  * See the file LICENSE for details.
4  */
5 /**
6  * \file zoom-c.c
7  * \brief Implements ZOOM C interface.
8  */
9
10 #include <assert.h>
11 #include <string.h>
12 #include <errno.h>
13 #include "zoom-p.h"
14
15 #include <yaz/yaz-util.h>
16 #include <yaz/xmalloc.h>
17 #include <yaz/otherinfo.h>
18 #include <yaz/log.h>
19 #include <yaz/diagbib1.h>
20 #include <yaz/charneg.h>
21 #include <yaz/query-charset.h>
22 #include <yaz/snprintf.h>
23 #include <yaz/facet.h>
24
25 #include <yaz/shptr.h>
26
27 #if SHPTR
28 YAZ_SHPTR_TYPE(WRBUF)
29 #endif
30
31 static int log_api0 = 0;
32 static int log_details0 = 0;
33
34 static void resultset_destroy(ZOOM_resultset r);
35 static zoom_ret do_write_ex(ZOOM_connection c, char *buf_out, int len_out);
36
37 ZOOM_API(const char *) ZOOM_get_event_str(int event)
38 {
39     static const char *ar[] = {
40         "NONE",
41         "CONNECT",
42         "SEND_DATA",
43         "RECV_DATA",
44         "TIMEOUT",
45         "UNKNOWN",
46         "SEND_APDU",
47         "RECV_APDU",
48         "RECV_RECORD",
49         "RECV_SEARCH",
50         "END"
51     };
52     return ar[event];
53 }
54
55 static void initlog(void)
56 {
57     static int log_level_initialized = 0;
58     if (!log_level_initialized)
59     {
60         log_api0 = yaz_log_module_level("zoom");
61         log_details0 = yaz_log_module_level("zoomdetails");
62         log_level_initialized = 1;
63     }
64 }
65
66 ZOOM_Event ZOOM_Event_create(int kind)
67 {
68     ZOOM_Event event = (ZOOM_Event) xmalloc(sizeof(*event));
69     event->kind = kind;
70     event->next = 0;
71     event->prev = 0;
72     return event;
73 }
74
75 static void ZOOM_Event_destroy(ZOOM_Event event)
76 {
77     xfree(event);
78 }
79
80 void ZOOM_connection_put_event(ZOOM_connection c, ZOOM_Event event)
81 {
82     if (c->m_queue_back)
83     {
84         c->m_queue_back->prev = event;
85         assert(c->m_queue_front);
86     }
87     else
88     {
89         assert(!c->m_queue_front);
90         c->m_queue_front = event;
91     }
92     event->next = c->m_queue_back;
93     event->prev = 0;
94     c->m_queue_back = event;
95 }
96
97 static ZOOM_Event ZOOM_connection_get_event(ZOOM_connection c)
98 {
99     ZOOM_Event event = c->m_queue_front;
100     if (!event)
101     {
102         c->last_event = ZOOM_EVENT_NONE;
103         return 0;
104     }
105     assert(c->m_queue_back);
106     c->m_queue_front = event->prev;
107     if (c->m_queue_front)
108     {
109         assert(c->m_queue_back);
110         c->m_queue_front->next = 0;
111     }
112     else
113         c->m_queue_back = 0;
114     c->last_event = event->kind;
115     return event;
116 }
117
118 static void ZOOM_connection_remove_events(ZOOM_connection c)
119 {
120     ZOOM_Event event;
121     while ((event = ZOOM_connection_get_event(c)))
122         ZOOM_Event_destroy(event);
123 }
124
125 ZOOM_API(int) ZOOM_connection_peek_event(ZOOM_connection c)
126 {
127     ZOOM_Event event = c->m_queue_front;
128
129     return event ? event->kind : ZOOM_EVENT_NONE;
130 }
131
132 void ZOOM_connection_remove_tasks(ZOOM_connection c);
133
134 void ZOOM_set_dset_error(ZOOM_connection c, int error,
135                          const char *dset,
136                          const char *addinfo, const char *addinfo2)
137 {
138     char *cp;
139
140     xfree(c->addinfo);
141     c->addinfo = 0;
142     c->error = error;
143     if (!c->diagset || strcmp(dset, c->diagset))
144     {
145         xfree(c->diagset);
146         c->diagset = xstrdup(dset);
147         /* remove integer part from SRW diagset .. */
148         if ((cp = strrchr(c->diagset, '/')))
149             *cp = '\0';
150     }
151     if (addinfo && addinfo2)
152     {
153         c->addinfo = (char*) xmalloc(strlen(addinfo) + strlen(addinfo2) + 2);
154         strcpy(c->addinfo, addinfo);
155         strcat(c->addinfo, addinfo2);
156     }
157     else if (addinfo)
158         c->addinfo = xstrdup(addinfo);
159     if (error != ZOOM_ERROR_NONE)
160     {
161         yaz_log(c->log_api, "%p set_dset_error %s %s:%d %s %s",
162                 c, c->host_port ? c->host_port : "<>", dset, error,
163                 addinfo ? addinfo : "",
164                 addinfo2 ? addinfo2 : "");
165         ZOOM_connection_remove_tasks(c);
166     }
167 }
168
169 int ZOOM_uri_to_code(const char *uri)
170 {
171     int code = 0;       
172     const char *cp;
173     if ((cp = strrchr(uri, '/')))
174         code = atoi(cp+1);
175     return code;
176 }
177
178
179
180 void ZOOM_set_error(ZOOM_connection c, int error, const char *addinfo)
181 {
182     ZOOM_set_dset_error(c, error, "ZOOM", addinfo, 0);
183 }
184
185 static void clear_error(ZOOM_connection c)
186 {
187     /*
188      * If an error is tied to an operation then it's ok to clear: for
189      * example, a diagnostic returned from a search is cleared by a
190      * subsequent search.  However, problems such as Connection Lost
191      * or Init Refused are not cleared, because they are not
192      * recoverable: doing another search doesn't help.
193      */
194
195     ZOOM_connection_remove_events(c);
196     switch (c->error)
197     {
198     case ZOOM_ERROR_CONNECT:
199     case ZOOM_ERROR_MEMORY:
200     case ZOOM_ERROR_DECODE:
201     case ZOOM_ERROR_CONNECTION_LOST:
202     case ZOOM_ERROR_INIT:
203     case ZOOM_ERROR_INTERNAL:
204     case ZOOM_ERROR_UNSUPPORTED_PROTOCOL:
205         break;
206     default:
207         ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
208     }
209 }
210
211 void ZOOM_connection_show_task(ZOOM_task task)
212 {
213     switch(task->which)
214     {
215     case ZOOM_TASK_SEARCH:
216         yaz_log(YLOG_LOG, "search p=%p", task);
217         break;
218     case ZOOM_TASK_RETRIEVE:
219         yaz_log(YLOG_LOG, "retrieve p=%p", task);
220         break;
221     case ZOOM_TASK_CONNECT:
222         yaz_log(YLOG_LOG, "connect p=%p", task);
223         break;
224     case ZOOM_TASK_SCAN:
225         yaz_log(YLOG_LOG, "scan p=%p", task);
226         break;
227     }
228 }
229
230 void ZOOM_connection_show_tasks(ZOOM_connection c)
231 {
232     ZOOM_task task;
233     yaz_log(YLOG_LOG, "connection p=%p tasks", c);
234     for (task = c->tasks; task; task = task->next)
235         ZOOM_connection_show_task(task);
236 }
237
238 ZOOM_task ZOOM_connection_add_task(ZOOM_connection c, int which)
239 {
240     ZOOM_task *taskp = &c->tasks;
241     while (*taskp)
242         taskp = &(*taskp)->next;
243     *taskp = (ZOOM_task) xmalloc(sizeof(**taskp));
244     (*taskp)->running = 0;
245     (*taskp)->which = which;
246     (*taskp)->next = 0;
247     clear_error(c);
248     return *taskp;
249 }
250
251 ZOOM_API(int) ZOOM_connection_is_idle(ZOOM_connection c)
252 {
253     return c->tasks ? 0 : 1;
254 }
255
256 ZOOM_task ZOOM_connection_insert_task(ZOOM_connection c, int which)
257 {
258     ZOOM_task task = (ZOOM_task) xmalloc(sizeof(*task));
259
260     task->next = c->tasks;
261     c->tasks = task;
262
263     task->running = 0;
264     task->which = which;
265     clear_error(c);
266     return task;
267 }
268
269 void ZOOM_connection_remove_task(ZOOM_connection c)
270 {
271     ZOOM_task task = c->tasks;
272
273     if (task)
274     {
275         c->tasks = task->next;
276         switch (task->which)
277         {
278         case ZOOM_TASK_SEARCH:
279             resultset_destroy(task->u.search.resultset);
280             xfree(task->u.search.syntax);
281             xfree(task->u.search.elementSetName);
282             break;
283         case ZOOM_TASK_RETRIEVE:
284             resultset_destroy(task->u.retrieve.resultset);
285             xfree(task->u.retrieve.syntax);
286             xfree(task->u.retrieve.elementSetName);
287             break;
288         case ZOOM_TASK_CONNECT:
289             break;
290         case ZOOM_TASK_SCAN:
291             ZOOM_scanset_destroy(task->u.scan.scan);
292             break;
293         case ZOOM_TASK_PACKAGE:
294             ZOOM_package_destroy(task->u.package);
295             break;
296         case ZOOM_TASK_SORT:
297             resultset_destroy(task->u.sort.resultset);
298             ZOOM_query_destroy(task->u.sort.q);
299             break;
300         default:
301             assert(0);
302         }
303         xfree(task);
304
305         if (!c->tasks)
306         {
307             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_END);
308             ZOOM_connection_put_event(c, event);
309         }
310     }
311 }
312
313 void ZOOM_connection_remove_tasks(ZOOM_connection c)
314 {
315     while (c->tasks)
316         ZOOM_connection_remove_task(c);
317 }
318
319
320 ZOOM_API(ZOOM_connection)
321     ZOOM_connection_create(ZOOM_options options)
322 {
323     ZOOM_connection c = (ZOOM_connection) xmalloc(sizeof(*c));
324
325     initlog();
326
327     c->log_api = log_api0;
328     c->log_details = log_details0;
329
330     yaz_log(c->log_api, "%p ZOOM_connection_create", c);
331
332     c->proto = PROTO_Z3950;
333     c->cs = 0;
334     ZOOM_connection_set_mask(c, 0);
335     c->reconnect_ok = 0;
336     c->state = STATE_IDLE;
337     c->addinfo = 0;
338     c->diagset = 0;
339     ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
340     c->buf_in = 0;
341     c->len_in = 0;
342     c->buf_out = 0;
343     c->len_out = 0;
344     c->resultsets = 0;
345
346     c->options = ZOOM_options_create_with_parent(options);
347
348     c->host_port = 0;
349     c->path = 0;
350     c->proxy = 0;
351     
352     c->charset = c->lang = 0;
353
354     c->cookie_out = 0;
355     c->cookie_in = 0;
356     c->client_IP = 0;
357     c->tasks = 0;
358
359     c->user = 0;
360     c->group = 0;
361     c->password = 0;
362
363     c->maximum_record_size = 0;
364     c->preferred_message_size = 0;
365
366     c->odr_in = odr_createmem(ODR_DECODE);
367     c->odr_out = odr_createmem(ODR_ENCODE);
368     c->odr_print = 0;
369
370     c->async = 0;
371     c->support_named_resultsets = 0;
372     c->last_event = ZOOM_EVENT_NONE;
373
374     c->m_queue_front = 0;
375     c->m_queue_back = 0;
376
377     c->sru_version = 0;
378     c->no_redirects = 0;
379     return c;
380 }
381
382
383 /* set database names. Take local databases (if set); otherwise
384    take databases given in ZURL (if set); otherwise use Default */
385 char **ZOOM_connection_get_databases(ZOOM_connection con, ZOOM_options options,
386                                      int *num, ODR odr)
387 {
388     char **databaseNames;
389     const char *cp = ZOOM_options_get(options, "databaseName");
390     
391     if ((!cp || !*cp) && con->host_port)
392     {
393         if (strncmp(con->host_port, "unix:", 5) == 0)
394             cp = strchr(con->host_port+5, ':');
395         else
396             cp = strchr(con->host_port, '/');
397         if (cp)
398             cp++;
399     }
400     if (!cp)
401         cp = "Default";
402     nmem_strsplit(odr_getmem(odr), "+", cp,  &databaseNames, num);
403     return databaseNames;
404 }
405
406 ZOOM_API(ZOOM_connection)
407     ZOOM_connection_new(const char *host, int portnum)
408 {
409     ZOOM_connection c = ZOOM_connection_create(0);
410
411     ZOOM_connection_connect(c, host, portnum);
412     return c;
413 }
414
415 static zoom_sru_mode get_sru_mode_from_string(const char *s)
416 {
417     if (!s || !*s)
418         return zoom_sru_soap;
419     if (!yaz_matchstr(s, "soap"))
420         return zoom_sru_soap;
421     else if (!yaz_matchstr(s, "get"))
422         return zoom_sru_get;
423     else if (!yaz_matchstr(s, "post"))
424         return zoom_sru_post;
425     return zoom_sru_error;
426 }
427
428 ZOOM_API(void)
429     ZOOM_connection_connect(ZOOM_connection c,
430                             const char *host, int portnum)
431 {
432     const char *val;
433     ZOOM_task task;
434
435     initlog();
436
437     yaz_log(c->log_api, "%p ZOOM_connection_connect host=%s portnum=%d",
438             c, host ? host : "null", portnum);
439
440     ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
441     ZOOM_connection_remove_tasks(c);
442
443     if (c->odr_print)
444     {
445         odr_setprint(c->odr_print, 0); /* prevent destroy from fclose'ing */
446         odr_destroy(c->odr_print);
447     }
448     if (ZOOM_options_get_bool(c->options, "apdulog", 0))
449     {
450         c->odr_print = odr_createmem(ODR_PRINT);
451         odr_setprint(c->odr_print, yaz_log_file());
452     }
453     else
454         c->odr_print = 0;
455
456     if (c->cs)
457     {
458         yaz_log(c->log_details, "%p ZOOM_connection_connect reconnect ok", c);
459         c->reconnect_ok = 1;
460         return;
461     }
462     yaz_log(c->log_details, "%p ZOOM_connection_connect connect", c);
463     xfree(c->proxy);
464     c->proxy = 0;
465     val = ZOOM_options_get(c->options, "proxy");
466     if (val && *val)
467     {
468         yaz_log(c->log_details, "%p ZOOM_connection_connect proxy=%s", c, val);
469         c->proxy = xstrdup(val);
470     }
471
472     xfree(c->charset);
473     c->charset = 0;
474     val = ZOOM_options_get(c->options, "charset");
475     if (val && *val)
476     {
477         yaz_log(c->log_details, "%p ZOOM_connection_connect charset=%s", c, val);
478         c->charset = xstrdup(val);
479     }
480
481     xfree(c->lang);
482     val = ZOOM_options_get(c->options, "lang");
483     if (val && *val)
484     {
485         yaz_log(c->log_details, "%p ZOOM_connection_connect lang=%s", c, val);
486         c->lang = xstrdup(val);
487     }
488     else
489         c->lang = 0;
490
491     if (host)
492     {
493         xfree(c->host_port);
494         if (portnum)
495         {
496             char hostn[128];
497             sprintf(hostn, "%.80s:%d", host, portnum);
498             c->host_port = xstrdup(hostn);
499         }
500         else
501             c->host_port = xstrdup(host);
502     }        
503
504     {
505         /*
506          * If the "<scheme>:" part of the host string is preceded by one
507          * or more comma-separated <name>=<value> pairs, these are taken
508          * to be options to be set on the connection object.  Among other
509          * applications, this facility can be used to embed authentication
510          * in a host string:
511          *          user=admin,password=secret,tcp:localhost:9999
512          */
513         char *remainder = c->host_port;
514         char *pcolon = strchr(remainder, ':');
515         char *pcomma;
516         char *pequals;
517         while ((pcomma = strchr(remainder, ',')) != 0 &&
518                (pcolon == 0 || pcomma < pcolon)) {
519             *pcomma = '\0';
520             if ((pequals = strchr(remainder, '=')) != 0) {
521                 *pequals = '\0';
522                 /*printf("# setting '%s'='%s'\n", remainder, pequals+1);*/
523                 ZOOM_connection_option_set(c, remainder, pequals+1);
524             }
525             remainder = pcomma+1;
526         }
527
528         if (remainder != c->host_port) {
529             xfree(c->host_port);
530             c->host_port = xstrdup(remainder);
531             /*printf("# reset hp='%s'\n", remainder);*/
532         }
533     }
534
535     val = ZOOM_options_get(c->options, "sru");
536     c->sru_mode = get_sru_mode_from_string(val);
537
538     xfree(c->sru_version);
539     val = ZOOM_options_get(c->options, "sru_version");
540     c->sru_version = xstrdup(val ? val : "1.2");
541
542     ZOOM_options_set(c->options, "host", c->host_port);
543
544     xfree(c->cookie_out);
545     c->cookie_out = 0;
546     val = ZOOM_options_get(c->options, "cookie");
547     if (val && *val)
548     { 
549         yaz_log(c->log_details, "%p ZOOM_connection_connect cookie=%s", c, val);
550         c->cookie_out = xstrdup(val);
551     }
552
553     xfree(c->client_IP);
554     c->client_IP = 0;
555     val = ZOOM_options_get(c->options, "clientIP");
556     if (val && *val)
557     {
558         yaz_log(c->log_details, "%p ZOOM_connection_connect clientIP=%s",
559                 c, val);
560         c->client_IP = xstrdup(val);
561     }
562
563     xfree(c->group);
564     c->group = 0;
565     val = ZOOM_options_get(c->options, "group");
566     if (val && *val)
567         c->group = xstrdup(val);
568
569     xfree(c->user);
570     c->user = 0;
571     val = ZOOM_options_get(c->options, "user");
572     if (val && *val)
573         c->user = xstrdup(val);
574
575     xfree(c->password);
576     c->password = 0;
577     val = ZOOM_options_get(c->options, "password");
578     if (!val)
579         val = ZOOM_options_get(c->options, "pass");
580
581     if (val && *val)
582         c->password = xstrdup(val);
583     
584     c->maximum_record_size =
585         ZOOM_options_get_int(c->options, "maximumRecordSize", 1024*1024);
586     c->preferred_message_size =
587         ZOOM_options_get_int(c->options, "preferredMessageSize", 1024*1024);
588
589     c->async = ZOOM_options_get_bool(c->options, "async", 0);
590     yaz_log(c->log_details, "%p ZOOM_connection_connect async=%d", c, c->async);
591  
592     task = ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
593
594     if (!c->async)
595     {
596         while (ZOOM_event(1, &c))
597             ;
598     }
599 }
600
601 ZOOM_API(void) ZOOM_resultset_release(ZOOM_resultset r)
602 {
603 #if ZOOM_RESULT_LISTS
604 #else
605     if (r->connection)
606     {
607         /* remove ourselves from the resultsets in connection */
608         ZOOM_resultset *rp = &r->connection->resultsets;
609         while (1)
610         {
611             assert(*rp);   /* we must be in this list!! */
612             if (*rp == r)
613             {   /* OK, we're here - take us out of it */
614                 *rp = (*rp)->next;
615                 break;
616             }
617             rp = &(*rp)->next;
618         }
619         r->connection = 0;
620     }
621 #endif
622 }
623
624 ZOOM_API(void)
625     ZOOM_connection_destroy(ZOOM_connection c)
626 {
627 #if ZOOM_RESULT_LISTS
628     ZOOM_resultsets list;
629 #else
630     ZOOM_resultset r;
631 #endif
632     if (!c)
633         return;
634     yaz_log(c->log_api, "%p ZOOM_connection_destroy", c);
635     if (c->cs)
636         cs_close(c->cs);
637
638 #if ZOOM_RESULT_LISTS
639     // Remove the connection's usage of resultsets
640     list = c->resultsets;
641     while (list) {
642         ZOOM_resultsets removed = list;
643         ZOOM_resultset_destroy(list->resultset);
644         list = list->next;
645         xfree(removed);
646     }
647 #else
648     for (r = c->resultsets; r; r = r->next)
649         r->connection = 0;
650 #endif
651
652     xfree(c->buf_in);
653     xfree(c->addinfo);
654     xfree(c->diagset);
655     odr_destroy(c->odr_in);
656     odr_destroy(c->odr_out);
657     if (c->odr_print)
658     {
659         odr_setprint(c->odr_print, 0); /* prevent destroy from fclose'ing */
660         odr_destroy(c->odr_print);
661     }
662     ZOOM_options_destroy(c->options);
663     ZOOM_connection_remove_tasks(c);
664     ZOOM_connection_remove_events(c);
665     xfree(c->host_port);
666     xfree(c->path);
667     xfree(c->proxy);
668     xfree(c->charset);
669     xfree(c->lang);
670     xfree(c->cookie_out);
671     xfree(c->cookie_in);
672     xfree(c->client_IP);
673     xfree(c->user);
674     xfree(c->group);
675     xfree(c->password);
676     xfree(c->sru_version);
677     xfree(c);
678 }
679
680 void ZOOM_resultset_addref(ZOOM_resultset r)
681 {
682     if (r)
683     {
684         yaz_mutex_enter(r->mutex);
685         (r->refcount)++;
686         yaz_log(log_details0, "%p ZOOM_resultset_addref count=%d",
687                 r, r->refcount);
688         yaz_mutex_leave(r->mutex);
689     }
690 }
691
692 ZOOM_resultset ZOOM_resultset_create(void)
693 {
694     int i;
695     ZOOM_resultset r = (ZOOM_resultset) xmalloc(sizeof(*r));
696
697     initlog();
698
699     yaz_log(log_details0, "%p ZOOM_resultset_create", r);
700     r->refcount = 1;
701     r->size = 0;
702     r->odr = odr_createmem(ODR_ENCODE);
703     r->piggyback = 1;
704     r->setname = 0;
705     r->schema = 0;
706     r->step = 0;
707     for (i = 0; i<RECORD_HASH_SIZE; i++)
708         r->record_hash[i] = 0;
709     r->r_sort_spec = 0;
710     r->query = 0;
711     r->connection = 0;
712     r->databaseNames = 0;
713     r->num_databaseNames = 0;
714     r->facets = 0;
715     r->num_facets = 0;
716     r->facets_names = 0;
717     r->mutex = 0;
718     yaz_mutex_create(&r->mutex);
719 #if SHPTR
720     {
721         WRBUF w = wrbuf_alloc();
722         YAZ_SHPTR_INIT(r->record_wrbuf, w);
723     }
724 #endif
725     return r;
726 }
727
728 ZOOM_API(ZOOM_resultset)
729     ZOOM_connection_search_pqf(ZOOM_connection c, const char *q)
730 {
731     ZOOM_resultset r;
732     ZOOM_query s = ZOOM_query_create();
733
734     ZOOM_query_prefix(s, q);
735
736     r = ZOOM_connection_search(c, s);
737     ZOOM_query_destroy(s);
738     return r;
739 }
740
741 ZOOM_API(ZOOM_resultset)
742     ZOOM_connection_search(ZOOM_connection c, ZOOM_query q)
743 {
744     ZOOM_resultset r = ZOOM_resultset_create();
745     ZOOM_task task;
746     const char *cp;
747     int start, count;
748     const char *syntax, *elementSetName;
749 #if ZOOM_RESULT_LISTS
750     ZOOM_resultsets set;
751 #endif
752
753     yaz_log(c->log_api, "%p ZOOM_connection_search set %p query %p", c, r, q);
754     r->r_sort_spec = ZOOM_query_get_sortspec(q);
755     r->query = q;
756
757     r->options = ZOOM_options_create_with_parent(c->options);
758     
759     start = ZOOM_options_get_int(r->options, "start", 0);
760     count = ZOOM_options_get_int(r->options, "count", 0);
761     {
762         /* If "presentChunk" is defined use that; otherwise "step" */
763         const char *cp = ZOOM_options_get(r->options, "presentChunk");
764         r->step = ZOOM_options_get_int(r->options,
765                                        (cp != 0 ? "presentChunk": "step"), 0);
766     }
767     r->piggyback = ZOOM_options_get_bool(r->options, "piggyback", 1);
768     cp = ZOOM_options_get(r->options, "setname");
769     if (cp)
770         r->setname = xstrdup(cp);
771     cp = ZOOM_options_get(r->options, "schema");
772     if (cp)
773         r->schema = xstrdup(cp);
774
775     r->databaseNames = ZOOM_connection_get_databases(c, c->options, &r->num_databaseNames,
776                                          r->odr);
777     
778     r->connection = c;
779
780 #if ZOOM_RESULT_LISTS
781     yaz_log(log_details, "%p ZOOM_connection_search: Adding new resultset (%p) to resultsets (%p) ", c, r, c->resultsets);
782     set = xmalloc(sizeof(*set));
783     ZOOM_resultset_addref(r);
784     set->resultset = r;
785     set->next = c->resultsets;
786     c->resultsets = set;
787 #else
788     r->next = c->resultsets;
789     c->resultsets = r;
790 #endif
791     if (c->host_port && c->proto == PROTO_HTTP)
792     {
793         if (!c->cs)
794         {
795             yaz_log(c->log_details, "ZOOM_connection_search: no comstack");
796             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
797         }
798         else
799         {
800             yaz_log(c->log_details, "ZOOM_connection_search: reconnect");
801             c->reconnect_ok = 1;
802         }
803     }
804
805     task = ZOOM_connection_add_task(c, ZOOM_TASK_SEARCH);
806     task->u.search.resultset = r;
807     task->u.search.start = start;
808     task->u.search.count = count;
809     task->u.search.recv_search_fired = 0;
810
811     syntax = ZOOM_options_get(r->options, "preferredRecordSyntax"); 
812     task->u.search.syntax = syntax ? xstrdup(syntax) : 0;
813     elementSetName = ZOOM_options_get(r->options, "elementSetName");
814     task->u.search.elementSetName = elementSetName 
815         ? xstrdup(elementSetName) : 0;
816    
817     ZOOM_resultset_addref(r);
818
819     ZOOM_query_addref(q);
820
821     if (!c->async)
822     {
823         while (ZOOM_event(1, &c))
824             ;
825     }
826     return r;
827 }
828
829 ZOOM_API(void)
830     ZOOM_resultset_sort(ZOOM_resultset r,
831                          const char *sort_type, const char *sort_spec)
832 {
833     (void) ZOOM_resultset_sort1(r, sort_type, sort_spec);
834 }
835
836 ZOOM_API(int)
837     ZOOM_resultset_sort1(ZOOM_resultset r,
838                          const char *sort_type, const char *sort_spec)
839 {
840     ZOOM_connection c = r->connection;
841     ZOOM_task task;
842     ZOOM_query newq;
843
844     newq = ZOOM_query_create();
845     if (ZOOM_query_sortby(newq, sort_spec) < 0)
846         return -1;
847
848     yaz_log(c->log_api, "%p ZOOM_resultset_sort r=%p sort_type=%s sort_spec=%s",
849             r, r, sort_type, sort_spec);
850     if (!c)
851         return 0;
852
853     if (c->host_port && c->proto == PROTO_HTTP)
854     {
855         if (!c->cs)
856         {
857             yaz_log(c->log_details, "%p ZOOM_resultset_sort: no comstack", r);
858             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
859         }
860         else
861         {
862             yaz_log(c->log_details, "%p ZOOM_resultset_sort: prepare reconnect",
863                     r);
864             c->reconnect_ok = 1;
865         }
866     }
867     
868     ZOOM_resultset_cache_reset(r);
869     task = ZOOM_connection_add_task(c, ZOOM_TASK_SORT);
870     task->u.sort.resultset = r;
871     task->u.sort.q = newq;
872
873     ZOOM_resultset_addref(r);  
874
875     if (!c->async)
876     {
877         while (ZOOM_event(1, &c))
878             ;
879     }
880
881     return 0;
882 }
883
884 ZOOM_API(void)
885     ZOOM_resultset_destroy(ZOOM_resultset r)
886 {
887     resultset_destroy(r);
888 }
889
890 static void resultset_destroy(ZOOM_resultset r)
891 {
892     if (!r)
893         return;
894     yaz_mutex_enter(r->mutex);
895     (r->refcount)--;
896     yaz_log(log_details0, "%p ZOOM_resultset_destroy r=%p count=%d",
897             r, r, r->refcount);
898     if (r->refcount == 0)
899     {
900         yaz_mutex_leave(r->mutex);
901
902         yaz_log(log_details0, "%p ZOOM_connection resultset_destroy: Deleting resultset (%p) ", r->connection, r);
903         ZOOM_resultset_cache_reset(r);
904         ZOOM_resultset_release(r);
905         ZOOM_query_destroy(r->query);
906         ZOOM_options_destroy(r->options);
907         odr_destroy(r->odr);
908         xfree(r->setname);
909         xfree(r->schema);
910         yaz_mutex_destroy(&r->mutex);
911 #if SHPTR
912         YAZ_SHPTR_DEC(r->record_wrbuf, wrbuf_destroy);
913 #endif
914         xfree(r);
915     }
916     else
917         yaz_mutex_leave(r->mutex);
918 }
919
920 ZOOM_API(size_t)
921     ZOOM_resultset_size(ZOOM_resultset r)
922 {
923     return r->size;
924 }
925
926 int ZOOM_test_reconnect(ZOOM_connection c)
927 {
928     ZOOM_Event event;
929
930     if (!c->reconnect_ok)
931         return 0;
932     ZOOM_connection_close(c);
933     c->reconnect_ok = 0;
934     c->tasks->running = 0;
935     ZOOM_connection_insert_task(c, ZOOM_TASK_CONNECT);
936
937     event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
938     ZOOM_connection_put_event(c, event);
939
940     return 1;
941 }
942
943 static void ZOOM_resultset_retrieve(ZOOM_resultset r,
944                                     int force_sync, int start, int count)
945 {
946     ZOOM_task task;
947     ZOOM_connection c;
948     const char *cp;
949     const char *syntax, *elementSetName;
950
951     if (!r)
952         return;
953     yaz_log(log_details0, "%p ZOOM_resultset_retrieve force_sync=%d start=%d"
954             " count=%d", r, force_sync, start, count);
955     c = r->connection;
956     if (!c)
957         return;
958
959     if (c->host_port && c->proto == PROTO_HTTP)
960     {
961         if (!c->cs)
962         {
963             yaz_log(log_details0, "%p ZOOM_resultset_retrieve: no comstack", r);
964             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
965         }
966         else
967         {
968             yaz_log(log_details0, "%p ZOOM_resultset_retrieve: prepare "
969                     "reconnect", r);
970             c->reconnect_ok = 1;
971         }
972     }
973     task = ZOOM_connection_add_task(c, ZOOM_TASK_RETRIEVE);
974     task->u.retrieve.resultset = r;
975     task->u.retrieve.start = start;
976     task->u.retrieve.count = count;
977
978     syntax = ZOOM_options_get(r->options, "preferredRecordSyntax"); 
979     task->u.retrieve.syntax = syntax ? xstrdup(syntax) : 0;
980     elementSetName = ZOOM_options_get(r->options, "elementSetName");
981     task->u.retrieve.elementSetName = elementSetName 
982         ? xstrdup(elementSetName) : 0;
983
984     cp = ZOOM_options_get(r->options, "schema");
985     if (cp)
986     {
987         if (!r->schema || strcmp(r->schema, cp))
988         {
989             xfree(r->schema);
990             r->schema = xstrdup(cp);
991         }
992     }
993
994     ZOOM_resultset_addref(r);
995
996     if (!r->connection->async || force_sync)
997         while (r->connection && ZOOM_event(1, &r->connection))
998             ;
999 }
1000
1001 ZOOM_API(void)
1002     ZOOM_resultset_records(ZOOM_resultset r, ZOOM_record *recs,
1003                            size_t start, size_t count)
1004 {
1005     int force_present = 0;
1006
1007     if (!r)
1008         return ;
1009     yaz_log(log_api0, "%p ZOOM_resultset_records r=%p start=%ld count=%ld",
1010             r, r, (long) start, (long) count);
1011     if (count && recs)
1012         force_present = 1;
1013     ZOOM_resultset_retrieve(r, force_present, start, count);
1014     if (force_present)
1015     {
1016         size_t i;
1017         for (i = 0; i< count; i++)
1018             recs[i] = ZOOM_resultset_record_immediate(r, i+start);
1019     }
1020 }
1021
1022 ZOOM_API(size_t)
1023     ZOOM_resultset_facets_size(ZOOM_resultset r) {
1024     return r->num_facets;
1025 }
1026
1027 ZOOM_API(ZOOM_facet_field)
1028     ZOOM_resultset_get_facet_field(ZOOM_resultset r, const char *name) {
1029     int num = r->num_facets;
1030     ZOOM_facet_field *facets = r->facets;
1031     int index;
1032     for (index = 0; index < num; index++) {
1033         if (!strcmp(facets[index]->facet_name, name)) {
1034             return facets[index];
1035         }
1036     }
1037     return 0;
1038 }
1039
1040
1041 ZOOM_API(ZOOM_facet_field *)
1042     ZOOM_resultset_facets(ZOOM_resultset r)
1043 {
1044     return r->facets;
1045 }
1046
1047 ZOOM_API(const char**)
1048     ZOOM_resultset_facet_names(ZOOM_resultset r)
1049 {
1050     return (const char **) r->facets_names;
1051 }
1052
1053 ZOOM_API(const char*)
1054     ZOOM_facet_field_name(ZOOM_facet_field field)
1055 {
1056     return field->facet_name;
1057 }
1058
1059 ZOOM_API(size_t)
1060     ZOOM_facet_field_term_count(ZOOM_facet_field field)
1061 {
1062     return field->num_terms;
1063 }
1064
1065 ZOOM_API(const char*)
1066     ZOOM_facet_field_get_term(ZOOM_facet_field field, size_t idx, int *freq) {
1067     *freq = field->facet_terms[idx].frequency;
1068     return field->facet_terms[idx].term;
1069 }
1070
1071
1072 static void get_cert(ZOOM_connection c)
1073 {
1074     char *cert_buf;
1075     int cert_len;
1076     
1077     if (cs_get_peer_certificate_x509(c->cs, &cert_buf, &cert_len))
1078     {
1079         ZOOM_connection_option_setl(c, "sslPeerCert",
1080                                     cert_buf, cert_len);
1081         xfree(cert_buf);
1082     }
1083 }
1084
1085 static zoom_ret do_connect_host(ZOOM_connection c,
1086                                 const char *effective_host,
1087                                 const char *logical_url);
1088
1089 static zoom_ret do_connect(ZOOM_connection c)
1090 {
1091     const char *effective_host;
1092
1093     if (c->proxy)
1094         effective_host = c->proxy;
1095     else
1096         effective_host = c->host_port;
1097     return do_connect_host(c, effective_host, c->host_port);
1098 }
1099
1100 static zoom_ret do_connect_host(ZOOM_connection c, const char *effective_host,
1101     const char *logical_url)
1102 {
1103     void *add;
1104
1105     yaz_log(c->log_details, "%p do_connect effective_host=%s", c, effective_host);
1106
1107     if (c->cs)
1108         cs_close(c->cs);
1109     c->cs = cs_create_host(effective_host, 0, &add);
1110
1111     if (c->cs && c->cs->protocol == PROTO_HTTP)
1112     {
1113 #if YAZ_HAVE_XML2
1114         if (logical_url)
1115         {
1116             const char *db = 0;
1117             
1118             c->proto = PROTO_HTTP;
1119             cs_get_host_args(logical_url, &db);
1120             xfree(c->path);
1121             
1122             c->path = xmalloc(strlen(db) * 3 + 2);
1123             yaz_encode_sru_dbpath_buf(c->path, db);
1124         }
1125 #else
1126         ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, "SRW");
1127         ZOOM_connection_close(c);
1128         return zoom_complete;
1129 #endif
1130     }
1131     if (c->cs)
1132     {
1133         int ret = cs_connect(c->cs, add);
1134         if (ret == 0)
1135         {
1136             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1137             ZOOM_connection_put_event(c, event);
1138             get_cert(c);
1139             if (c->proto == PROTO_Z3950)
1140                 ZOOM_connection_Z3950_send_init(c);
1141             else
1142             {
1143                 /* no init request for SRW .. */
1144                 assert(c->tasks->which == ZOOM_TASK_CONNECT);
1145                 ZOOM_connection_remove_task(c);
1146                 ZOOM_connection_set_mask(c, 0);
1147                 ZOOM_connection_exec_task(c);
1148             }
1149             c->state = STATE_ESTABLISHED;
1150             return zoom_pending;
1151         }
1152         else if (ret > 0)
1153         {
1154             int mask = ZOOM_SELECT_EXCEPT;
1155             if (c->cs->io_pending & CS_WANT_WRITE)
1156                 mask += ZOOM_SELECT_WRITE;
1157             if (c->cs->io_pending & CS_WANT_READ)
1158                 mask += ZOOM_SELECT_READ;
1159             ZOOM_connection_set_mask(c, mask);
1160             c->state = STATE_CONNECTING; 
1161             return zoom_pending;
1162         }
1163     }
1164     c->state = STATE_IDLE;
1165     ZOOM_set_error(c, ZOOM_ERROR_CONNECT, logical_url);
1166     return zoom_complete;
1167 }
1168
1169 /* returns 1 if PDU was sent OK (still pending )
1170    0 if PDU was not sent OK (nothing to wait for) 
1171 */
1172
1173 ZOOM_API(ZOOM_record)
1174     ZOOM_resultset_record_immediate(ZOOM_resultset s,size_t pos)
1175 {
1176     const char *syntax =
1177         ZOOM_options_get(s->options, "preferredRecordSyntax"); 
1178     const char *elementSetName =
1179         ZOOM_options_get(s->options, "elementSetName");
1180
1181     return ZOOM_record_cache_lookup(s, pos, syntax, elementSetName);
1182 }
1183
1184 ZOOM_API(ZOOM_record)
1185     ZOOM_resultset_record(ZOOM_resultset r, size_t pos)
1186 {
1187     ZOOM_record rec = ZOOM_resultset_record_immediate(r, pos);
1188
1189     if (!rec)
1190     {
1191         /*
1192          * MIKE: I think force_sync should always be zero, but I don't
1193          * want to make this change until I get the go-ahead from
1194          * Adam, in case something depends on the old synchronous
1195          * behaviour.
1196          */
1197         int force_sync = 1;
1198         if (getenv("ZOOM_RECORD_NO_FORCE_SYNC")) force_sync = 0;
1199         ZOOM_resultset_retrieve(r, force_sync, pos, 1);
1200         rec = ZOOM_resultset_record_immediate(r, pos);
1201     }
1202     return rec;
1203 }
1204
1205 ZOOM_API(ZOOM_scanset)
1206     ZOOM_connection_scan(ZOOM_connection c, const char *start)
1207 {
1208     ZOOM_scanset s;
1209     ZOOM_query q = ZOOM_query_create();
1210
1211     ZOOM_query_prefix(q, start);
1212
1213     s = ZOOM_connection_scan1(c, q);
1214     ZOOM_query_destroy(q);
1215     return s;
1216
1217 }
1218
1219 ZOOM_API(ZOOM_scanset)
1220     ZOOM_connection_scan1(ZOOM_connection c, ZOOM_query q)
1221 {
1222     ZOOM_scanset scan = 0;
1223     Z_Query *z_query = ZOOM_query_get_Z_Query(q);
1224
1225     if (!z_query)
1226         return 0;
1227     scan = (ZOOM_scanset) xmalloc(sizeof(*scan));
1228     scan->connection = c;
1229     scan->odr = odr_createmem(ODR_DECODE);
1230     scan->options = ZOOM_options_create_with_parent(c->options);
1231     scan->refcount = 1;
1232     scan->scan_response = 0;
1233     scan->srw_scan_response = 0;
1234
1235     scan->query = q;
1236     ZOOM_query_addref(q);
1237     scan->databaseNames = ZOOM_connection_get_databases(c, c->options,
1238                                             &scan->num_databaseNames,
1239                                             scan->odr);
1240
1241     if (1)
1242     {
1243         ZOOM_task task = ZOOM_connection_add_task(c, ZOOM_TASK_SCAN);
1244         task->u.scan.scan = scan;
1245         
1246         (scan->refcount)++;
1247         if (!c->async)
1248         {
1249             while (ZOOM_event(1, &c))
1250                 ;
1251         }
1252     }
1253     return scan;
1254 }
1255
1256 ZOOM_API(void)
1257     ZOOM_scanset_destroy(ZOOM_scanset scan)
1258 {
1259     if (!scan)
1260         return;
1261     (scan->refcount)--;
1262     if (scan->refcount == 0)
1263     {
1264         ZOOM_query_destroy(scan->query);
1265
1266         odr_destroy(scan->odr);
1267         
1268         ZOOM_options_destroy(scan->options);
1269         xfree(scan);
1270     }
1271 }
1272
1273 static zoom_ret send_package(ZOOM_connection c)
1274 {
1275     ZOOM_Event event;
1276
1277     yaz_log(c->log_details, "%p send_package", c);
1278     if (!c->tasks)
1279         return zoom_complete;
1280     assert (c->tasks->which == ZOOM_TASK_PACKAGE);
1281     
1282     event = ZOOM_Event_create(ZOOM_EVENT_SEND_APDU);
1283     ZOOM_connection_put_event(c, event);
1284     
1285     c->buf_out = c->tasks->u.package->buf_out;
1286     c->len_out = c->tasks->u.package->len_out;
1287
1288     return ZOOM_send_buf(c);
1289 }
1290
1291 ZOOM_API(size_t)
1292     ZOOM_scanset_size(ZOOM_scanset scan)
1293 {
1294     if (!scan)
1295         return 0;
1296
1297     if (scan->scan_response && scan->scan_response->entries)
1298         return scan->scan_response->entries->num_entries;
1299     else if (scan->srw_scan_response)
1300         return scan->srw_scan_response->num_terms;
1301     return 0;
1302 }
1303
1304 static void ZOOM_scanset_term_x(ZOOM_scanset scan, size_t pos,
1305                                 size_t *occ,
1306                                 const char **value_term, size_t *value_len,
1307                                 const char **disp_term, size_t *disp_len)
1308 {
1309     size_t noent = ZOOM_scanset_size(scan);
1310     
1311     *value_term = 0;
1312     *value_len = 0;
1313
1314     *disp_term = 0;
1315     *disp_len = 0;
1316
1317     *occ = 0;
1318     if (pos >= noent)
1319         return;
1320     if (scan->scan_response)
1321     {
1322         Z_ScanResponse *res = scan->scan_response;
1323         if (res->entries->entries[pos]->which == Z_Entry_termInfo)
1324         {
1325             Z_TermInfo *t = res->entries->entries[pos]->u.termInfo;
1326             
1327             *value_term = (const char *) t->term->u.general->buf;
1328             *value_len = t->term->u.general->len;
1329             if (t->displayTerm)
1330             {
1331                 *disp_term = t->displayTerm;
1332                 *disp_len = strlen(*disp_term);
1333             }
1334             else if (t->term->which == Z_Term_general)
1335             {
1336                 *disp_term = (const char *) t->term->u.general->buf;
1337                 *disp_len = t->term->u.general->len;
1338             }
1339             *occ = t->globalOccurrences ? *t->globalOccurrences : 0;
1340         }
1341     }
1342     if (scan->srw_scan_response)
1343     {
1344         Z_SRW_scanResponse *res = scan->srw_scan_response;
1345         Z_SRW_scanTerm *t = res->terms + pos;
1346         if (t)
1347         {
1348             *value_term = t->value;
1349             *value_len = strlen(*value_term);
1350
1351             if (t->displayTerm)
1352                 *disp_term = t->displayTerm;
1353             else
1354                 *disp_term = t->value;
1355             *disp_len = strlen(*disp_term);
1356             *occ = t->numberOfRecords ? *t->numberOfRecords : 0;
1357         }
1358     }
1359 }
1360
1361 ZOOM_API(const char *)
1362     ZOOM_scanset_term(ZOOM_scanset scan, size_t pos,
1363                       size_t *occ, size_t *len)
1364 {
1365     const char *value_term = 0;
1366     size_t value_len = 0;
1367     const char *disp_term = 0;
1368     size_t disp_len = 0;
1369
1370     ZOOM_scanset_term_x(scan, pos, occ, &value_term, &value_len,
1371                         &disp_term, &disp_len);
1372     
1373     *len = value_len;
1374     return value_term;
1375 }
1376
1377 ZOOM_API(const char *)
1378     ZOOM_scanset_display_term(ZOOM_scanset scan, size_t pos,
1379                               size_t *occ, size_t *len)
1380 {
1381     const char *value_term = 0;
1382     size_t value_len = 0;
1383     const char *disp_term = 0;
1384     size_t disp_len = 0;
1385
1386     ZOOM_scanset_term_x(scan, pos, occ, &value_term, &value_len,
1387                         &disp_term, &disp_len);
1388     
1389     *len = disp_len;
1390     return disp_term;
1391 }
1392
1393 ZOOM_API(const char *)
1394     ZOOM_scanset_option_get(ZOOM_scanset scan, const char *key)
1395 {
1396     return ZOOM_options_get(scan->options, key);
1397 }
1398
1399 ZOOM_API(void)
1400     ZOOM_scanset_option_set(ZOOM_scanset scan, const char *key,
1401                             const char *val)
1402 {
1403     ZOOM_options_set(scan->options, key, val);
1404 }
1405
1406
1407 ZOOM_API(ZOOM_package)
1408     ZOOM_connection_package(ZOOM_connection c, ZOOM_options options)
1409 {
1410     ZOOM_package p = (ZOOM_package) xmalloc(sizeof(*p));
1411
1412     p->connection = c;
1413     p->odr_out = odr_createmem(ODR_ENCODE);
1414     p->options = ZOOM_options_create_with_parent2(options, c->options);
1415     p->refcount = 1;
1416     p->buf_out = 0;
1417     p->len_out = 0;
1418     return p;
1419 }
1420
1421 ZOOM_API(void)
1422     ZOOM_package_destroy(ZOOM_package p)
1423 {
1424     if (!p)
1425         return;
1426     (p->refcount)--;
1427     if (p->refcount == 0)
1428     {
1429         odr_destroy(p->odr_out);
1430         xfree(p->buf_out);
1431         
1432         ZOOM_options_destroy(p->options);
1433         xfree(p);
1434     }
1435 }
1436
1437 ZOOM_API(const char *)
1438     ZOOM_package_option_get(ZOOM_package p, const char *key)
1439 {
1440     return ZOOM_options_get(p->options, key);
1441 }
1442
1443 ZOOM_API(const char *)
1444     ZOOM_package_option_getl(ZOOM_package p, const char *key, int *lenp)
1445 {
1446     return ZOOM_options_getl(p->options, key, lenp);
1447 }
1448
1449 ZOOM_API(void)
1450     ZOOM_package_option_set(ZOOM_package p, const char *key,
1451                             const char *val)
1452 {
1453     ZOOM_options_set(p->options, key, val);
1454 }
1455
1456 ZOOM_API(void)
1457     ZOOM_package_option_setl(ZOOM_package p, const char *key,
1458                              const char *val, int len)
1459 {
1460     ZOOM_options_setl(p->options, key, val, len);
1461 }
1462
1463 ZOOM_API(int)
1464     ZOOM_connection_exec_task(ZOOM_connection c)
1465 {
1466     ZOOM_task task = c->tasks;
1467     zoom_ret ret = zoom_complete;
1468
1469     if (!task)
1470         return 0;
1471     yaz_log(c->log_details, "%p ZOOM_connection_exec_task type=%d run=%d",
1472             c, task->which, task->running);
1473     if (c->error != ZOOM_ERROR_NONE)
1474     {
1475         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1476                 "removing tasks because of error = %d", c, c->error);
1477         ZOOM_connection_remove_tasks(c);
1478         return 0;
1479     }
1480     if (task->running)
1481     {
1482         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1483                 "task already running", c);
1484         return 0;
1485     }
1486     task->running = 1;
1487     ret = zoom_complete;
1488     if (c->cs || task->which == ZOOM_TASK_CONNECT)
1489     {
1490         switch (task->which)
1491         {
1492         case ZOOM_TASK_SEARCH:
1493             if (c->proto == PROTO_HTTP)
1494                 ret = ZOOM_connection_srw_send_search(c);
1495             else
1496                 ret = ZOOM_connection_Z3950_send_search(c);
1497             break;
1498         case ZOOM_TASK_RETRIEVE:
1499             if (c->proto == PROTO_HTTP)
1500                 ret = ZOOM_connection_srw_send_search(c);
1501             else
1502                 ret = send_Z3950_present(c);
1503             break;
1504         case ZOOM_TASK_CONNECT:
1505             ret = do_connect(c);
1506             break;
1507         case ZOOM_TASK_SCAN:
1508             if (c->proto == PROTO_HTTP)
1509                 ret = ZOOM_connection_srw_send_scan(c);
1510             else
1511                 ret = ZOOM_connection_Z3950_send_scan(c);
1512             break;
1513         case ZOOM_TASK_PACKAGE:
1514             ret = send_package(c);
1515             break;
1516         case ZOOM_TASK_SORT:
1517             c->tasks->u.sort.resultset->r_sort_spec = 
1518                 ZOOM_query_get_sortspec(c->tasks->u.sort.q);
1519             ret = send_Z3950_sort(c, c->tasks->u.sort.resultset);
1520             break;
1521         }
1522     }
1523     else
1524     {
1525         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1526                 "remove tasks because no connection exist", c);
1527         ZOOM_connection_remove_tasks(c);
1528     }
1529     if (ret == zoom_complete)
1530     {
1531         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1532                 "task removed (complete)", c);
1533         ZOOM_connection_remove_task(c);
1534         return 0;
1535     }
1536     yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1537             "task pending", c);
1538     return 1;
1539 }
1540
1541 #if YAZ_HAVE_XML2
1542 static Z_GDU *get_HTTP_Request_url(ODR odr, const char *url)
1543 {
1544     Z_GDU *p = z_get_HTTP_Request(odr);
1545     const char *host = url;
1546     const char *cp0 = strstr(host, "://");
1547     const char *cp1 = 0;
1548     if (cp0)
1549         cp0 = cp0+3;
1550     else
1551         cp0 = host;
1552     
1553     cp1 = strchr(cp0, '/');
1554     if (!cp1)
1555         cp1 = cp0 + strlen(cp0);
1556     
1557     if (cp0 && cp1)
1558     {
1559         char *h = (char*) odr_malloc(odr, cp1 - cp0 + 1);
1560         memcpy (h, cp0, cp1 - cp0);
1561         h[cp1-cp0] = '\0';
1562         z_HTTP_header_add(odr, &p->u.HTTP_Request->headers, "Host", h);
1563     }
1564     p->u.HTTP_Request->path = odr_strdup(odr, *cp1 ? cp1 : "/");
1565     return p;
1566 }
1567
1568 static zoom_ret send_HTTP_redirect(ZOOM_connection c, const char *uri,
1569                                   Z_HTTP_Response *cookie_hres)
1570 {
1571     struct Z_HTTP_Header *h;
1572     Z_GDU *gdu = get_HTTP_Request_url(c->odr_out, uri);
1573     char *combined_cookies;
1574     int combined_cookies_len = 0;
1575
1576     gdu->u.HTTP_Request->method = odr_strdup(c->odr_out, "GET");
1577     z_HTTP_header_add(c->odr_out, &gdu->u.HTTP_Request->headers, "Accept",
1578                       "text/xml");
1579
1580     for (h = cookie_hres->headers; h; h = h->next)
1581     {
1582         if (!strcmp(h->name, "Set-Cookie"))
1583         {
1584             char *cp;
1585
1586             if (!(cp = strchr(h->value, ';')))
1587                 cp = h->value + strlen(h->value);
1588             if (cp - h->value >= 1) {
1589                 combined_cookies = xrealloc(combined_cookies, combined_cookies_len + cp - h->value + 3);
1590                 memcpy(combined_cookies+combined_cookies_len, h->value, cp - h->value);
1591                 combined_cookies[combined_cookies_len + cp - h->value] = '\0';
1592                 strcat(combined_cookies,"; ");
1593                 combined_cookies_len = strlen(combined_cookies);
1594             }
1595         }
1596     }
1597
1598     if (combined_cookies_len)
1599     {
1600         z_HTTP_header_add(c->odr_out, &gdu->u.HTTP_Request->headers,
1601                           "Cookie", combined_cookies);
1602         xfree(combined_cookies);
1603     }
1604
1605     if (c->user && c->password)
1606     {
1607         z_HTTP_header_add_basic_auth(c->odr_out, &gdu->u.HTTP_Request->headers,
1608                                      c->user, c->password);
1609     }
1610     if (!z_GDU(c->odr_out, &gdu, 0, 0))
1611         return zoom_complete;
1612     if (c->odr_print)
1613         z_GDU(c->odr_print, &gdu, 0, 0);
1614     c->buf_out = odr_getbuf(c->odr_out, &c->len_out, 0);
1615
1616     odr_reset(c->odr_out);
1617     return ZOOM_send_buf(c);
1618 }
1619
1620 #if YAZ_HAVE_XML2
1621 void ZOOM_set_HTTP_error(ZOOM_connection c, int error,
1622                          const char *addinfo, const char *addinfo2)
1623 {
1624     ZOOM_set_dset_error(c, error, "HTTP", addinfo, addinfo2);
1625 }
1626 #endif
1627
1628
1629 static void handle_http(ZOOM_connection c, Z_HTTP_Response *hres)
1630 {
1631     zoom_ret cret = zoom_complete;
1632     int ret = -1;
1633     const char *addinfo = 0;
1634     const char *connection_head = z_HTTP_header_lookup(hres->headers,
1635                                                        "Connection");
1636     const char *location;
1637
1638     ZOOM_connection_set_mask(c, 0);
1639     yaz_log(c->log_details, "%p handle_http", c);
1640     
1641     if ((hres->code == 301 || hres->code == 302) && c->sru_mode == zoom_sru_get
1642         && (location = z_HTTP_header_lookup(hres->headers, "Location")))
1643     {
1644         c->no_redirects++;
1645         if (c->no_redirects > 10)
1646         {
1647             ZOOM_set_HTTP_error(c, hres->code, 0, 0);
1648             c->no_redirects = 0;
1649             ZOOM_connection_close(c);
1650         }
1651         else
1652         {
1653             /* since redirect may change host we just reconnect. A smarter
1654                implementation might check whether it's the same server */
1655             do_connect_host(c, location, 0);
1656             send_HTTP_redirect(c, location, hres);
1657             /* we're OK for now. Operation is not really complete */
1658             ret = 0;
1659             cret = zoom_pending;
1660         }
1661     }
1662     else 
1663     {  
1664         ret = ZOOM_handle_sru(c, hres, &cret);
1665         if (ret == 0)
1666         {
1667             if (c->no_redirects) /* end of redirect. change hosts again */
1668                 ZOOM_connection_close(c);
1669         }
1670         c->no_redirects = 0;
1671     }
1672     if (ret)
1673     {
1674         if (hres->code != 200)
1675             ZOOM_set_HTTP_error(c, hres->code, 0, 0);
1676         else
1677             ZOOM_set_error(c, ZOOM_ERROR_DECODE, addinfo);
1678         ZOOM_connection_close(c);
1679     }
1680     if (cret == zoom_complete)
1681     {
1682         yaz_log(YLOG_LOG, "removing tasks in handle_http");
1683         ZOOM_connection_remove_task(c);
1684     }
1685     {
1686         int must_close = 0;
1687         if (!strcmp(hres->version, "1.0"))
1688         {
1689             /* HTTP 1.0: only if Keep-Alive we stay alive.. */
1690             if (!connection_head || strcmp(connection_head, "Keep-Alive"))
1691                 must_close = 1;
1692         }
1693         else
1694         {
1695             /* HTTP 1.1: only if no close we stay alive.. */
1696             if (connection_head && !strcmp(connection_head, "close"))
1697                 must_close = 1;
1698         }
1699         if (must_close)
1700         {
1701             ZOOM_connection_close(c);
1702             if (c->tasks)
1703             {
1704                 c->tasks->running = 0;
1705                 ZOOM_connection_insert_task(c, ZOOM_TASK_CONNECT);
1706                 c->reconnect_ok = 0;
1707             }
1708         }
1709     }
1710 }
1711 #endif
1712
1713 static int do_read(ZOOM_connection c)
1714 {
1715     int r, more;
1716     ZOOM_Event event;
1717     
1718     event = ZOOM_Event_create(ZOOM_EVENT_RECV_DATA);
1719     ZOOM_connection_put_event(c, event);
1720     
1721     r = cs_get(c->cs, &c->buf_in, &c->len_in);
1722     more = cs_more(c->cs);
1723     yaz_log(c->log_details, "%p do_read len=%d more=%d", c, r, more);
1724     if (r == 1)
1725         return 0;
1726     if (r <= 0)
1727     {
1728         if (!ZOOM_test_reconnect(c))
1729         {
1730             ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
1731             ZOOM_connection_close(c);
1732         }
1733     }
1734     else
1735     {
1736         Z_GDU *gdu;
1737         ZOOM_Event event;
1738
1739         odr_reset(c->odr_in);
1740         odr_setbuf(c->odr_in, c->buf_in, r, 0);
1741         event = ZOOM_Event_create(ZOOM_EVENT_RECV_APDU);
1742         ZOOM_connection_put_event(c, event);
1743
1744         if (!z_GDU(c->odr_in, &gdu, 0, 0))
1745         {
1746             int x;
1747             int err = odr_geterrorx(c->odr_in, &x);
1748             char msg[100];
1749             const char *element = odr_getelement(c->odr_in);
1750             yaz_snprintf(msg, sizeof(msg),
1751                     "ODR code %d:%d element=%s offset=%d",
1752                     err, x, element ? element : "<unknown>",
1753                     odr_offset(c->odr_in));
1754             ZOOM_set_error(c, ZOOM_ERROR_DECODE, msg);
1755             if (c->log_api)
1756             {
1757                 FILE *ber_file = yaz_log_file();
1758                 if (ber_file)
1759                     odr_dumpBER(ber_file, c->buf_in, r);
1760             }
1761             ZOOM_connection_close(c);
1762         }
1763         else
1764         {
1765             if (c->odr_print)
1766                 z_GDU(c->odr_print, &gdu, 0, 0);
1767             if (gdu->which == Z_GDU_Z3950)
1768                 ZOOM_handle_Z3950_apdu(c, gdu->u.z3950);
1769             else if (gdu->which == Z_GDU_HTTP_Response)
1770             {
1771 #if YAZ_HAVE_XML2
1772                 handle_http(c, gdu->u.HTTP_Response);
1773 #else
1774                 ZOOM_set_error(c, ZOOM_ERROR_DECODE, 0);
1775                 ZOOM_connection_close(c);
1776 #endif
1777             }
1778         }
1779         c->reconnect_ok = 0;
1780     }
1781     return 1;
1782 }
1783
1784 static zoom_ret do_write_ex(ZOOM_connection c, char *buf_out, int len_out)
1785 {
1786     int r;
1787     ZOOM_Event event;
1788     
1789     event = ZOOM_Event_create(ZOOM_EVENT_SEND_DATA);
1790     ZOOM_connection_put_event(c, event);
1791
1792     yaz_log(c->log_details, "%p do_write_ex len=%d", c, len_out);
1793     if ((r = cs_put(c->cs, buf_out, len_out)) < 0)
1794     {
1795         yaz_log(c->log_details, "%p do_write_ex write failed", c);
1796         if (ZOOM_test_reconnect(c))
1797         {
1798             return zoom_pending;
1799         }
1800         if (c->state == STATE_CONNECTING)
1801             ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
1802         else
1803             ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
1804         ZOOM_connection_close(c);
1805         return zoom_complete;
1806     }
1807     else if (r == 1)
1808     {    
1809         int mask = ZOOM_SELECT_EXCEPT;
1810         if (c->cs->io_pending & CS_WANT_WRITE)
1811             mask += ZOOM_SELECT_WRITE;
1812         if (c->cs->io_pending & CS_WANT_READ)
1813             mask += ZOOM_SELECT_READ;
1814         ZOOM_connection_set_mask(c, mask);
1815         yaz_log(c->log_details, "%p do_write_ex write incomplete mask=%d",
1816                 c, c->mask);
1817     }
1818     else
1819     {
1820         ZOOM_connection_set_mask(c, ZOOM_SELECT_READ|ZOOM_SELECT_EXCEPT);
1821         yaz_log(c->log_details, "%p do_write_ex write complete mask=%d",
1822                 c, c->mask);
1823     }
1824     return zoom_pending;
1825 }
1826
1827 zoom_ret ZOOM_send_buf(ZOOM_connection c)
1828 {
1829     return do_write_ex(c, c->buf_out, c->len_out);
1830 }
1831
1832
1833 ZOOM_API(const char *)
1834     ZOOM_connection_option_get(ZOOM_connection c, const char *key)
1835 {
1836     return ZOOM_options_get(c->options, key);
1837 }
1838
1839 ZOOM_API(const char *)
1840     ZOOM_connection_option_getl(ZOOM_connection c, const char *key, int *lenp)
1841 {
1842     return ZOOM_options_getl(c->options, key, lenp);
1843 }
1844
1845 ZOOM_API(void)
1846     ZOOM_connection_option_set(ZOOM_connection c, const char *key,
1847                                const char *val)
1848 {
1849     ZOOM_options_set(c->options, key, val);
1850 }
1851
1852 ZOOM_API(void)
1853     ZOOM_connection_option_setl(ZOOM_connection c, const char *key,
1854                                 const char *val, int len)
1855 {
1856     ZOOM_options_setl(c->options, key, val, len);
1857 }
1858
1859 ZOOM_API(const char *)
1860     ZOOM_resultset_option_get(ZOOM_resultset r, const char *key)
1861 {
1862     return ZOOM_options_get(r->options, key);
1863 }
1864
1865 ZOOM_API(void)
1866     ZOOM_resultset_option_set(ZOOM_resultset r, const char *key,
1867                               const char *val)
1868 {
1869     ZOOM_options_set(r->options, key, val);
1870 }
1871
1872
1873 ZOOM_API(int)
1874     ZOOM_connection_errcode(ZOOM_connection c)
1875 {
1876     return ZOOM_connection_error(c, 0, 0);
1877 }
1878
1879 ZOOM_API(const char *)
1880     ZOOM_connection_errmsg(ZOOM_connection c)
1881 {
1882     const char *msg;
1883     ZOOM_connection_error(c, &msg, 0);
1884     return msg;
1885 }
1886
1887 ZOOM_API(const char *)
1888     ZOOM_connection_addinfo(ZOOM_connection c)
1889 {
1890     const char *addinfo;
1891     ZOOM_connection_error(c, 0, &addinfo);
1892     return addinfo;
1893 }
1894
1895 ZOOM_API(const char *)
1896     ZOOM_connection_diagset(ZOOM_connection c)
1897 {
1898     const char *diagset;
1899     ZOOM_connection_error_x(c, 0, 0, &diagset);
1900     return diagset;
1901 }
1902
1903 ZOOM_API(const char *)
1904     ZOOM_diag_str(int error)
1905 {
1906     switch (error)
1907     {
1908     case ZOOM_ERROR_NONE:
1909         return "No error";
1910     case ZOOM_ERROR_CONNECT:
1911         return "Connect failed";
1912     case ZOOM_ERROR_MEMORY:
1913         return "Out of memory";
1914     case ZOOM_ERROR_ENCODE:
1915         return "Encoding failed";
1916     case ZOOM_ERROR_DECODE:
1917         return "Decoding failed";
1918     case ZOOM_ERROR_CONNECTION_LOST:
1919         return "Connection lost";
1920     case ZOOM_ERROR_INIT:
1921         return "Init rejected";
1922     case ZOOM_ERROR_INTERNAL:
1923         return "Internal failure";
1924     case ZOOM_ERROR_TIMEOUT:
1925         return "Timeout";
1926     case ZOOM_ERROR_UNSUPPORTED_PROTOCOL:
1927         return "Unsupported protocol";
1928     case ZOOM_ERROR_UNSUPPORTED_QUERY:
1929         return "Unsupported query type";
1930     case ZOOM_ERROR_INVALID_QUERY:
1931         return "Invalid query";
1932     case ZOOM_ERROR_CQL_PARSE:
1933         return "CQL parsing error";
1934     case ZOOM_ERROR_CQL_TRANSFORM:
1935         return "CQL transformation error";
1936     case ZOOM_ERROR_CCL_CONFIG:
1937         return "CCL configuration error";
1938     case ZOOM_ERROR_CCL_PARSE:
1939         return "CCL parsing error";
1940     default:
1941         return diagbib1_str(error);
1942     }
1943 }
1944
1945 ZOOM_API(int)
1946     ZOOM_connection_error_x(ZOOM_connection c, const char **cp,
1947                             const char **addinfo, const char **diagset)
1948 {
1949     int error = c->error;
1950     if (cp)
1951     {
1952         if (!c->diagset || !strcmp(c->diagset, "ZOOM"))
1953             *cp = ZOOM_diag_str(error);
1954         else if (!strcmp(c->diagset, "HTTP"))
1955             *cp = z_HTTP_errmsg(c->error);
1956         else if (!strcmp(c->diagset, "Bib-1"))
1957             *cp = ZOOM_diag_str(error);
1958         else if (!strcmp(c->diagset, "info:srw/diagnostic/1"))
1959             *cp = yaz_diag_srw_str(c->error);
1960         else
1961             *cp = "Unknown error and diagnostic set";
1962     }
1963     if (addinfo)
1964         *addinfo = c->addinfo ? c->addinfo : "";
1965     if (diagset)
1966         *diagset = c->diagset ? c->diagset : "";
1967     return c->error;
1968 }
1969
1970 ZOOM_API(int)
1971     ZOOM_connection_error(ZOOM_connection c, const char **cp,
1972                           const char **addinfo)
1973 {
1974     return ZOOM_connection_error_x(c, cp, addinfo, 0);
1975 }
1976
1977 static void ZOOM_connection_do_io(ZOOM_connection c, int mask)
1978 {
1979     ZOOM_Event event = 0;
1980     int r = cs_look(c->cs);
1981     yaz_log(c->log_details, "%p ZOOM_connection_do_io mask=%d cs_look=%d",
1982             c, mask, r);
1983     
1984     if (r == CS_NONE)
1985     {
1986         event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1987         ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
1988         ZOOM_connection_close(c);
1989         ZOOM_connection_put_event(c, event);
1990     }
1991     else if (r == CS_CONNECT)
1992     {
1993         int ret = ret = cs_rcvconnect(c->cs);
1994         yaz_log(c->log_details, "%p ZOOM_connection_do_io "
1995                 "cs_rcvconnect returned %d", c, ret);
1996         if (ret == 1)
1997         {
1998             int mask = ZOOM_SELECT_EXCEPT;
1999             if (c->cs->io_pending & CS_WANT_WRITE)
2000                 mask += ZOOM_SELECT_WRITE;
2001             if (c->cs->io_pending & CS_WANT_READ)
2002                 mask += ZOOM_SELECT_READ;
2003             ZOOM_connection_set_mask(c, mask);
2004             event = ZOOM_Event_create(ZOOM_EVENT_NONE);
2005             ZOOM_connection_put_event(c, event);
2006         }
2007         else if (ret == 0)
2008         {
2009             event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
2010             ZOOM_connection_put_event(c, event);
2011             get_cert(c);
2012             if (c->proto == PROTO_Z3950)
2013                 ZOOM_connection_Z3950_send_init(c);
2014             else
2015             {
2016                 /* no init request for SRW .. */
2017                 assert(c->tasks->which == ZOOM_TASK_CONNECT);
2018                 ZOOM_connection_remove_task(c);
2019                 ZOOM_connection_set_mask(c, 0);
2020                 ZOOM_connection_exec_task(c);
2021             }
2022             c->state = STATE_ESTABLISHED;
2023         }
2024         else
2025         {
2026             ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
2027             ZOOM_connection_close(c);
2028         }
2029     }
2030     else
2031     {
2032         if (mask & ZOOM_SELECT_EXCEPT)
2033         {
2034             if (!ZOOM_test_reconnect(c))
2035             {
2036                 ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
2037                 ZOOM_connection_close(c);
2038             }
2039             return;
2040         }
2041         if (mask & ZOOM_SELECT_READ)
2042             do_read(c);
2043         if (c->cs && (mask & ZOOM_SELECT_WRITE))
2044             ZOOM_send_buf(c);
2045     }
2046 }
2047
2048 ZOOM_API(int)
2049     ZOOM_connection_last_event(ZOOM_connection cs)
2050 {
2051     if (!cs)
2052         return ZOOM_EVENT_NONE;
2053     return cs->last_event;
2054 }
2055
2056
2057 ZOOM_API(int) ZOOM_connection_fire_event_timeout(ZOOM_connection c)
2058 {
2059     if (c->mask)
2060     {
2061         ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_TIMEOUT);
2062         /* timeout and this connection was waiting */
2063         ZOOM_set_error(c, ZOOM_ERROR_TIMEOUT, 0);
2064         ZOOM_connection_close(c);
2065         ZOOM_connection_put_event(c, event);
2066     }
2067     return 0;
2068 }
2069
2070 ZOOM_API(int)
2071     ZOOM_connection_process(ZOOM_connection c)
2072 {
2073     ZOOM_Event event;
2074     if (!c)
2075         return 0;
2076
2077     event = ZOOM_connection_get_event(c);
2078     if (event)
2079     {
2080         ZOOM_Event_destroy(event);
2081         return 1;
2082     }
2083     ZOOM_connection_exec_task(c);
2084     event = ZOOM_connection_get_event(c);
2085     if (event)
2086     {
2087         ZOOM_Event_destroy(event);
2088         return 1;
2089     }
2090     return 0;
2091 }
2092
2093 ZOOM_API(int)
2094     ZOOM_event_nonblock(int no, ZOOM_connection *cs)
2095 {
2096     int i;
2097
2098     yaz_log(log_details0, "ZOOM_process_event(no=%d,cs=%p)", no, cs);
2099     
2100     for (i = 0; i<no; i++)
2101     {
2102         ZOOM_connection c = cs[i];
2103
2104         if (c && ZOOM_connection_process(c))
2105             return i+1;
2106     }
2107     return 0;
2108 }
2109
2110 ZOOM_API(int) ZOOM_connection_fire_event_socket(ZOOM_connection c, int mask)
2111 {
2112     if (c->mask && mask)
2113         ZOOM_connection_do_io(c, mask);
2114     return 0;
2115 }
2116
2117 ZOOM_API(int) ZOOM_connection_get_socket(ZOOM_connection c)
2118 {
2119     if (c->cs)
2120         return cs_fileno(c->cs);
2121     return -1;
2122 }
2123
2124 ZOOM_API(int) ZOOM_connection_set_mask(ZOOM_connection c, int mask)
2125 {
2126     c->mask = mask;
2127     if (!c->cs)
2128         return -1; 
2129     return 0;
2130 }
2131
2132 ZOOM_API(int) ZOOM_connection_get_mask(ZOOM_connection c)
2133 {
2134     if (c->cs)
2135         return c->mask;
2136     return 0;
2137 }
2138
2139 ZOOM_API(int) ZOOM_connection_get_timeout(ZOOM_connection c)
2140 {
2141     return ZOOM_options_get_int(c->options, "timeout", 30);
2142 }
2143
2144 ZOOM_API(void) ZOOM_connection_close(ZOOM_connection c)
2145 {
2146     if (c->cs)
2147         cs_close(c->cs);
2148     c->cs = 0;
2149     ZOOM_connection_set_mask(c, 0);
2150     c->state = STATE_IDLE;
2151 }
2152
2153 /*
2154  * Local variables:
2155  * c-basic-offset: 4
2156  * c-file-style: "Stroustrup"
2157  * indent-tabs-mode: nil
2158  * End:
2159  * vim: shiftwidth=4 tabstop=8 expandtab
2160  */
2161