Extend get_org_info (snippets) to return original string YAZ-836
[yaz-moved-to-github.git] / src / zoom-c.c
1 /* This file is part of the YAZ toolkit.
2  * Copyright (C) Index Data
3  * See the file LICENSE for details.
4  */
5 /**
6  * \file zoom-c.c
7  * \brief Implements ZOOM C interface.
8  */
9 #if HAVE_CONFIG_H
10 #include <config.h>
11 #endif
12
13 #include <assert.h>
14 #include <string.h>
15 #include <errno.h>
16 #include "zoom-p.h"
17
18 #include <yaz/yaz-util.h>
19 #include <yaz/xmalloc.h>
20 #include <yaz/otherinfo.h>
21 #include <yaz/log.h>
22 #include <yaz/diagbib1.h>
23 #include <yaz/charneg.h>
24 #include <yaz/query-charset.h>
25 #include <yaz/snprintf.h>
26 #include <yaz/facet.h>
27
28 #include <yaz/shptr.h>
29
30 #if SHPTR
31 YAZ_SHPTR_TYPE(WRBUF)
32 #endif
33
34 static int log_api0 = 0;
35 static int log_details0 = 0;
36
37 static void resultset_destroy(ZOOM_resultset r);
38 static zoom_ret do_write_ex(ZOOM_connection c, char *buf_out, int len_out);
39
40 static void initlog(void)
41 {
42     static int log_level_initialized = 0;
43
44     if (!log_level_initialized)
45     {
46         log_api0 = yaz_log_module_level("zoom");
47         log_details0 = yaz_log_module_level("zoomdetails");
48         log_level_initialized = 1;
49     }
50 }
51
52 void ZOOM_connection_remove_tasks(ZOOM_connection c);
53 static zoom_ret send_HTTP_redirect(ZOOM_connection c, const char *uri);
54
55 void ZOOM_set_dset_error(ZOOM_connection c, int error,
56                          const char *dset,
57                          const char *addinfo, const char *addinfo2)
58 {
59     char *cp;
60
61     xfree(c->addinfo);
62     c->addinfo = 0;
63     c->error = error;
64     if (!c->diagset || strcmp(dset, c->diagset))
65     {
66         xfree(c->diagset);
67         c->diagset = xstrdup(dset);
68         /* remove integer part from SRW diagset .. */
69         if ((cp = strrchr(c->diagset, '/')))
70             *cp = '\0';
71     }
72     if (addinfo && addinfo2)
73     {
74         c->addinfo = (char*) xmalloc(strlen(addinfo) + strlen(addinfo2) + 3);
75         strcpy(c->addinfo, addinfo);
76         strcat(c->addinfo, ": ");
77         strcat(c->addinfo, addinfo2);
78     }
79     else if (addinfo)
80         c->addinfo = xstrdup(addinfo);
81     if (error != ZOOM_ERROR_NONE)
82     {
83         yaz_log(c->log_api, "%p set_dset_error %s %s:%d %s %s",
84                 c, c->host_port ? c->host_port : "<>", dset, error,
85                 addinfo ? addinfo : "",
86                 addinfo2 ? addinfo2 : "");
87         ZOOM_connection_remove_tasks(c);
88     }
89 }
90
91 int ZOOM_uri_to_code(const char *uri)
92 {
93     int code = 0;
94     const char *cp;
95     if ((cp = strrchr(uri, '/')))
96         code = atoi(cp+1);
97     return code;
98 }
99
100 void ZOOM_set_error(ZOOM_connection c, int error, const char *addinfo)
101 {
102     ZOOM_set_dset_error(c, error, "ZOOM", addinfo, 0);
103 }
104
105 static void clear_error(ZOOM_connection c)
106 {
107     /*
108      * If an error is tied to an operation then it's ok to clear: for
109      * example, a diagnostic returned from a search is cleared by a
110      * subsequent search.  However, problems such as Connection Lost
111      * or Init Refused are not cleared, because they are not
112      * recoverable: doing another search doesn't help.
113      */
114
115     ZOOM_connection_remove_events(c);
116     switch (c->error)
117     {
118     case ZOOM_ERROR_CONNECT:
119     case ZOOM_ERROR_MEMORY:
120     case ZOOM_ERROR_DECODE:
121     case ZOOM_ERROR_CONNECTION_LOST:
122     case ZOOM_ERROR_INIT:
123     case ZOOM_ERROR_INTERNAL:
124     case ZOOM_ERROR_UNSUPPORTED_PROTOCOL:
125         break;
126     default:
127         ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
128     }
129 }
130
131 void ZOOM_connection_show_task(ZOOM_task task)
132 {
133     switch(task->which)
134     {
135     case ZOOM_TASK_SEARCH:
136         yaz_log(YLOG_LOG, "search p=%p", task);
137         break;
138     case ZOOM_TASK_CONNECT:
139         yaz_log(YLOG_LOG, "connect p=%p", task);
140         break;
141     case ZOOM_TASK_SCAN:
142         yaz_log(YLOG_LOG, "scan p=%p", task);
143         break;
144     }
145 }
146
147 void ZOOM_connection_show_tasks(ZOOM_connection c)
148 {
149     ZOOM_task task;
150     yaz_log(YLOG_LOG, "connection p=%p tasks", c);
151     for (task = c->tasks; task; task = task->next)
152         ZOOM_connection_show_task(task);
153 }
154
155 ZOOM_task ZOOM_connection_add_task(ZOOM_connection c, int which)
156 {
157     ZOOM_task *taskp = &c->tasks;
158     while (*taskp)
159         taskp = &(*taskp)->next;
160     *taskp = (ZOOM_task) xmalloc(sizeof(**taskp));
161     (*taskp)->running = 0;
162     (*taskp)->which = which;
163     (*taskp)->next = 0;
164     clear_error(c);
165     return *taskp;
166 }
167
168 ZOOM_API(int) ZOOM_connection_is_idle(ZOOM_connection c)
169 {
170     return c->tasks ? 0 : 1;
171 }
172
173 ZOOM_task ZOOM_connection_insert_task(ZOOM_connection c, int which)
174 {
175     ZOOM_task task = (ZOOM_task) xmalloc(sizeof(*task));
176
177     task->next = c->tasks;
178     c->tasks = task;
179
180     task->running = 0;
181     task->which = which;
182     return task;
183 }
184
185 void ZOOM_connection_remove_task(ZOOM_connection c)
186 {
187     ZOOM_task task = c->tasks;
188
189     if (task)
190     {
191         c->tasks = task->next;
192         switch (task->which)
193         {
194         case ZOOM_TASK_SEARCH:
195             resultset_destroy(task->u.search.resultset);
196             xfree(task->u.search.syntax);
197             xfree(task->u.search.elementSetName);
198             xfree(task->u.search.schema);
199             break;
200         case ZOOM_TASK_CONNECT:
201             break;
202         case ZOOM_TASK_SCAN:
203             ZOOM_scanset_destroy(task->u.scan.scan);
204             break;
205         case ZOOM_TASK_PACKAGE:
206             ZOOM_package_destroy(task->u.package);
207             break;
208         case ZOOM_TASK_SORT:
209             resultset_destroy(task->u.sort.resultset);
210             ZOOM_query_destroy(task->u.sort.q);
211             break;
212         default:
213             assert(0);
214         }
215         xfree(task);
216
217         if (!c->tasks)
218         {
219             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_END);
220             ZOOM_connection_put_event(c, event);
221         }
222     }
223 }
224
225 void ZOOM_connection_remove_tasks(ZOOM_connection c)
226 {
227     while (c->tasks)
228         ZOOM_connection_remove_task(c);
229 }
230
231 static void odr_wrbuf_write(ODR o, void *handle, int type,
232                             const char *buf, int len)
233 {
234     WRBUF w = (WRBUF) handle;
235     wrbuf_write(w, buf, len);
236 }
237
238 ZOOM_API(ZOOM_connection)
239     ZOOM_connection_create(ZOOM_options options)
240 {
241     ZOOM_connection c = (ZOOM_connection) xmalloc(sizeof(*c));
242
243     initlog();
244
245     c->log_api = log_api0;
246     c->log_details = log_details0;
247
248     yaz_log(c->log_api, "%p ZOOM_connection_create", c);
249
250     c->proto = PROTO_Z3950;
251     c->cs = 0;
252     ZOOM_connection_set_mask(c, 0);
253     c->reconnect_ok = 0;
254     c->state = STATE_IDLE;
255     c->addinfo = 0;
256     c->diagset = 0;
257     ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
258     c->buf_in = 0;
259     c->len_in = 0;
260     c->buf_out = 0;
261     c->len_out = 0;
262     c->resultsets = 0;
263
264     c->options = ZOOM_options_create_with_parent(options);
265
266     c->host_port = 0;
267     c->proxy = 0;
268     c->tproxy = 0;
269     c->proxy_mode = 0;
270
271     c->charset = c->lang = 0;
272
273     c->cookie_out = 0;
274     c->cookie_in = 0;
275     c->client_IP = 0;
276     c->tasks = 0;
277
278     c->user = 0;
279     c->group = 0;
280     c->password = 0;
281     c->url_authentication = 0;
282
283     c->maximum_record_size = 0;
284     c->preferred_message_size = 0;
285
286     c->odr_in = odr_createmem(ODR_DECODE);
287     c->odr_out = odr_createmem(ODR_ENCODE);
288     c->odr_print = 0;
289     c->odr_save = 0;
290
291     c->async = 0;
292     c->support_named_resultsets = 0;
293     c->last_event = ZOOM_EVENT_NONE;
294
295     c->m_queue_front = 0;
296     c->m_queue_back = 0;
297
298     c->sru_version = 0;
299     c->no_redirects = 0;
300     c->cookies = 0;
301     c->location = 0;
302     c->saveAPDU_wrbuf = 0;
303
304     ZOOM_memcached_init(c);
305     return c;
306 }
307
308 ZOOM_API(void) ZOOM_connection_save_apdu_wrbuf(ZOOM_connection c, WRBUF w)
309 {
310     if (c->odr_save)
311     {
312         odr_destroy(c->odr_save);
313         c->odr_save = 0;
314     }
315     if (w)
316     {
317         c->odr_save = odr_createmem(ODR_PRINT);
318         odr_set_stream(c->odr_save, w, odr_wrbuf_write, 0);
319     }
320 }
321
322 /* set database names. Take local databases (if set); otherwise
323    take databases given in ZURL (if set); otherwise use Default */
324 char **ZOOM_connection_get_databases(ZOOM_connection con, ZOOM_options options,
325                                      int *num, ODR odr)
326 {
327     char **databaseNames;
328     const char *cp = ZOOM_options_get(options, "databaseName");
329
330     if ((!cp || !*cp) && con->host_port)
331         cs_get_host_args(con->host_port, &cp);
332     if (!cp || !*cp)
333         cp = "Default";
334     nmem_strsplit(odr_getmem(odr), "+", cp,  &databaseNames, num);
335     return databaseNames;
336 }
337
338 ZOOM_API(ZOOM_connection)
339     ZOOM_connection_new(const char *host, int portnum)
340 {
341     ZOOM_connection c = ZOOM_connection_create(0);
342
343     ZOOM_connection_connect(c, host, portnum);
344     return c;
345 }
346
347 static zoom_sru_mode get_sru_mode_from_string(const char *s)
348 {
349     if (!s || !*s)
350         return zoom_sru_soap;
351     if (!yaz_matchstr(s, "soap"))
352         return zoom_sru_soap;
353     else if (!yaz_matchstr(s, "get"))
354         return zoom_sru_get;
355     else if (!yaz_matchstr(s, "post"))
356         return zoom_sru_post;
357     else if (!yaz_matchstr(s, "solr"))
358         return zoom_sru_solr;
359     return zoom_sru_error;
360 }
361
362 ZOOM_API(void)
363     ZOOM_connection_connect(ZOOM_connection c,
364                             const char *host, int portnum)
365 {
366     const char *val;
367
368     initlog();
369
370     yaz_log(c->log_api, "%p ZOOM_connection_connect host=%s portnum=%d",
371             c, host ? host : "null", portnum);
372
373     ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
374     ZOOM_connection_remove_tasks(c);
375
376     if (c->cs)
377     {
378         yaz_log(c->log_details, "%p ZOOM_connection_connect reconnect ok", c);
379         c->reconnect_ok = 1;
380         return;
381     }
382     yaz_log(c->log_details, "%p ZOOM_connection_connect connect", c);
383     xfree(c->proxy);
384     c->proxy = 0;
385     val = ZOOM_options_get(c->options, "proxy");
386     if (val && *val)
387     {
388         yaz_log(c->log_details, "%p ZOOM_connection_connect proxy=%s", c, val);
389         c->proxy = xstrdup(val);
390     }
391
392     xfree(c->tproxy);
393     c->tproxy = 0;
394     val = ZOOM_options_get(c->options, "tproxy");
395     if (val && *val)
396     {
397         yaz_log(c->log_details, "%p ZOOM_connection_connect tproxy=%s", c, val);
398         c->tproxy = xstrdup(val);
399     }
400
401     xfree(c->charset);
402     c->charset = 0;
403     val = ZOOM_options_get(c->options, "charset");
404     if (val && *val)
405     {
406         yaz_log(c->log_details, "%p ZOOM_connection_connect charset=%s", c, val);
407         c->charset = xstrdup(val);
408     }
409
410     xfree(c->lang);
411     val = ZOOM_options_get(c->options, "lang");
412     if (val && *val)
413     {
414         yaz_log(c->log_details, "%p ZOOM_connection_connect lang=%s", c, val);
415         c->lang = xstrdup(val);
416     }
417     else
418         c->lang = 0;
419
420     if (host)
421     {
422         char hostn[128];
423         const char *http_lead;
424
425         val = ZOOM_options_get(c->options, "sru");
426         if (val && *val && !strstr(host, "://"))
427             http_lead = "http://";
428         else
429             http_lead = "";
430         c->sru_mode = get_sru_mode_from_string(val);
431
432         xfree(c->host_port);
433         if (portnum)
434         {
435             sprintf(hostn, "%.80s:%d", host, portnum);
436             host = hostn;
437         }
438         c->host_port = xmalloc(strlen(host) + strlen(http_lead) + 1);
439         strcpy(c->host_port, http_lead);
440         strcat(c->host_port, host);
441     }
442
443     {
444         /*
445          * If the "<scheme>:" part of the host string is preceded by one
446          * or more comma-separated <name>=<value> pairs, these are taken
447          * to be options to be set on the connection object.  Among other
448          * applications, this facility can be used to embed authentication
449          * in a host string:
450          *          user=admin,password=secret,tcp:localhost:9999
451          */
452         char *remainder = c->host_port;
453         char *pcolon = strchr(remainder, ':');
454         char *pcomma;
455         char *pequals;
456         while ((pcomma = strchr(remainder, ',')) != 0 &&
457                (pcolon == 0 || pcomma < pcolon))
458         {
459             *pcomma = '\0';
460             if ((pequals = strchr(remainder, '=')) != 0)
461             {
462                 *pequals = '\0';
463                 ZOOM_connection_option_set(c, remainder, pequals+1);
464             }
465             remainder = pcomma+1;
466         }
467
468         if (remainder != c->host_port)
469         {
470             remainder = xstrdup(remainder);
471             xfree(c->host_port);
472             c->host_port = remainder;
473         }
474     }
475
476     xfree(c->sru_version);
477     val = ZOOM_options_get(c->options, "sru_version");
478     c->sru_version = xstrdup(val ? val : "1.2");
479
480     ZOOM_options_set(c->options, "host", c->host_port);
481
482     xfree(c->cookie_out);
483     c->cookie_out = 0;
484     val = ZOOM_options_get(c->options, "cookie");
485     if (val && *val)
486     {
487         yaz_log(c->log_details, "%p ZOOM_connection_connect cookie=%s", c, val);
488         c->cookie_out = xstrdup(val);
489     }
490
491     xfree(c->client_IP);
492     c->client_IP = 0;
493     val = ZOOM_options_get(c->options, "clientIP");
494     if (val && *val)
495     {
496         yaz_log(c->log_details, "%p ZOOM_connection_connect clientIP=%s",
497                 c, val);
498         c->client_IP = xstrdup(val);
499     }
500
501     xfree(c->group);
502     c->group = 0;
503     val = ZOOM_options_get(c->options, "group");
504     if (val && *val)
505         c->group = xstrdup(val);
506
507     xfree(c->user);
508     c->user = 0;
509     val = ZOOM_options_get(c->options, "user");
510     if (val && *val)
511         c->user = xstrdup(val);
512
513     xfree(c->password);
514     c->password = 0;
515     val = ZOOM_options_get(c->options, "password");
516     if (!val)
517         val = ZOOM_options_get(c->options, "pass");
518     if (val && *val)
519         c->password = xstrdup(val);
520
521     val = ZOOM_options_get(c->options, "authenticationMode");
522     if (val && !strcmp(val, "url"))
523         c->url_authentication = 1;
524     else
525         c->url_authentication = 0;
526
527     c->maximum_record_size =
528         ZOOM_options_get_int(c->options, "maximumRecordSize", 64*1024*1024);
529     c->preferred_message_size =
530         ZOOM_options_get_int(c->options, "preferredMessageSize", 64*1024*1024);
531
532     c->async = ZOOM_options_get_bool(c->options, "async", 0);
533
534     yaz_cookies_destroy(c->cookies);
535     c->cookies = yaz_cookies_create();
536
537     if (ZOOM_memcached_configure(c))
538     {
539         ZOOM_connection_remove_tasks(c);
540         return;
541     }
542     if (c->sru_mode == zoom_sru_error)
543     {
544         ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, val);
545         ZOOM_connection_remove_tasks(c);
546         return;
547     }
548
549     if (c->odr_print)
550         odr_destroy(c->odr_print);
551     c->odr_print = 0;
552     val = ZOOM_options_get(c->options, "apdufile");
553     if (val)
554     {
555         c->odr_print = odr_createmem(ODR_PRINT);
556         if (strcmp(val, "-"))
557         {
558             FILE *f = fopen(val, "a");
559             if (!f)
560             {
561                 WRBUF w = wrbuf_alloc();
562                 wrbuf_printf(w, "fopen: %s", val);
563                 ZOOM_set_error(c, ZOOM_ERROR_INTERNAL, wrbuf_cstr(w));
564                 wrbuf_destroy(w);
565                 return;
566             }
567             odr_setprint(c->odr_print, f);
568         }
569     }
570     else if (ZOOM_options_get_bool(c->options, "apdulog", 0))
571     {
572         c->odr_print = odr_createmem(ODR_PRINT);
573         odr_setprint_noclose(c->odr_print, yaz_log_file());
574     }
575
576     yaz_log(c->log_details, "%p ZOOM_connection_connect async=%d", c, c->async);
577     ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
578
579     if (!c->async)
580     {
581         while (ZOOM_event(1, &c))
582             ;
583     }
584 }
585
586 ZOOM_API(void) ZOOM_resultset_release(ZOOM_resultset r)
587 {
588     if (r->connection)
589     {
590         /* remove ourselves from the resultsets in connection */
591         ZOOM_resultset *rp = &r->connection->resultsets;
592         while (1)
593         {
594             assert(*rp);   /* we must be in this list!! */
595             if (*rp == r)
596             {   /* OK, we're here - take us out of it */
597                 *rp = (*rp)->next;
598                 break;
599             }
600             rp = &(*rp)->next;
601         }
602         r->connection = 0;
603     }
604 }
605
606 ZOOM_API(void)
607     ZOOM_connection_destroy(ZOOM_connection c)
608 {
609     ZOOM_resultset r;
610     if (!c)
611         return;
612     yaz_log(c->log_api, "%p ZOOM_connection_destroy", c);
613
614     ZOOM_memcached_destroy(c);
615     if (c->cs)
616         cs_close(c->cs);
617
618     for (r = c->resultsets; r; r = r->next)
619         r->connection = 0;
620
621     xfree(c->buf_in);
622     xfree(c->addinfo);
623     xfree(c->diagset);
624     odr_destroy(c->odr_in);
625     odr_destroy(c->odr_out);
626     if (c->odr_save)
627         odr_destroy(c->odr_save);
628     if (c->odr_print)
629         odr_destroy(c->odr_print);
630     ZOOM_options_destroy(c->options);
631     ZOOM_connection_remove_tasks(c);
632     ZOOM_connection_remove_events(c);
633     xfree(c->host_port);
634     xfree(c->proxy);
635     xfree(c->tproxy);
636     xfree(c->charset);
637     xfree(c->lang);
638     xfree(c->cookie_out);
639     xfree(c->cookie_in);
640     xfree(c->client_IP);
641     xfree(c->user);
642     xfree(c->group);
643     xfree(c->password);
644     xfree(c->sru_version);
645     xfree(c->location);
646     yaz_cookies_destroy(c->cookies);
647     wrbuf_destroy(c->saveAPDU_wrbuf);
648     xfree(c);
649 }
650
651 void ZOOM_resultset_addref(ZOOM_resultset r)
652 {
653     if (r)
654     {
655         yaz_mutex_enter(r->mutex);
656         (r->refcount)++;
657         yaz_log(log_details0, "%p ZOOM_resultset_addref count=%d",
658                 r, r->refcount);
659         yaz_mutex_leave(r->mutex);
660     }
661 }
662
663 static int g_resultsets = 0;
664 static YAZ_MUTEX g_resultset_mutex = 0;
665
666 /* TODO We need to initialize this before running threaded:
667  * call resultset_use(0)  */
668
669 static int resultset_use(int delta) {
670     int resultset_count;
671     if (g_resultset_mutex == 0)
672         yaz_mutex_create(&g_resultset_mutex);
673     yaz_mutex_enter(g_resultset_mutex);
674     g_resultsets += delta;
675     resultset_count = g_resultsets;
676     yaz_mutex_leave(g_resultset_mutex);
677     return resultset_count;
678 }
679
680 int resultsets_count(void) {
681     return resultset_use(0);
682 }
683
684 ZOOM_resultset ZOOM_resultset_create(void)
685 {
686     int i;
687     ZOOM_resultset r = (ZOOM_resultset) xmalloc(sizeof(*r));
688
689     initlog();
690
691     yaz_log(log_details0, "%p ZOOM_resultset_create", r);
692     r->refcount = 1;
693     r->size = 0;
694     r->odr = odr_createmem(ODR_DECODE);
695     r->piggyback = 1;
696     r->setname = 0;
697     r->step = 0;
698     for (i = 0; i<RECORD_HASH_SIZE; i++)
699         r->record_hash[i] = 0;
700     r->r_sort_spec = 0;
701     r->query = 0;
702     r->connection = 0;
703     r->databaseNames = 0;
704     r->num_databaseNames = 0;
705     r->req_facets = 0;
706     r->res_facets = 0;
707     r->num_res_facets = 0;
708     r->facets_names = 0;
709     r->mutex = 0;
710     yaz_mutex_create(&r->mutex);
711 #if SHPTR
712     {
713         WRBUF w = wrbuf_alloc();
714         YAZ_SHPTR_INIT(r->record_wrbuf, w);
715     }
716 #endif
717     resultset_use(1);
718     r->mc_key = 0;
719     r->live_set = 0;
720     return r;
721 }
722
723 ZOOM_API(ZOOM_resultset)
724     ZOOM_connection_search_pqf(ZOOM_connection c, const char *q)
725 {
726     ZOOM_resultset r;
727     ZOOM_query s = ZOOM_query_create();
728
729     ZOOM_query_prefix(s, q);
730
731     r = ZOOM_connection_search(c, s);
732     ZOOM_query_destroy(s);
733     return r;
734 }
735
736 ZOOM_API(ZOOM_resultset)
737     ZOOM_connection_search(ZOOM_connection c, ZOOM_query q)
738 {
739     ZOOM_resultset r = ZOOM_resultset_create();
740     ZOOM_task task;
741     int start, count;
742     const char *syntax, *elementSetName, *schema;
743     yaz_log(c->log_api, "%p ZOOM_connection_search set %p query %p", c, r, q);
744     r->r_sort_spec = ZOOM_query_get_sortspec(q);
745     r->query = q;
746     ZOOM_query_addref(q);
747
748     r->options = ZOOM_options_create_with_parent(c->options);
749
750     r->req_facets = odr_strdup_null(r->odr,
751                                     ZOOM_options_get(r->options, "facets"));
752     start = ZOOM_options_get_int(r->options, "start", 0);
753     count = ZOOM_options_get_int(r->options, "count", 0);
754     {
755         /* If "presentChunk" is defined use that; otherwise "step" */
756         const char *cp = ZOOM_options_get(r->options, "presentChunk");
757         r->step = ZOOM_options_get_int(r->options,
758                                        (cp != 0 ? "presentChunk": "step"), 0);
759     }
760     r->piggyback = ZOOM_options_get_bool(r->options, "piggyback", 1);
761     r->setname = odr_strdup_null(r->odr,
762                                  ZOOM_options_get(r->options, "setname"));
763     r->databaseNames = ZOOM_connection_get_databases(c, c->options,
764                                                      &r->num_databaseNames,
765                                                      r->odr);
766     r->connection = c;
767     r->next = c->resultsets;
768     c->resultsets = r;
769
770     ZOOM_memcached_resultset(r, q);
771
772     if (c->host_port && c->proto == PROTO_HTTP)
773     {
774         if (!c->cs)
775         {
776             yaz_log(c->log_details, "ZOOM_connection_search: no comstack");
777             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
778         }
779         else
780         {
781             yaz_log(c->log_details, "ZOOM_connection_search: reconnect");
782             c->reconnect_ok = 1;
783         }
784     }
785
786     task = ZOOM_connection_add_task(c, ZOOM_TASK_SEARCH);
787     task->u.search.resultset = r;
788     task->u.search.start = start;
789     task->u.search.count = count;
790
791     syntax = ZOOM_options_get(r->options, "preferredRecordSyntax");
792     task->u.search.syntax = syntax ? xstrdup(syntax) : 0;
793     elementSetName = ZOOM_options_get(r->options, "elementSetName");
794     task->u.search.elementSetName = elementSetName ?
795         xstrdup(elementSetName) : 0;
796     schema = ZOOM_options_get(r->options, "schema");
797     task->u.search.schema = schema ? xstrdup(schema) : 0;
798
799     ZOOM_resultset_addref(r);
800
801     if (!c->async)
802     {
803         while (ZOOM_event(1, &c))
804             ;
805     }
806     return r;
807 }
808
809 ZOOM_API(void)
810     ZOOM_resultset_sort(ZOOM_resultset r,
811                          const char *sort_type, const char *sort_spec)
812 {
813     (void) ZOOM_resultset_sort1(r, sort_type, sort_spec);
814 }
815
816 ZOOM_API(int)
817     ZOOM_resultset_sort1(ZOOM_resultset r,
818                          const char *sort_type, const char *sort_spec)
819 {
820     ZOOM_connection c = r->connection;
821     ZOOM_task task;
822     ZOOM_query newq;
823
824     newq = ZOOM_query_create();
825     if (ZOOM_query_sortby(newq, sort_spec) < 0)
826         return -1;
827
828     yaz_log(c->log_api, "%p ZOOM_resultset_sort r=%p sort_type=%s sort_spec=%s",
829             r, r, sort_type, sort_spec);
830     if (!c)
831         return 0;
832
833     if (c->host_port && c->proto == PROTO_HTTP)
834     {
835         if (!c->cs)
836         {
837             yaz_log(c->log_details, "%p ZOOM_resultset_sort: no comstack", r);
838             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
839         }
840         else
841         {
842             yaz_log(c->log_details, "%p ZOOM_resultset_sort: prepare reconnect",
843                     r);
844             c->reconnect_ok = 1;
845         }
846     }
847
848     ZOOM_resultset_cache_reset(r);
849     task = ZOOM_connection_add_task(c, ZOOM_TASK_SORT);
850     task->u.sort.resultset = r;
851     task->u.sort.q = newq;
852
853     ZOOM_resultset_addref(r);
854
855     if (!c->async)
856     {
857         while (ZOOM_event(1, &c))
858             ;
859     }
860
861     return 0;
862 }
863
864 ZOOM_API(void)
865     ZOOM_resultset_destroy(ZOOM_resultset r)
866 {
867     resultset_destroy(r);
868 }
869
870 static void resultset_destroy(ZOOM_resultset r)
871 {
872     if (!r)
873         return;
874     yaz_mutex_enter(r->mutex);
875     (r->refcount)--;
876     yaz_log(log_details0, "%p ZOOM_resultset_destroy r=%p count=%d",
877             r, r, r->refcount);
878     if (r->refcount == 0)
879     {
880         yaz_mutex_leave(r->mutex);
881
882         yaz_log(log_details0, "%p ZOOM_connection resultset_destroy: Deleting resultset (%p) ", r->connection, r);
883         ZOOM_resultset_cache_reset(r);
884         ZOOM_resultset_release(r);
885         ZOOM_query_destroy(r->query);
886         ZOOM_options_destroy(r->options);
887         odr_destroy(r->odr);
888         yaz_mutex_destroy(&r->mutex);
889 #if SHPTR
890         YAZ_SHPTR_DEC(r->record_wrbuf, wrbuf_destroy);
891 #endif
892         wrbuf_destroy(r->mc_key);
893         resultset_use(-1);
894         xfree(r);
895     }
896     else
897         yaz_mutex_leave(r->mutex);
898 }
899
900 ZOOM_API(size_t)
901     ZOOM_resultset_size(ZOOM_resultset r)
902 {
903     return r->size;
904 }
905
906 int ZOOM_test_reconnect(ZOOM_connection c)
907 {
908     ZOOM_Event event;
909
910     if (!c->reconnect_ok)
911         return 0;
912     ZOOM_connection_close(c);
913     c->reconnect_ok = 0;
914     c->tasks->running = 0;
915     ZOOM_connection_insert_task(c, ZOOM_TASK_CONNECT);
916
917     event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
918     ZOOM_connection_put_event(c, event);
919
920     return 1;
921 }
922
923 static void ZOOM_resultset_retrieve(ZOOM_resultset r,
924                                     int force_sync, int start, int count)
925 {
926     ZOOM_task task;
927     ZOOM_connection c;
928     const char *cp;
929     const char *syntax, *elementSetName;
930
931     if (!r)
932         return;
933     yaz_log(log_details0, "%p ZOOM_resultset_retrieve force_sync=%d start=%d"
934             " count=%d", r, force_sync, start, count);
935     c = r->connection;
936     if (!c)
937         return;
938
939     if (c->host_port && c->proto == PROTO_HTTP)
940     {
941         if (!c->cs)
942         {
943             yaz_log(log_details0, "%p ZOOM_resultset_retrieve: no comstack", r);
944             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
945         }
946         else
947         {
948             yaz_log(log_details0, "%p ZOOM_resultset_retrieve: prepare "
949                     "reconnect", r);
950             c->reconnect_ok = 1;
951         }
952     }
953     task = ZOOM_connection_add_task(c, ZOOM_TASK_SEARCH);
954     task->u.search.resultset = r;
955     task->u.search.start = start;
956     task->u.search.count = count;
957
958     syntax = ZOOM_options_get(r->options, "preferredRecordSyntax");
959     task->u.search.syntax = syntax ? xstrdup(syntax) : 0;
960     elementSetName = ZOOM_options_get(r->options, "elementSetName");
961     task->u.search.elementSetName = elementSetName
962         ? xstrdup(elementSetName) : 0;
963
964     cp = ZOOM_options_get(r->options, "schema");
965     task->u.search.schema = cp ? xstrdup(cp) : 0;
966
967     ZOOM_resultset_addref(r);
968
969     if (!r->connection->async || force_sync)
970         while (r->connection && ZOOM_event(1, &r->connection))
971             ;
972 }
973
974 ZOOM_API(void)
975     ZOOM_resultset_records(ZOOM_resultset r, ZOOM_record *recs,
976                            size_t start, size_t count)
977 {
978     int force_present = 0;
979
980     if (!r)
981         return ;
982     yaz_log(log_api0, "%p ZOOM_resultset_records r=%p start=%ld count=%ld",
983             r, r, (long) start, (long) count);
984     if (count && recs)
985         force_present = 1;
986     ZOOM_resultset_retrieve(r, force_present, start, count);
987     if (force_present)
988     {
989         size_t i;
990         for (i = 0; i< count; i++)
991             recs[i] = ZOOM_resultset_record_immediate(r, i+start);
992     }
993 }
994
995 ZOOM_API(size_t)
996     ZOOM_resultset_facets_size(ZOOM_resultset r)
997 {
998     return r->num_res_facets;
999 }
1000
1001 ZOOM_API(ZOOM_facet_field)
1002     ZOOM_resultset_get_facet_field(ZOOM_resultset r, const char *name)
1003 {
1004     int num = r->num_res_facets;
1005     ZOOM_facet_field *facets = r->res_facets;
1006     int i;
1007     for (i = 0; i < num; i++)
1008         if (!strcmp(facets[i]->facet_name, name))
1009             return facets[i];
1010     return 0;
1011 }
1012
1013 ZOOM_API(ZOOM_facet_field)
1014     ZOOM_resultset_get_facet_field_by_index(ZOOM_resultset r, int idx)
1015 {
1016     int num = r->num_res_facets;
1017     ZOOM_facet_field *facets = r->res_facets;
1018     if (idx >= 0 && idx < num)
1019         return facets[idx];
1020     return 0;
1021 }
1022
1023 ZOOM_API(ZOOM_facet_field *)
1024     ZOOM_resultset_facets(ZOOM_resultset r)
1025 {
1026     return r->res_facets;
1027 }
1028
1029 ZOOM_API(const char**)
1030     ZOOM_resultset_facets_names(ZOOM_resultset r)
1031 {
1032     return (const char **) r->facets_names;
1033 }
1034
1035 ZOOM_API(const char*)
1036     ZOOM_facet_field_name(ZOOM_facet_field field)
1037 {
1038     return field->facet_name;
1039 }
1040
1041 ZOOM_API(size_t)
1042     ZOOM_facet_field_term_count(ZOOM_facet_field field)
1043 {
1044     return field->num_terms;
1045 }
1046
1047 ZOOM_API(const char*)
1048     ZOOM_facet_field_get_term(ZOOM_facet_field field, size_t idx, int *freq)
1049 {
1050     *freq = field->facet_terms[idx].frequency;
1051     return field->facet_terms[idx].term;
1052 }
1053
1054
1055 static void get_cert(ZOOM_connection c)
1056 {
1057     char *cert_buf;
1058     int cert_len;
1059
1060     if (cs_get_peer_certificate_x509(c->cs, &cert_buf, &cert_len))
1061     {
1062         ZOOM_connection_option_setl(c, "sslPeerCert",
1063                                     cert_buf, cert_len);
1064         xfree(cert_buf);
1065     }
1066 }
1067
1068 static zoom_ret do_connect_host(ZOOM_connection c,
1069                                 const char *logical_url);
1070
1071 static zoom_ret do_connect(ZOOM_connection c)
1072 {
1073     return do_connect_host(c, c->host_port);
1074 }
1075
1076 static zoom_ret do_connect_host(ZOOM_connection c, const char *logical_url)
1077 {
1078     void *add;
1079
1080     if (c->cs)
1081         cs_close(c->cs);
1082     c->cs = cs_create_host2(logical_url, CS_FLAGS_DNS_NO_BLOCK, &add,
1083                             c->tproxy ? c->tproxy : c->proxy,
1084                             &c->proxy_mode);
1085     if (!c->proxy)
1086         c->proxy_mode = 0;
1087
1088     if (c->cs && c->cs->protocol == PROTO_HTTP)
1089     {
1090 #if YAZ_HAVE_XML2
1091         c->proto = PROTO_HTTP;
1092 #else
1093         ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, "SRW");
1094         ZOOM_connection_close(c);
1095         return zoom_complete;
1096 #endif
1097     }
1098     if (c->cs)
1099     {
1100         int ret = cs_connect(c->cs, add);
1101         if (ret == 0)
1102         {
1103             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1104             ZOOM_connection_put_event(c, event);
1105             get_cert(c);
1106             if (c->proto == PROTO_Z3950)
1107                 ZOOM_connection_Z3950_send_init(c);
1108             else
1109             {
1110                 /* no init request for SRW .. */
1111                 assert(c->tasks->which == ZOOM_TASK_CONNECT);
1112                 ZOOM_connection_remove_task(c);
1113                 ZOOM_connection_set_mask(c, 0);
1114
1115                 if (c->cs && c->location)
1116                     send_HTTP_redirect(c, c->location);
1117                 else
1118                     ZOOM_connection_exec_task(c);
1119             }
1120             c->state = STATE_ESTABLISHED;
1121             return zoom_pending;
1122         }
1123         else if (ret > 0)
1124         {
1125             int mask = ZOOM_SELECT_EXCEPT;
1126             if (c->cs->io_pending & CS_WANT_WRITE)
1127                 mask += ZOOM_SELECT_WRITE;
1128             if (c->cs->io_pending & CS_WANT_READ)
1129                 mask += ZOOM_SELECT_READ;
1130             ZOOM_connection_set_mask(c, mask);
1131             c->state = STATE_CONNECTING;
1132             return zoom_pending;
1133         }
1134     }
1135     c->state = STATE_IDLE;
1136     ZOOM_set_error(c, ZOOM_ERROR_CONNECT, logical_url);
1137     return zoom_complete;
1138 }
1139
1140 /* returns 1 if PDU was sent OK (still pending )
1141    0 if PDU was not sent OK (nothing to wait for)
1142 */
1143
1144 ZOOM_API(ZOOM_record)
1145     ZOOM_resultset_record_immediate(ZOOM_resultset s,size_t pos)
1146 {
1147     const char *syntax =
1148         ZOOM_options_get(s->options, "preferredRecordSyntax");
1149     const char *elementSetName =
1150         ZOOM_options_get(s->options, "elementSetName");
1151     const char *schema =
1152         ZOOM_options_get(s->options, "schema");
1153
1154     return ZOOM_record_cache_lookup_i(s, pos, syntax, elementSetName, schema);
1155 }
1156
1157 ZOOM_API(ZOOM_record)
1158     ZOOM_resultset_record(ZOOM_resultset r, size_t pos)
1159 {
1160     ZOOM_record rec = ZOOM_resultset_record_immediate(r, pos);
1161
1162     if (!rec)
1163     {
1164         /*
1165          * MIKE: I think force_sync should always be zero, but I don't
1166          * want to make this change until I get the go-ahead from
1167          * Adam, in case something depends on the old synchronous
1168          * behaviour.
1169          */
1170         int force_sync = 1;
1171         if (getenv("ZOOM_RECORD_NO_FORCE_SYNC")) force_sync = 0;
1172         ZOOM_resultset_retrieve(r, force_sync, pos, 1);
1173         rec = ZOOM_resultset_record_immediate(r, pos);
1174     }
1175     return rec;
1176 }
1177
1178 ZOOM_API(ZOOM_scanset)
1179     ZOOM_connection_scan(ZOOM_connection c, const char *start)
1180 {
1181     ZOOM_scanset s;
1182     ZOOM_query q = ZOOM_query_create();
1183
1184     ZOOM_query_prefix(q, start);
1185
1186     s = ZOOM_connection_scan1(c, q);
1187     ZOOM_query_destroy(q);
1188     return s;
1189
1190 }
1191
1192 ZOOM_API(ZOOM_scanset)
1193     ZOOM_connection_scan1(ZOOM_connection c, ZOOM_query q)
1194 {
1195     ZOOM_scanset scan = 0;
1196     Z_Query *z_query = ZOOM_query_get_Z_Query(q);
1197
1198     if (!z_query)
1199         return 0;
1200     scan = (ZOOM_scanset) xmalloc(sizeof(*scan));
1201     scan->connection = c;
1202     scan->odr = odr_createmem(ODR_DECODE);
1203     scan->options = ZOOM_options_create_with_parent(c->options);
1204     scan->refcount = 1;
1205     scan->scan_response = 0;
1206     scan->srw_scan_response = 0;
1207
1208     scan->query = q;
1209     ZOOM_query_addref(q);
1210     scan->databaseNames = ZOOM_connection_get_databases(c, c->options,
1211                                             &scan->num_databaseNames,
1212                                             scan->odr);
1213
1214     if (1)
1215     {
1216         ZOOM_task task = ZOOM_connection_add_task(c, ZOOM_TASK_SCAN);
1217         task->u.scan.scan = scan;
1218
1219         (scan->refcount)++;
1220         if (!c->async)
1221         {
1222             while (ZOOM_event(1, &c))
1223                 ;
1224         }
1225     }
1226     return scan;
1227 }
1228
1229 ZOOM_API(void)
1230     ZOOM_scanset_destroy(ZOOM_scanset scan)
1231 {
1232     if (!scan)
1233         return;
1234     (scan->refcount)--;
1235     if (scan->refcount == 0)
1236     {
1237         ZOOM_query_destroy(scan->query);
1238
1239         odr_destroy(scan->odr);
1240
1241         ZOOM_options_destroy(scan->options);
1242         xfree(scan);
1243     }
1244 }
1245
1246 static zoom_ret send_package(ZOOM_connection c)
1247 {
1248     ZOOM_Event event;
1249
1250     yaz_log(c->log_details, "%p send_package", c);
1251     if (!c->tasks)
1252         return zoom_complete;
1253     assert (c->tasks->which == ZOOM_TASK_PACKAGE);
1254
1255     event = ZOOM_Event_create(ZOOM_EVENT_SEND_APDU);
1256     ZOOM_connection_put_event(c, event);
1257
1258     c->buf_out = c->tasks->u.package->buf_out;
1259     c->len_out = c->tasks->u.package->len_out;
1260
1261     return ZOOM_send_buf(c);
1262 }
1263
1264 ZOOM_API(size_t)
1265     ZOOM_scanset_size(ZOOM_scanset scan)
1266 {
1267     if (!scan)
1268         return 0;
1269
1270     if (scan->scan_response && scan->scan_response->entries)
1271         return scan->scan_response->entries->num_entries;
1272     else if (scan->srw_scan_response)
1273         return scan->srw_scan_response->num_terms;
1274     return 0;
1275 }
1276
1277 static void ZOOM_scanset_term_x(ZOOM_scanset scan, size_t pos,
1278                                 size_t *occ,
1279                                 const char **value_term, size_t *value_len,
1280                                 const char **disp_term, size_t *disp_len)
1281 {
1282     size_t noent = ZOOM_scanset_size(scan);
1283
1284     *value_term = 0;
1285     *value_len = 0;
1286
1287     *disp_term = 0;
1288     *disp_len = 0;
1289
1290     *occ = 0;
1291     if (pos >= noent)
1292         return;
1293     if (scan->scan_response)
1294     {
1295         Z_ScanResponse *res = scan->scan_response;
1296         if (res->entries->entries[pos]->which == Z_Entry_termInfo)
1297         {
1298             Z_TermInfo *t = res->entries->entries[pos]->u.termInfo;
1299
1300             *value_term = (const char *) t->term->u.general->buf;
1301             *value_len = t->term->u.general->len;
1302             if (t->displayTerm)
1303             {
1304                 *disp_term = t->displayTerm;
1305                 *disp_len = strlen(*disp_term);
1306             }
1307             else if (t->term->which == Z_Term_general)
1308             {
1309                 *disp_term = (const char *) t->term->u.general->buf;
1310                 *disp_len = t->term->u.general->len;
1311             }
1312             *occ = t->globalOccurrences ? *t->globalOccurrences : 0;
1313         }
1314     }
1315     if (scan->srw_scan_response)
1316     {
1317         Z_SRW_scanResponse *res = scan->srw_scan_response;
1318         Z_SRW_scanTerm *t = res->terms + pos;
1319         if (t)
1320         {
1321             *value_term = t->value;
1322             *value_len = strlen(*value_term);
1323
1324             if (t->displayTerm)
1325                 *disp_term = t->displayTerm;
1326             else
1327                 *disp_term = t->value;
1328             *disp_len = strlen(*disp_term);
1329             *occ = t->numberOfRecords ? *t->numberOfRecords : 0;
1330         }
1331     }
1332 }
1333
1334 ZOOM_API(const char *)
1335     ZOOM_scanset_term(ZOOM_scanset scan, size_t pos,
1336                       size_t *occ, size_t *len)
1337 {
1338     const char *value_term = 0;
1339     size_t value_len = 0;
1340     const char *disp_term = 0;
1341     size_t disp_len = 0;
1342
1343     ZOOM_scanset_term_x(scan, pos, occ, &value_term, &value_len,
1344                         &disp_term, &disp_len);
1345
1346     *len = value_len;
1347     return value_term;
1348 }
1349
1350 ZOOM_API(const char *)
1351     ZOOM_scanset_display_term(ZOOM_scanset scan, size_t pos,
1352                               size_t *occ, size_t *len)
1353 {
1354     const char *value_term = 0;
1355     size_t value_len = 0;
1356     const char *disp_term = 0;
1357     size_t disp_len = 0;
1358
1359     ZOOM_scanset_term_x(scan, pos, occ, &value_term, &value_len,
1360                         &disp_term, &disp_len);
1361
1362     *len = disp_len;
1363     return disp_term;
1364 }
1365
1366 ZOOM_API(const char *)
1367     ZOOM_scanset_option_get(ZOOM_scanset scan, const char *key)
1368 {
1369     return ZOOM_options_get(scan->options, key);
1370 }
1371
1372 ZOOM_API(void)
1373     ZOOM_scanset_option_set(ZOOM_scanset scan, const char *key,
1374                             const char *val)
1375 {
1376     ZOOM_options_set(scan->options, key, val);
1377 }
1378
1379
1380 ZOOM_API(ZOOM_package)
1381     ZOOM_connection_package(ZOOM_connection c, ZOOM_options options)
1382 {
1383     ZOOM_package p = (ZOOM_package) xmalloc(sizeof(*p));
1384
1385     p->connection = c;
1386     p->odr_out = odr_createmem(ODR_ENCODE);
1387     p->options = ZOOM_options_create_with_parent2(options, c->options);
1388     p->refcount = 1;
1389     p->buf_out = 0;
1390     p->len_out = 0;
1391     return p;
1392 }
1393
1394 ZOOM_API(void)
1395     ZOOM_package_destroy(ZOOM_package p)
1396 {
1397     if (!p)
1398         return;
1399     (p->refcount)--;
1400     if (p->refcount == 0)
1401     {
1402         odr_destroy(p->odr_out);
1403         xfree(p->buf_out);
1404
1405         ZOOM_options_destroy(p->options);
1406         xfree(p);
1407     }
1408 }
1409
1410 ZOOM_API(const char *)
1411     ZOOM_package_option_get(ZOOM_package p, const char *key)
1412 {
1413     return ZOOM_options_get(p->options, key);
1414 }
1415
1416 ZOOM_API(const char *)
1417     ZOOM_package_option_getl(ZOOM_package p, const char *key, int *lenp)
1418 {
1419     return ZOOM_options_getl(p->options, key, lenp);
1420 }
1421
1422 ZOOM_API(void)
1423     ZOOM_package_option_set(ZOOM_package p, const char *key,
1424                             const char *val)
1425 {
1426     ZOOM_options_set(p->options, key, val);
1427 }
1428
1429 ZOOM_API(void)
1430     ZOOM_package_option_setl(ZOOM_package p, const char *key,
1431                              const char *val, int len)
1432 {
1433     ZOOM_options_setl(p->options, key, val, len);
1434 }
1435
1436 ZOOM_API(int)
1437     ZOOM_connection_exec_task(ZOOM_connection c)
1438 {
1439     ZOOM_task task = c->tasks;
1440     zoom_ret ret = zoom_complete;
1441
1442     if (!task)
1443         return 0;
1444     yaz_log(c->log_details, "%p ZOOM_connection_exec_task type=%d run=%d",
1445             c, task->which, task->running);
1446     if (c->error != ZOOM_ERROR_NONE)
1447     {
1448         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1449                 "removing tasks because of error = %d", c, c->error);
1450         ZOOM_connection_remove_tasks(c);
1451         return 0;
1452     }
1453     if (task->running)
1454     {
1455         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1456                 "task already running", c);
1457         return 0;
1458     }
1459     task->running = 1;
1460     ret = zoom_complete;
1461     if (c->cs || task->which == ZOOM_TASK_CONNECT)
1462     {
1463         switch (task->which)
1464         {
1465         case ZOOM_TASK_SEARCH:
1466             if (c->proto == PROTO_HTTP)
1467                 ret = ZOOM_connection_srw_send_search(c);
1468             else
1469                 ret = ZOOM_connection_Z3950_search(c);
1470             break;
1471         case ZOOM_TASK_CONNECT:
1472             ret = do_connect(c);
1473             break;
1474         case ZOOM_TASK_SCAN:
1475             if (c->proto == PROTO_HTTP)
1476                 ret = ZOOM_connection_srw_send_scan(c);
1477             else
1478                 ret = ZOOM_connection_Z3950_send_scan(c);
1479             break;
1480         case ZOOM_TASK_PACKAGE:
1481             ret = send_package(c);
1482             break;
1483         case ZOOM_TASK_SORT:
1484             c->tasks->u.sort.resultset->r_sort_spec =
1485                 ZOOM_query_get_sortspec(c->tasks->u.sort.q);
1486             ret = send_Z3950_sort(c, c->tasks->u.sort.resultset);
1487             break;
1488         }
1489     }
1490     else
1491     {
1492         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1493                 "remove tasks because no connection exist", c);
1494         ZOOM_connection_remove_tasks(c);
1495     }
1496     if (ret == zoom_complete)
1497     {
1498         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1499                 "task removed (complete)", c);
1500         ZOOM_connection_remove_task(c);
1501         return 0;
1502     }
1503     yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1504             "task pending", c);
1505     return 1;
1506 }
1507
1508
1509 static zoom_ret send_HTTP_redirect(ZOOM_connection c, const char *uri)
1510 {
1511     Z_GDU *gdu = z_get_HTTP_Request_uri(c->odr_out, uri, 0, c->proxy ? 1 : 0);
1512
1513     gdu->u.HTTP_Request->method = odr_strdup(c->odr_out, "GET");
1514     z_HTTP_header_add(c->odr_out, &gdu->u.HTTP_Request->headers, "Accept",
1515                       "text/xml");
1516     yaz_cookies_request(c->cookies, c->odr_out, gdu->u.HTTP_Request);
1517     if (c->user && c->password)
1518     {
1519         z_HTTP_header_add_basic_auth(c->odr_out, &gdu->u.HTTP_Request->headers,
1520                                      c->user, c->password);
1521     }
1522     xfree(c->location);
1523     c->location = 0;
1524     return ZOOM_send_GDU(c, gdu);
1525 }
1526
1527
1528 zoom_ret ZOOM_send_GDU(ZOOM_connection c, Z_GDU *gdu)
1529 {
1530     ZOOM_Event event;
1531
1532     int r = z_GDU(c->odr_out, &gdu, 0, 0);
1533     if (!r)
1534         return zoom_complete;
1535     if (c->odr_print)
1536         z_GDU(c->odr_print, &gdu, 0, 0);
1537     if (c->odr_save)
1538         z_GDU(c->odr_save, &gdu, 0, 0);
1539     c->buf_out = odr_getbuf(c->odr_out, &c->len_out, 0);
1540     odr_reset(c->odr_out);
1541
1542     event = ZOOM_Event_create(ZOOM_EVENT_SEND_APDU);
1543     ZOOM_connection_put_event(c, event);
1544
1545     return ZOOM_send_buf(c);
1546 }
1547
1548 void ZOOM_set_HTTP_error(ZOOM_connection c, int error,
1549                          const char *addinfo, const char *addinfo2)
1550 {
1551     ZOOM_set_dset_error(c, error, "HTTP", addinfo, addinfo2);
1552 }
1553
1554 #if YAZ_HAVE_XML2
1555 static void handle_http(ZOOM_connection c, Z_HTTP_Response *hres)
1556 {
1557     zoom_ret cret = zoom_complete;
1558     int ret = -1;
1559     char *addinfo = 0;
1560     const char *connection_head = z_HTTP_header_lookup(hres->headers,
1561                                                        "Connection");
1562     const char *location;
1563
1564     ZOOM_connection_set_mask(c, 0);
1565     yaz_log(c->log_details, "%p handle_http", c);
1566
1567     yaz_cookies_response(c->cookies, hres);
1568     if ((hres->code == 301 || hres->code == 302) && c->sru_mode == zoom_sru_get
1569         && (location = z_HTTP_header_lookup(hres->headers, "Location")))
1570     {
1571         c->no_redirects++;
1572         if (c->no_redirects > 10)
1573         {
1574             ZOOM_set_HTTP_error(c, hres->code, 0, 0);
1575             c->no_redirects = 0;
1576             ZOOM_connection_close(c);
1577         }
1578         else
1579         {
1580             /* since redirect may change host we just reconnect. A smarter
1581                implementation might check whether it's the same server */
1582
1583             int host_change = 0;
1584             location = yaz_check_location(c->odr_in, c->host_port,
1585                                           location, &host_change);
1586             xfree(c->location);
1587             c->location = xstrdup(location);
1588             do_connect_host(c, location);
1589             return;
1590         }
1591     }
1592     else
1593     {
1594         ret = ZOOM_handle_sru(c, hres, &cret, &addinfo);
1595         if (ret == 0)
1596         {
1597             if (c->no_redirects) /* end of redirect. change hosts again */
1598                 ZOOM_connection_close(c);
1599         }
1600         c->no_redirects = 0;
1601     }
1602     if (ret)
1603     {
1604         if (hres->code != 200)
1605             ZOOM_set_HTTP_error(c, hres->code, 0, 0);
1606         else
1607         {
1608             yaz_log(YLOG_LOG, "set error... addinfo=%s", addinfo ?
1609                     addinfo : "NULL");
1610             ZOOM_set_error(c, ZOOM_ERROR_DECODE, addinfo);
1611         }
1612         ZOOM_connection_close(c);
1613     }
1614     if (cret == zoom_complete)
1615     {
1616         yaz_log(c->log_details, "removing tasks in handle_http");
1617         ZOOM_connection_remove_task(c);
1618     }
1619     {
1620         int must_close = 0;
1621         if (!strcmp(hres->version, "1.0"))
1622         {
1623             /* HTTP 1.0: only if Keep-Alive we stay alive.. */
1624             if (!connection_head || strcmp(connection_head, "Keep-Alive"))
1625                 must_close = 1;
1626         }
1627         else
1628         {
1629             /* HTTP 1.1: only if no close we stay alive.. */
1630             if (connection_head && !strcmp(connection_head, "close"))
1631                 must_close = 1;
1632         }
1633         if (must_close)
1634         {
1635             ZOOM_connection_close(c);
1636             if (c->tasks)
1637             {
1638                 c->tasks->running = 0;
1639                 ZOOM_connection_insert_task(c, ZOOM_TASK_CONNECT);
1640                 c->reconnect_ok = 0;
1641             }
1642         }
1643         else
1644             c->reconnect_ok = 1; /* if the server closes anyway */
1645     }
1646 }
1647 #endif
1648
1649 static int do_read(ZOOM_connection c)
1650 {
1651     int r, more;
1652     ZOOM_Event event;
1653
1654     event = ZOOM_Event_create(ZOOM_EVENT_RECV_DATA);
1655     ZOOM_connection_put_event(c, event);
1656
1657     r = cs_get(c->cs, &c->buf_in, &c->len_in);
1658     more = cs_more(c->cs);
1659     yaz_log(c->log_details, "%p do_read len=%d more=%d", c, r, more);
1660     if (r == 1)
1661         return 0;
1662     if (r <= 0)
1663     {
1664         if (!ZOOM_test_reconnect(c))
1665         {
1666             ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
1667             ZOOM_connection_close(c);
1668         }
1669     }
1670     else
1671     {
1672         Z_GDU *gdu;
1673         ZOOM_Event event;
1674
1675         odr_reset(c->odr_in);
1676         odr_setbuf(c->odr_in, c->buf_in, r, 0);
1677         event = ZOOM_Event_create(ZOOM_EVENT_RECV_APDU);
1678         ZOOM_connection_put_event(c, event);
1679
1680         if (!z_GDU(c->odr_in, &gdu, 0, 0))
1681         {
1682             int x;
1683             int err = odr_geterrorx(c->odr_in, &x);
1684             char msg[100];
1685             const char *element = odr_getelement(c->odr_in);
1686             yaz_snprintf(msg, sizeof(msg),
1687                     "ODR code %d:%d element=%s offset=%d",
1688                     err, x, element ? element : "<unknown>",
1689                     odr_offset(c->odr_in));
1690             ZOOM_set_error(c, ZOOM_ERROR_DECODE, msg);
1691             if (c->log_api)
1692             {
1693                 FILE *ber_file = yaz_log_file();
1694                 if (ber_file)
1695                     odr_dumpBER(ber_file, c->buf_in, r);
1696             }
1697             ZOOM_connection_close(c);
1698         }
1699         else
1700         {
1701             if (c->odr_print)
1702                 z_GDU(c->odr_print, &gdu, 0, 0);
1703             if (c->odr_save)
1704                 z_GDU(c->odr_save, &gdu, 0, 0);
1705             if (gdu->which == Z_GDU_Z3950)
1706                 ZOOM_handle_Z3950_apdu(c, gdu->u.z3950);
1707             else if (gdu->which == Z_GDU_HTTP_Response)
1708             {
1709 #if YAZ_HAVE_XML2
1710                 handle_http(c, gdu->u.HTTP_Response);
1711 #else
1712                 ZOOM_set_error(c, ZOOM_ERROR_DECODE, 0);
1713                 ZOOM_connection_close(c);
1714 #endif
1715             }
1716         }
1717     }
1718     return 1;
1719 }
1720
1721 static zoom_ret do_write_ex(ZOOM_connection c, char *buf_out, int len_out)
1722 {
1723     int r;
1724     ZOOM_Event event;
1725
1726     event = ZOOM_Event_create(ZOOM_EVENT_SEND_DATA);
1727     ZOOM_connection_put_event(c, event);
1728
1729     yaz_log(c->log_details, "%p do_write_ex len=%d", c, len_out);
1730     if ((r = cs_put(c->cs, buf_out, len_out)) < 0)
1731     {
1732         yaz_log(c->log_details, "%p do_write_ex write failed", c);
1733         if (ZOOM_test_reconnect(c))
1734         {
1735             return zoom_pending;
1736         }
1737         if (c->state == STATE_CONNECTING)
1738             ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
1739         else
1740             ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
1741         ZOOM_connection_close(c);
1742         return zoom_complete;
1743     }
1744     else if (r == 1)
1745     {
1746         int mask = ZOOM_SELECT_EXCEPT;
1747         if (c->cs->io_pending & CS_WANT_WRITE)
1748             mask += ZOOM_SELECT_WRITE;
1749         if (c->cs->io_pending & CS_WANT_READ)
1750             mask += ZOOM_SELECT_READ;
1751         ZOOM_connection_set_mask(c, mask);
1752         yaz_log(c->log_details, "%p do_write_ex write incomplete mask=%d",
1753                 c, c->mask);
1754     }
1755     else
1756     {
1757         ZOOM_connection_set_mask(c, ZOOM_SELECT_READ|ZOOM_SELECT_EXCEPT);
1758         yaz_log(c->log_details, "%p do_write_ex write complete mask=%d",
1759                 c, c->mask);
1760     }
1761     return zoom_pending;
1762 }
1763
1764 zoom_ret ZOOM_send_buf(ZOOM_connection c)
1765 {
1766     return do_write_ex(c, c->buf_out, c->len_out);
1767 }
1768
1769
1770 ZOOM_API(const char *)
1771     ZOOM_connection_option_get(ZOOM_connection c, const char *key)
1772 {
1773     if (!strcmp(key, "APDU"))
1774     {
1775         return c->saveAPDU_wrbuf ? wrbuf_cstr(c->saveAPDU_wrbuf) : "";
1776     }
1777     else
1778         return ZOOM_options_get(c->options, key);
1779 }
1780
1781 ZOOM_API(const char *)
1782     ZOOM_connection_option_getl(ZOOM_connection c, const char *key, int *lenp)
1783 {
1784     if (!strcmp(key, "APDU"))
1785     {
1786         if (c->saveAPDU_wrbuf)
1787         {
1788             *lenp = wrbuf_len(c->saveAPDU_wrbuf);
1789             return wrbuf_cstr(c->saveAPDU_wrbuf);
1790         }
1791         else
1792         {
1793             *lenp = 0;
1794             return "";
1795         }
1796     }
1797     else
1798         return ZOOM_options_getl(c->options, key, lenp);
1799 }
1800
1801 ZOOM_API(void)
1802     ZOOM_connection_option_set(ZOOM_connection c, const char *key,
1803                                const char *val)
1804 {
1805     if (!strcmp(key, "saveAPDU"))
1806     {
1807         if (val && strcmp(val, "0"))
1808         {
1809             if (!c->saveAPDU_wrbuf)
1810                 c->saveAPDU_wrbuf = wrbuf_alloc();
1811             else
1812                 wrbuf_rewind(c->saveAPDU_wrbuf);
1813         }
1814         else
1815         {
1816             wrbuf_destroy(c->saveAPDU_wrbuf);
1817             c->saveAPDU_wrbuf = 0;
1818         }
1819         ZOOM_connection_save_apdu_wrbuf(c, c->saveAPDU_wrbuf);
1820     }
1821     else
1822         ZOOM_options_set(c->options, key, val);
1823 }
1824
1825 ZOOM_API(void)
1826     ZOOM_connection_option_setl(ZOOM_connection c, const char *key,
1827                                 const char *val, int len)
1828 {
1829     ZOOM_options_setl(c->options, key, val, len);
1830 }
1831
1832 ZOOM_API(const char *)
1833     ZOOM_resultset_option_get(ZOOM_resultset r, const char *key)
1834 {
1835     return ZOOM_options_get(r->options, key);
1836 }
1837
1838 ZOOM_API(void)
1839     ZOOM_resultset_option_set(ZOOM_resultset r, const char *key,
1840                               const char *val)
1841 {
1842     ZOOM_options_set(r->options, key, val);
1843 }
1844
1845
1846 ZOOM_API(int)
1847     ZOOM_connection_errcode(ZOOM_connection c)
1848 {
1849     return ZOOM_connection_error(c, 0, 0);
1850 }
1851
1852 ZOOM_API(const char *)
1853     ZOOM_connection_errmsg(ZOOM_connection c)
1854 {
1855     const char *msg;
1856     ZOOM_connection_error(c, &msg, 0);
1857     return msg;
1858 }
1859
1860 ZOOM_API(const char *)
1861     ZOOM_connection_addinfo(ZOOM_connection c)
1862 {
1863     const char *addinfo;
1864     ZOOM_connection_error(c, 0, &addinfo);
1865     return addinfo;
1866 }
1867
1868 ZOOM_API(const char *)
1869     ZOOM_connection_diagset(ZOOM_connection c)
1870 {
1871     const char *diagset;
1872     ZOOM_connection_error_x(c, 0, 0, &diagset);
1873     return diagset;
1874 }
1875
1876 ZOOM_API(const char *)
1877     ZOOM_diag_str(int error)
1878 {
1879     switch (error)
1880     {
1881     case ZOOM_ERROR_NONE:
1882         return "No error";
1883     case ZOOM_ERROR_CONNECT:
1884         return "Connect failed";
1885     case ZOOM_ERROR_MEMORY:
1886         return "Out of memory";
1887     case ZOOM_ERROR_ENCODE:
1888         return "Encoding failed";
1889     case ZOOM_ERROR_DECODE:
1890         return "Decoding failed";
1891     case ZOOM_ERROR_CONNECTION_LOST:
1892         return "Connection lost";
1893     case ZOOM_ERROR_INIT:
1894         return "Init rejected";
1895     case ZOOM_ERROR_INTERNAL:
1896         return "Internal failure";
1897     case ZOOM_ERROR_TIMEOUT:
1898         return "Timeout";
1899     case ZOOM_ERROR_UNSUPPORTED_PROTOCOL:
1900         return "Unsupported protocol";
1901     case ZOOM_ERROR_UNSUPPORTED_QUERY:
1902         return "Unsupported query type";
1903     case ZOOM_ERROR_INVALID_QUERY:
1904         return "Invalid query";
1905     case ZOOM_ERROR_CQL_PARSE:
1906         return "CQL parsing error";
1907     case ZOOM_ERROR_CQL_TRANSFORM:
1908         return "CQL transformation error";
1909     case ZOOM_ERROR_CCL_CONFIG:
1910         return "CCL configuration error";
1911     case ZOOM_ERROR_CCL_PARSE:
1912         return "CCL parsing error";
1913     case ZOOM_ERROR_ES_INVALID_ACTION:
1914         return "Extended Service. invalid action";
1915     case ZOOM_ERROR_ES_INVALID_VERSION:
1916         return "Extended Service. invalid version";
1917     case ZOOM_ERROR_ES_INVALID_SYNTAX:
1918         return "Extended Service. invalid syntax";
1919     case ZOOM_ERROR_MEMCACHED:
1920         return "Memcached";
1921     default:
1922         return diagbib1_str(error);
1923     }
1924 }
1925
1926 ZOOM_API(int)
1927     ZOOM_connection_error_x(ZOOM_connection c, const char **cp,
1928                             const char **addinfo, const char **diagset)
1929 {
1930     int error = c->error;
1931     if (cp)
1932     {
1933         if (!c->diagset || !strcmp(c->diagset, "ZOOM"))
1934             *cp = ZOOM_diag_str(error);
1935         else if (!strcmp(c->diagset, "HTTP"))
1936             *cp = z_HTTP_errmsg(c->error);
1937         else if (!strcmp(c->diagset, "Bib-1"))
1938             *cp = ZOOM_diag_str(error);
1939         else if (!strcmp(c->diagset, "info:srw/diagnostic/1"))
1940             *cp = yaz_diag_srw_str(c->error);
1941         else
1942             *cp = "Unknown error and diagnostic set";
1943     }
1944     if (addinfo)
1945         *addinfo = c->addinfo ? c->addinfo : "";
1946     if (diagset)
1947         *diagset = c->diagset ? c->diagset : "";
1948     return c->error;
1949 }
1950
1951 ZOOM_API(int)
1952     ZOOM_connection_error(ZOOM_connection c, const char **cp,
1953                           const char **addinfo)
1954 {
1955     return ZOOM_connection_error_x(c, cp, addinfo, 0);
1956 }
1957
1958 static void ZOOM_connection_do_io(ZOOM_connection c, int mask)
1959 {
1960     ZOOM_Event event = 0;
1961     int r = cs_look(c->cs);
1962     yaz_log(c->log_details, "%p ZOOM_connection_do_io mask=%d cs_look=%d",
1963             c, mask, r);
1964
1965     if (r == CS_NONE)
1966     {
1967         event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1968         ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
1969         ZOOM_connection_close(c);
1970         ZOOM_connection_put_event(c, event);
1971     }
1972     else if (r == CS_CONNECT)
1973     {
1974         int ret = ret = cs_rcvconnect(c->cs);
1975         yaz_log(c->log_details, "%p ZOOM_connection_do_io "
1976                 "cs_rcvconnect returned %d", c, ret);
1977         if (ret == 1)
1978         {
1979             int mask = ZOOM_SELECT_EXCEPT;
1980             if (c->cs->io_pending & CS_WANT_WRITE)
1981                 mask += ZOOM_SELECT_WRITE;
1982             if (c->cs->io_pending & CS_WANT_READ)
1983                 mask += ZOOM_SELECT_READ;
1984             ZOOM_connection_set_mask(c, mask);
1985             event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1986             ZOOM_connection_put_event(c, event);
1987         }
1988         else if (ret == 0)
1989         {
1990             event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1991             ZOOM_connection_put_event(c, event);
1992             get_cert(c);
1993             if (c->proto == PROTO_Z3950)
1994                 ZOOM_connection_Z3950_send_init(c);
1995             else
1996             {
1997                 /* no init request for SRW .. */
1998                 if (c->tasks->which == ZOOM_TASK_CONNECT)
1999                 {
2000                     ZOOM_connection_remove_task(c);
2001                     ZOOM_connection_set_mask(c, 0);
2002                 }
2003                 if (c->cs && c->location)
2004                     send_HTTP_redirect(c, c->location);
2005                 else
2006                     ZOOM_connection_exec_task(c);
2007             }
2008             c->state = STATE_ESTABLISHED;
2009         }
2010         else
2011         {
2012             ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
2013             ZOOM_connection_close(c);
2014         }
2015     }
2016     else
2017     {
2018         if (mask & ZOOM_SELECT_EXCEPT)
2019         {
2020             if (!ZOOM_test_reconnect(c))
2021             {
2022                 ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
2023                 ZOOM_connection_close(c);
2024             }
2025             return;
2026         }
2027         if (mask & ZOOM_SELECT_READ)
2028             do_read(c);
2029         if (c->cs && (mask & ZOOM_SELECT_WRITE))
2030             ZOOM_send_buf(c);
2031     }
2032 }
2033
2034 ZOOM_API(int)
2035     ZOOM_connection_last_event(ZOOM_connection cs)
2036 {
2037     if (!cs)
2038         return ZOOM_EVENT_NONE;
2039     return cs->last_event;
2040 }
2041
2042
2043 ZOOM_API(int) ZOOM_connection_fire_event_timeout(ZOOM_connection c)
2044 {
2045     if (c->mask)
2046     {
2047         ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_TIMEOUT);
2048         /* timeout and this connection was waiting */
2049         ZOOM_set_error(c, ZOOM_ERROR_TIMEOUT, 0);
2050         ZOOM_connection_close(c);
2051         ZOOM_connection_put_event(c, event);
2052     }
2053     return 0;
2054 }
2055
2056 ZOOM_API(int)
2057     ZOOM_connection_process(ZOOM_connection c)
2058 {
2059     ZOOM_Event event;
2060     if (!c)
2061         return 0;
2062
2063     event = ZOOM_connection_get_event(c);
2064     if (event)
2065     {
2066         ZOOM_Event_destroy(event);
2067         return 1;
2068     }
2069     ZOOM_connection_exec_task(c);
2070     event = ZOOM_connection_get_event(c);
2071     if (event)
2072     {
2073         ZOOM_Event_destroy(event);
2074         return 1;
2075     }
2076     return 0;
2077 }
2078
2079 ZOOM_API(int)
2080     ZOOM_event_nonblock(int no, ZOOM_connection *cs)
2081 {
2082     int i;
2083
2084     yaz_log(log_details0, "ZOOM_process_event(no=%d,cs=%p)", no, cs);
2085
2086     for (i = 0; i<no; i++)
2087     {
2088         ZOOM_connection c = cs[i];
2089
2090         if (c && ZOOM_connection_process(c))
2091             return i+1;
2092     }
2093     return 0;
2094 }
2095
2096 ZOOM_API(int) ZOOM_connection_fire_event_socket(ZOOM_connection c, int mask)
2097 {
2098     if (c->mask && mask)
2099         ZOOM_connection_do_io(c, mask);
2100     return 0;
2101 }
2102
2103 ZOOM_API(int) ZOOM_connection_get_socket(ZOOM_connection c)
2104 {
2105     if (c->cs)
2106         return cs_fileno(c->cs);
2107     return -1;
2108 }
2109
2110 ZOOM_API(int) ZOOM_connection_set_mask(ZOOM_connection c, int mask)
2111 {
2112     c->mask = mask;
2113     if (!c->cs)
2114         return -1;
2115     return 0;
2116 }
2117
2118 ZOOM_API(int) ZOOM_connection_get_mask(ZOOM_connection c)
2119 {
2120     if (c->cs)
2121         return c->mask;
2122     return 0;
2123 }
2124
2125 ZOOM_API(int) ZOOM_connection_get_timeout(ZOOM_connection c)
2126 {
2127     return ZOOM_options_get_int(c->options, "timeout", 30);
2128 }
2129
2130 ZOOM_API(void) ZOOM_connection_close(ZOOM_connection c)
2131 {
2132     if (c->cs)
2133         cs_close(c->cs);
2134     c->cs = 0;
2135     ZOOM_connection_set_mask(c, 0);
2136     c->state = STATE_IDLE;
2137 }
2138
2139 /*
2140  * Local variables:
2141  * c-basic-offset: 4
2142  * c-file-style: "Stroustrup"
2143  * indent-tabs-mode: nil
2144  * End:
2145  * vim: shiftwidth=4 tabstop=8 expandtab
2146  */
2147