Changed task queue management a little for ZOOM C.
[yaz-moved-to-github.git] / src / zoom-c.c
1 /*
2  * Copyright (C) 1995-2007, Index Data ApS
3  * See the file LICENSE for details.
4  *
5  * $Id: zoom-c.c,v 1.112 2007-02-21 09:17:51 adam Exp $
6  */
7 /**
8  * \file zoom-c.c
9  * \brief Implements ZOOM C interface.
10  */
11
12 #include <assert.h>
13 #include <string.h>
14 #include <errno.h>
15 #include "zoom-p.h"
16
17 #include <yaz/yaz-util.h>
18 #include <yaz/xmalloc.h>
19 #include <yaz/otherinfo.h>
20 #include <yaz/log.h>
21 #include <yaz/pquery.h>
22 #include <yaz/marcdisp.h>
23 #include <yaz/diagbib1.h>
24 #include <yaz/charneg.h>
25 #include <yaz/ill.h>
26 #include <yaz/srw.h>
27 #include <yaz/cql.h>
28 #include <yaz/ccl.h>
29
30 #define TASK_FIX 1
31
32 static int log_api = 0;
33 static int log_details = 0;
34
35 typedef enum {
36     zoom_pending,
37     zoom_complete
38 } zoom_ret;
39
40 static zoom_ret ZOOM_connection_send_init(ZOOM_connection c);
41 static zoom_ret do_write_ex(ZOOM_connection c, char *buf_out, int len_out);
42 static char *cql2pqf(ZOOM_connection c, const char *cql);
43
44
45 /*
46  * This wrapper is just for logging failed lookups.  It would be nicer
47  * if it could cause failure when a lookup fails, but that's hard.
48  */
49 static Odr_oid *zoom_yaz_str_to_z3950oid(ZOOM_connection c,
50                                          int oid_class, const char *str) {
51     Odr_oid *res = yaz_str_to_z3950oid(c->odr_out, oid_class, str);
52     if (res == 0)
53         yaz_log(YLOG_WARN, "%p OID lookup (%d, '%s') failed",
54                 c, (int) oid_class, str);
55     return res;
56 }
57
58
59 static void initlog(void)
60 {
61     static int log_level_initialized = 0;
62     if (!log_level_initialized)
63     {
64         log_api = yaz_log_module_level("zoom");
65         log_details = yaz_log_module_level("zoomdetails");
66         log_level_initialized = 1;
67     }
68 }
69
70 static ZOOM_Event ZOOM_Event_create(int kind)
71 {
72     ZOOM_Event event = (ZOOM_Event) xmalloc(sizeof(*event));
73     event->kind = kind;
74     event->next = 0;
75     event->prev = 0;
76     yaz_log(log_details, "ZOOM_Event_create(kind=%d)", kind);
77     return event;
78 }
79
80 static void ZOOM_Event_destroy(ZOOM_Event event)
81 {
82     xfree(event);
83 }
84
85 static void ZOOM_connection_put_event(ZOOM_connection c, ZOOM_Event event)
86 {
87     if (c->m_queue_back)
88     {
89         c->m_queue_back->prev = event;
90         assert(c->m_queue_front);
91     }
92     else
93     {
94         assert(!c->m_queue_front);
95         c->m_queue_front = event;
96     }
97     event->next = c->m_queue_back;
98     event->prev = 0;
99     c->m_queue_back = event;
100 }
101
102 static ZOOM_Event ZOOM_connection_get_event(ZOOM_connection c)
103 {
104     ZOOM_Event event = c->m_queue_front;
105     if (!event)
106     {
107         c->last_event = ZOOM_EVENT_NONE;
108         return 0;
109     }
110     assert(c->m_queue_back);
111     c->m_queue_front = event->prev;
112     if (c->m_queue_front)
113     {
114         assert(c->m_queue_back);
115         c->m_queue_front->next = 0;
116     }
117     else
118         c->m_queue_back = 0;
119     c->last_event = event->kind;
120     return event;
121 }
122
123 ZOOM_API(int) ZOOM_connection_peek_event(ZOOM_connection c)
124 {
125     ZOOM_Event event = c->m_queue_front;
126
127     return event ? event->kind : ZOOM_EVENT_NONE;
128 }
129
130 static void set_dset_error(ZOOM_connection c, int error,
131                            const char *dset,
132                            const char *addinfo, const char *addinfo2)
133 {
134     char *cp;
135
136     xfree(c->addinfo);
137     c->addinfo = 0;
138     c->error = error;
139     if (!c->diagset || strcmp(dset, c->diagset))
140     {
141         xfree(c->diagset);
142         c->diagset = xstrdup(dset);
143         /* remove integer part from SRW diagset .. */
144         if ((cp = strrchr(c->diagset, '/')))
145             *cp = '\0';
146     }
147     if (addinfo && addinfo2)
148     {
149         c->addinfo = (char*) xmalloc(strlen(addinfo) + strlen(addinfo2) + 2);
150         strcpy(c->addinfo, addinfo);
151         strcat(c->addinfo, addinfo2);
152     }
153     else if (addinfo)
154         c->addinfo = xstrdup(addinfo);
155     if (error != ZOOM_ERROR_NONE)
156     {
157         yaz_log(log_api, "%p set_dset_error %s %s:%d %s %s",
158                 c, c->host_port ? c->host_port : "<>", dset, error,
159                 addinfo ? addinfo : "",
160                 addinfo2 ? addinfo2 : "");
161 #if TASK_FIX
162         ZOOM_connection_remove_tasks(c);
163 #endif
164     }
165 }
166
167 #if YAZ_HAVE_XML2
168 static void set_HTTP_error(ZOOM_connection c, int error,
169                            const char *addinfo, const char *addinfo2)
170 {
171     set_dset_error(c, error, "HTTP", addinfo, addinfo2);
172 }
173 #endif
174
175 void ZOOM_connection_remove_tasks(ZOOM_connection c);
176
177 static void set_ZOOM_error(ZOOM_connection c, int error,
178                            const char *addinfo)
179 {
180     set_dset_error(c, error, "ZOOM", addinfo, 0);
181 }
182
183 static void clear_error(ZOOM_connection c)
184 {
185     /*
186      * If an error is tied to an operation then it's ok to clear: for
187      * example, a diagnostic returned from a search is cleared by a
188      * subsequent search.  However, problems such as Connection Lost
189      * or Init Refused are not cleared, because they are not
190      * recoverable: doing another search doesn't help.
191      */
192     switch (c->error)
193     {
194     case ZOOM_ERROR_CONNECT:
195     case ZOOM_ERROR_MEMORY:
196     case ZOOM_ERROR_DECODE:
197     case ZOOM_ERROR_CONNECTION_LOST:
198     case ZOOM_ERROR_INIT:
199     case ZOOM_ERROR_INTERNAL:
200     case ZOOM_ERROR_UNSUPPORTED_PROTOCOL:
201         break;
202     default:
203         set_ZOOM_error(c, ZOOM_ERROR_NONE, 0);
204     }
205 }
206
207 void ZOOM_connection_show_task(ZOOM_task task)
208 {
209     switch(task->which)
210     {
211     case ZOOM_TASK_SEARCH:
212         yaz_log(YLOG_LOG, "search p=%p", task);
213         break;
214     case ZOOM_TASK_RETRIEVE:
215         yaz_log(YLOG_LOG, "retrieve p=%p", task);
216         break;
217     case ZOOM_TASK_CONNECT:
218         yaz_log(YLOG_LOG, "connect p=%p", task);
219         break;
220     case ZOOM_TASK_SCAN:
221         yaz_log(YLOG_LOG, "scant p=%p", task);
222         break;
223     }
224 }
225
226 void ZOOM_connection_show_tasks(ZOOM_connection c)
227 {
228     ZOOM_task task;
229     yaz_log(YLOG_LOG, "connection p=%p tasks", c);
230     for (task = c->tasks; task; task = task->next)
231         ZOOM_connection_show_task(task);
232 }
233
234 ZOOM_task ZOOM_connection_add_task(ZOOM_connection c, int which)
235 {
236     ZOOM_task *taskp = &c->tasks;
237     while (*taskp)
238         taskp = &(*taskp)->next;
239     *taskp = (ZOOM_task) xmalloc(sizeof(**taskp));
240     (*taskp)->running = 0;
241     (*taskp)->which = which;
242     (*taskp)->next = 0;
243     clear_error(c);
244     return *taskp;
245 }
246
247 ZOOM_API(int) ZOOM_connection_is_idle(ZOOM_connection c)
248 {
249     return c->tasks ? 0 : 1;
250 }
251
252 ZOOM_task ZOOM_connection_insert_task(ZOOM_connection c, int which)
253 {
254     ZOOM_task task = (ZOOM_task) xmalloc(sizeof(*task));
255
256     task->next = c->tasks;
257     c->tasks = task;
258
259     task->running = 0;
260     task->which = which;
261     clear_error(c);
262     return task;
263 }
264
265 void ZOOM_connection_remove_task(ZOOM_connection c)
266 {
267     ZOOM_task task = c->tasks;
268
269     if (task)
270     {
271         c->tasks = task->next;
272         switch (task->which)
273         {
274         case ZOOM_TASK_SEARCH:
275             ZOOM_resultset_destroy(task->u.search.resultset);
276             xfree(task->u.search.syntax);
277             xfree(task->u.search.elementSetName);
278             break;
279         case ZOOM_TASK_RETRIEVE:
280             ZOOM_resultset_destroy(task->u.retrieve.resultset);
281             xfree(task->u.retrieve.syntax);
282             xfree(task->u.retrieve.elementSetName);
283             break;
284         case ZOOM_TASK_CONNECT:
285             break;
286         case ZOOM_TASK_SCAN:
287             ZOOM_scanset_destroy(task->u.scan.scan);
288             break;
289         case ZOOM_TASK_PACKAGE:
290             ZOOM_package_destroy(task->u.package);
291             break;
292         case ZOOM_TASK_SORT:
293             ZOOM_resultset_destroy(task->u.sort.resultset);
294             ZOOM_query_destroy(task->u.sort.q);
295             break;
296         default:
297             assert(0);
298         }
299         xfree(task);
300
301         if (!c->tasks)
302         {
303             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_END);
304             ZOOM_connection_put_event(c, event);
305         }
306     }
307 }
308
309 static int ZOOM_connection_exec_task(ZOOM_connection c);
310
311 void ZOOM_connection_remove_tasks(ZOOM_connection c)
312 {
313     while (c->tasks)
314         ZOOM_connection_remove_task(c);
315 }
316
317 static ZOOM_record record_cache_lookup(ZOOM_resultset r, int pos,
318                                        const char *syntax,
319                                        const char *elementSetName);
320
321 ZOOM_API(ZOOM_connection)
322     ZOOM_connection_create(ZOOM_options options)
323 {
324     ZOOM_connection c = (ZOOM_connection) xmalloc(sizeof(*c));
325
326     initlog();
327
328     yaz_log(log_api, "%p ZOOM_connection_create", c);
329
330     c->proto = PROTO_Z3950;
331     c->cs = 0;
332     ZOOM_connection_set_mask(c, 0);
333     c->reconnect_ok = 0;
334     c->state = STATE_IDLE;
335     c->addinfo = 0;
336     c->diagset = 0;
337     set_ZOOM_error(c, ZOOM_ERROR_NONE, 0);
338     c->buf_in = 0;
339     c->len_in = 0;
340     c->buf_out = 0;
341     c->len_out = 0;
342     c->resultsets = 0;
343
344     c->options = ZOOM_options_create_with_parent(options);
345
346     c->host_port = 0;
347     c->path = 0;
348     c->proxy = 0;
349     
350     c->charset = c->lang = 0;
351
352     c->cookie_out = 0;
353     c->cookie_in = 0;
354     c->client_IP = 0;
355     c->tasks = 0;
356
357     c->odr_in = odr_createmem(ODR_DECODE);
358     c->odr_out = odr_createmem(ODR_ENCODE);
359
360     c->async = 0;
361     c->support_named_resultsets = 0;
362     c->last_event = ZOOM_EVENT_NONE;
363
364     c->m_queue_front = 0;
365     c->m_queue_back = 0;
366     return c;
367 }
368
369
370 /* set database names. Take local databases (if set); otherwise
371    take databases given in ZURL (if set); otherwise use Default */
372 static char **set_DatabaseNames(ZOOM_connection con, ZOOM_options options,
373                                 int *num, ODR odr)
374 {
375     char **databaseNames;
376     const char *cp = ZOOM_options_get(options, "databaseName");
377     
378     if ((!cp || !*cp) && con->host_port)
379     {
380         if (strncmp(con->host_port, "unix:", 5) == 0)
381             cp = strchr(con->host_port+5, ':');
382         else
383             cp = strchr(con->host_port, '/');
384         if (cp)
385             cp++;
386     }
387     if (!cp)
388         cp = "Default";
389     nmem_strsplit(odr->mem, "+", cp,  &databaseNames, num);
390     return databaseNames;
391 }
392
393 ZOOM_API(ZOOM_connection)
394     ZOOM_connection_new(const char *host, int portnum)
395 {
396     ZOOM_connection c = ZOOM_connection_create(0);
397
398     ZOOM_connection_connect(c, host, portnum);
399     return c;
400 }
401
402 static zoom_sru_mode get_sru_mode_from_string(const char *s)
403 {
404     if (!s || !*s)
405         return zoom_sru_soap;
406     if (!yaz_matchstr(s, "soap"))
407         return zoom_sru_soap;
408     else if (!yaz_matchstr(s, "get"))
409         return zoom_sru_get;
410     else if (!yaz_matchstr(s, "post"))
411         return zoom_sru_post;
412     return zoom_sru_error;
413 }
414
415 ZOOM_API(void)
416     ZOOM_connection_connect(ZOOM_connection c,
417                             const char *host, int portnum)
418 {
419     const char *val;
420     ZOOM_task task;
421
422     initlog();
423
424     yaz_log(log_api, "%p ZOOM_connection_connect host=%s portnum=%d",
425             c, host, portnum);
426
427     set_ZOOM_error(c, ZOOM_ERROR_NONE, 0);
428     ZOOM_connection_remove_tasks(c);
429
430     if (c->cs)
431     {
432         yaz_log(log_details, "%p ZOOM_connection_connect reconnect ok", c);
433         c->reconnect_ok = 1;
434         return;
435     }
436     yaz_log(log_details, "%p ZOOM_connection_connect connect", c);
437     xfree(c->proxy);
438     val = ZOOM_options_get(c->options, "proxy");
439     if (val && *val)
440     {
441         yaz_log(log_details, "%p ZOOM_connection_connect proxy=%s", c, val);
442         c->proxy = xstrdup(val);
443     }
444     else
445         c->proxy = 0;
446
447     xfree(c->charset);
448     val = ZOOM_options_get(c->options, "charset");
449     if (val && *val)
450     {
451         yaz_log(log_details, "%p ZOOM_connection_connect charset=%s", c, val);
452         c->charset = xstrdup(val);
453     }
454     else
455         c->charset = 0;
456
457     xfree(c->lang);
458     val = ZOOM_options_get(c->options, "lang");
459     if (val && *val)
460     {
461         yaz_log(log_details, "%p ZOOM_connection_connect lang=%s", c, val);
462         c->lang = xstrdup(val);
463     }
464     else
465         c->lang = 0;
466
467     xfree(c->host_port);
468     if (portnum)
469     {
470         char hostn[128];
471         sprintf(hostn, "%.80s:%d", host, portnum);
472         c->host_port = xstrdup(hostn);
473     }
474     else
475         c->host_port = xstrdup(host);
476
477     {
478         /*
479          * If the "<scheme>:" part of the host string is preceded by one
480          * or more comma-separated <name>=<value> pairs, these are taken
481          * to be options to be set on the connection object.  Among other
482          * applications, this facility can be used to embed authentication
483          * in a host string:
484          *          user=admin,password=secret,tcp:localhost:9999
485          */
486         char *remainder = c->host_port;
487         char *pcolon = strchr(remainder, ':');
488         char *pcomma;
489         char *pequals;
490         while ((pcomma = strchr(remainder, ',')) != 0 &&
491                (pcolon == 0 || pcomma < pcolon)) {
492             *pcomma = '\0';
493             if ((pequals = strchr(remainder, '=')) != 0) {
494                 *pequals = '\0';
495                 /*printf("# setting '%s'='%s'\n", remainder, pequals+1);*/
496                 ZOOM_connection_option_set(c, remainder, pequals+1);
497             }
498             remainder = pcomma+1;
499         }
500
501         if (remainder != c->host_port) {
502             xfree(c->host_port);
503             c->host_port = xstrdup(remainder);
504             /*printf("# reset hp='%s'\n", remainder);*/
505         }
506     }
507
508     val = ZOOM_options_get(c->options, "sru");
509     c->sru_mode = get_sru_mode_from_string(val);
510
511     ZOOM_options_set(c->options, "host", c->host_port);
512
513     val = ZOOM_options_get(c->options, "cookie");
514     if (val && *val)
515     { 
516         yaz_log(log_details, "%p ZOOM_connection_connect cookie=%s", c, val);
517         c->cookie_out = xstrdup(val);
518     }
519
520     val = ZOOM_options_get(c->options, "clientIP");
521     if (val && *val)
522     {
523         yaz_log(log_details, "%p ZOOM_connection_connect clientIP=%s",
524                 c, val);
525         c->client_IP = xstrdup(val);
526     }
527
528     c->async = ZOOM_options_get_bool(c->options, "async", 0);
529     yaz_log(log_details, "%p ZOOM_connection_connect async=%d", c, c->async);
530  
531     task = ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
532
533     if (!c->async)
534     {
535         while (ZOOM_event(1, &c))
536             ;
537     }
538 }
539
540 ZOOM_API(ZOOM_query)
541     ZOOM_query_create(void)
542 {
543     ZOOM_query s = (ZOOM_query) xmalloc(sizeof(*s));
544
545     yaz_log(log_details, "%p ZOOM_query_create", s);
546     s->refcount = 1;
547     s->z_query = 0;
548     s->sort_spec = 0;
549     s->odr = odr_createmem(ODR_ENCODE);
550     s->query_string = 0;
551
552     return s;
553 }
554
555 ZOOM_API(void)
556     ZOOM_query_destroy(ZOOM_query s)
557 {
558     if (!s)
559         return;
560
561     (s->refcount)--;
562     yaz_log(log_details, "%p ZOOM_query_destroy count=%d", s, s->refcount);
563     if (s->refcount == 0)
564     {
565         odr_destroy(s->odr);
566         xfree(s);
567     }
568 }
569
570 ZOOM_API(int)
571     ZOOM_query_prefix(ZOOM_query s, const char *str)
572 {
573     s->query_string = odr_strdup(s->odr, str);
574     s->z_query = (Z_Query *) odr_malloc(s->odr, sizeof(*s->z_query));
575     s->z_query->which = Z_Query_type_1;
576     s->z_query->u.type_1 =  p_query_rpn(s->odr, PROTO_Z3950, str);
577     if (!s->z_query->u.type_1)
578     {
579         yaz_log(log_details, "%p ZOOM_query_prefix str=%s failed", s, str);
580         s->z_query = 0;
581         return -1;
582     }
583     yaz_log(log_details, "%p ZOOM_query_prefix str=%s", s, str);
584     return 0;
585 }
586
587 ZOOM_API(int)
588     ZOOM_query_cql(ZOOM_query s, const char *str)
589 {
590     Z_External *ext;
591
592     s->query_string = odr_strdup(s->odr, str);
593
594     ext = (Z_External *) odr_malloc(s->odr, sizeof(*ext));
595     ext->direct_reference = odr_getoidbystr(s->odr, "1.2.840.10003.16.2");
596     ext->indirect_reference = 0;
597     ext->descriptor = 0;
598     ext->which = Z_External_CQL;
599     ext->u.cql = s->query_string;
600     
601     s->z_query = (Z_Query *) odr_malloc(s->odr, sizeof(*s->z_query));
602     s->z_query->which = Z_Query_type_104;
603     s->z_query->u.type_104 =  ext;
604
605     yaz_log(log_details, "%p ZOOM_query_cql str=%s", s, str);
606
607     return 0;
608 }
609
610 /*
611  * Translate the CQL string client-side into RPN which is passed to
612  * the server.  This is useful for server's that don't themselves
613  * support CQL, for which ZOOM_query_cql() is useless.  `conn' is used
614  * only as a place to stash diagnostics if compilation fails; if this
615  * information is not needed, a null pointer may be used.
616  */
617 ZOOM_API(int)
618     ZOOM_query_cql2rpn(ZOOM_query s, const char *str, ZOOM_connection conn)
619 {
620     char *rpn;
621     int ret;
622     ZOOM_connection freeme = 0;
623
624     yaz_log(log_details, "%p ZOOM_query_cql2rpn str=%s conn=%p", s, str, conn);
625     if (conn == 0)
626         conn = freeme = ZOOM_connection_create(0);
627
628     rpn = cql2pqf(conn, str);
629     if (freeme != 0)
630         ZOOM_connection_destroy(freeme);
631     if (rpn == 0)
632         return -1;
633
634     ret = ZOOM_query_prefix(s, rpn);
635     xfree(rpn);
636     return ret;
637 }
638
639 /*
640  * Analogous in every way to ZOOM_query_cql2rpn(), except that there
641  * is no analogous ZOOM_query_ccl() that just sends uninterpreted CCL
642  * to the server, as the YAZ GFS doesn't know how to handle this.
643  */
644 ZOOM_API(int)
645     ZOOM_query_ccl2rpn(ZOOM_query s, const char *str, const char *config,
646                        int *ccl_error, const char **error_string,
647                        int *error_pos)
648 {
649     int ret;
650     struct ccl_rpn_node *rpn;
651     CCL_bibset bibset = ccl_qual_mk();
652
653     if (config)
654         ccl_qual_buf(bibset, config);
655
656     rpn = ccl_find_str(bibset, str, ccl_error, error_pos);
657     if (!rpn)
658     {
659         *error_string = ccl_err_msg(*ccl_error);
660         ret = -1;
661     }
662     else
663     {
664         WRBUF wr = wrbuf_alloc();
665         ccl_pquery(wr, rpn);
666         ccl_rpn_delete(rpn);
667         ret = ZOOM_query_prefix(s, wrbuf_buf(wr));
668         wrbuf_free(wr, 1);
669     }
670     ccl_qual_rm(&bibset);
671     return ret;
672 }
673
674 ZOOM_API(int)
675     ZOOM_query_sortby(ZOOM_query s, const char *criteria)
676 {
677     s->sort_spec = yaz_sort_spec(s->odr, criteria);
678     if (!s->sort_spec)
679     {
680         yaz_log(log_details, "%p ZOOM_query_sortby criteria=%s failed",
681                 s, criteria);
682         return -1;
683     }
684     yaz_log(log_details, "%p ZOOM_query_sortby criteria=%s", s, criteria);
685     return 0;
686 }
687
688 static zoom_ret do_write(ZOOM_connection c);
689
690 ZOOM_API(void)
691     ZOOM_connection_destroy(ZOOM_connection c)
692 {
693     ZOOM_resultset r;
694     if (!c)
695         return;
696     yaz_log(log_api, "%p ZOOM_connection_destroy", c);
697     if (c->cs)
698         cs_close(c->cs);
699     for (r = c->resultsets; r; r = r->next)
700         r->connection = 0;
701
702     xfree(c->buf_in);
703     xfree(c->addinfo);
704     xfree(c->diagset);
705     odr_destroy(c->odr_in);
706     odr_destroy(c->odr_out);
707     ZOOM_options_destroy(c->options);
708     ZOOM_connection_remove_tasks(c);
709     xfree(c->host_port);
710     xfree(c->path);
711     xfree(c->proxy);
712     xfree(c->charset);
713     xfree(c->lang);
714     xfree(c->cookie_out);
715     xfree(c->cookie_in);
716     xfree(c->client_IP);
717     xfree(c);
718 }
719
720 void ZOOM_resultset_addref(ZOOM_resultset r)
721 {
722     if (r)
723     {
724         (r->refcount)++;
725         yaz_log(log_details, "%p ZOOM_resultset_addref count=%d",
726                 r, r->refcount);
727     }
728 }
729
730 ZOOM_resultset ZOOM_resultset_create(void)
731 {
732     int i;
733     ZOOM_resultset r = (ZOOM_resultset) xmalloc(sizeof(*r));
734
735     initlog();
736
737     yaz_log(log_details, "%p ZOOM_resultset_create", r);
738     r->refcount = 1;
739     r->size = 0;
740     r->odr = odr_createmem(ODR_ENCODE);
741     r->piggyback = 1;
742     r->setname = 0;
743     r->schema = 0;
744     r->step = 0;
745     for (i = 0; i<RECORD_HASH_SIZE; i++)
746         r->record_hash[i] = 0;
747     r->r_sort_spec = 0;
748     r->query = 0;
749     r->connection = 0;
750     r->next = 0;
751     r->databaseNames = 0;
752     r->num_databaseNames = 0;
753     return r;
754 }
755
756 ZOOM_API(ZOOM_resultset)
757     ZOOM_connection_search_pqf(ZOOM_connection c, const char *q)
758 {
759     ZOOM_resultset r;
760     ZOOM_query s = ZOOM_query_create();
761
762     ZOOM_query_prefix(s, q);
763
764     r = ZOOM_connection_search(c, s);
765     ZOOM_query_destroy(s);
766     return r;
767 }
768
769 ZOOM_API(ZOOM_resultset)
770     ZOOM_connection_search(ZOOM_connection c, ZOOM_query q)
771 {
772     ZOOM_resultset r = ZOOM_resultset_create();
773     ZOOM_task task;
774     const char *cp;
775     int start, count;
776     const char *syntax, *elementSetName;
777
778     yaz_log(log_api, "%p ZOOM_connection_search set %p query %p", c, r, q);
779     r->r_sort_spec = q->sort_spec;
780     r->query = q;
781
782     r->options = ZOOM_options_create_with_parent(c->options);
783
784     start = ZOOM_options_get_int(r->options, "start", 0);
785     count = ZOOM_options_get_int(r->options, "count", 0);
786     {
787         /* If "presentChunk" is defined use that; otherwise "step" */
788         const char *cp = ZOOM_options_get(r->options, "presentChunk");
789         r->step = ZOOM_options_get_int(r->options,
790                                        (cp != 0 ? "presentChunk": "step"), 0);
791     }
792     r->piggyback = ZOOM_options_get_bool(r->options, "piggyback", 1);
793     cp = ZOOM_options_get(r->options, "setname");
794     if (cp)
795         r->setname = xstrdup(cp);
796     cp = ZOOM_options_get(r->options, "schema");
797     if (cp)
798         r->schema = xstrdup(cp);
799
800     r->databaseNames = set_DatabaseNames(c, c->options, &r->num_databaseNames,
801                                          r->odr);
802     
803     r->connection = c;
804
805     r->next = c->resultsets;
806     c->resultsets = r;
807
808     if (c->host_port && c->proto == PROTO_HTTP)
809     {
810         if (!c->cs)
811         {
812             yaz_log(log_details, "ZOOM_connection_search: no comstack");
813             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
814         }
815         else
816         {
817             yaz_log(log_details, "ZOOM_connection_search: reconnect");
818             c->reconnect_ok = 1;
819         }
820     }
821
822     task = ZOOM_connection_add_task(c, ZOOM_TASK_SEARCH);
823     task->u.search.resultset = r;
824     task->u.search.start = start;
825     task->u.search.count = count;
826
827     syntax = ZOOM_options_get(r->options, "preferredRecordSyntax"); 
828     task->u.search.syntax = syntax ? xstrdup(syntax) : 0;
829     elementSetName = ZOOM_options_get(r->options, "elementSetName");
830     task->u.search.elementSetName = elementSetName 
831         ? xstrdup(elementSetName) : 0;
832    
833     ZOOM_resultset_addref(r);
834
835     (q->refcount)++;
836
837     if (!c->async)
838     {
839         while (ZOOM_event(1, &c))
840             ;
841     }
842     return r;
843 }
844
845 /*
846  * This is the old result-set sorting API, which is maintained only
847  * for the sake of binary compatibility.  There is no reason ever to
848  * use this rather than ZOOM_resultset_sort1().
849  */
850 ZOOM_API(void)
851     ZOOM_resultset_sort(ZOOM_resultset r,
852                         const char *sort_type, const char *sort_spec)
853 {
854     (void) ZOOM_resultset_sort1(r, sort_type, sort_spec);
855 }
856
857 ZOOM_API(int)
858     ZOOM_resultset_sort1(ZOOM_resultset r,
859                          const char *sort_type, const char *sort_spec)
860 {
861     ZOOM_connection c = r->connection;
862     ZOOM_task task;
863     ZOOM_query newq;
864
865     newq = ZOOM_query_create();
866     if (ZOOM_query_sortby(newq, sort_spec) < 0)
867         return -1;
868
869     yaz_log(log_api, "%p ZOOM_resultset_sort r=%p sort_type=%s sort_spec=%s",
870             r, r, sort_type, sort_spec);
871     if (!c)
872         return 0;
873
874     if (c->host_port && c->proto == PROTO_HTTP)
875     {
876         if (!c->cs)
877         {
878             yaz_log(log_details, "%p ZOOM_resultset_sort: no comstack", r);
879             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
880         }
881         else
882         {
883             yaz_log(log_details, "%p ZOOM_resultset_sort: prepare reconnect",
884                     r);
885             c->reconnect_ok = 1;
886         }
887     }
888     
889     ZOOM_resultset_cache_reset(r);
890     task = ZOOM_connection_add_task(c, ZOOM_TASK_SORT);
891     task->u.sort.resultset = r;
892     task->u.sort.q = newq;
893
894     ZOOM_resultset_addref(r);  
895
896     if (!c->async)
897     {
898         while (ZOOM_event(1, &c))
899             ;
900     }
901
902     return 0;
903 }
904
905 ZOOM_API(void)
906     ZOOM_resultset_cache_reset(ZOOM_resultset r)
907 {
908     int i;
909     for (i = 0; i<RECORD_HASH_SIZE; i++)
910     {
911         ZOOM_record_cache rc;
912         for (rc = r->record_hash[i]; rc; rc = rc->next)
913         {
914             if (rc->rec.wrbuf_marc)
915                 wrbuf_free(rc->rec.wrbuf_marc, 1);
916             if (rc->rec.wrbuf_iconv)
917                 wrbuf_free(rc->rec.wrbuf_iconv, 1);
918             if (rc->rec.wrbuf_opac)
919                 wrbuf_free(rc->rec.wrbuf_opac, 1);
920         }
921         r->record_hash[i] = 0;
922     }
923 }
924
925 ZOOM_API(void)
926     ZOOM_resultset_destroy(ZOOM_resultset r)
927 {
928     if (!r)
929         return;
930     (r->refcount)--;
931     yaz_log(log_details, "%p ZOOM_resultset_destroy r=%p count=%d",
932             r, r, r->refcount);
933     if (r->refcount == 0)
934     {
935         ZOOM_resultset_cache_reset(r);
936
937         if (r->connection)
938         {
939             /* remove ourselves from the resultsets in connection */
940             ZOOM_resultset *rp = &r->connection->resultsets;
941             while (1)
942             {
943                 assert(*rp);   /* we must be in this list!! */
944                 if (*rp == r)
945                 {   /* OK, we're here - take us out of it */
946                     *rp = (*rp)->next;
947                     break;
948                 }
949                 rp = &(*rp)->next;
950             }
951         }
952         ZOOM_query_destroy(r->query);
953         ZOOM_options_destroy(r->options);
954         odr_destroy(r->odr);
955         xfree(r->setname);
956         xfree(r->schema);
957         xfree(r);
958     }
959 }
960
961 ZOOM_API(size_t)
962     ZOOM_resultset_size(ZOOM_resultset r)
963 {
964     yaz_log(log_details, "ZOOM_resultset_size r=%p count=%d",
965             r, r->size);
966     return r->size;
967 }
968
969 static void do_close(ZOOM_connection c)
970 {
971     if (c->cs)
972         cs_close(c->cs);
973     c->cs = 0;
974     ZOOM_connection_set_mask(c, 0);
975     c->state = STATE_IDLE;
976 }
977
978 static int ZOOM_test_reconnect(ZOOM_connection c)
979 {
980     if (!c->reconnect_ok)
981         return 0;
982     do_close(c);
983     c->reconnect_ok = 0;
984     c->tasks->running = 0;
985     ZOOM_connection_insert_task(c, ZOOM_TASK_CONNECT);
986     return 1;
987 }
988
989 static void ZOOM_resultset_retrieve(ZOOM_resultset r,
990                                     int force_sync, int start, int count)
991 {
992     ZOOM_task task;
993     ZOOM_connection c;
994     const char *cp;
995     const char *syntax, *elementSetName;
996
997     if (!r)
998         return;
999     yaz_log(log_details, "%p ZOOM_resultset_retrieve force_sync=%d start=%d"
1000             " count=%d", r, force_sync, start, count);
1001     c = r->connection;
1002     if (!c)
1003         return;
1004
1005     if (c->host_port && c->proto == PROTO_HTTP)
1006     {
1007         if (!c->cs)
1008         {
1009             yaz_log(log_details, "%p ZOOM_resultset_retrieve: no comstack", r);
1010             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
1011         }
1012         else
1013         {
1014             yaz_log(log_details, "%p ZOOM_resultset_retrieve: prepare "
1015                     "reconnect", r);
1016             c->reconnect_ok = 1;
1017         }
1018     }
1019     task = ZOOM_connection_add_task(c, ZOOM_TASK_RETRIEVE);
1020     task->u.retrieve.resultset = r;
1021     task->u.retrieve.start = start;
1022     task->u.retrieve.count = count;
1023
1024     syntax = ZOOM_options_get(r->options, "preferredRecordSyntax"); 
1025     task->u.retrieve.syntax = syntax ? xstrdup(syntax) : 0;
1026     elementSetName = ZOOM_options_get(r->options, "elementSetName");
1027     task->u.retrieve.elementSetName = elementSetName 
1028         ? xstrdup(elementSetName) : 0;
1029
1030     cp = ZOOM_options_get(r->options, "schema");
1031     if (cp)
1032     {
1033         if (!r->schema || strcmp(r->schema, cp))
1034         {
1035             xfree(r->schema);
1036             r->schema = xstrdup(cp);
1037         }
1038     }
1039
1040     ZOOM_resultset_addref(r);
1041
1042     if (!r->connection->async || force_sync)
1043         while (r->connection && ZOOM_event(1, &r->connection))
1044             ;
1045 }
1046
1047 ZOOM_API(void)
1048     ZOOM_resultset_records(ZOOM_resultset r, ZOOM_record *recs,
1049                            size_t start, size_t count)
1050 {
1051     int force_present = 0;
1052
1053     if (!r)
1054         return ;
1055     yaz_log(log_api, "%p ZOOM_resultset_records r=%p start=%ld count=%ld",
1056             r, r, (long) start, (long) count);
1057     if (count && recs)
1058         force_present = 1;
1059     ZOOM_resultset_retrieve(r, force_present, start, count);
1060     if (force_present)
1061     {
1062         size_t i;
1063         for (i = 0; i< count; i++)
1064             recs[i] = ZOOM_resultset_record_immediate(r, i+start);
1065     }
1066 }
1067
1068 static void get_cert(ZOOM_connection c)
1069 {
1070     char *cert_buf;
1071     int cert_len;
1072     
1073     if (cs_get_peer_certificate_x509(c->cs, &cert_buf, &cert_len))
1074     {
1075         ZOOM_connection_option_setl(c, "sslPeerCert",
1076                                     cert_buf, cert_len);
1077         xfree(cert_buf);
1078     }
1079 }
1080
1081 static zoom_ret do_connect(ZOOM_connection c)
1082 {
1083     void *add;
1084     const char *effective_host;
1085
1086     if (c->proxy)
1087         effective_host = c->proxy;
1088     else
1089         effective_host = c->host_port;
1090
1091     yaz_log(log_details, "%p do_connect effective_host=%s", c, effective_host);
1092
1093     if (c->cs)
1094         cs_close(c->cs);
1095     c->cs = cs_create_host(effective_host, 0, &add);
1096
1097     if (c->cs && c->cs->protocol == PROTO_HTTP)
1098     {
1099 #if YAZ_HAVE_XML2
1100         const char *path = 0;
1101
1102         c->proto = PROTO_HTTP;
1103         cs_get_host_args(c->host_port, &path);
1104         xfree(c->path);
1105         c->path = (char*) xmalloc(strlen(path)+2);
1106         c->path[0] = '/';
1107         strcpy(c->path+1, path);
1108 #else
1109         set_ZOOM_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, "SRW");
1110         do_close(c);
1111         return zoom_complete;
1112 #endif
1113     }
1114     if (c->cs)
1115     {
1116         int ret = cs_connect(c->cs, add);
1117         if (ret == 0)
1118         {
1119             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1120             ZOOM_connection_put_event(c, event);
1121             get_cert(c);
1122             if (c->proto == PROTO_Z3950)
1123                 ZOOM_connection_send_init(c);
1124             else
1125             {
1126                 /* no init request for SRW .. */
1127                 assert(c->tasks->which == ZOOM_TASK_CONNECT);
1128                 ZOOM_connection_remove_task(c);
1129                 ZOOM_connection_set_mask(c, 0);
1130                 ZOOM_connection_exec_task(c);
1131             }
1132             c->state = STATE_ESTABLISHED;
1133             return zoom_pending;
1134         }
1135         else if (ret > 0)
1136         {
1137             int mask = ZOOM_SELECT_EXCEPT;
1138             if (c->cs->io_pending & CS_WANT_WRITE)
1139                 mask += ZOOM_SELECT_WRITE;
1140             if (c->cs->io_pending & CS_WANT_READ)
1141                 mask += ZOOM_SELECT_READ;
1142             ZOOM_connection_set_mask(c, mask);
1143             c->state = STATE_CONNECTING; 
1144             return zoom_pending;
1145         }
1146     }
1147     c->state = STATE_IDLE;
1148     set_ZOOM_error(c, ZOOM_ERROR_CONNECT, c->host_port);
1149     return zoom_complete;
1150 }
1151
1152 static void otherInfo_attach(ZOOM_connection c, Z_APDU *a, ODR out)
1153 {
1154     int i;
1155     for (i = 0; i<200; i++)
1156     {
1157         size_t len;
1158         Z_OtherInformation **oi;
1159         char buf[80];
1160         const char *val;
1161         const char *cp;
1162         int oidval;
1163
1164         sprintf(buf, "otherInfo%d", i);
1165         val = ZOOM_options_get(c->options, buf);
1166         if (!val)
1167             break;
1168         cp = strchr(val, ':');
1169         if (!cp)
1170             continue;
1171         len = cp - val;
1172         if (len >= sizeof(buf))
1173             len = sizeof(buf)-1;
1174         memcpy(buf, val, len);
1175         buf[len] = '\0';
1176         oidval = oid_getvalbyname(buf);
1177         if (oidval == VAL_NONE)
1178             continue;
1179         
1180         yaz_oi_APDU(a, &oi);
1181         yaz_oi_set_string_oidval(oi, out, oidval, 1, cp+1);
1182     }
1183 }
1184
1185 static int encode_APDU(ZOOM_connection c, Z_APDU *a, ODR out)
1186 {
1187     assert(a);
1188     if (c->cookie_out)
1189     {
1190         Z_OtherInformation **oi;
1191         yaz_oi_APDU(a, &oi);
1192         yaz_oi_set_string_oidval(oi, out, VAL_COOKIE, 1, c->cookie_out);
1193     }
1194     if (c->client_IP)
1195     {
1196         Z_OtherInformation **oi;
1197         yaz_oi_APDU(a, &oi);
1198         yaz_oi_set_string_oidval(oi, out, VAL_CLIENT_IP, 1, c->client_IP);
1199     }
1200     otherInfo_attach(c, a, out);
1201     if (!z_APDU(out, &a, 0, 0))
1202     {
1203         FILE *outf = fopen("/tmp/apdu.txt", "a");
1204         if (a && outf)
1205         {
1206             ODR odr_pr = odr_createmem(ODR_PRINT);
1207             fprintf(outf, "a=%p\n", a);
1208             odr_setprint(odr_pr, outf);
1209             z_APDU(odr_pr, &a, 0, 0);
1210             odr_destroy(odr_pr);
1211         }
1212         yaz_log(log_api, "%p encoding_APDU: encoding failed", c);
1213         set_ZOOM_error(c, ZOOM_ERROR_ENCODE, 0);
1214         odr_reset(out);
1215         return -1;
1216     }
1217     yaz_log(log_details, "%p encoding_APDU encoding OK", c);
1218     return 0;
1219 }
1220
1221 static zoom_ret send_APDU(ZOOM_connection c, Z_APDU *a)
1222 {
1223     ZOOM_Event event;
1224     assert(a);
1225     if (encode_APDU(c, a, c->odr_out))
1226         return zoom_complete;
1227     yaz_log(log_details, "%p send APDU type=%d", c, a->which);
1228     c->buf_out = odr_getbuf(c->odr_out, &c->len_out, 0);
1229     event = ZOOM_Event_create(ZOOM_EVENT_SEND_APDU);
1230     ZOOM_connection_put_event(c, event);
1231     odr_reset(c->odr_out);
1232     return do_write(c);
1233 }
1234
1235 /* returns 1 if PDU was sent OK (still pending )
1236    0 if PDU was not sent OK (nothing to wait for) 
1237 */
1238
1239 static zoom_ret ZOOM_connection_send_init(ZOOM_connection c)
1240 {
1241     Z_APDU *apdu = zget_APDU(c->odr_out, Z_APDU_initRequest);
1242     Z_InitRequest *ireq = apdu->u.initRequest;
1243     Z_IdAuthentication *auth = (Z_IdAuthentication *)
1244         odr_malloc(c->odr_out, sizeof(*auth));
1245     const char *auth_groupId = ZOOM_options_get(c->options, "group");
1246     const char *auth_userId = ZOOM_options_get(c->options, "user");
1247     const char *auth_password = ZOOM_options_get(c->options, "password");
1248     char *version;
1249
1250     /* support the pass for backwards compatibility */
1251     if (!auth_password)
1252         auth_password = ZOOM_options_get(c->options, "pass");
1253         
1254     ODR_MASK_SET(ireq->options, Z_Options_search);
1255     ODR_MASK_SET(ireq->options, Z_Options_present);
1256     ODR_MASK_SET(ireq->options, Z_Options_scan);
1257     ODR_MASK_SET(ireq->options, Z_Options_sort);
1258     ODR_MASK_SET(ireq->options, Z_Options_extendedServices);
1259     ODR_MASK_SET(ireq->options, Z_Options_namedResultSets);
1260     
1261     ODR_MASK_SET(ireq->protocolVersion, Z_ProtocolVersion_1);
1262     ODR_MASK_SET(ireq->protocolVersion, Z_ProtocolVersion_2);
1263     ODR_MASK_SET(ireq->protocolVersion, Z_ProtocolVersion_3);
1264     
1265     /* Index Data's Z39.50 Implementor Id is 81 */
1266     ireq->implementationId =
1267         odr_prepend(c->odr_out,
1268                     ZOOM_options_get(c->options, "implementationId"),
1269                     odr_prepend(c->odr_out, "81", ireq->implementationId));
1270     
1271     ireq->implementationName = 
1272         odr_prepend(c->odr_out,
1273                     ZOOM_options_get(c->options, "implementationName"),
1274                     odr_prepend(c->odr_out, "ZOOM-C",
1275                                 ireq->implementationName));
1276     
1277     version = odr_strdup(c->odr_out, "$Revision: 1.112 $");
1278     if (strlen(version) > 10)   /* check for unexpanded CVS strings */
1279         version[strlen(version)-2] = '\0';
1280     ireq->implementationVersion = 
1281         odr_prepend(c->odr_out,
1282                     ZOOM_options_get(c->options, "implementationVersion"),
1283                     odr_prepend(c->odr_out, &version[11],
1284                                 ireq->implementationVersion));
1285     
1286     *ireq->maximumRecordSize =
1287         ZOOM_options_get_int(c->options, "maximumRecordSize", 1024*1024);
1288     *ireq->preferredMessageSize =
1289         ZOOM_options_get_int(c->options, "preferredMessageSize", 1024*1024);
1290     
1291     if (auth_groupId || auth_password)
1292     {
1293         Z_IdPass *pass = (Z_IdPass *) odr_malloc(c->odr_out, sizeof(*pass));
1294         int i = 0;
1295         pass->groupId = 0;
1296         if (auth_groupId && *auth_groupId)
1297         {
1298             pass->groupId = (char *)
1299                 odr_malloc(c->odr_out, strlen(auth_groupId)+1);
1300             strcpy(pass->groupId, auth_groupId);
1301             i++;
1302         }
1303         pass->userId = 0;
1304         if (auth_userId && *auth_userId)
1305         {
1306             pass->userId = (char *)
1307                 odr_malloc(c->odr_out, strlen(auth_userId)+1);
1308             strcpy(pass->userId, auth_userId);
1309             i++;
1310         }
1311         pass->password = 0;
1312         if (auth_password && *auth_password)
1313         {
1314             pass->password = (char *)
1315                 odr_malloc(c->odr_out, strlen(auth_password)+1);
1316             strcpy(pass->password, auth_password);
1317             i++;
1318         }
1319         if (i)
1320         {
1321             auth->which = Z_IdAuthentication_idPass;
1322             auth->u.idPass = pass;
1323             ireq->idAuthentication = auth;
1324         }
1325     }
1326     else if (auth_userId)
1327     {
1328         auth->which = Z_IdAuthentication_open;
1329         auth->u.open = (char *)
1330             odr_malloc(c->odr_out, strlen(auth_userId)+1);
1331         strcpy(auth->u.open, auth_userId);
1332         ireq->idAuthentication = auth;
1333     }
1334     if (c->proxy)
1335         yaz_oi_set_string_oidval(&ireq->otherInfo, c->odr_out,
1336                                  VAL_PROXY, 1, c->host_port);
1337     if (c->charset || c->lang)
1338     {
1339         Z_OtherInformation **oi;
1340         Z_OtherInformationUnit *oi_unit;
1341         
1342         yaz_oi_APDU(apdu, &oi);
1343         
1344         if ((oi_unit = yaz_oi_update(oi, c->odr_out, NULL, 0, 0)))
1345         {
1346             ODR_MASK_SET(ireq->options, Z_Options_negotiationModel);
1347             oi_unit->which = Z_OtherInfo_externallyDefinedInfo;
1348             oi_unit->information.externallyDefinedInfo =
1349                 yaz_set_proposal_charneg_list(c->odr_out, " ",
1350                                               c->charset, c->lang, 1);
1351         }
1352     }
1353     assert(apdu);
1354     return send_APDU(c, apdu);
1355 }
1356
1357 #if YAZ_HAVE_XML2
1358 static zoom_ret send_srw(ZOOM_connection c, Z_SRW_PDU *sr)
1359 {
1360     Z_GDU *gdu;
1361     ZOOM_Event event;
1362
1363     gdu = z_get_HTTP_Request_host_path(c->odr_out, c->host_port, c->path);
1364
1365     if (c->sru_mode == zoom_sru_get)
1366     {
1367         yaz_sru_get_encode(gdu->u.HTTP_Request, sr, c->odr_out, c->charset);
1368     }
1369     else if (c->sru_mode == zoom_sru_post)
1370     {
1371         yaz_sru_post_encode(gdu->u.HTTP_Request, sr, c->odr_out, c->charset);
1372     }
1373     else if (c->sru_mode == zoom_sru_soap)
1374     {
1375         yaz_sru_soap_encode(gdu->u.HTTP_Request, sr, c->odr_out, c->charset);
1376     }
1377     if (!z_GDU(c->odr_out, &gdu, 0, 0))
1378         return zoom_complete;
1379     c->buf_out = odr_getbuf(c->odr_out, &c->len_out, 0);
1380         
1381     event = ZOOM_Event_create(ZOOM_EVENT_SEND_APDU);
1382     ZOOM_connection_put_event(c, event);
1383     odr_reset(c->odr_out);
1384     return do_write(c);
1385 }
1386 #endif
1387
1388 #if YAZ_HAVE_XML2
1389 static zoom_ret ZOOM_connection_srw_send_search(ZOOM_connection c)
1390 {
1391     int i;
1392     int *start, *count;
1393     ZOOM_resultset resultset = 0;
1394     Z_SRW_PDU *sr = 0;
1395     const char *option_val = 0;
1396
1397     if (c->error)                  /* don't continue on error */
1398         return zoom_complete;
1399     assert(c->tasks);
1400     switch(c->tasks->which)
1401     {
1402     case ZOOM_TASK_SEARCH:
1403         resultset = c->tasks->u.search.resultset;
1404         resultset->setname = xstrdup("default");
1405         ZOOM_options_set(resultset->options, "setname", resultset->setname);
1406         start = &c->tasks->u.search.start;
1407         count = &c->tasks->u.search.count;
1408         break;
1409     case ZOOM_TASK_RETRIEVE:
1410         resultset = c->tasks->u.retrieve.resultset;
1411
1412         start = &c->tasks->u.retrieve.start;
1413         count = &c->tasks->u.retrieve.count;
1414         
1415         if (*start >= resultset->size)
1416             return zoom_complete;
1417         if (*start + *count > resultset->size)
1418             *count = resultset->size - *start;
1419
1420         for (i = 0; i < *count; i++)
1421         {
1422             ZOOM_record rec =
1423                 record_cache_lookup(resultset, i + *start,
1424                                     c->tasks->u.retrieve.syntax,
1425                                     c->tasks->u.retrieve.elementSetName);
1426             if (!rec)
1427                 break;
1428             else
1429             {
1430                 ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_RECV_RECORD);
1431                 ZOOM_connection_put_event(c, event);
1432             }
1433         }
1434         *start += i;
1435         *count -= i;
1436
1437         if (*count == 0)
1438             return zoom_complete;
1439         break;
1440     default:
1441         return zoom_complete;
1442     }
1443     assert(resultset->query);
1444         
1445     sr = yaz_srw_get(c->odr_out, Z_SRW_searchRetrieve_request);
1446
1447     if (resultset->query->z_query->which == Z_Query_type_104
1448         && resultset->query->z_query->u.type_104->which == Z_External_CQL)
1449     {
1450         sr->u.request->query_type = Z_SRW_query_type_cql;
1451         sr->u.request->query.cql =resultset->query->z_query->u.type_104->u.cql;
1452     }
1453     else if (resultset->query->z_query->which == Z_Query_type_1 &&
1454              resultset->query->z_query->u.type_1)
1455     {
1456         sr->u.request->query_type = Z_SRW_query_type_pqf;
1457         sr->u.request->query.pqf = resultset->query->query_string;
1458     }
1459     else
1460     {
1461         set_ZOOM_error(c, ZOOM_ERROR_UNSUPPORTED_QUERY, 0);
1462         return zoom_complete;
1463     }
1464     sr->u.request->startRecord = odr_intdup(c->odr_out, *start + 1);
1465     sr->u.request->maximumRecords = odr_intdup(
1466         c->odr_out, resultset->step>0 ? resultset->step : *count);
1467     sr->u.request->recordSchema = resultset->schema;
1468     
1469     option_val = ZOOM_resultset_option_get(resultset, "recordPacking");
1470     if (option_val)
1471         sr->u.request->recordPacking = odr_strdup(c->odr_out, option_val);
1472
1473     option_val = ZOOM_resultset_option_get(resultset, "extraArgs");
1474     if (option_val)
1475         sr->extra_args = odr_strdup(c->odr_out, option_val);
1476     return send_srw(c, sr);
1477 }
1478 #else
1479 static zoom_ret ZOOM_connection_srw_send_search(ZOOM_connection c)
1480 {
1481     return zoom_complete;
1482 }
1483 #endif
1484
1485 static zoom_ret ZOOM_connection_send_search(ZOOM_connection c)
1486 {
1487     ZOOM_resultset r;
1488     int lslb, ssub, mspn;
1489     const char *syntax;
1490     Z_APDU *apdu = zget_APDU(c->odr_out, Z_APDU_searchRequest);
1491     Z_SearchRequest *search_req = apdu->u.searchRequest;
1492     const char *elementSetName;
1493     const char *smallSetElementSetName;
1494     const char *mediumSetElementSetName;
1495
1496     assert(c->tasks);
1497     assert(c->tasks->which == ZOOM_TASK_SEARCH);
1498
1499     r = c->tasks->u.search.resultset;
1500
1501     yaz_log(log_details, "%p ZOOM_connection_send_search set=%p", c, r);
1502
1503     elementSetName =
1504         ZOOM_options_get(r->options, "elementSetName");
1505     smallSetElementSetName  =
1506         ZOOM_options_get(r->options, "smallSetElementSetName");
1507     mediumSetElementSetName =
1508         ZOOM_options_get(r->options, "mediumSetElementSetName");
1509
1510     if (!smallSetElementSetName)
1511         smallSetElementSetName = elementSetName;
1512
1513     if (!mediumSetElementSetName)
1514         mediumSetElementSetName = elementSetName;
1515
1516     assert(r);
1517     assert(r->query);
1518
1519     /* prepare query for the search request */
1520     search_req->query = r->query->z_query;
1521     if (!search_req->query)
1522     {
1523         set_ZOOM_error(c, ZOOM_ERROR_INVALID_QUERY, 0);
1524         return zoom_complete;
1525     }
1526
1527     search_req->databaseNames = r->databaseNames;
1528     search_req->num_databaseNames = r->num_databaseNames;
1529
1530     /* get syntax (no need to provide unless piggyback is in effect) */
1531     syntax = c->tasks->u.search.syntax;
1532
1533     lslb = ZOOM_options_get_int(r->options, "largeSetLowerBound", -1);
1534     ssub = ZOOM_options_get_int(r->options, "smallSetUpperBound", -1);
1535     mspn = ZOOM_options_get_int(r->options, "mediumSetPresentNumber", -1);
1536     if (lslb != -1 && ssub != -1 && mspn != -1)
1537     {
1538         /* So're a Z39.50 expert? Let's hope you don't do sort */
1539         *search_req->largeSetLowerBound = lslb;
1540         *search_req->smallSetUpperBound = ssub;
1541         *search_req->mediumSetPresentNumber = mspn;
1542     }
1543     else if (c->tasks->u.search.start == 0 && c->tasks->u.search.count > 0
1544              && r->piggyback && !r->r_sort_spec && !r->schema)
1545     {
1546         /* Regular piggyback - do it unless we're going to do sort */
1547         *search_req->largeSetLowerBound = 2000000000;
1548         *search_req->smallSetUpperBound = 1;
1549         *search_req->mediumSetPresentNumber = 
1550             r->step>0 ? r->step : c->tasks->u.search.count;
1551     }
1552     else
1553     {
1554         /* non-piggyback. Need not provide elementsets or syntaxes .. */
1555         smallSetElementSetName = 0;
1556         mediumSetElementSetName = 0;
1557         syntax = 0;
1558     }
1559     if (smallSetElementSetName && *smallSetElementSetName)
1560     {
1561         Z_ElementSetNames *esn = (Z_ElementSetNames *)
1562             odr_malloc(c->odr_out, sizeof(*esn));
1563         
1564         esn->which = Z_ElementSetNames_generic;
1565         esn->u.generic = odr_strdup(c->odr_out, smallSetElementSetName);
1566         search_req->smallSetElementSetNames = esn;
1567     }
1568     if (mediumSetElementSetName && *mediumSetElementSetName)
1569     {
1570         Z_ElementSetNames *esn =(Z_ElementSetNames *)
1571             odr_malloc(c->odr_out, sizeof(*esn));
1572         
1573         esn->which = Z_ElementSetNames_generic;
1574         esn->u.generic = odr_strdup(c->odr_out, mediumSetElementSetName);
1575         search_req->mediumSetElementSetNames = esn;
1576     }
1577     if (syntax)
1578         search_req->preferredRecordSyntax =
1579             zoom_yaz_str_to_z3950oid(c, CLASS_RECSYN, syntax);
1580     
1581     if (!r->setname)
1582     {
1583         if (c->support_named_resultsets)
1584         {
1585             char setname[14];
1586             int ord;
1587             /* find the lowest unused ordinal so that we re-use
1588                result sets on the server. */
1589             for (ord = 1; ; ord++)
1590             {
1591                 ZOOM_resultset rp;
1592                 sprintf(setname, "%d", ord);
1593                 for (rp = c->resultsets; rp; rp = rp->next)
1594                     if (rp->setname && !strcmp(rp->setname, setname))
1595                         break;
1596                 if (!rp)
1597                     break;
1598             }
1599             r->setname = xstrdup(setname);
1600             yaz_log(log_details, "%p ZOOM_connection_send_search: allocating "
1601                     "set %s", c, r->setname);
1602         }
1603         else
1604         {
1605             yaz_log(log_details, "%p ZOOM_connection_send_search: using "
1606                     "default set", c);
1607             r->setname = xstrdup("default");
1608         }
1609         ZOOM_options_set(r->options, "setname", r->setname);
1610     }
1611     search_req->resultSetName = odr_strdup(c->odr_out, r->setname);
1612     return send_APDU(c, apdu);
1613 }
1614
1615 static void response_default_diag(ZOOM_connection c, Z_DefaultDiagFormat *r)
1616 {
1617     int oclass;
1618     char *addinfo = 0;
1619
1620     switch (r->which)
1621     {
1622     case Z_DefaultDiagFormat_v2Addinfo:
1623         addinfo = r->u.v2Addinfo;
1624         break;
1625     case Z_DefaultDiagFormat_v3Addinfo:
1626         addinfo = r->u.v3Addinfo;
1627         break;
1628     }
1629     xfree(c->addinfo);
1630     c->addinfo = 0;
1631     set_dset_error(c, *r->condition,
1632                    yaz_z3950oid_to_str(r->diagnosticSetId, &oclass),
1633                    addinfo, 0);
1634 }
1635
1636 static void response_diag(ZOOM_connection c, Z_DiagRec *p)
1637 {
1638     if (p->which != Z_DiagRec_defaultFormat)
1639         set_ZOOM_error(c, ZOOM_ERROR_DECODE, 0);
1640     else
1641         response_default_diag(c, p->u.defaultFormat);
1642 }
1643
1644 ZOOM_API(ZOOM_record)
1645     ZOOM_record_clone(ZOOM_record srec)
1646 {
1647     char *buf;
1648     int size;
1649     ODR odr_enc;
1650     ZOOM_record nrec;
1651
1652     odr_enc = odr_createmem(ODR_ENCODE);
1653     if (!z_NamePlusRecord(odr_enc, &srec->npr, 0, 0))
1654         return 0;
1655     buf = odr_getbuf(odr_enc, &size, 0);
1656     
1657     nrec = (ZOOM_record) xmalloc(sizeof(*nrec));
1658     nrec->odr = odr_createmem(ODR_DECODE);
1659     nrec->wrbuf_marc = 0;
1660     nrec->wrbuf_iconv = 0;
1661     nrec->wrbuf_opac = 0;
1662     odr_setbuf(nrec->odr, buf, size, 0);
1663     z_NamePlusRecord(nrec->odr, &nrec->npr, 0, 0);
1664     
1665     odr_destroy(odr_enc);
1666     return nrec;
1667 }
1668
1669 ZOOM_API(ZOOM_record)
1670     ZOOM_resultset_record_immediate(ZOOM_resultset s,size_t pos)
1671 {
1672     const char *syntax =
1673         ZOOM_options_get(s->options, "preferredRecordSyntax"); 
1674     const char *elementSetName =
1675         ZOOM_options_get(s->options, "elementSetName");
1676
1677     return record_cache_lookup(s, pos, syntax, elementSetName);
1678 }
1679
1680 ZOOM_API(ZOOM_record)
1681     ZOOM_resultset_record(ZOOM_resultset r, size_t pos)
1682 {
1683     ZOOM_record rec = ZOOM_resultset_record_immediate(r, pos);
1684
1685     if (!rec)
1686     {
1687         /*
1688          * MIKE: I think force_sync should always be zero, but I don't
1689          * want to make this change until I get the go-ahead from
1690          * Adam, in case something depends on the old synchronous
1691          * behaviour.
1692          */
1693         int force_sync = 1;
1694         if (getenv("ZOOM_RECORD_NO_FORCE_SYNC")) force_sync = 0;
1695         ZOOM_resultset_retrieve(r, force_sync, pos, 1);
1696         rec = ZOOM_resultset_record_immediate(r, pos);
1697     }
1698     return rec;
1699 }
1700
1701 ZOOM_API(void)
1702     ZOOM_record_destroy(ZOOM_record rec)
1703 {
1704     if (!rec)
1705         return;
1706     if (rec->wrbuf_marc)
1707         wrbuf_free(rec->wrbuf_marc, 1);
1708     if (rec->wrbuf_iconv)
1709         wrbuf_free(rec->wrbuf_iconv, 1);
1710     if (rec->wrbuf_opac)
1711         wrbuf_free(rec->wrbuf_opac, 1);
1712     odr_destroy(rec->odr);
1713     xfree(rec);
1714 }
1715
1716 static const char *marc_iconv_return(ZOOM_record rec, int marc_type,
1717                                      int *len,
1718                                      const char *buf, int sz,
1719                                      const char *record_charset)
1720 {
1721     char to[40];
1722     char from[40];
1723     yaz_iconv_t cd = 0;
1724     yaz_marc_t mt = yaz_marc_create();
1725
1726     *from = '\0';
1727     strcpy(to, "UTF-8");
1728     if (record_charset && *record_charset)
1729     {
1730         /* Use "from,to" or just "from" */
1731         const char *cp = strchr(record_charset, ',');
1732         int clen = strlen(record_charset);
1733         if (cp && cp[1])
1734         {
1735             strncpy( to, cp+1, sizeof(to)-1);
1736             to[sizeof(to)-1] = '\0';
1737             clen = cp - record_charset;
1738         }
1739         if (clen > sizeof(from)-1)
1740             clen = sizeof(from)-1;
1741         
1742         if (clen)
1743             strncpy(from, record_charset, clen);
1744         from[clen] = '\0';
1745     }
1746
1747     if (*from && *to)
1748     {
1749         cd = yaz_iconv_open(to, from);
1750         yaz_marc_iconv(mt, cd);
1751     }
1752
1753     yaz_marc_xml(mt, marc_type);
1754     if (!rec->wrbuf_marc)
1755         rec->wrbuf_marc = wrbuf_alloc();
1756     wrbuf_rewind(rec->wrbuf_marc);
1757     if (yaz_marc_decode_wrbuf(mt, buf, sz, rec->wrbuf_marc) > 0)
1758     {
1759         yaz_marc_destroy(mt);
1760         if (cd)
1761             yaz_iconv_close(cd);
1762         if (len)
1763             *len = wrbuf_len(rec->wrbuf_marc);
1764         return wrbuf_buf(rec->wrbuf_marc);
1765     }
1766     yaz_marc_destroy(mt);
1767     if (cd)
1768         yaz_iconv_close(cd);
1769     return 0;
1770 }
1771
1772 static const char *record_iconv_return(ZOOM_record rec, int *len,
1773                                        const char *buf, int sz,
1774                                        const char *record_charset)
1775 {
1776     char to[40];
1777     char from[40];
1778     yaz_iconv_t cd = 0;
1779
1780     *from = '\0';
1781     strcpy(to, "UTF-8");
1782     if (record_charset && *record_charset)
1783     {
1784         /* Use "from,to" or just "from" */
1785         const char *cp = strchr(record_charset, ',');
1786         int clen = strlen(record_charset);
1787         if (cp && cp[1])
1788         {
1789             strncpy( to, cp+1, sizeof(to)-1);
1790             to[sizeof(to)-1] = '\0';
1791             clen = cp - record_charset;
1792         }
1793         if (clen > sizeof(from)-1)
1794             clen = sizeof(from)-1;
1795         
1796         if (clen)
1797             strncpy(from, record_charset, clen);
1798         from[clen] = '\0';
1799     }
1800
1801     if (*from && *to && (cd = yaz_iconv_open(to, from)))
1802     {
1803         char outbuf[12];
1804         size_t inbytesleft = sz;
1805         const char *inp = buf;
1806         
1807         if (!rec->wrbuf_iconv)
1808             rec->wrbuf_iconv = wrbuf_alloc();
1809
1810         wrbuf_rewind(rec->wrbuf_iconv);
1811
1812         while (inbytesleft)
1813         {
1814             size_t outbytesleft = sizeof(outbuf);
1815             char *outp = outbuf;
1816             size_t r = yaz_iconv(cd, (char**) &inp,
1817                                  &inbytesleft, 
1818                                  &outp, &outbytesleft);
1819             if (r == (size_t) (-1))
1820             {
1821                 int e = yaz_iconv_error(cd);
1822                 if (e != YAZ_ICONV_E2BIG)
1823                     break;
1824             }
1825             wrbuf_write(rec->wrbuf_iconv, outbuf, outp - outbuf);
1826         }
1827         wrbuf_puts(rec->wrbuf_iconv, "");
1828         buf = wrbuf_buf(rec->wrbuf_iconv);
1829         sz = wrbuf_len(rec->wrbuf_iconv);
1830         yaz_iconv_close(cd);
1831     }
1832     if (len)
1833         *len = sz;
1834     return buf;
1835 }
1836
1837 ZOOM_API(int)
1838     ZOOM_record_error(ZOOM_record rec, const char **cp,
1839                       const char **addinfo, const char **diagset)
1840 {
1841     Z_NamePlusRecord *npr;
1842     
1843     if (!rec)
1844         return 0;
1845     npr = rec->npr;
1846     if (npr && npr->which == Z_NamePlusRecord_surrogateDiagnostic)
1847     {
1848         Z_DiagRec *diag_rec = npr->u.surrogateDiagnostic;
1849         int error = YAZ_BIB1_UNSPECIFIED_ERROR;
1850         const char *add = 0;
1851
1852         if (diag_rec->which == Z_DiagRec_defaultFormat)
1853         {
1854             Z_DefaultDiagFormat *ddf = diag_rec->u.defaultFormat;
1855             int oclass;
1856     
1857             error = *ddf->condition;
1858             switch (ddf->which)
1859             {
1860             case Z_DefaultDiagFormat_v2Addinfo:
1861                 add = ddf->u.v2Addinfo;
1862                 break;
1863             case Z_DefaultDiagFormat_v3Addinfo:
1864                 add = ddf->u.v3Addinfo;
1865                 break;
1866             }
1867             if (diagset)
1868                 *diagset = yaz_z3950oid_to_str(ddf->diagnosticSetId, &oclass);
1869         }
1870         else
1871         {
1872             if (diagset)
1873                 *diagset = "Bib-1";
1874         }
1875         if (addinfo)
1876             *addinfo = add ? add : "";
1877         if (cp)
1878             *cp = diagbib1_str(error);
1879         return error;
1880     }
1881     return 0;
1882 }
1883
1884 ZOOM_API(const char *)
1885     ZOOM_record_get(ZOOM_record rec, const char *type_spec, int *len)
1886 {
1887     char type[40];
1888     char charset[40];
1889     char xpath[512];
1890     const char *cp;
1891     int i;
1892     Z_NamePlusRecord *npr;
1893     
1894     if (len)
1895         *len = 0; /* default return */
1896         
1897     if (!rec)
1898         return 0;
1899     npr = rec->npr;
1900     if (!npr)
1901         return 0;
1902
1903     cp = type_spec;
1904     for (i = 0; cp[i] && i < sizeof(type)-1; i++)
1905     {
1906         if (cp[i] == ';' || cp[i] == ' ')
1907             break;
1908         type[i] = cp[i];
1909     }
1910     type[i] = '\0';
1911     charset[0] = '\0';
1912     while (type_spec[i] == ';')
1913     {
1914         i++;
1915         while (type_spec[i] == ' ')
1916             i++;
1917         if (!strncmp(type_spec+i, "charset=", 8))
1918         {
1919             int j = 0;
1920             i = i + 8; /* skip charset= */
1921             for (j = 0; type_spec[i]  && j < sizeof(charset)-1; i++, j++)
1922             {
1923                 if (type_spec[i] == ';' || type_spec[i] == ' ')
1924                     break;
1925                 charset[j] = cp[i];
1926             }
1927             charset[j] = '\0';
1928         }
1929         else if (!strncmp(type_spec+i, "xpath=", 6))
1930         {
1931             int j = 0; 
1932             i = i + 6;
1933             for (j = 0; type_spec[i] && j < sizeof(xpath)-1; i++, j++)
1934                 xpath[j] = cp[i];
1935             xpath[j] = '\0';
1936         } 
1937         while (type_spec[i] == ' ')
1938             i++;
1939     }
1940     if (!strcmp(type, "database"))
1941     {
1942         if (len)
1943             *len = (npr->databaseName ? strlen(npr->databaseName) : 0);
1944         return npr->databaseName;
1945     }
1946     else if (!strcmp(type, "syntax"))
1947     {
1948         const char *desc = 0;   
1949         if (npr->which == Z_NamePlusRecord_databaseRecord)
1950         {
1951             Z_External *r = (Z_External *) npr->u.databaseRecord;
1952             oident *ent = oid_getentbyoid(r->direct_reference);
1953             if (ent)
1954                 desc = ent->desc;
1955         }
1956         if (!desc)
1957             desc = "none";
1958         if (len)
1959             *len = strlen(desc);
1960         return desc;
1961     }
1962     if (npr->which != Z_NamePlusRecord_databaseRecord)
1963         return 0;
1964
1965     /* from now on - we have a database record .. */
1966     if (!strcmp(type, "render"))
1967     {
1968         Z_External *r = (Z_External *) npr->u.databaseRecord;
1969         oident *ent = oid_getentbyoid(r->direct_reference);
1970
1971         /* render bibliographic record .. */
1972         if (r->which == Z_External_OPAC)
1973         {
1974             r = r->u.opac->bibliographicRecord;
1975             if (!r)
1976                 return 0;
1977             ent = oid_getentbyoid(r->direct_reference);
1978         }
1979         if (r->which == Z_External_sutrs)
1980             return record_iconv_return(rec, len,
1981                                        (char*) r->u.sutrs->buf,
1982                                        r->u.sutrs->len,
1983                                        charset);
1984         else if (r->which == Z_External_octet)
1985         {
1986             const char *ret_buf;
1987             switch (ent->value)
1988             {
1989             case VAL_SOIF:
1990             case VAL_HTML:
1991             case VAL_SUTRS:
1992                 break;
1993             case VAL_TEXT_XML:
1994             case VAL_APPLICATION_XML:
1995                 break;
1996             default:
1997                 ret_buf = marc_iconv_return(
1998                     rec, YAZ_MARC_LINE, len,
1999                     (const char *) r->u.octet_aligned->buf,
2000                     r->u.octet_aligned->len,
2001                     charset);
2002                 if (ret_buf)
2003                     return ret_buf;
2004             }
2005             return record_iconv_return(rec, len,
2006                                        (const char *) r->u.octet_aligned->buf,
2007                                        r->u.octet_aligned->len,
2008                                        charset);
2009         }
2010         else if (r->which == Z_External_grs1)
2011         {
2012             if (!rec->wrbuf_marc)
2013                 rec->wrbuf_marc = wrbuf_alloc();
2014             wrbuf_rewind(rec->wrbuf_marc);
2015             yaz_display_grs1(rec->wrbuf_marc, r->u.grs1, 0);
2016             return record_iconv_return(rec, len,
2017                                        wrbuf_buf(rec->wrbuf_marc),
2018                                        wrbuf_len(rec->wrbuf_marc),
2019                                        charset);
2020         }
2021         return 0;
2022     }
2023     else if (!strcmp(type, "xml"))
2024     {
2025         Z_External *r = (Z_External *) npr->u.databaseRecord;
2026         oident *ent = oid_getentbyoid(r->direct_reference);
2027
2028         /* render bibliographic record .. */
2029         if (r->which == Z_External_OPAC)
2030         {
2031             r = r->u.opac->bibliographicRecord;
2032             if (!r)
2033                 return 0;
2034             ent = oid_getentbyoid(r->direct_reference);
2035         }
2036         
2037         if (r->which == Z_External_sutrs)
2038             return record_iconv_return(rec, len,
2039                                        (const char *) r->u.sutrs->buf,
2040                                        r->u.sutrs->len,
2041                                        charset);
2042         else if (r->which == Z_External_octet)
2043         {
2044             const char *ret_buf;
2045             int marc_decode_type = YAZ_MARC_MARCXML;
2046
2047             switch (ent->value)
2048             {
2049             case VAL_SOIF:
2050             case VAL_HTML:
2051             case VAL_SUTRS:
2052                 break;
2053             case VAL_TEXT_XML:
2054             case VAL_APPLICATION_XML:
2055                 break;
2056             default:
2057                 ret_buf = marc_iconv_return(
2058                     rec, marc_decode_type, len,
2059                     (const char *) r->u.octet_aligned->buf,
2060                     r->u.octet_aligned->len,
2061                     charset);
2062                 if (ret_buf)
2063                     return ret_buf;
2064             }
2065             return record_iconv_return(rec, len,
2066                                        (const char *) r->u.octet_aligned->buf,
2067                                        r->u.octet_aligned->len,
2068                                        charset);
2069         }
2070         else if (r->which == Z_External_grs1)
2071         {
2072             if (len) *len = 5;
2073             return "GRS-1";
2074         }
2075         return 0;
2076     }
2077     else if (!strcmp(type, "raw"))
2078     {
2079         Z_External *r = (Z_External *) npr->u.databaseRecord;
2080         
2081         if (r->which == Z_External_sutrs)
2082         {
2083             if (len) *len = r->u.sutrs->len;
2084             return (const char *) r->u.sutrs->buf;
2085         }
2086         else if (r->which == Z_External_octet)
2087         {
2088             if (len) *len = r->u.octet_aligned->len;
2089             return (const char *) r->u.octet_aligned->buf;
2090         }
2091         else /* grs-1, explain, OPAC, ... */
2092         {
2093             if (len) *len = -1;
2094             return (const char *) npr->u.databaseRecord;
2095         }
2096         return 0;
2097     }
2098     else if (!strcmp (type, "ext"))
2099     {
2100         if (len) *len = -1;
2101         return (const char *) npr->u.databaseRecord;
2102     }
2103     else if (!strcmp (type, "opac"))
2104              
2105     {
2106         Z_External *r = (Z_External *) npr->u.databaseRecord;
2107         if (r->which == Z_External_OPAC)
2108         {
2109             if (!rec->wrbuf_opac)
2110                 rec->wrbuf_opac = wrbuf_alloc();
2111             wrbuf_rewind(rec->wrbuf_opac);
2112             yaz_display_OPAC(rec->wrbuf_opac, r->u.opac, 0);
2113             return record_iconv_return(rec, len,
2114                                        wrbuf_buf(rec->wrbuf_opac),
2115                                        wrbuf_len(rec->wrbuf_opac),
2116                                        charset);
2117         }
2118     }
2119     return 0;
2120 }
2121
2122 static int strcmp_null(const char *v1, const char *v2)
2123 {
2124     if (!v1 && !v2)
2125         return 0;
2126     if (!v1 || !v2)
2127         return -1;
2128     return strcmp(v1, v2);
2129 }
2130
2131 static size_t record_hash(int pos)
2132 {
2133     if (pos < 0)
2134         pos = 0;
2135     return pos % RECORD_HASH_SIZE;
2136 }
2137
2138 static void record_cache_add(ZOOM_resultset r, Z_NamePlusRecord *npr, 
2139                              int pos,
2140                              const char *syntax, const char *elementSetName)
2141 {
2142     ZOOM_record_cache rc;
2143     
2144     ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_RECV_RECORD);
2145     ZOOM_connection_put_event(r->connection, event);
2146
2147     for (rc = r->record_hash[record_hash(pos)]; rc; rc = rc->next)
2148     {
2149         if (pos == rc->pos)
2150         {
2151             if (strcmp_null(r->schema, rc->schema))
2152                 continue;
2153             if (strcmp_null(elementSetName,rc->elementSetName))
2154                 continue;
2155             if (strcmp_null(syntax, rc->syntax))
2156                 continue;
2157             /* not destroying rc->npr (it's handled by nmem )*/
2158             rc->rec.npr = npr;
2159             /* keeping wrbuf_marc too */
2160             return;
2161         }
2162     }
2163     rc = (ZOOM_record_cache) odr_malloc(r->odr, sizeof(*rc));
2164     rc->rec.npr = npr; 
2165     rc->rec.odr = 0;
2166     rc->rec.wrbuf_marc = 0;
2167     rc->rec.wrbuf_iconv = 0;
2168     rc->rec.wrbuf_opac = 0;
2169     if (elementSetName)
2170         rc->elementSetName = odr_strdup(r->odr, elementSetName);
2171     else
2172         rc->elementSetName = 0;
2173
2174     if (syntax)
2175         rc->syntax = odr_strdup(r->odr, syntax);
2176     else
2177         rc->syntax = 0;
2178
2179     if (r->schema)
2180         rc->schema = odr_strdup(r->odr, r->schema);
2181     else
2182         rc->schema = 0;
2183
2184     rc->pos = pos;
2185     rc->next = r->record_hash[record_hash(pos)];
2186     r->record_hash[record_hash(pos)] = rc;
2187 }
2188
2189 static ZOOM_record record_cache_lookup(ZOOM_resultset r, int pos,
2190                                        const char *syntax,
2191                                        const char *elementSetName)
2192 {
2193     ZOOM_record_cache rc;
2194     
2195     for (rc = r->record_hash[record_hash(pos)]; rc; rc = rc->next)
2196     {
2197         if (pos == rc->pos)
2198         {
2199             if (strcmp_null(r->schema, rc->schema))
2200                 continue;
2201             if (strcmp_null(elementSetName,rc->elementSetName))
2202                 continue;
2203             if (strcmp_null(syntax, rc->syntax))
2204                 continue;
2205             return &rc->rec;
2206         }
2207     }
2208     return 0;
2209 }
2210                                              
2211 static void handle_records(ZOOM_connection c, Z_Records *sr,
2212                            int present_phase)
2213 {
2214     ZOOM_resultset resultset;
2215     int *start, *count;
2216     const char *syntax = 0, *elementSetName = 0;
2217
2218     if (!c->tasks)
2219         return ;
2220     switch (c->tasks->which)
2221     {
2222     case ZOOM_TASK_SEARCH:
2223         resultset = c->tasks->u.search.resultset;
2224         start = &c->tasks->u.search.start;
2225         count = &c->tasks->u.search.count;
2226         syntax = c->tasks->u.search.syntax;
2227         elementSetName = c->tasks->u.search.elementSetName;
2228         break;
2229     case ZOOM_TASK_RETRIEVE:
2230         resultset = c->tasks->u.retrieve.resultset;        
2231         start = &c->tasks->u.retrieve.start;
2232         count = &c->tasks->u.retrieve.count;
2233         syntax = c->tasks->u.retrieve.syntax;
2234         elementSetName = c->tasks->u.retrieve.elementSetName;
2235         break;
2236     default:
2237         return;
2238     }
2239     if (sr && sr->which == Z_Records_NSD)
2240         response_default_diag(c, sr->u.nonSurrogateDiagnostic);
2241     else if (sr && sr->which == Z_Records_multipleNSD)
2242     {
2243         if (sr->u.multipleNonSurDiagnostics->num_diagRecs >= 1)
2244             response_diag(c, sr->u.multipleNonSurDiagnostics->diagRecs[0]);
2245         else
2246             set_ZOOM_error(c, ZOOM_ERROR_DECODE, 0);
2247     }
2248     else 
2249     {
2250         if (*count + *start > resultset->size)
2251             *count = resultset->size - *start;
2252         if (*count < 0)
2253             *count = 0;
2254         if (sr && sr->which == Z_Records_DBOSD)
2255         {
2256             int i;
2257             NMEM nmem = odr_extract_mem(c->odr_in);
2258             Z_NamePlusRecordList *p =
2259                 sr->u.databaseOrSurDiagnostics;
2260             for (i = 0; i<p->num_records; i++)
2261             {
2262                 record_cache_add(resultset, p->records[i], i + *start,
2263                                  syntax, elementSetName);
2264             }
2265             *count -= i;
2266             if (*count < 0)
2267                 *count = 0;
2268             *start += i;
2269             yaz_log(log_details, 
2270                     "handle_records resultset=%p start=%d count=%d",
2271                     resultset, *start, *count);
2272
2273             /* transfer our response to search_nmem .. we need it later */
2274             nmem_transfer(resultset->odr->mem, nmem);
2275             nmem_destroy(nmem);
2276             if (present_phase && p->num_records == 0)
2277             {
2278                 /* present response and we didn't get any records! */
2279                 Z_NamePlusRecord *myrec = 
2280                     zget_surrogateDiagRec(resultset->odr, 0, 14, 0);
2281                 record_cache_add(resultset, myrec, *start,
2282                                  syntax, elementSetName);
2283             }
2284         }
2285         else if (present_phase)
2286         {
2287             /* present response and we didn't get any records! */
2288             Z_NamePlusRecord *myrec = 
2289                 zget_surrogateDiagRec(resultset->odr, 0, 14, 0);
2290             record_cache_add(resultset, myrec, *start, syntax, elementSetName);
2291         }
2292     }
2293 }
2294
2295 static void handle_present_response(ZOOM_connection c, Z_PresentResponse *pr)
2296 {
2297     handle_records(c, pr->records, 1);
2298 }
2299
2300 static void handle_queryExpressionTerm(ZOOM_options opt, const char *name,
2301                                        Z_Term *term)
2302 {
2303     switch (term->which)
2304     {
2305     case Z_Term_general:
2306         ZOOM_options_setl(opt, name,
2307                           (const char *)(term->u.general->buf), 
2308                           term->u.general->len);
2309         break;
2310     case Z_Term_characterString:
2311         ZOOM_options_set(opt, name, term->u.characterString);
2312         break;
2313     case Z_Term_numeric:
2314         ZOOM_options_set_int(opt, name, *term->u.numeric);
2315         break;
2316     }
2317 }
2318
2319 static void handle_queryExpression(ZOOM_options opt, const char *name,
2320                                    Z_QueryExpression *exp)
2321 {
2322     char opt_name[80];
2323     
2324     switch (exp->which)
2325     {
2326     case Z_QueryExpression_term:
2327         if (exp->u.term && exp->u.term->queryTerm)
2328         {
2329             sprintf(opt_name, "%s.term", name);
2330             handle_queryExpressionTerm(opt, opt_name, exp->u.term->queryTerm);
2331         }
2332         break;
2333     case Z_QueryExpression_query:
2334         break;
2335     }
2336 }
2337
2338 static void handle_searchResult(ZOOM_connection c, ZOOM_resultset resultset,
2339                                 Z_OtherInformation *o)
2340 {
2341     int i;
2342     for (i = 0; o && i < o->num_elements; i++)
2343     {
2344         if (o->list[i]->which == Z_OtherInfo_externallyDefinedInfo)
2345         {
2346             Z_External *ext = o->list[i]->information.externallyDefinedInfo;
2347             
2348             if (ext->which == Z_External_searchResult1)
2349             {
2350                 int j;
2351                 Z_SearchInfoReport *sr = ext->u.searchResult1;
2352                 
2353                 if (sr->num)
2354                     ZOOM_options_set_int(
2355                         resultset->options, "searchresult.size", sr->num);
2356
2357                 for (j = 0; j < sr->num; j++)
2358                 {
2359                     Z_SearchInfoReport_s *ent =
2360                         ext->u.searchResult1->elements[j];
2361                     char pref[80];
2362                     
2363                     sprintf(pref, "searchresult.%d", j);
2364
2365                     if (ent->subqueryId)
2366                     {
2367                         char opt_name[80];
2368                         sprintf(opt_name, "%s.id", pref);
2369                         ZOOM_options_set(resultset->options, opt_name,
2370                                          ent->subqueryId);
2371                     }
2372                     if (ent->subqueryExpression)
2373                     {
2374                         char opt_name[80];
2375                         sprintf(opt_name, "%s.subquery", pref);
2376                         handle_queryExpression(resultset->options, opt_name,
2377                                                ent->subqueryExpression);
2378                     }
2379                     if (ent->subqueryInterpretation)
2380                     {
2381                         char opt_name[80];
2382                         sprintf(opt_name, "%s.interpretation", pref);
2383                         handle_queryExpression(resultset->options, opt_name,
2384                                                ent->subqueryInterpretation);
2385                     }
2386                     if (ent->subqueryRecommendation)
2387                     {
2388                         char opt_name[80];
2389                         sprintf(opt_name, "%s.recommendation", pref);
2390                         handle_queryExpression(resultset->options, opt_name,
2391                                                ent->subqueryRecommendation);
2392                     }
2393                     if (ent->subqueryCount)
2394                     {
2395                         char opt_name[80];
2396                         sprintf(opt_name, "%s.count", pref);
2397                         ZOOM_options_set_int(resultset->options, opt_name,
2398                                              *ent->subqueryCount);
2399                     }                                             
2400                 }
2401             }
2402         }
2403     }
2404 }
2405
2406 static void handle_search_response(ZOOM_connection c, Z_SearchResponse *sr)
2407 {
2408     ZOOM_resultset resultset;
2409     ZOOM_Event event;
2410
2411     if (!c->tasks || c->tasks->which != ZOOM_TASK_SEARCH)
2412         return ;
2413
2414     event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH);
2415     ZOOM_connection_put_event(c, event);
2416
2417     resultset = c->tasks->u.search.resultset;
2418
2419     if (sr->resultSetStatus)
2420     {
2421         ZOOM_options_set_int(resultset->options, "resultSetStatus",
2422                              *sr->resultSetStatus);
2423     }
2424     if (sr->presentStatus)
2425     {
2426         ZOOM_options_set_int(resultset->options, "presentStatus",
2427                              *sr->presentStatus);
2428     }
2429     handle_searchResult(c, resultset, sr->additionalSearchInfo);
2430
2431     resultset->size = *sr->resultCount;
2432     handle_records(c, sr->records, 0);
2433 }
2434
2435 static void sort_response(ZOOM_connection c, Z_SortResponse *res)
2436 {
2437     if (res->diagnostics && res->num_diagnostics > 0)
2438         response_diag(c, res->diagnostics[0]);
2439 }
2440
2441 static int scan_response(ZOOM_connection c, Z_ScanResponse *res)
2442 {
2443     NMEM nmem = odr_extract_mem(c->odr_in);
2444     ZOOM_scanset scan;
2445
2446     if (!c->tasks || c->tasks->which != ZOOM_TASK_SCAN)
2447         return 0;
2448     scan = c->tasks->u.scan.scan;
2449
2450     if (res->entries && res->entries->nonsurrogateDiagnostics)
2451         response_diag(c, res->entries->nonsurrogateDiagnostics[0]);
2452     scan->scan_response = res;
2453     nmem_transfer(scan->odr->mem, nmem);
2454     if (res->stepSize)
2455         ZOOM_options_set_int(scan->options, "stepSize", *res->stepSize);
2456     if (res->positionOfTerm)
2457         ZOOM_options_set_int(scan->options, "position", *res->positionOfTerm);
2458     if (res->scanStatus)
2459         ZOOM_options_set_int(scan->options, "scanStatus", *res->scanStatus);
2460     if (res->numberOfEntriesReturned)
2461         ZOOM_options_set_int(scan->options, "number",
2462                              *res->numberOfEntriesReturned);
2463     nmem_destroy(nmem);
2464     return 1;
2465 }
2466
2467 static zoom_ret send_sort(ZOOM_connection c,
2468                           ZOOM_resultset resultset)
2469 {
2470     if (c->error)
2471         resultset->r_sort_spec = 0;
2472     if (resultset->r_sort_spec)
2473     {
2474         Z_APDU *apdu = zget_APDU(c->odr_out, Z_APDU_sortRequest);
2475         Z_SortRequest *req = apdu->u.sortRequest;
2476         
2477         req->num_inputResultSetNames = 1;
2478         req->inputResultSetNames = (Z_InternationalString **)
2479             odr_malloc(c->odr_out, sizeof(*req->inputResultSetNames));
2480         req->inputResultSetNames[0] =
2481             odr_strdup(c->odr_out, resultset->setname);
2482         req->sortedResultSetName = odr_strdup(c->odr_out, resultset->setname);
2483         req->sortSequence = resultset->r_sort_spec;
2484         resultset->r_sort_spec = 0;
2485         return send_APDU(c, apdu);
2486     }
2487     return zoom_complete;
2488 }
2489
2490 static zoom_ret send_present(ZOOM_connection c)
2491 {
2492     Z_APDU *apdu = 0;
2493     Z_PresentRequest *req = 0;
2494     int i = 0;
2495     const char *syntax = 0;
2496     const char *elementSetName = 0;
2497     ZOOM_resultset  resultset;
2498     int *start, *count;
2499
2500     if (!c->tasks)
2501     {
2502         yaz_log(log_details, "%p send_present no tasks", c);
2503         return zoom_complete;
2504     }
2505     
2506     switch (c->tasks->which)
2507     {
2508     case ZOOM_TASK_SEARCH:
2509         resultset = c->tasks->u.search.resultset;
2510         start = &c->tasks->u.search.start;
2511         count = &c->tasks->u.search.count;
2512         syntax = c->tasks->u.search.syntax;
2513         elementSetName = c->tasks->u.search.elementSetName;
2514         break;
2515     case ZOOM_TASK_RETRIEVE:
2516         resultset = c->tasks->u.retrieve.resultset;
2517         start = &c->tasks->u.retrieve.start;
2518         count = &c->tasks->u.retrieve.count;
2519         syntax = c->tasks->u.retrieve.syntax;
2520         elementSetName = c->tasks->u.retrieve.elementSetName;
2521         break;
2522     default:
2523         return zoom_complete;
2524     }
2525     yaz_log(log_details, "%p send_present start=%d count=%d",
2526             c, *start, *count);
2527
2528     if (*start < 0 || *count < 0 || *start + *count > resultset->size)
2529     {
2530         set_dset_error(c, YAZ_BIB1_PRESENT_REQUEST_OUT_OF_RANGE, "Bib-1",
2531                        "", 0);
2532     }
2533     if (c->error)                  /* don't continue on error */
2534         return zoom_complete;
2535     yaz_log(log_details, "send_present resultset=%p start=%d count=%d",
2536             resultset, *start, *count);
2537
2538     for (i = 0; i < *count; i++)
2539     {
2540         ZOOM_record rec =
2541             record_cache_lookup(resultset, i + *start, syntax, elementSetName);
2542         if (!rec)
2543             break;
2544         else
2545         {
2546             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_RECV_RECORD);
2547             ZOOM_connection_put_event(c, event);
2548         }
2549     }
2550     *start += i;
2551     *count -= i;
2552
2553     if (*count == 0)
2554     {
2555         yaz_log(log_details, "%p send_present skip=%d no more to fetch", c, i);
2556         return zoom_complete;
2557     }
2558
2559     apdu = zget_APDU(c->odr_out, Z_APDU_presentRequest);
2560     req = apdu->u.presentRequest;
2561
2562     if (i)
2563         yaz_log(log_details, "%p send_present skip=%d", c, i);
2564
2565     *req->resultSetStartPoint = *start + 1;
2566     *req->numberOfRecordsRequested = resultset->step>0 ?
2567         resultset->step : *count;
2568     if (*req->numberOfRecordsRequested + *start > resultset->size)
2569         *req->numberOfRecordsRequested = resultset->size - *start;
2570     assert(*req->numberOfRecordsRequested > 0);
2571
2572     if (syntax && *syntax)
2573         req->preferredRecordSyntax =
2574             zoom_yaz_str_to_z3950oid(c, CLASS_RECSYN, syntax);
2575
2576     if (resultset->schema && *resultset->schema)
2577     {
2578         Z_RecordComposition *compo = (Z_RecordComposition *)
2579             odr_malloc(c->odr_out, sizeof(*compo));
2580
2581         req->recordComposition = compo;
2582         compo->which = Z_RecordComp_complex;
2583         compo->u.complex = (Z_CompSpec *)
2584             odr_malloc(c->odr_out, sizeof(*compo->u.complex));
2585         compo->u.complex->selectAlternativeSyntax = (bool_t *) 
2586             odr_malloc(c->odr_out, sizeof(bool_t));
2587         *compo->u.complex->selectAlternativeSyntax = 0;
2588
2589         compo->u.complex->generic = (Z_Specification *)
2590             odr_malloc(c->odr_out, sizeof(*compo->u.complex->generic));
2591
2592         compo->u.complex->generic->which = Z_Schema_oid;
2593         compo->u.complex->generic->schema.oid = (Odr_oid *)
2594             zoom_yaz_str_to_z3950oid (c, CLASS_SCHEMA, resultset->schema);
2595
2596         if (!compo->u.complex->generic->schema.oid)
2597         {
2598             /* OID wasn't a schema! Try record syntax instead. */
2599
2600             compo->u.complex->generic->schema.oid = (Odr_oid *)
2601                 zoom_yaz_str_to_z3950oid (c, CLASS_RECSYN, resultset->schema);
2602         }
2603         if (elementSetName && *elementSetName)
2604         {
2605             compo->u.complex->generic->elementSpec = (Z_ElementSpec *)
2606                 odr_malloc(c->odr_out, sizeof(Z_ElementSpec));
2607             compo->u.complex->generic->elementSpec->which =
2608                 Z_ElementSpec_elementSetName;
2609             compo->u.complex->generic->elementSpec->u.elementSetName =
2610                 odr_strdup(c->odr_out, elementSetName);
2611         }
2612         else
2613             compo->u.complex->generic->elementSpec = 0;
2614         compo->u.complex->num_dbSpecific = 0;
2615         compo->u.complex->dbSpecific = 0;
2616         compo->u.complex->num_recordSyntax = 0;
2617         compo->u.complex->recordSyntax = 0;
2618     }
2619     else if (elementSetName && *elementSetName)
2620     {
2621         Z_ElementSetNames *esn = (Z_ElementSetNames *)
2622             odr_malloc(c->odr_out, sizeof(*esn));
2623         Z_RecordComposition *compo = (Z_RecordComposition *)
2624             odr_malloc(c->odr_out, sizeof(*compo));
2625         
2626         esn->which = Z_ElementSetNames_generic;
2627         esn->u.generic = odr_strdup(c->odr_out, elementSetName);
2628         compo->which = Z_RecordComp_simple;
2629         compo->u.simple = esn;
2630         req->recordComposition = compo;
2631     }
2632     req->resultSetId = odr_strdup(c->odr_out, resultset->setname);
2633     return send_APDU(c, apdu);
2634 }
2635
2636 ZOOM_API(ZOOM_scanset)
2637     ZOOM_connection_scan(ZOOM_connection c, const char *start)
2638 {
2639     ZOOM_scanset s;
2640     ZOOM_query q = ZOOM_query_create();
2641
2642     ZOOM_query_prefix(q, start);
2643
2644     s = ZOOM_connection_scan1(c, q);
2645     ZOOM_query_destroy(q);
2646     return s;
2647
2648 }
2649
2650 ZOOM_API(ZOOM_scanset)
2651     ZOOM_connection_scan1(ZOOM_connection c, ZOOM_query q)
2652 {
2653     char *start;
2654     char *freeme = 0;
2655     ZOOM_scanset scan = 0;
2656
2657     /*
2658      * We need to check the query-type, so we can recognise CQL and
2659      * CCL and compile them into a form that we can use here.  The
2660      * ZOOM_query structure has no explicit `type' member, but
2661      * inspection of the ZOOM_query_prefix() and ZOOM_query_cql()
2662      * functions shows how the structure is set up in each case.
2663      */
2664     if (!q->z_query)
2665         return 0;
2666     else if (q->z_query->which == Z_Query_type_1) 
2667     {
2668         yaz_log(log_api, "%p ZOOM_connection_scan1 q=%p PQF '%s'",
2669                 c, q, q->query_string);
2670         start = q->query_string;
2671     } 
2672     else if (q->z_query->which == Z_Query_type_104)
2673     {
2674         yaz_log(log_api, "%p ZOOM_connection_scan1 q=%p CQL '%s'",
2675                 c, q, q->query_string);
2676         start = freeme = cql2pqf(c, q->query_string);
2677         if (start == 0)
2678             return 0;
2679     } 
2680     else
2681     {
2682         yaz_log(YLOG_FATAL, "%p ZOOM_connection_scan1 q=%p unknown type '%s'",
2683                 c, q, q->query_string);
2684         abort();
2685     }
2686     
2687     scan = (ZOOM_scanset) xmalloc(sizeof(*scan));
2688     scan->connection = c;
2689     scan->odr = odr_createmem(ODR_DECODE);
2690     scan->options = ZOOM_options_create_with_parent(c->options);
2691     scan->refcount = 1;
2692     scan->scan_response = 0;
2693     scan->termListAndStartPoint =
2694         p_query_scan(scan->odr, PROTO_Z3950, &scan->attributeSet, start);
2695     xfree(freeme);
2696
2697     scan->databaseNames = set_DatabaseNames(c, c->options,
2698                                             &scan->num_databaseNames,
2699                                             scan->odr);
2700     if (scan->termListAndStartPoint != 0)
2701     {
2702         ZOOM_task task = ZOOM_connection_add_task(c, ZOOM_TASK_SCAN);
2703         task->u.scan.scan = scan;
2704         
2705         (scan->refcount)++;
2706         if (!c->async)
2707         {
2708             while (ZOOM_event(1, &c))
2709                 ;
2710         }
2711     }
2712     return scan;
2713 }
2714
2715 ZOOM_API(void)
2716     ZOOM_scanset_destroy(ZOOM_scanset scan)
2717 {
2718     if (!scan)
2719         return;
2720     (scan->refcount)--;
2721     if (scan->refcount == 0)
2722     {
2723         odr_destroy(scan->odr);
2724         
2725         ZOOM_options_destroy(scan->options);
2726         xfree(scan);
2727     }
2728 }
2729
2730 static zoom_ret send_package(ZOOM_connection c)
2731 {
2732     ZOOM_Event event;
2733
2734     yaz_log(log_details, "%p send_package", c);
2735     if (!c->tasks)
2736         return zoom_complete;
2737     assert (c->tasks->which == ZOOM_TASK_PACKAGE);
2738     
2739     event = ZOOM_Event_create(ZOOM_EVENT_SEND_APDU);
2740     ZOOM_connection_put_event(c, event);
2741     
2742     c->buf_out = c->tasks->u.package->buf_out;
2743     c->len_out = c->tasks->u.package->len_out;
2744
2745     return do_write(c);
2746 }
2747
2748 static zoom_ret send_scan(ZOOM_connection c)
2749 {
2750     ZOOM_scanset scan;
2751     Z_APDU *apdu = zget_APDU(c->odr_out, Z_APDU_scanRequest);
2752     Z_ScanRequest *req = apdu->u.scanRequest;
2753
2754     yaz_log(log_details, "%p send_scan", c);
2755     if (!c->tasks)
2756         return zoom_complete;
2757     assert (c->tasks->which == ZOOM_TASK_SCAN);
2758     scan = c->tasks->u.scan.scan;
2759
2760     req->termListAndStartPoint = scan->termListAndStartPoint;
2761     req->attributeSet = scan->attributeSet;
2762
2763     *req->numberOfTermsRequested =
2764         ZOOM_options_get_int(scan->options, "number", 10);
2765
2766     req->preferredPositionInResponse =
2767         odr_intdup(c->odr_out,
2768                    ZOOM_options_get_int(scan->options, "position", 1));
2769
2770     req->stepSize =
2771         odr_intdup(c->odr_out,
2772                    ZOOM_options_get_int(scan->options, "stepSize", 0));
2773     
2774     req->databaseNames = scan->databaseNames;
2775     req->num_databaseNames = scan->num_databaseNames;
2776
2777     return send_APDU(c, apdu);
2778 }
2779
2780 ZOOM_API(size_t)
2781     ZOOM_scanset_size(ZOOM_scanset scan)
2782 {
2783     if (!scan || !scan->scan_response || !scan->scan_response->entries)
2784         return 0;
2785     return scan->scan_response->entries->num_entries;
2786 }
2787
2788 ZOOM_API(const char *)
2789     ZOOM_scanset_term(ZOOM_scanset scan, size_t pos,
2790                       int *occ, int *len)
2791 {
2792     const char *term = 0;
2793     size_t noent = ZOOM_scanset_size(scan);
2794     Z_ScanResponse *res = scan->scan_response;
2795     
2796     *len = 0;
2797     *occ = 0;
2798     if (pos >= noent)
2799         return 0;
2800     if (res->entries->entries[pos]->which == Z_Entry_termInfo)
2801     {
2802         Z_TermInfo *t = res->entries->entries[pos]->u.termInfo;
2803         
2804         if (t->term->which == Z_Term_general)
2805         {
2806             term = (const char *) t->term->u.general->buf;
2807             *len = t->term->u.general->len;
2808         }
2809         *occ = t->globalOccurrences ? *t->globalOccurrences : 0;
2810     }
2811     return term;
2812 }
2813
2814 ZOOM_API(const char *)
2815     ZOOM_scanset_display_term(ZOOM_scanset scan, size_t pos,
2816                               int *occ, int *len)
2817 {
2818     const char *term = 0;
2819     size_t noent = ZOOM_scanset_size(scan);
2820     Z_ScanResponse *res = scan->scan_response;
2821     
2822     *len = 0;
2823     *occ = 0;
2824     if (pos >= noent)
2825         return 0;
2826     if (res->entries->entries[pos]->which == Z_Entry_termInfo)
2827     {
2828         Z_TermInfo *t = res->entries->entries[pos]->u.termInfo;
2829
2830         if (t->displayTerm)
2831         {
2832             term = t->displayTerm;
2833             *len = strlen(term);
2834         }
2835         else if (t->term->which == Z_Term_general)
2836         {
2837             term = (const char *) t->term->u.general->buf;
2838             *len = t->term->u.general->len;
2839         }
2840         *occ = t->globalOccurrences ? *t->globalOccurrences : 0;
2841     }
2842     return term;
2843 }
2844
2845 ZOOM_API(const char *)
2846     ZOOM_scanset_option_get(ZOOM_scanset scan, const char *key)
2847 {
2848     return ZOOM_options_get(scan->options, key);
2849 }
2850
2851 ZOOM_API(void)
2852     ZOOM_scanset_option_set(ZOOM_scanset scan, const char *key,
2853                             const char *val)
2854 {
2855     ZOOM_options_set(scan->options, key, val);
2856 }
2857
2858 static Z_APDU *create_es_package(ZOOM_package p, int type)
2859 {
2860     const char *str;
2861     Z_APDU *apdu = zget_APDU(p->odr_out, Z_APDU_extendedServicesRequest);
2862     Z_ExtendedServicesRequest *req = apdu->u.extendedServicesRequest;
2863     
2864     *req->function = Z_ExtendedServicesRequest_create;
2865     
2866     str = ZOOM_options_get(p->options, "package-name");
2867     if (str && *str)
2868         req->packageName = nmem_strdup(p->odr_out->mem, str);
2869     
2870     str = ZOOM_options_get(p->options, "user-id");
2871     if (str)
2872         req->userId = nmem_strdup(p->odr_out->mem, str);
2873     
2874     req->packageType = yaz_oidval_to_z3950oid(p->odr_out, CLASS_EXTSERV,
2875                                               type);
2876
2877     str = ZOOM_options_get(p->options, "function");
2878     if (str)
2879     {
2880         if (!strcmp (str, "create"))
2881             *req->function = 1;
2882         if (!strcmp (str, "delete"))
2883             *req->function = 2;
2884         if (!strcmp (str, "modify"))
2885             *req->function = 3;
2886     }
2887     return apdu;
2888 }
2889
2890 static const char *ill_array_lookup(void *clientData, const char *idx)
2891 {
2892     ZOOM_package p = (ZOOM_package) clientData;
2893     return ZOOM_options_get(p->options, idx+4);
2894 }
2895
2896 static Z_External *encode_ill_request(ZOOM_package p)
2897 {
2898     ODR out = p->odr_out;
2899     ILL_Request *req;
2900     Z_External *r = 0;
2901     struct ill_get_ctl ctl;
2902         
2903     ctl.odr = p->odr_out;
2904     ctl.clientData = p;
2905     ctl.f = ill_array_lookup;
2906         
2907     req = ill_get_ILLRequest(&ctl, "ill", 0);
2908         
2909     if (!ill_Request(out, &req, 0, 0))
2910     {
2911         int ill_request_size;
2912         char *ill_request_buf = odr_getbuf(out, &ill_request_size, 0);
2913         if (ill_request_buf)
2914             odr_setbuf(out, ill_request_buf, ill_request_size, 1);
2915         return 0;
2916     }
2917     else
2918     {
2919         oident oid;
2920         int illRequest_size = 0;
2921         char *illRequest_buf = odr_getbuf(out, &illRequest_size, 0);
2922                 
2923         oid.proto = PROTO_GENERAL;
2924         oid.oclass = CLASS_GENERAL;
2925         oid.value = VAL_ISO_ILL_1;
2926                 
2927         r = (Z_External *) odr_malloc(out, sizeof(*r));
2928         r->direct_reference = odr_oiddup(out,oid_getoidbyent(&oid)); 
2929         r->indirect_reference = 0;
2930         r->descriptor = 0;
2931         r->which = Z_External_single;
2932                 
2933         r->u.single_ASN1_type =
2934             odr_create_Odr_oct(out,
2935                                (unsigned char *)illRequest_buf,
2936                                illRequest_size);
2937     }
2938     return r;
2939 }
2940
2941 static Z_ItemOrder *encode_item_order(ZOOM_package p)
2942 {
2943     Z_ItemOrder *req = (Z_ItemOrder *) odr_malloc(p->odr_out, sizeof(*req));
2944     const char *str;
2945     
2946     req->which = Z_IOItemOrder_esRequest;
2947     req->u.esRequest = (Z_IORequest *) 
2948         odr_malloc(p->odr_out,sizeof(Z_IORequest));
2949
2950     /* to keep part ... */
2951     req->u.esRequest->toKeep = (Z_IOOriginPartToKeep *)
2952         odr_malloc(p->odr_out,sizeof(Z_IOOriginPartToKeep));
2953     req->u.esRequest->toKeep->supplDescription = 0;
2954     req->u.esRequest->toKeep->contact = (Z_IOContact *)
2955         odr_malloc(p->odr_out, sizeof(*req->u.esRequest->toKeep->contact));
2956         
2957     str = ZOOM_options_get(p->options, "contact-name");
2958     req->u.esRequest->toKeep->contact->name = str ?
2959         nmem_strdup(p->odr_out->mem, str) : 0;
2960         
2961     str = ZOOM_options_get(p->options, "contact-phone");
2962     req->u.esRequest->toKeep->contact->phone = str ?
2963         nmem_strdup(p->odr_out->mem, str) : 0;
2964         
2965     str = ZOOM_options_get(p->options, "contact-email");
2966     req->u.esRequest->toKeep->contact->email = str ?
2967         nmem_strdup(p->odr_out->mem, str) : 0;
2968         
2969     req->u.esRequest->toKeep->addlBilling = 0;
2970         
2971     /* not to keep part ... */
2972     req->u.esRequest->notToKeep = (Z_IOOriginPartNotToKeep *)
2973         odr_malloc(p->odr_out,sizeof(Z_IOOriginPartNotToKeep));
2974         
2975     str = ZOOM_options_get(p->options, "itemorder-setname");
2976     if (!str)
2977         str = "default";
2978
2979     if (!*str) 
2980         req->u.esRequest->notToKeep->resultSetItem = 0;
2981     else
2982     {
2983         req->u.esRequest->notToKeep->resultSetItem = (Z_IOResultSetItem *)
2984             odr_malloc(p->odr_out, sizeof(Z_IOResultSetItem));
2985
2986         req->u.esRequest->notToKeep->resultSetItem->resultSetId =
2987             nmem_strdup(p->odr_out->mem, str);
2988         req->u.esRequest->notToKeep->resultSetItem->item =
2989             (int *) odr_malloc(p->odr_out, sizeof(int));
2990         
2991         str = ZOOM_options_get(p->options, "itemorder-item");
2992         *req->u.esRequest->notToKeep->resultSetItem->item =
2993             (str ? atoi(str) : 1);
2994     }
2995
2996     str = ZOOM_options_get(p->options, "doc");
2997     if (str)
2998         req->u.esRequest->notToKeep->itemRequest =
2999             z_ext_record(p->odr_out, VAL_TEXT_XML, str, strlen(str));
3000     else
3001         req->u.esRequest->notToKeep->itemRequest = encode_ill_request(p);
3002     
3003     return req;
3004 }
3005
3006 Z_APDU *create_admin_package(ZOOM_package p, int type, 
3007                              Z_ESAdminOriginPartToKeep **toKeepP,
3008                              Z_ESAdminOriginPartNotToKeep **notToKeepP)
3009 {
3010     Z_APDU *apdu = create_es_package(p, VAL_ADMINSERVICE);
3011     if (apdu)
3012     {
3013         Z_ESAdminOriginPartToKeep  *toKeep;
3014         Z_ESAdminOriginPartNotToKeep  *notToKeep;
3015         Z_External *r = (Z_External *) odr_malloc(p->odr_out, sizeof(*r));
3016         const char *first_db = "Default";
3017         int num_db;
3018         char **db = set_DatabaseNames(p->connection, p->options, &num_db,
3019                                       p->odr_out);
3020         if (num_db > 0)
3021             first_db = db[0];
3022             
3023         r->direct_reference =
3024             yaz_oidval_to_z3950oid(p->odr_out, CLASS_EXTSERV,
3025                                    VAL_ADMINSERVICE);
3026         r->descriptor = 0;
3027         r->indirect_reference = 0;
3028         r->which = Z_External_ESAdmin;
3029         
3030         r->u.adminService = (Z_Admin *)
3031             odr_malloc(p->odr_out, sizeof(*r->u.adminService));
3032         r->u.adminService->which = Z_Admin_esRequest;
3033         r->u.adminService->u.esRequest = (Z_AdminEsRequest *)
3034             odr_malloc(p->odr_out, sizeof(*r->u.adminService->u.esRequest));
3035         
3036         toKeep = r->u.adminService->u.esRequest->toKeep =
3037             (Z_ESAdminOriginPartToKeep *) 
3038             odr_malloc(p->odr_out, sizeof(*r->u.adminService->u.esRequest->toKeep));
3039         toKeep->which = type;
3040         toKeep->databaseName = odr_strdup(p->odr_out, first_db);
3041         toKeep->u.create = odr_nullval();
3042         apdu->u.extendedServicesRequest->taskSpecificParameters = r;
3043         
3044         r->u.adminService->u.esRequest->notToKeep = notToKeep =
3045             (Z_ESAdminOriginPartNotToKeep *)
3046             odr_malloc(p->odr_out,
3047                        sizeof(*r->u.adminService->u.esRequest->notToKeep));
3048         notToKeep->which = Z_ESAdminOriginPartNotToKeep_recordsWillFollow;
3049         notToKeep->u.recordsWillFollow = odr_nullval();
3050         if (toKeepP)
3051             *toKeepP = toKeep;
3052         if (notToKeepP)
3053             *notToKeepP = notToKeep;
3054     }
3055     return apdu;
3056 }
3057
3058 static Z_APDU *create_xmlupdate_package(ZOOM_package p)
3059 {
3060     Z_APDU *apdu = create_es_package(p, VAL_XMLES);
3061     Z_ExtendedServicesRequest *req = apdu->u.extendedServicesRequest;
3062     Z_External *ext = (Z_External *) odr_malloc(p->odr_out, sizeof(*ext));
3063     const char *doc = ZOOM_options_get(p->options, "doc");
3064
3065     if (!doc)
3066         doc = "";
3067
3068     req->taskSpecificParameters = ext;
3069     ext->direct_reference = req->packageType;
3070     ext->descriptor = 0;
3071     ext->indirect_reference = 0;
3072     
3073     ext->which = Z_External_octet;
3074     ext->u.single_ASN1_type =
3075         odr_create_Odr_oct(p->odr_out, (const unsigned char *) doc,
3076                            strlen(doc));
3077     return apdu;
3078 }
3079
3080 static Z_APDU *create_update_package(ZOOM_package p)
3081 {
3082     Z_APDU *apdu = 0;
3083     const char *first_db = "Default";
3084     int num_db;
3085     char **db = set_DatabaseNames(p->connection, p->options, &num_db, p->odr_out);
3086     const char *action = ZOOM_options_get(p->options, "action");
3087     const char *recordIdOpaque = ZOOM_options_get(p->options, "recordIdOpaque");
3088     const char *recordIdNumber = ZOOM_options_get(p->options, "recordIdNumber");
3089     const char *record_buf = ZOOM_options_get(p->options, "record");
3090     const char *syntax_str = ZOOM_options_get(p->options, "syntax");
3091     int syntax_oid = VAL_NONE;
3092     int action_no = -1;
3093     
3094     if (syntax_str)
3095         syntax_oid = oid_getvalbyname(syntax_str);
3096     if (!record_buf)
3097     {
3098         record_buf = "void";
3099         syntax_oid = VAL_SUTRS;
3100     }
3101     if (syntax_oid == VAL_NONE)
3102         syntax_oid = VAL_TEXT_XML;
3103     
3104     if (num_db > 0)
3105         first_db = db[0];
3106     
3107     if (!action)
3108         action = "specialUpdate";
3109     
3110     if (!strcmp(action, "recordInsert"))
3111         action_no = Z_IUOriginPartToKeep_recordInsert;
3112     else if (!strcmp(action, "recordReplace"))
3113         action_no = Z_IUOriginPartToKeep_recordReplace;
3114     else if (!strcmp(action, "recordDelete"))
3115         action_no = Z_IUOriginPartToKeep_recordDelete;
3116     else if (!strcmp(action, "elementUpdate"))
3117         action_no = Z_IUOriginPartToKeep_elementUpdate;
3118     else if (!strcmp(action, "specialUpdate"))
3119         action_no = Z_IUOriginPartToKeep_specialUpdate;
3120     else
3121         return 0;
3122
3123     apdu = create_es_package(p, VAL_DBUPDATE);
3124     if (apdu)
3125     {
3126         Z_IUOriginPartToKeep *toKeep;
3127         Z_IUSuppliedRecords *notToKeep;
3128         Z_External *r = (Z_External *)
3129             odr_malloc(p->odr_out, sizeof(*r));
3130         
3131         apdu->u.extendedServicesRequest->taskSpecificParameters = r;
3132         
3133         r->direct_reference =
3134             yaz_oidval_to_z3950oid(p->odr_out, CLASS_EXTSERV,
3135                                    VAL_DBUPDATE);
3136         r->descriptor = 0;
3137         r->which = Z_External_update;
3138         r->indirect_reference = 0;
3139         r->u.update = (Z_IUUpdate *)
3140             odr_malloc(p->odr_out, sizeof(*r->u.update));
3141         
3142         r->u.update->which = Z_IUUpdate_esRequest;
3143         r->u.update->u.esRequest = (Z_IUUpdateEsRequest *)
3144             odr_malloc(p->odr_out, sizeof(*r->u.update->u.esRequest));
3145         toKeep = r->u.update->u.esRequest->toKeep = 
3146             (Z_IUOriginPartToKeep *)
3147             odr_malloc(p->odr_out, sizeof(*toKeep));
3148         
3149         toKeep->databaseName = odr_strdup(p->odr_out, first_db);
3150         toKeep->schema = 0;
3151         toKeep->elementSetName = 0;
3152         toKeep->actionQualifier = 0;
3153         toKeep->action = odr_intdup(p->odr_out, action_no);
3154         
3155         notToKeep = r->u.update->u.esRequest->notToKeep = 
3156             (Z_IUSuppliedRecords *)
3157             odr_malloc(p->odr_out, sizeof(*notToKeep));
3158         notToKeep->num = 1;
3159         notToKeep->elements = (Z_IUSuppliedRecords_elem **)
3160             odr_malloc(p->odr_out, sizeof(*notToKeep->elements));
3161         notToKeep->elements[0] = (Z_IUSuppliedRecords_elem *)
3162             odr_malloc(p->odr_out, sizeof(**notToKeep->elements));
3163         notToKeep->elements[0]->which = Z_IUSuppliedRecords_elem_opaque;
3164         if (recordIdOpaque)
3165         {
3166             notToKeep->elements[0]->u.opaque = 
3167                 odr_create_Odr_oct(p->odr_out,
3168                                    (const unsigned char *) recordIdOpaque,
3169                                    strlen(recordIdOpaque));
3170         }
3171         else if (recordIdNumber)
3172         {
3173             notToKeep->elements[0]->which = Z_IUSuppliedRecords_elem_number;
3174             
3175             notToKeep->elements[0]->u.number =
3176                 odr_intdup(p->odr_out, atoi(recordIdNumber));
3177         }
3178         else
3179             notToKeep->elements[0]->u.opaque = 0;
3180         notToKeep->elements[0]->supplementalId = 0;
3181         notToKeep->elements[0]->correlationInfo = 0;
3182         notToKeep->elements[0]->record =
3183             z_ext_record(p->odr_out, syntax_oid,
3184                          record_buf, strlen(record_buf));
3185     }
3186     if (0 && apdu)
3187     {
3188         ODR print = odr_createmem(ODR_PRINT);
3189
3190         z_APDU(print, &apdu, 0, 0);
3191         odr_destroy(print);
3192     }
3193     return apdu;
3194 }
3195
3196 ZOOM_API(void)
3197     ZOOM_package_send(ZOOM_package p, const char *type)
3198 {
3199     Z_APDU *apdu = 0;
3200     ZOOM_connection c;
3201     if (!p)
3202         return;
3203     c = p->connection;
3204     odr_reset(p->odr_out);
3205     xfree(p->buf_out);
3206     p->buf_out = 0;
3207     if (!strcmp(type, "itemorder"))
3208     {
3209         apdu = create_es_package(p, VAL_ITEMORDER);
3210         if (apdu)
3211         {
3212             Z_External *r = (Z_External *) odr_malloc(p->odr_out, sizeof(*r));
3213             
3214             r->direct_reference =
3215                 yaz_oidval_to_z3950oid(p->odr_out, CLASS_EXTSERV,
3216                                        VAL_ITEMORDER);
3217             r->descriptor = 0;
3218             r->which = Z_External_itemOrder;
3219             r->indirect_reference = 0;
3220             r->u.itemOrder = encode_item_order(p);
3221
3222             apdu->u.extendedServicesRequest->taskSpecificParameters = r;
3223         }
3224     }
3225     else if (!strcmp(type, "create"))  /* create database */
3226     {
3227         apdu = create_admin_package(p, Z_ESAdminOriginPartToKeep_create,
3228                                     0, 0);
3229     }   
3230     else if (!strcmp(type, "drop"))  /* drop database */
3231     {
3232         apdu = create_admin_package(p, Z_ESAdminOriginPartToKeep_drop,
3233                                     0, 0);
3234     }
3235     else if (!strcmp(type, "commit"))  /* commit changes */
3236     {
3237         apdu = create_admin_package(p, Z_ESAdminOriginPartToKeep_commit,
3238                                     0, 0);
3239     }
3240     else if (!strcmp(type, "update")) /* update record(s) */
3241     {
3242         apdu = create_update_package(p);
3243     }
3244     else if (!strcmp(type, "xmlupdate"))
3245     {
3246         apdu = create_xmlupdate_package(p);
3247     }
3248     if (apdu)
3249     {
3250         if (encode_APDU(p->connection, apdu, p->odr_out) == 0)
3251         {
3252             char *buf;
3253
3254             ZOOM_task task = ZOOM_connection_add_task(c, ZOOM_TASK_PACKAGE);
3255             task->u.package = p;
3256             buf = odr_getbuf(p->odr_out, &p->len_out, 0);
3257             p->buf_out = (char *) xmalloc(p->len_out);
3258             memcpy(p->buf_out, buf, p->len_out);
3259             
3260             (p->refcount)++;
3261             if (!c->async)
3262             {
3263                 while (ZOOM_event(1, &c))
3264                     ;
3265             }
3266         }
3267     }
3268 }
3269
3270 ZOOM_API(ZOOM_package)
3271     ZOOM_connection_package(ZOOM_connection c, ZOOM_options options)
3272 {
3273     ZOOM_package p = (ZOOM_package) xmalloc(sizeof(*p));
3274
3275     p->connection = c;
3276     p->odr_out = odr_createmem(ODR_ENCODE);
3277     p->options = ZOOM_options_create_with_parent2(options, c->options);
3278     p->refcount = 1;
3279     p->buf_out = 0;
3280     p->len_out = 0;
3281     return p;
3282 }
3283
3284 ZOOM_API(void)
3285     ZOOM_package_destroy(ZOOM_package p)
3286 {
3287     if (!p)
3288         return;
3289     (p->refcount)--;
3290     if (p->refcount == 0)
3291     {
3292         odr_destroy(p->odr_out);
3293         xfree(p->buf_out);
3294         
3295         ZOOM_options_destroy(p->options);
3296         xfree(p);
3297     }
3298 }
3299
3300 ZOOM_API(const char *)
3301     ZOOM_package_option_get(ZOOM_package p, const char *key)
3302 {
3303     return ZOOM_options_get(p->options, key);
3304 }
3305
3306
3307 ZOOM_API(void)
3308     ZOOM_package_option_set(ZOOM_package p, const char *key,
3309                             const char *val)
3310 {
3311     ZOOM_options_set(p->options, key, val);
3312 }
3313
3314 static int ZOOM_connection_exec_task(ZOOM_connection c)
3315 {
3316     ZOOM_task task = c->tasks;
3317     zoom_ret ret = zoom_complete;
3318
3319     if (!task)
3320         return 0;
3321     yaz_log(log_details, "%p ZOOM_connection_exec_task type=%d run=%d",
3322             c, task->which, task->running);
3323     if (c->error != ZOOM_ERROR_NONE)
3324     {
3325         yaz_log(log_details, "%p ZOOM_connection_exec_task "
3326                 "removing tasks because of error = %d", c, c->error);
3327         ZOOM_connection_remove_tasks(c);
3328         return 0;
3329     }
3330     if (task->running)
3331     {
3332         yaz_log(log_details, "%p ZOOM_connection_exec_task "
3333                 "task already running", c);
3334         return 0;
3335     }
3336     task->running = 1;
3337     ret = zoom_complete;
3338     if (c->cs || task->which == ZOOM_TASK_CONNECT)
3339     {
3340         switch (task->which)
3341         {
3342         case ZOOM_TASK_SEARCH:
3343             if (c->proto == PROTO_HTTP)
3344                 ret = ZOOM_connection_srw_send_search(c);
3345             else
3346                 ret = ZOOM_connection_send_search(c);
3347             break;
3348         case ZOOM_TASK_RETRIEVE:
3349             if (c->proto == PROTO_HTTP)
3350                 ret = ZOOM_connection_srw_send_search(c);
3351             else
3352                 ret = send_present(c);
3353             break;
3354         case ZOOM_TASK_CONNECT:
3355             ret = do_connect(c);
3356             break;
3357         case ZOOM_TASK_SCAN:
3358             ret = send_scan(c);
3359             break;
3360         case ZOOM_TASK_PACKAGE:
3361             ret = send_package(c);
3362             break;
3363         case ZOOM_TASK_SORT:
3364             c->tasks->u.sort.resultset->r_sort_spec = 
3365                 c->tasks->u.sort.q->sort_spec;
3366             ret = send_sort(c, c->tasks->u.sort.resultset);
3367             break;
3368         }
3369     }
3370     else
3371     {
3372         yaz_log(log_details, "%p ZOOM_connection_exec_task "
3373                 "remove tasks because no connection exist", c);
3374         ZOOM_connection_remove_tasks(c);
3375     }
3376     if (ret == zoom_complete)
3377     {
3378         yaz_log(log_details, "%p ZOOM_connection_exec_task "
3379                 "task removed (complete)", c);
3380         ZOOM_connection_remove_task(c);
3381         return 0;
3382     }
3383     yaz_log(log_details, "%p ZOOM_connection_exec_task "
3384             "task pending", c);
3385     return 1;
3386 }
3387
3388 static zoom_ret send_sort_present(ZOOM_connection c)
3389 {
3390     zoom_ret r = zoom_complete;
3391
3392     if (c->tasks && c->tasks->which == ZOOM_TASK_SEARCH)
3393         r = send_sort(c, c->tasks->u.search.resultset);
3394     if (r == zoom_complete)
3395         r = send_present(c);
3396     return r;
3397 }
3398
3399 static int es_response(ZOOM_connection c,
3400                        Z_ExtendedServicesResponse *res)
3401 {
3402     if (!c->tasks || c->tasks->which != ZOOM_TASK_PACKAGE)
3403         return 0;
3404     if (res->diagnostics && res->num_diagnostics > 0)
3405         response_diag(c, res->diagnostics[0]);
3406     if (res->taskPackage &&
3407         res->taskPackage->which == Z_External_extendedService)
3408     {
3409         Z_TaskPackage *taskPackage = res->taskPackage->u.extendedService;
3410         Odr_oct *id = taskPackage->targetReference;
3411         
3412         if (id)
3413             ZOOM_options_setl(c->tasks->u.package->options,
3414                               "targetReference", (char*) id->buf, id->len);
3415     }
3416     if (res->taskPackage && 
3417         res->taskPackage->which == Z_External_octet)
3418     {
3419         Odr_oct *doc = res->taskPackage->u.octet_aligned;
3420         ZOOM_options_setl(c->tasks->u.package->options,
3421                           "xmlUpdateDoc", (char*) doc->buf, doc->len);
3422     }
3423     return 1;
3424 }
3425
3426 static void interpret_init_diag(ZOOM_connection c,
3427                                 Z_DiagnosticFormat *diag)
3428 {
3429     if (diag->num > 0)
3430     {
3431         Z_DiagnosticFormat_s *ds = diag->elements[0];
3432         if (ds->which == Z_DiagnosticFormat_s_defaultDiagRec)
3433             response_default_diag(c, ds->u.defaultDiagRec);
3434     }
3435 }
3436
3437
3438 static void interpret_otherinformation_field(ZOOM_connection c,
3439                                              Z_OtherInformation *ui)
3440 {
3441     int i;
3442     for (i = 0; i < ui->num_elements; i++)
3443     {
3444         Z_OtherInformationUnit *unit = ui->list[i];
3445         if (unit->which == Z_OtherInfo_externallyDefinedInfo &&
3446             unit->information.externallyDefinedInfo &&
3447             unit->information.externallyDefinedInfo->which ==
3448             Z_External_diag1) 
3449         {
3450             interpret_init_diag(c, unit->information.externallyDefinedInfo->u.diag1);
3451         } 
3452     }
3453 }
3454
3455
3456 static void set_init_option(const char *name, void *clientData) {
3457     ZOOM_connection c = clientData;
3458     char buf[80];
3459
3460     sprintf(buf, "init_opt_%.70s", name);
3461     ZOOM_connection_option_set(c, buf, "1");
3462 }
3463
3464
3465 static void recv_apdu(ZOOM_connection c, Z_APDU *apdu)
3466 {
3467     Z_InitResponse *initrs;
3468     
3469     ZOOM_connection_set_mask(c, 0);
3470     yaz_log(log_details, "%p recv_apdu apdu->which=%d", c, apdu->which);
3471     switch(apdu->which)
3472     {
3473     case Z_APDU_initResponse:
3474         yaz_log(log_api, "%p recv_apd: Received Init response", c);
3475         initrs = apdu->u.initResponse;
3476         ZOOM_connection_option_set(c, "serverImplementationId",
3477                                    initrs->implementationId ?
3478                                    initrs->implementationId : "");
3479         ZOOM_connection_option_set(c, "serverImplementationName",
3480                                    initrs->implementationName ?
3481                                    initrs->implementationName : "");
3482         ZOOM_connection_option_set(c, "serverImplementationVersion",
3483                                    initrs->implementationVersion ?
3484                                    initrs->implementationVersion : "");
3485         /* Set the three old options too, for old applications */
3486         ZOOM_connection_option_set(c, "targetImplementationId",
3487                                    initrs->implementationId ?
3488                                    initrs->implementationId : "");
3489         ZOOM_connection_option_set(c, "targetImplementationName",
3490                                    initrs->implementationName ?
3491                                    initrs->implementationName : "");
3492         ZOOM_connection_option_set(c, "targetImplementationVersion",
3493                                    initrs->implementationVersion ?
3494                                    initrs->implementationVersion : "");
3495
3496         /* Make initrs->options available as ZOOM-level options */
3497         yaz_init_opt_decode(initrs->options, set_init_option, (void*) c);
3498
3499         if (!*initrs->result)
3500         {
3501             Z_External *uif = initrs->userInformationField;
3502
3503             set_ZOOM_error(c, ZOOM_ERROR_INIT, 0); /* default error */
3504
3505             if (uif && uif->which == Z_External_userInfo1)
3506                 interpret_otherinformation_field(c, uif->u.userInfo1);
3507         }
3508         else
3509         {
3510             char *cookie =
3511                 yaz_oi_get_string_oidval(&apdu->u.initResponse->otherInfo,
3512                                          VAL_COOKIE, 1, 0);
3513             xfree(c->cookie_in);
3514             c->cookie_in = 0;
3515             if (cookie)
3516                 c->cookie_in = xstrdup(cookie);
3517             if (ODR_MASK_GET(initrs->options, Z_Options_namedResultSets) &&
3518                 ODR_MASK_GET(initrs->protocolVersion, Z_ProtocolVersion_3))
3519                 c->support_named_resultsets = 1;
3520             if (c->tasks)
3521             {
3522                 assert(c->tasks->which == ZOOM_TASK_CONNECT);
3523                 ZOOM_connection_remove_task(c);
3524             }
3525             ZOOM_connection_exec_task(c);
3526         }
3527         if (ODR_MASK_GET(initrs->options, Z_Options_negotiationModel))
3528         {
3529             NMEM tmpmem = nmem_create();
3530             Z_CharSetandLanguageNegotiation *p =
3531                 yaz_get_charneg_record(initrs->otherInfo);
3532             
3533             if (p)
3534             {
3535                 char *charset = NULL, *lang = NULL;
3536                 int sel;
3537                 
3538                 yaz_get_response_charneg(tmpmem, p, &charset, &lang, &sel);
3539                 yaz_log(log_details, "%p recv_apdu target accepted: "
3540                         "charset %s, language %s, select %d",
3541                         c,
3542                         charset ? charset : "none", lang ? lang : "none", sel);
3543                 if (charset)
3544                     ZOOM_connection_option_set(c, "negotiation-charset",
3545                                                charset);
3546                 if (lang)
3547                     ZOOM_connection_option_set(c, "negotiation-lang",
3548                                                lang);
3549
3550                 ZOOM_connection_option_set(
3551                     c,  "negotiation-charset-in-effect-for-records",
3552                     (sel != 0) ? "1" : "0");
3553                 nmem_destroy(tmpmem);
3554             }
3555         }       
3556         break;
3557     case Z_APDU_searchResponse:
3558         yaz_log(log_api, "%p recv_apdu Search response", c);
3559         handle_search_response(c, apdu->u.searchResponse);
3560         if (send_sort_present(c) == zoom_complete)
3561             ZOOM_connection_remove_task(c);
3562         break;
3563     case Z_APDU_presentResponse:
3564         yaz_log(log_api, "%p recv_apdu Present response", c);
3565         handle_present_response(c, apdu->u.presentResponse);
3566         if (send_present(c) == zoom_complete)
3567             ZOOM_connection_remove_task(c);
3568         break;
3569     case Z_APDU_sortResponse:
3570         yaz_log(log_api, "%p recv_apdu Sort response", c);
3571         sort_response(c, apdu->u.sortResponse);
3572         if (send_present(c) == zoom_complete)
3573             ZOOM_connection_remove_task(c);
3574         break;
3575     case Z_APDU_scanResponse:
3576         yaz_log(log_api, "%p recv_apdu Scan response", c);
3577         scan_response(c, apdu->u.scanResponse);
3578         ZOOM_connection_remove_task(c);
3579         break;
3580     case Z_APDU_extendedServicesResponse:
3581         yaz_log(log_api, "%p recv_apdu Extended Services response", c);
3582         es_response(c, apdu->u.extendedServicesResponse);
3583         ZOOM_connection_remove_task(c);
3584         break;
3585     case Z_APDU_close:
3586         yaz_log(log_api, "%p recv_apdu Close PDU", c);
3587         if (!ZOOM_test_reconnect(c))
3588         {
3589             set_ZOOM_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
3590             do_close(c);
3591         }
3592         break;
3593     default:
3594         yaz_log(log_api, "%p Received unknown PDU", c);
3595         set_ZOOM_error(c, ZOOM_ERROR_DECODE, 0);
3596         do_close(c);
3597     }
3598 }
3599
3600 #if YAZ_HAVE_XML2
3601 static void handle_srw_response(ZOOM_connection c,
3602                                 Z_SRW_searchRetrieveResponse *res)
3603 {
3604     ZOOM_resultset resultset = 0;
3605     int i;
3606     NMEM nmem;
3607     ZOOM_Event event;
3608     int *start;
3609     const char *syntax, *elementSetName;
3610
3611     if (!c->tasks)
3612         return;
3613
3614     switch(c->tasks->which)
3615     {
3616     case ZOOM_TASK_SEARCH:
3617         resultset = c->tasks->u.search.resultset;
3618         start = &c->tasks->u.search.start;
3619         syntax = c->tasks->u.search.syntax;
3620         elementSetName = c->tasks->u.search.elementSetName;        
3621         break;
3622     case ZOOM_TASK_RETRIEVE:
3623         resultset = c->tasks->u.retrieve.resultset;
3624         start = &c->tasks->u.retrieve.start;
3625         syntax = c->tasks->u.retrieve.syntax;
3626         elementSetName = c->tasks->u.retrieve.elementSetName;
3627         break;
3628     default:
3629         return;
3630     }
3631     event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH);
3632     ZOOM_connection_put_event(c, event);
3633
3634     resultset->size = 0;
3635
3636     yaz_log(log_details, "%p handle_srw_response got SRW response OK", c);
3637     
3638     if (res->numberOfRecords)
3639         resultset->size = *res->numberOfRecords;
3640
3641     for (i = 0; i<res->num_records; i++)
3642     {
3643         int pos;
3644
3645         Z_NamePlusRecord *npr = (Z_NamePlusRecord *)
3646             odr_malloc(c->odr_in, sizeof(Z_NamePlusRecord));
3647
3648         if (res->records[i].recordPosition && 
3649             *res->records[i].recordPosition > 0)
3650             pos = *res->records[i].recordPosition - 1;
3651         else
3652             pos = *start + i;
3653         
3654         npr->databaseName = 0;
3655         npr->which = Z_NamePlusRecord_databaseRecord;
3656         npr->u.databaseRecord = (Z_External *)
3657             odr_malloc(c->odr_in, sizeof(Z_External));
3658         npr->u.databaseRecord->descriptor = 0;