Added SRU-GET and SRU-POST support for ZOOM C. Option "sru" specifies
[yaz-moved-to-github.git] / src / zoom-c.c
1 /*
2  * Copyright (C) 1995-2005, Index Data ApS
3  * See the file LICENSE for details.
4  *
5  * $Id: zoom-c.c,v 1.64 2006-03-01 23:24:25 adam Exp $
6  */
7 /**
8  * \file zoom-c.c
9  * \brief Implements ZOOM C interface.
10  */
11
12 #include <assert.h>
13 #include <string.h>
14 #include <errno.h>
15 #include "zoom-p.h"
16
17 #include <yaz/yaz-util.h>
18 #include <yaz/xmalloc.h>
19 #include <yaz/otherinfo.h>
20 #include <yaz/log.h>
21 #include <yaz/pquery.h>
22 #include <yaz/marcdisp.h>
23 #include <yaz/diagbib1.h>
24 #include <yaz/charneg.h>
25 #include <yaz/ill.h>
26 #include <yaz/srw.h>
27 #include <yaz/cql.h>
28
29 #if HAVE_SYS_TYPES_H
30 #include <sys/types.h>
31 #endif
32 #if HAVE_SYS_TIME_H
33 #include <sys/time.h>
34 #endif
35 #if HAVE_SYS_POLL_H
36 #include <sys/poll.h>
37 #endif
38 #if HAVE_SYS_SELECT_H
39 #include <sys/select.h>
40 #endif
41 #ifdef WIN32
42 #include <winsock.h>
43 #endif
44
45 static int log_api = 0;
46 static int log_details = 0;
47
48 typedef enum {
49     zoom_pending,
50     zoom_complete
51 } zoom_ret;
52
53 static zoom_ret ZOOM_connection_send_init (ZOOM_connection c);
54 static zoom_ret do_write_ex (ZOOM_connection c, char *buf_out, int len_out);
55 static char *cql2pqf(ZOOM_connection c, const char *cql);
56
57 static void initlog()
58 {
59     static int log_level_initialized = 0;
60     if (!log_level_initialized)
61     {
62         log_api = yaz_log_module_level("zoom");
63         log_details = yaz_log_module_level("zoomdetails");
64         log_level_initialized = 1;
65     }
66 }
67
68 static ZOOM_Event ZOOM_Event_create (int kind)
69 {
70     ZOOM_Event event = (ZOOM_Event) xmalloc (sizeof(*event));
71     event->kind = kind;
72     event->next = 0;
73     event->prev = 0;
74     return event;
75 }
76
77 static void ZOOM_Event_destroy (ZOOM_Event event)
78 {
79     xfree (event);
80 }
81
82 static void ZOOM_connection_put_event (ZOOM_connection c, ZOOM_Event event)
83 {
84     if (c->m_queue_back)
85     {
86         c->m_queue_back->prev = event;
87         assert (c->m_queue_front);
88     }
89     else
90     {
91         assert (!c->m_queue_front);
92         c->m_queue_front = event;
93     }
94     event->next = c->m_queue_back;
95     event->prev = 0;
96     c->m_queue_back = event;
97 }
98
99 static ZOOM_Event ZOOM_connection_get_event(ZOOM_connection c)
100 {
101     ZOOM_Event event = c->m_queue_front;
102     if (!event)
103     {
104         c->last_event = ZOOM_EVENT_NONE;
105         return 0;
106     }
107     assert (c->m_queue_back);
108     c->m_queue_front = event->prev;
109     if (c->m_queue_front)
110     {
111         assert (c->m_queue_back);
112         c->m_queue_front->next = 0;
113     }
114     else
115         c->m_queue_back = 0;
116     c->last_event = event->kind;
117     return event;
118 }
119
120
121 static void set_dset_error (ZOOM_connection c, int error,
122                             const char *dset,
123                             const char *addinfo, const char *addinfo2)
124 {
125     char *cp;
126
127     xfree (c->addinfo);
128     c->addinfo = 0;
129     c->error = error;
130     if (!c->diagset || strcmp(dset, c->diagset))
131     {
132         xfree(c->diagset);
133         c->diagset = xstrdup(dset);
134         /* remove integer part from SRW diagset .. */
135         if ((cp = strrchr(c->diagset, '/')))
136            *cp = '\0';
137     }
138     if (addinfo && addinfo2)
139     {
140         c->addinfo = (char*) xmalloc(strlen(addinfo) + strlen(addinfo2) + 2);
141         strcpy(c->addinfo, addinfo);
142         strcat(c->addinfo, addinfo2);
143     }
144     else if (addinfo)
145         c->addinfo = xstrdup(addinfo);
146     if (error)
147         yaz_log(log_api, "%p set_dset_error %s %s:%d %s %s",
148                 c, c->host_port ? c->host_port : "<>", dset, error,
149                 addinfo ? addinfo : "",
150                 addinfo2 ? addinfo2 : "");
151 }
152
153 #if HAVE_XML2
154 static void set_HTTP_error (ZOOM_connection c, int error,
155                             const char *addinfo, const char *addinfo2)
156 {
157     set_dset_error(c, error, "HTTP", addinfo, addinfo2);
158 }
159 #endif
160
161 static void set_ZOOM_error (ZOOM_connection c, int error,
162                             const char *addinfo)
163 {
164     set_dset_error(c, error, "ZOOM", addinfo, 0);
165 }
166
167 static void clear_error (ZOOM_connection c)
168 {
169     /*
170      * If an error is tied to an operation then it's ok to clear: for
171      * example, a diagnostic returned from a search is cleared by a
172      * subsequent search.  However, problems such as Connection Lost
173      * or Init Refused are not cleared, because they are not
174      * recoverable: doing another search doesn't help.
175      */
176     switch (c->error)
177     {
178     case ZOOM_ERROR_CONNECT:
179     case ZOOM_ERROR_MEMORY:
180     case ZOOM_ERROR_DECODE:
181     case ZOOM_ERROR_CONNECTION_LOST:
182     case ZOOM_ERROR_INIT:
183     case ZOOM_ERROR_INTERNAL:
184     case ZOOM_ERROR_UNSUPPORTED_PROTOCOL:
185         break;
186     default:
187         set_ZOOM_error(c, ZOOM_ERROR_NONE, 0);
188     }
189 }
190
191 ZOOM_task ZOOM_connection_add_task (ZOOM_connection c, int which)
192 {
193     ZOOM_task *taskp = &c->tasks;
194     while (*taskp)
195         taskp = &(*taskp)->next;
196     *taskp = (ZOOM_task) xmalloc (sizeof(**taskp));
197     (*taskp)->running = 0;
198     (*taskp)->which = which;
199     (*taskp)->next = 0;
200     clear_error (c);
201     return *taskp;
202 }
203
204 ZOOM_task ZOOM_connection_insert_task (ZOOM_connection c, int which)
205 {
206     ZOOM_task task = (ZOOM_task) xmalloc (sizeof(*task));
207
208     task->next = c->tasks;
209     c->tasks = task;
210
211     task->running = 0;
212     task->which = which;
213     clear_error (c);
214     return task;
215 }
216
217 void ZOOM_connection_remove_task (ZOOM_connection c)
218 {
219     ZOOM_task task = c->tasks;
220
221     if (task)
222     {
223         c->tasks = task->next;
224         switch (task->which)
225         {
226         case ZOOM_TASK_SEARCH:
227             ZOOM_resultset_destroy (task->u.search.resultset);
228             break;
229         case ZOOM_TASK_RETRIEVE:
230             ZOOM_resultset_destroy (task->u.retrieve.resultset);
231             break;
232         case ZOOM_TASK_CONNECT:
233             break;
234         case ZOOM_TASK_SCAN:
235             ZOOM_scanset_destroy (task->u.scan.scan);
236             break;
237         case ZOOM_TASK_PACKAGE:
238             ZOOM_package_destroy (task->u.package);
239             break;
240         case ZOOM_TASK_SORT:
241             ZOOM_resultset_destroy (task->u.sort.resultset);
242             ZOOM_query_destroy(task->u.sort.q);
243             break;
244         default:
245             assert (0);
246         }
247         xfree (task);
248     }
249 }
250
251 static int ZOOM_connection_exec_task (ZOOM_connection c);
252
253 void ZOOM_connection_remove_tasks (ZOOM_connection c)
254 {
255     while (c->tasks)
256         ZOOM_connection_remove_task(c);
257 }
258
259 static ZOOM_record record_cache_lookup (ZOOM_resultset r, int pos);
260
261 ZOOM_API(ZOOM_connection)
262 ZOOM_connection_create (ZOOM_options options)
263 {
264     ZOOM_connection c = (ZOOM_connection) xmalloc (sizeof(*c));
265
266     initlog();
267
268     yaz_log(log_api, "%p ZOOM_connection_create", c);
269
270     c->proto = PROTO_Z3950;
271     c->cs = 0;
272     c->mask = 0;
273     c->reconnect_ok = 0;
274     c->state = STATE_IDLE;
275     c->addinfo = 0;
276     c->diagset = 0;
277     set_ZOOM_error(c, ZOOM_ERROR_NONE, 0);
278     c->buf_in = 0;
279     c->len_in = 0;
280     c->buf_out = 0;
281     c->len_out = 0;
282     c->resultsets = 0;
283
284     c->options = ZOOM_options_create_with_parent(options);
285
286     c->host_port = 0;
287     c->path = 0;
288     c->proxy = 0;
289     
290     c->charset = c->lang = 0;
291
292     c->cookie_out = 0;
293     c->cookie_in = 0;
294     c->client_IP = 0;
295     c->tasks = 0;
296
297     c->odr_in = odr_createmem (ODR_DECODE);
298     c->odr_out = odr_createmem (ODR_ENCODE);
299
300     c->async = 0;
301     c->support_named_resultsets = 0;
302     c->last_event = ZOOM_EVENT_NONE;
303
304     c->m_queue_front = 0;
305     c->m_queue_back = 0;
306     return c;
307 }
308
309 /* set database names. Take local databases (if set); otherwise
310    take databases given in ZURL (if set); otherwise use Default */
311 static char **set_DatabaseNames (ZOOM_connection con, ZOOM_options options,
312                                  int *num)
313 {
314     char **databaseNames;
315     const char *cp = ZOOM_options_get (options, "databaseName");
316     
317     if (!cp || !*cp)
318     {
319         if (strncmp (con->host_port, "unix:", 5) == 0)
320             cp = strchr(con->host_port+5, ':');
321         else
322             cp = strchr(con->host_port, '/');
323         if (cp)
324             cp++;
325     }
326     if (!cp)
327         cp = "Default";
328     nmem_strsplit(con->odr_out->mem, "+", cp,  &databaseNames, num);
329     return databaseNames;
330 }
331
332 ZOOM_API(ZOOM_connection)
333 ZOOM_connection_new (const char *host, int portnum)
334 {
335     ZOOM_connection c = ZOOM_connection_create (0);
336
337     ZOOM_connection_connect (c, host, portnum);
338     return c;
339 }
340
341 static zoom_sru_mode get_sru_mode_from_string(const char *s)
342 {
343     if (!s || !*s)
344         return zoom_sru_soap;
345     if (!yaz_matchstr(s, "soap"))
346         return zoom_sru_soap;
347     else if (!yaz_matchstr(s, "get"))
348         return zoom_sru_get;
349     else if (!yaz_matchstr(s, "post"))
350         return zoom_sru_post;
351     return zoom_sru_error;
352 }
353
354 ZOOM_API(void)
355 ZOOM_connection_connect(ZOOM_connection c,
356                         const char *host, int portnum)
357 {
358     const char *val;
359     ZOOM_task task;
360
361     initlog();
362
363     yaz_log(log_api, "%p ZOOM_connection_connect host=%s portnum=%d",
364             c, host, portnum);
365
366     if (c->cs)
367     {
368         yaz_log(log_details, "%p ZOOM_connection_connect reconnect ok", c);
369         c->reconnect_ok = 1;
370         return;
371     }
372     yaz_log(log_details, "%p ZOOM_connection_connect connect", c);
373     xfree (c->proxy);
374     val = ZOOM_options_get (c->options, "proxy");
375     if (val && *val)
376     {
377         yaz_log(log_details, "%p ZOOM_connection_connect proxy=%s", c, val);
378         c->proxy = xstrdup (val);
379     }
380     else
381         c->proxy = 0;
382
383     xfree (c->charset);
384     val = ZOOM_options_get (c->options, "charset");
385     if (val && *val)
386     {
387         yaz_log(log_details, "%p ZOOM_connection_connect charset=%s", c, val);
388         c->charset = xstrdup (val);
389     }
390     else
391         c->charset = 0;
392
393     xfree (c->lang);
394     val = ZOOM_options_get (c->options, "lang");
395     if (val && *val)
396     {
397         yaz_log(log_details, "%p ZOOM_connection_connect lang=%s", c, val);
398         c->lang = xstrdup (val);
399     }
400     else
401         c->lang = 0;
402
403     val = ZOOM_options_get (c->options, "sru");
404     c->sru_mode = get_sru_mode_from_string(val);
405
406     xfree (c->host_port);
407     if (portnum)
408     {
409         char hostn[128];
410         sprintf (hostn, "%.80s:%d", host, portnum);
411         c->host_port = xstrdup(hostn);
412     }
413     else
414         c->host_port = xstrdup(host);
415
416     {
417         /*
418          * If the "<scheme>:" part of the host string is preceded by one
419          * or more comma-separated <name>=<value> pairs, these are taken
420          * to be options to be set on the connection object.  Among other
421          * applications, this facility can be used to embed authentication
422          * in a host string:
423          *          user=admin,password=secret,tcp:localhost:9999
424          */
425         char *remainder = c->host_port;
426         char *pcolon = strchr(remainder, ':');
427         char *pcomma;
428         char *pequals;
429         while ((pcomma = strchr(remainder, ',')) != 0 &&
430                (pcolon == 0 || pcomma < pcolon)) {
431             *pcomma = '\0';
432             if ((pequals = strchr(remainder, '=')) != 0) {
433                 *pequals = '\0';
434                 /*printf("# setting '%s'='%s'\n", remainder, pequals+1);*/
435                 ZOOM_connection_option_set(c, remainder, pequals+1);
436             }
437             remainder = pcomma+1;
438         }
439
440         if (remainder != c->host_port) {
441             xfree(c->host_port);
442             c->host_port = xstrdup(remainder);
443             /*printf("# reset hp='%s'\n", remainder);*/
444         }
445     }
446
447     ZOOM_options_set(c->options, "host", c->host_port);
448
449     val = ZOOM_options_get (c->options, "cookie");
450     if (val && *val)
451     { 
452         yaz_log(log_details, "%p ZOOM_connection_connect cookie=%s", c, val);
453         c->cookie_out = xstrdup (val);
454     }
455
456     val = ZOOM_options_get (c->options, "clientIP");
457     if (val && *val)
458     {
459         yaz_log(log_details, "%p ZOOM_connection_connect clientIP=%s",
460                 c, val);
461         c->client_IP = xstrdup (val);
462     }
463
464     c->async = ZOOM_options_get_bool (c->options, "async", 0);
465     yaz_log(log_details, "%p ZOOM_connection_connect async=%d", c, c->async);
466  
467     set_ZOOM_error(c, ZOOM_ERROR_NONE, 0);
468
469     task = ZOOM_connection_add_task (c, ZOOM_TASK_CONNECT);
470
471     if (!c->async)
472     {
473         while (ZOOM_event (1, &c))
474             ;
475     }
476 }
477
478 ZOOM_API(ZOOM_query)
479 ZOOM_query_create(void)
480 {
481     ZOOM_query s = (ZOOM_query) xmalloc (sizeof(*s));
482
483     yaz_log(log_details, "%p ZOOM_query_create", s);
484     s->refcount = 1;
485     s->z_query = 0;
486     s->sort_spec = 0;
487     s->odr = odr_createmem (ODR_ENCODE);
488     s->query_string = 0;
489
490     return s;
491 }
492
493 ZOOM_API(void)
494 ZOOM_query_destroy(ZOOM_query s)
495 {
496     if (!s)
497         return;
498
499     (s->refcount)--;
500     yaz_log(log_details, "%p ZOOM_query_destroy count=%d", s, s->refcount);
501     if (s->refcount == 0)
502     {
503         odr_destroy (s->odr);
504         xfree (s);
505     }
506 }
507
508 ZOOM_API(int)
509 ZOOM_query_prefix(ZOOM_query s, const char *str)
510 {
511     s->query_string = odr_strdup(s->odr, str);
512     s->z_query = (Z_Query *) odr_malloc (s->odr, sizeof(*s->z_query));
513     s->z_query->which = Z_Query_type_1;
514     s->z_query->u.type_1 =  p_query_rpn(s->odr, PROTO_Z3950, str);
515     if (!s->z_query->u.type_1)
516     {
517         yaz_log(log_details, "%p ZOOM_query_prefix str=%s failed", s, str);
518         s->z_query = 0;
519         return -1;
520     }
521     yaz_log(log_details, "%p ZOOM_query_prefix str=%s", s, str);
522     return 0;
523 }
524
525 ZOOM_API(int)
526 ZOOM_query_cql(ZOOM_query s, const char *str)
527 {
528     Z_External *ext;
529
530     s->query_string = odr_strdup(s->odr, str);
531
532     ext = (Z_External *) odr_malloc(s->odr, sizeof(*ext));
533     ext->direct_reference = odr_getoidbystr(s->odr, "1.2.840.10003.16.2");
534     ext->indirect_reference = 0;
535     ext->descriptor = 0;
536     ext->which = Z_External_CQL;
537     ext->u.cql = s->query_string;
538     
539     s->z_query = (Z_Query *) odr_malloc (s->odr, sizeof(*s->z_query));
540     s->z_query->which = Z_Query_type_104;
541     s->z_query->u.type_104 =  ext;
542
543     yaz_log(log_details, "%p ZOOM_query_cql str=%s", s, str);
544
545     return 0;
546 }
547
548 /*
549  * Translate the CQL string client-side into RPN which is passed to
550  * the server.  This is useful for server's that don't themselves
551  * support CQL, for which ZOOM_query_cql() is useless.  `conn' is used
552  * only as a place to stash diagnostics if compilation fails; if this
553  * information is not needed, a null pointer may be used.
554  */
555 ZOOM_API(int)
556 ZOOM_query_cql2rpn(ZOOM_query s, const char *str, ZOOM_connection conn)
557 {
558     char *rpn;
559     int ret;
560
561     yaz_log(log_details, "%p ZOOM_query_cql2rpn str=%s conn=%p", s, str, conn);
562     if (conn == 0)
563         conn = ZOOM_connection_create(0);
564
565     if ((rpn = cql2pqf(conn, str)) == 0)
566         return -1;
567
568     ret = ZOOM_query_prefix(s, rpn);
569     xfree(rpn);
570     return ret;
571 }
572
573 ZOOM_API(int)
574 ZOOM_query_sortby(ZOOM_query s, const char *criteria)
575 {
576     s->sort_spec = yaz_sort_spec (s->odr, criteria);
577     if (!s->sort_spec)
578     {
579         yaz_log(log_details, "%p ZOOM_query_sortby criteria=%s failed",
580                 s, criteria);
581         return -1;
582     }
583     yaz_log(log_details, "%p ZOOM_query_sortby criteria=%s", s, criteria);
584     return 0;
585 }
586
587 static zoom_ret do_write(ZOOM_connection c);
588
589 ZOOM_API(void)
590 ZOOM_connection_destroy(ZOOM_connection c)
591 {
592     ZOOM_resultset r;
593     if (!c)
594         return;
595     yaz_log(log_api, "%p ZOOM_connection_destroy", c);
596     if (c->cs)
597         cs_close (c->cs);
598     for (r = c->resultsets; r; r = r->next)
599         r->connection = 0;
600
601     xfree (c->buf_in);
602     xfree (c->addinfo);
603     xfree (c->diagset);
604     odr_destroy (c->odr_in);
605     odr_destroy (c->odr_out);
606     ZOOM_options_destroy (c->options);
607     ZOOM_connection_remove_tasks (c);
608     xfree (c->host_port);
609     xfree (c->path);
610     xfree (c->proxy);
611     xfree (c->charset);
612     xfree (c->lang);
613     xfree (c->cookie_out);
614     xfree (c->cookie_in);
615     xfree (c->client_IP);
616     xfree (c);
617 }
618
619 void ZOOM_resultset_addref (ZOOM_resultset r)
620 {
621     if (r)
622     {
623         (r->refcount)++;
624         yaz_log(log_details, "%p ZOOM_resultset_addref count=%d",
625                 r, r->refcount);
626     }
627 }
628
629 ZOOM_resultset ZOOM_resultset_create ()
630 {
631     ZOOM_resultset r = (ZOOM_resultset) xmalloc (sizeof(*r));
632
633     initlog();
634
635     yaz_log(log_details, "%p ZOOM_resultset_create", r);
636     r->refcount = 1;
637     r->size = 0;
638     r->odr = odr_createmem (ODR_ENCODE);
639     r->start = 0;
640     r->piggyback = 1;
641     r->setname = 0;
642     r->schema = 0;
643     r->count = 0;
644     r->step = 0;
645     r->record_cache = 0;
646     r->r_sort_spec = 0;
647     r->query = 0;
648     r->connection = 0;
649     r->next = 0;
650     return r;
651 }
652
653 ZOOM_API(ZOOM_resultset)
654 ZOOM_connection_search_pqf(ZOOM_connection c, const char *q)
655 {
656     ZOOM_resultset r;
657     ZOOM_query s = ZOOM_query_create();
658
659     ZOOM_query_prefix (s, q);
660
661     r = ZOOM_connection_search (c, s);
662     ZOOM_query_destroy (s);
663     return r;
664 }
665
666 ZOOM_API(ZOOM_resultset)
667 ZOOM_connection_search(ZOOM_connection c, ZOOM_query q)
668 {
669     ZOOM_resultset r = ZOOM_resultset_create ();
670     ZOOM_task task;
671     const char *cp;
672
673     yaz_log(log_api, "%p ZOOM_connection_search set %p query %p", c, r, q);
674     r->r_sort_spec = q->sort_spec;
675     r->query = q;
676
677     r->options = ZOOM_options_create_with_parent(c->options);
678
679     r->start = ZOOM_options_get_int(r->options, "start", 0);
680     r->count = ZOOM_options_get_int(r->options, "count", 0);
681     {
682         /* If "presentChunk" is defined use that; otherwise "step" */
683         const char *cp = ZOOM_options_get (r->options, "presentChunk");
684         r->step = ZOOM_options_get_int(r->options,
685                                        (cp != 0 ? "presentChunk": "step"), 0);
686     }
687     r->piggyback = ZOOM_options_get_bool (r->options, "piggyback", 1);
688     cp = ZOOM_options_get (r->options, "setname");
689     if (cp)
690         r->setname = xstrdup(cp);
691     cp = ZOOM_options_get (r->options, "schema");
692     if (cp)
693         r->schema = xstrdup(cp);
694     
695     r->connection = c;
696
697     r->next = c->resultsets;
698     c->resultsets = r;
699
700     if (c->host_port && c->proto == PROTO_HTTP)
701     {
702         if (!c->cs)
703         {
704             yaz_log(log_details, "ZOOM_connection_search: no comstack");
705             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
706         }
707         else
708         {
709             yaz_log(log_details, "ZOOM_connection_search: reconnect");
710             c->reconnect_ok = 1;
711         }
712     }
713
714     task = ZOOM_connection_add_task (c, ZOOM_TASK_SEARCH);
715     task->u.search.resultset = r;
716     ZOOM_resultset_addref (r);  
717
718     (q->refcount)++;
719
720     if (!c->async)
721     {
722         while (ZOOM_event (1, &c))
723             ;
724     }
725     return r;
726 }
727
728 /*
729  * This is the old result-set sorting API, which is maintained only
730  * for the sake of binary compatibility.  There is no reason ever to
731  * use this rather than ZOOM_resultset_sort1().
732  */
733 ZOOM_API(void)
734 ZOOM_resultset_sort(ZOOM_resultset r,
735                     const char *sort_type, const char *sort_spec)
736 {
737     (void) ZOOM_resultset_sort1(r, sort_type, sort_spec);
738 }
739
740 ZOOM_API(int)
741 ZOOM_resultset_sort1(ZOOM_resultset r,
742                      const char *sort_type, const char *sort_spec)
743 {
744     ZOOM_connection c = r->connection;
745     ZOOM_task task;
746     ZOOM_query newq;
747
748     newq = ZOOM_query_create();
749     if (ZOOM_query_sortby(newq, sort_spec) < 0)
750         return -1;
751
752     yaz_log(log_api, "%p ZOOM_resultset_sort r=%p sort_type=%s sort_spec=%s",
753             r, r, sort_type, sort_spec);
754     if (!c)
755         return 0;
756
757     if (c->host_port && c->proto == PROTO_HTTP)
758     {
759         if (!c->cs)
760         {
761             yaz_log(log_details, "%p ZOOM_resultset_sort: no comstack", r);
762             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
763         }
764         else
765         {
766             yaz_log(log_details, "%p ZOOM_resultset_sort: prepare reconnect",
767                     r);
768             c->reconnect_ok = 1;
769         }
770     }
771     
772     ZOOM_resultset_cache_reset(r);
773     task = ZOOM_connection_add_task (c, ZOOM_TASK_SORT);
774     task->u.sort.resultset = r;
775     task->u.sort.q = newq;
776
777     ZOOM_resultset_addref (r);  
778
779     if (!c->async)
780     {
781         while (ZOOM_event (1, &c))
782             ;
783     }
784
785     return 0;
786 }
787
788 ZOOM_API(void)
789     ZOOM_resultset_cache_reset(ZOOM_resultset r)
790 {
791     ZOOM_record_cache rc;
792     
793     for (rc = r->record_cache; rc; rc = rc->next)
794     {
795         if (rc->rec.wrbuf_marc)
796             wrbuf_free (rc->rec.wrbuf_marc, 1);
797         if (rc->rec.wrbuf_iconv)
798             wrbuf_free (rc->rec.wrbuf_iconv, 1);
799         if (rc->rec.wrbuf_opac)
800             wrbuf_free (rc->rec.wrbuf_opac, 1);
801     }
802     r->record_cache = 0;
803 }
804
805 ZOOM_API(void)
806 ZOOM_resultset_destroy(ZOOM_resultset r)
807 {
808     if (!r)
809         return;
810     (r->refcount)--;
811     yaz_log(log_details, "%p ZOOM_resultset_destroy r=%p count=%d",
812             r, r, r->refcount);
813     if (r->refcount == 0)
814     {
815         ZOOM_resultset_cache_reset(r);
816
817         if (r->connection)
818         {
819             /* remove ourselves from the resultsets in connection */
820             ZOOM_resultset *rp = &r->connection->resultsets;
821             while (1)
822             {
823                 assert (*rp);   /* we must be in this list!! */
824                 if (*rp == r)
825                 {   /* OK, we're here - take us out of it */
826                     *rp = (*rp)->next;
827                     break;
828                 }
829                 rp = &(*rp)->next;
830             }
831         }
832         ZOOM_query_destroy (r->query);
833         ZOOM_options_destroy (r->options);
834         odr_destroy (r->odr);
835         xfree (r->setname);
836         xfree (r->schema);
837         xfree (r);
838     }
839 }
840
841 ZOOM_API(size_t)
842 ZOOM_resultset_size (ZOOM_resultset r)
843 {
844     return r->size;
845 }
846
847 static void do_close (ZOOM_connection c)
848 {
849     if (c->cs)
850         cs_close(c->cs);
851     c->cs = 0;
852     c->mask = 0;
853     c->state = STATE_IDLE;
854 }
855
856 static void ZOOM_resultset_retrieve (ZOOM_resultset r,
857                                      int force_sync, int start, int count)
858 {
859     ZOOM_task task;
860     ZOOM_connection c;
861     const char *cp;
862
863     if (!r)
864         return;
865     yaz_log(log_details, "%p ZOOM_resultset_retrieve force_sync=%d start=%d"
866             " count=%d", r, force_sync, start, count);
867     c = r->connection;
868     if (!c)
869         return;
870
871     if (c->host_port && c->proto == PROTO_HTTP)
872     {
873         if (!c->cs)
874         {
875             yaz_log(log_details, "%p ZOOM_resultset_retrieve: no comstack", r);
876             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
877         }
878         else
879         {
880             yaz_log(log_details, "%p ZOOM_resultset_retrieve: prepare "
881                     "reconnect", r);
882             c->reconnect_ok = 1;
883         }
884     }
885     task = ZOOM_connection_add_task (c, ZOOM_TASK_RETRIEVE);
886     task->u.retrieve.resultset = r;
887     task->u.retrieve.start = start;
888     task->u.retrieve.count = count;
889
890     cp = ZOOM_options_get (r->options, "schema");
891     if (cp)
892     {
893         if (!r->schema || strcmp(r->schema, cp))
894         {
895             xfree(r->schema);
896             r->schema = xstrdup(cp);
897         }
898     }
899
900     ZOOM_resultset_addref (r);
901
902     if (!r->connection->async || force_sync)
903         while (r->connection && ZOOM_event (1, &r->connection))
904             ;
905 }
906
907 ZOOM_API(void)
908 ZOOM_resultset_records (ZOOM_resultset r, ZOOM_record *recs,
909                         size_t start, size_t count)
910 {
911     int force_present = 0;
912
913     if (!r)
914         return ;
915     yaz_log(log_api, "%p ZOOM_resultset_records r=%p start=%d count=%d",
916             r, r, start, count);
917     if (count && recs)
918         force_present = 1;
919     ZOOM_resultset_retrieve (r, force_present, start, count);
920     if (force_present)
921     {
922         size_t i;
923         for (i = 0; i< count; i++)
924             recs[i] = ZOOM_resultset_record_immediate (r, i+start);
925     }
926 }
927
928 static void get_cert(ZOOM_connection c)
929 {
930     char *cert_buf;
931     int cert_len;
932     
933     if (cs_get_peer_certificate_x509(c->cs, &cert_buf, &cert_len))
934     {
935         ZOOM_connection_option_setl(c, "sslPeerCert",
936                                     cert_buf, cert_len);
937         xfree(cert_buf);
938     }
939 }
940
941 static zoom_ret do_connect (ZOOM_connection c)
942 {
943     void *add;
944     const char *effective_host;
945
946     if (c->proxy)
947         effective_host = c->proxy;
948     else
949         effective_host = c->host_port;
950
951     yaz_log(log_details, "%p do_connect effective_host=%s", c, effective_host);
952
953     if (c->cs)
954         cs_close(c->cs);
955     c->cs = cs_create_host (effective_host, 0, &add);
956
957     if (c->cs && c->cs->protocol == PROTO_HTTP)
958     {
959 #if HAVE_XML2
960         const char *path = 0;
961
962         c->proto = PROTO_HTTP;
963         cs_get_host_args(c->host_port, &path);
964         xfree(c->path);
965         c->path = (char*) xmalloc(strlen(path)+2);
966         c->path[0] = '/';
967         strcpy (c->path+1, path);
968 #else
969         set_ZOOM_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, "SRW");
970         do_close(c);
971         return zoom_complete;
972 #endif
973     }
974     if (c->cs)
975     {
976         int ret = cs_connect (c->cs, add);
977         if (ret == 0)
978         {
979             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
980             ZOOM_connection_put_event(c, event);
981             get_cert(c);
982             if (c->proto == PROTO_Z3950)
983                 ZOOM_connection_send_init(c);
984             else
985             {
986                 /* no init request for SRW .. */
987                 assert (c->tasks->which == ZOOM_TASK_CONNECT);
988                 ZOOM_connection_remove_task (c);
989                 c->mask = 0;
990                 ZOOM_connection_exec_task (c);
991             }
992             c->state = STATE_ESTABLISHED;
993             return zoom_pending;
994         }
995         else if (ret > 0)
996         {
997             c->state = STATE_CONNECTING; 
998             c->mask = ZOOM_SELECT_EXCEPT;
999             if (c->cs->io_pending & CS_WANT_WRITE)
1000                 c->mask += ZOOM_SELECT_WRITE;
1001             if (c->cs->io_pending & CS_WANT_READ)
1002                 c->mask += ZOOM_SELECT_READ;
1003             return zoom_pending;
1004         }
1005     }
1006     c->state = STATE_IDLE;
1007     set_ZOOM_error(c, ZOOM_ERROR_CONNECT, c->host_port);
1008     return zoom_complete;
1009 }
1010
1011 int z3950_connection_socket(ZOOM_connection c)
1012 {
1013     if (c->cs)
1014         return cs_fileno(c->cs);
1015     return -1;
1016 }
1017
1018 int z3950_connection_mask(ZOOM_connection c)
1019 {
1020     if (c->cs)
1021         return c->mask;
1022     return 0;
1023 }
1024
1025 static void otherInfo_attach (ZOOM_connection c, Z_APDU *a, ODR out)
1026 {
1027     int i;
1028     for (i = 0; i<200; i++)
1029     {
1030         size_t len;
1031         Z_OtherInformation **oi;
1032         char buf[80];
1033         const char *val;
1034         const char *cp;
1035         int oidval;
1036
1037         sprintf (buf, "otherInfo%d", i);
1038         val = ZOOM_options_get (c->options, buf);
1039         if (!val)
1040             break;
1041         cp = strchr(val, ':');
1042         if (!cp)
1043             continue;
1044         len = cp - val;
1045         if (len >= sizeof(buf))
1046             len = sizeof(buf)-1;
1047         memcpy (buf, val, len);
1048         buf[len] = '\0';
1049         oidval = oid_getvalbyname (buf);
1050         if (oidval == VAL_NONE)
1051             continue;
1052         
1053         yaz_oi_APDU(a, &oi);
1054         yaz_oi_set_string_oidval(oi, out, oidval, 1, cp+1);
1055     }
1056 }
1057
1058 static int encode_APDU(ZOOM_connection c, Z_APDU *a, ODR out)
1059 {
1060     assert (a);
1061     if (c->cookie_out)
1062     {
1063         Z_OtherInformation **oi;
1064         yaz_oi_APDU(a, &oi);
1065         yaz_oi_set_string_oidval(oi, out, VAL_COOKIE, 1, c->cookie_out);
1066     }
1067     if (c->client_IP)
1068     {
1069         Z_OtherInformation **oi;
1070         yaz_oi_APDU(a, &oi);
1071         yaz_oi_set_string_oidval(oi, out, VAL_CLIENT_IP, 1, c->client_IP);
1072     }
1073     otherInfo_attach (c, a, out);
1074     if (!z_APDU(out, &a, 0, 0))
1075     {
1076         FILE *outf = fopen("/tmp/apdu.txt", "a");
1077         if (a && outf)
1078         {
1079             ODR odr_pr = odr_createmem(ODR_PRINT);
1080             fprintf (outf, "a=%p\n", a);
1081             odr_setprint(odr_pr, outf);
1082             z_APDU(odr_pr, &a, 0, 0);
1083             odr_destroy(odr_pr);
1084         }
1085         yaz_log(log_api, "%p encoding_APDU: encoding failed", c);
1086         set_ZOOM_error(c, ZOOM_ERROR_ENCODE, 0);
1087         odr_reset(out);
1088         return -1;
1089     }
1090     yaz_log(log_details, "%p encoding_APDU encoding OK", c);
1091     return 0;
1092 }
1093
1094 static zoom_ret send_APDU (ZOOM_connection c, Z_APDU *a)
1095 {
1096     ZOOM_Event event;
1097     assert (a);
1098     if (encode_APDU(c, a, c->odr_out))
1099         return zoom_complete;
1100     yaz_log(log_details, "%p send APDU type=%d", c, a->which);
1101     c->buf_out = odr_getbuf(c->odr_out, &c->len_out, 0);
1102     event = ZOOM_Event_create (ZOOM_EVENT_SEND_APDU);
1103     ZOOM_connection_put_event (c, event);
1104     odr_reset(c->odr_out);
1105     return do_write (c);
1106 }
1107
1108 /* returns 1 if PDU was sent OK (still pending )
1109            0 if PDU was not sent OK (nothing to wait for) 
1110 */
1111
1112 static zoom_ret ZOOM_connection_send_init (ZOOM_connection c)
1113 {
1114     Z_APDU *apdu = zget_APDU(c->odr_out, Z_APDU_initRequest);
1115     Z_InitRequest *ireq = apdu->u.initRequest;
1116     Z_IdAuthentication *auth = (Z_IdAuthentication *)
1117         odr_malloc(c->odr_out, sizeof(*auth));
1118     const char *auth_groupId = ZOOM_options_get (c->options, "group");
1119     const char *auth_userId = ZOOM_options_get (c->options, "user");
1120     const char *auth_password = ZOOM_options_get (c->options, "password");
1121     char *version;
1122
1123     /* support the pass for backwards compatibility */
1124     if (!auth_password)
1125         auth_password = ZOOM_options_get (c->options, "pass");
1126         
1127     ODR_MASK_SET(ireq->options, Z_Options_search);
1128     ODR_MASK_SET(ireq->options, Z_Options_present);
1129     ODR_MASK_SET(ireq->options, Z_Options_scan);
1130     ODR_MASK_SET(ireq->options, Z_Options_sort);
1131     ODR_MASK_SET(ireq->options, Z_Options_extendedServices);
1132     ODR_MASK_SET(ireq->options, Z_Options_namedResultSets);
1133     
1134     ODR_MASK_SET(ireq->protocolVersion, Z_ProtocolVersion_1);
1135     ODR_MASK_SET(ireq->protocolVersion, Z_ProtocolVersion_2);
1136     ODR_MASK_SET(ireq->protocolVersion, Z_ProtocolVersion_3);
1137
1138     /* Index Data's Z39.50 Implementor Id is 81 */
1139     ireq->implementationId = odr_prepend(c->odr_out,
1140         ZOOM_options_get(c->options, "implementationId"),
1141         odr_prepend(c->odr_out, "81", ireq->implementationId));
1142
1143     ireq->implementationName = odr_prepend(c->odr_out,
1144         ZOOM_options_get(c->options, "implementationName"),
1145         odr_prepend(c->odr_out, "ZOOM-C", ireq->implementationName));
1146
1147     version = odr_strdup(c->odr_out, "$Revision: 1.64 $");
1148     if (strlen(version) > 10)   /* check for unexpanded CVS strings */
1149         version[strlen(version)-2] = '\0';
1150     ireq->implementationVersion = odr_prepend(c->odr_out,
1151         ZOOM_options_get(c->options, "implementationVersion"),
1152         odr_prepend(c->odr_out, &version[11], ireq->implementationVersion));
1153
1154     *ireq->maximumRecordSize =
1155         ZOOM_options_get_int (c->options, "maximumRecordSize", 1024*1024);
1156     *ireq->preferredMessageSize =
1157         ZOOM_options_get_int (c->options, "preferredMessageSize", 1024*1024);
1158     
1159     if (auth_groupId || auth_password)
1160     {
1161         Z_IdPass *pass = (Z_IdPass *) odr_malloc(c->odr_out, sizeof(*pass));
1162         int i = 0;
1163         pass->groupId = 0;
1164         if (auth_groupId && *auth_groupId)
1165         {
1166             pass->groupId = (char *)
1167                 odr_malloc(c->odr_out, strlen(auth_groupId)+1);
1168             strcpy(pass->groupId, auth_groupId);
1169             i++;
1170         }
1171         pass->userId = 0;
1172         if (auth_userId && *auth_userId)
1173         {
1174             pass->userId = (char *)
1175                 odr_malloc(c->odr_out, strlen(auth_userId)+1);
1176             strcpy(pass->userId, auth_userId);
1177             i++;
1178         }
1179         pass->password = 0;
1180         if (auth_password && *auth_password)
1181         {
1182             pass->password = (char *)
1183                 odr_malloc(c->odr_out, strlen(auth_password)+1);
1184             strcpy(pass->password, auth_password);
1185             i++;
1186         }
1187         if (i)
1188         {
1189             auth->which = Z_IdAuthentication_idPass;
1190             auth->u.idPass = pass;
1191             ireq->idAuthentication = auth;
1192         }
1193     }
1194     else if (auth_userId)
1195     {
1196         auth->which = Z_IdAuthentication_open;
1197         auth->u.open = (char *)
1198             odr_malloc(c->odr_out, strlen(auth_userId)+1);
1199         strcpy(auth->u.open, auth_userId);
1200         ireq->idAuthentication = auth;
1201     }
1202     if (c->proxy)
1203         yaz_oi_set_string_oidval(&ireq->otherInfo, c->odr_out,
1204                                  VAL_PROXY, 1, c->host_port);
1205     if (c->charset || c->lang)
1206     {
1207         Z_OtherInformation **oi;
1208         Z_OtherInformationUnit *oi_unit;
1209         
1210         yaz_oi_APDU(apdu, &oi);
1211         
1212         if ((oi_unit = yaz_oi_update(oi, c->odr_out, NULL, 0, 0)))
1213         {
1214             char **charsets_addresses = 0;
1215             char **langs_addresses = 0;
1216             int charsets_count = 0;
1217             int langs_count = 0;
1218            
1219             if (c->charset)
1220                 nmem_strsplit_blank(c->odr_out->mem, c->charset,
1221                                     &charsets_addresses, &charsets_count);
1222             if (c->lang)
1223                 nmem_strsplit_blank(c->odr_out->mem, c->lang,
1224                                     &langs_addresses, &langs_count);
1225             ODR_MASK_SET(ireq->options, Z_Options_negotiationModel);
1226             oi_unit->which = Z_OtherInfo_externallyDefinedInfo;
1227             oi_unit->information.externallyDefinedInfo =
1228                 yaz_set_proposal_charneg(c->odr_out,
1229                                          (const char **) charsets_addresses,
1230                                          charsets_count,
1231                                          (const char **) langs_addresses,
1232                                          langs_count, 
1233                                          1);
1234         }
1235     }
1236     assert (apdu);
1237     return send_APDU (c, apdu);
1238 }
1239
1240 #if HAVE_XML2
1241 static zoom_ret send_srw (ZOOM_connection c, Z_SRW_PDU *sr)
1242 {
1243     Z_SOAP_Handler h[2] = {
1244         {"http://www.loc.gov/zing/srw/", 0, (Z_SOAP_fun) yaz_srw_codec},
1245         {0, 0, 0}
1246     };
1247     ODR o = odr_createmem(ODR_ENCODE);
1248     int ret;
1249     Z_SOAP *p = (Z_SOAP*) odr_malloc(o, sizeof(*p));
1250     Z_GDU *gdu;
1251     ZOOM_Event event;
1252
1253     gdu = z_get_HTTP_Request(c->odr_out);
1254     gdu->u.HTTP_Request->path = c->path;
1255
1256     if (c->host_port)
1257     {
1258         const char *cp0 = strstr(c->host_port, "://");
1259         const char *cp1 = 0;
1260         if (cp0)
1261             cp0 = cp0+3;
1262         else
1263             cp0 = c->host_port;
1264
1265         cp1 = strchr(cp0, '/');
1266         if (!cp1)
1267             cp1 = cp0+strlen(cp0);
1268
1269         if (cp0 && cp1)
1270         {
1271             char *h = (char*) odr_malloc(c->odr_out, cp1 - cp0 + 1);
1272             memcpy (h, cp0, cp1 - cp0);
1273             h[cp1-cp0] = '\0';
1274             z_HTTP_header_add(c->odr_out, &gdu->u.HTTP_Request->headers,
1275                               "Host", h);
1276         }
1277     }
1278
1279     if (c->sru_mode == zoom_sru_get)
1280     {
1281         yaz_sru_get_encode(gdu->u.HTTP_Request, sr, c->odr_out, c->charset);
1282     }
1283     else if (c->sru_mode == zoom_sru_post)
1284     {
1285         yaz_sru_post_encode(gdu->u.HTTP_Request, sr, c->odr_out, c->charset);
1286     }
1287     else if (c->sru_mode == zoom_sru_soap)
1288     {
1289         z_HTTP_header_add_content_type(c->odr_out,
1290                                        &gdu->u.HTTP_Request->headers,
1291                                        "text/xml", c->charset);
1292
1293         z_HTTP_header_add(c->odr_out, &gdu->u.HTTP_Request->headers,
1294                           "SOAPAction", "\"\"");
1295         p->which = Z_SOAP_generic;
1296         p->u.generic = (Z_SOAP_Generic *) odr_malloc(o, sizeof(*p->u.generic));
1297         p->u.generic->no = 0;
1298         p->u.generic->ns = 0;
1299         p->u.generic->p = sr;
1300         p->ns = "http://schemas.xmlsoap.org/soap/envelope/";
1301         
1302         ret = z_soap_codec_enc(o, &p,
1303                                &gdu->u.HTTP_Request->content_buf,
1304                                &gdu->u.HTTP_Request->content_len, h,
1305                                c->charset);
1306
1307     }
1308     if (!z_GDU(c->odr_out, &gdu, 0, 0))
1309         return zoom_complete;
1310     c->buf_out = odr_getbuf(c->odr_out, &c->len_out, 0);
1311     odr_destroy(o);
1312         
1313     event = ZOOM_Event_create (ZOOM_EVENT_SEND_APDU);
1314     ZOOM_connection_put_event (c, event);
1315     odr_reset(c->odr_out);
1316     return do_write (c);
1317 }
1318 #endif
1319
1320 #if HAVE_XML2
1321 static zoom_ret ZOOM_connection_srw_send_search(ZOOM_connection c)
1322 {
1323     int i;
1324     ZOOM_resultset resultset = 0;
1325     Z_SRW_PDU *sr = 0;
1326     const char *recordPacking = 0;
1327
1328     if (c->error)                  /* don't continue on error */
1329         return zoom_complete;
1330     assert (c->tasks);
1331     if (c->tasks->which == ZOOM_TASK_SEARCH)
1332     {
1333         resultset = c->tasks->u.search.resultset;
1334         resultset->setname = xstrdup ("default");
1335         ZOOM_options_set (resultset->options, "setname", resultset->setname);
1336     }
1337     else if(c->tasks->which == ZOOM_TASK_RETRIEVE)
1338     {
1339         resultset = c->tasks->u.retrieve.resultset;
1340
1341         resultset->start = c->tasks->u.retrieve.start;
1342         resultset->count = c->tasks->u.retrieve.count;
1343         
1344         if (resultset->start >= resultset->size)
1345             return zoom_complete;
1346         if (resultset->start + resultset->count > resultset->size)
1347             resultset->count = resultset->size - resultset->start;
1348
1349         for (i = 0; i<resultset->count; i++)
1350         {
1351             ZOOM_record rec =
1352                 record_cache_lookup (resultset, i + resultset->start);
1353             if (!rec)
1354                 break;
1355         }
1356         if (i == resultset->count)
1357             return zoom_complete;
1358     }
1359     assert(resultset->query);
1360         
1361     sr = yaz_srw_get(c->odr_out, Z_SRW_searchRetrieve_request);
1362
1363     if (resultset->query->z_query->which == Z_Query_type_104
1364         && resultset->query->z_query->u.type_104->which == Z_External_CQL)
1365     {
1366
1367         sr->u.request->query_type = Z_SRW_query_type_cql;
1368         sr->u.request->query.cql =resultset->query->z_query->u.type_104->u.cql;
1369     }
1370     else if (resultset->query->z_query->which == Z_Query_type_1 &&
1371              resultset->query->z_query->u.type_1)
1372     {
1373         sr->u.request->query_type = Z_SRW_query_type_pqf;
1374         sr->u.request->query.pqf = resultset->query->query_string;
1375     }
1376     else
1377     {
1378         set_ZOOM_error(c, ZOOM_ERROR_UNSUPPORTED_QUERY, 0);
1379         return zoom_complete;
1380     }
1381     sr->u.request->startRecord = odr_intdup (c->odr_out, resultset->start + 1);
1382     sr->u.request->maximumRecords = odr_intdup (
1383         c->odr_out, resultset->step>0 ? resultset->step : resultset->count);
1384     sr->u.request->recordSchema = resultset->schema;
1385
1386     recordPacking = ZOOM_resultset_option_get (resultset, "recordPacking");
1387
1388     if (recordPacking)
1389         sr->u.request->recordPacking = odr_strdup(c->odr_out, recordPacking);
1390     
1391     return send_srw(c, sr);
1392 }
1393 #else
1394 static zoom_ret ZOOM_connection_srw_send_search(ZOOM_connection c)
1395 {
1396     return zoom_complete;
1397 }
1398 #endif
1399
1400 static zoom_ret ZOOM_connection_send_search (ZOOM_connection c)
1401 {
1402     ZOOM_resultset r;
1403     int lslb, ssub, mspn;
1404     const char *syntax;
1405     Z_APDU *apdu = zget_APDU(c->odr_out, Z_APDU_searchRequest);
1406     Z_SearchRequest *search_req = apdu->u.searchRequest;
1407     const char *elementSetName;
1408     const char *smallSetElementSetName;
1409     const char *mediumSetElementSetName;
1410
1411     assert (c->tasks);
1412     assert (c->tasks->which == ZOOM_TASK_SEARCH);
1413
1414     r = c->tasks->u.search.resultset;
1415
1416     yaz_log(log_details, "%p ZOOM_connection_send_search set=%p", c, r);
1417
1418     elementSetName =
1419         ZOOM_options_get (r->options, "elementSetName");
1420     smallSetElementSetName  =
1421         ZOOM_options_get (r->options, "smallSetElementSetName");
1422     mediumSetElementSetName =
1423         ZOOM_options_get (r->options, "mediumSetElementSetName");
1424
1425     if (!smallSetElementSetName)
1426         smallSetElementSetName = elementSetName;
1427
1428     if (!mediumSetElementSetName)
1429         mediumSetElementSetName = elementSetName;
1430
1431     assert (r);
1432     assert (r->query);
1433
1434     /* prepare query for the search request */
1435     search_req->query = r->query->z_query;
1436     if (!search_req->query)
1437     {
1438         set_ZOOM_error(c, ZOOM_ERROR_INVALID_QUERY, 0);
1439         return zoom_complete;
1440     }
1441
1442     search_req->databaseNames =
1443         set_DatabaseNames (c, r->options, &search_req->num_databaseNames);
1444
1445     /* get syntax (no need to provide unless piggyback is in effect) */
1446     syntax = ZOOM_options_get (r->options, "preferredRecordSyntax");
1447
1448     lslb = ZOOM_options_get_int (r->options, "largeSetLowerBound", -1);
1449     ssub = ZOOM_options_get_int (r->options, "smallSetUpperBound", -1);
1450     mspn = ZOOM_options_get_int (r->options, "mediumSetPresentNumber", -1);
1451     if (lslb != -1 && ssub != -1 && mspn != -1)
1452     {
1453         /* So're a Z39.50 expert? Let's hope you don't do sort */
1454         *search_req->largeSetLowerBound = lslb;
1455         *search_req->smallSetUpperBound = ssub;
1456         *search_req->mediumSetPresentNumber = mspn;
1457     }
1458     else if (r->start == 0 && r->count > 0
1459              && r->piggyback && !r->r_sort_spec && !r->schema)
1460     {
1461         /* Regular piggyback - do it unless we're going to do sort */
1462         *search_req->largeSetLowerBound = 2000000000;
1463         *search_req->smallSetUpperBound = 1;
1464         *search_req->mediumSetPresentNumber = r->step>0 ? r->step : r->count;
1465     }
1466     else
1467     {
1468         /* non-piggyback. Need not provide elementsets or syntaxes .. */
1469         smallSetElementSetName = 0;
1470         mediumSetElementSetName = 0;
1471         syntax = 0;
1472     }
1473     if (smallSetElementSetName && *smallSetElementSetName)
1474     {
1475         Z_ElementSetNames *esn = (Z_ElementSetNames *)
1476             odr_malloc (c->odr_out, sizeof(*esn));
1477         
1478         esn->which = Z_ElementSetNames_generic;
1479         esn->u.generic = odr_strdup (c->odr_out, smallSetElementSetName);
1480         search_req->smallSetElementSetNames = esn;
1481     }
1482     if (mediumSetElementSetName && *mediumSetElementSetName)
1483     {
1484         Z_ElementSetNames *esn = (Z_ElementSetNames *)
1485             odr_malloc (c->odr_out, sizeof(*esn));
1486         
1487         esn->which = Z_ElementSetNames_generic;
1488         esn->u.generic = odr_strdup (c->odr_out, mediumSetElementSetName);
1489         search_req->mediumSetElementSetNames = esn;
1490     }
1491     if (syntax)
1492         search_req->preferredRecordSyntax =
1493             yaz_str_to_z3950oid (c->odr_out, CLASS_RECSYN, syntax);
1494     
1495     if (!r->setname)
1496     {
1497         if (c->support_named_resultsets)
1498         {
1499             char setname[14];
1500             int ord;
1501             /* find the lowest unused ordinal so that we re-use
1502                result sets on the server. */
1503             for (ord = 1; ; ord++)
1504             {
1505                 ZOOM_resultset rp;
1506                 sprintf (setname, "%d", ord);
1507                 for (rp = c->resultsets; rp; rp = rp->next)
1508                     if (rp->setname && !strcmp (rp->setname, setname))
1509                         break;
1510                 if (!rp)
1511                     break;
1512             }
1513             r->setname = xstrdup (setname);
1514             yaz_log(log_details, "%p ZOOM_connection_send_search: allocating "
1515                     "set %s", c, r->setname);
1516         }
1517         else
1518         {
1519             yaz_log(log_details, "%p ZOOM_connection_send_search: using "
1520                     "default set", c);
1521             r->setname = xstrdup ("default");
1522         }
1523         ZOOM_options_set (r->options, "setname", r->setname);
1524     }
1525     search_req->resultSetName = odr_strdup(c->odr_out, r->setname);
1526     return send_APDU (c, apdu);
1527 }
1528
1529 static void response_default_diag(ZOOM_connection c, Z_DefaultDiagFormat *r)
1530 {
1531     int oclass;
1532     char *addinfo = 0;
1533
1534     switch (r->which)
1535     {
1536     case Z_DefaultDiagFormat_v2Addinfo:
1537         addinfo = r->u.v2Addinfo;
1538         break;
1539     case Z_DefaultDiagFormat_v3Addinfo:
1540         addinfo = r->u.v3Addinfo;
1541         break;
1542     }
1543     xfree (c->addinfo);
1544     c->addinfo = 0;
1545     set_dset_error(c, *r->condition,
1546                    yaz_z3950oid_to_str(r->diagnosticSetId, &oclass),
1547                    addinfo, 0);
1548 }
1549
1550 static void response_diag(ZOOM_connection c, Z_DiagRec *p)
1551 {
1552     if (p->which != Z_DiagRec_defaultFormat)
1553         set_ZOOM_error(c, ZOOM_ERROR_DECODE, 0);
1554     else
1555         response_default_diag(c, p->u.defaultFormat);
1556 }
1557
1558 ZOOM_API(ZOOM_record)
1559 ZOOM_record_clone (ZOOM_record srec)
1560 {
1561     char *buf;
1562     int size;
1563     ODR odr_enc;
1564     ZOOM_record nrec;
1565
1566     odr_enc = odr_createmem(ODR_ENCODE);
1567     if (!z_NamePlusRecord (odr_enc, &srec->npr, 0, 0))
1568         return 0;
1569     buf = odr_getbuf (odr_enc, &size, 0);
1570     
1571     nrec = (ZOOM_record) xmalloc (sizeof(*nrec));
1572     nrec->odr = odr_createmem(ODR_DECODE);
1573     nrec->wrbuf_marc = 0;
1574     nrec->wrbuf_iconv = 0;
1575     nrec->wrbuf_opac = 0;
1576     odr_setbuf (nrec->odr, buf, size, 0);
1577     z_NamePlusRecord (nrec->odr, &nrec->npr, 0, 0);
1578     
1579     odr_destroy (odr_enc);
1580     return nrec;
1581 }
1582
1583 ZOOM_API(ZOOM_record)
1584 ZOOM_resultset_record_immediate (ZOOM_resultset s,size_t pos)
1585 {
1586     return record_cache_lookup (s, pos);
1587 }
1588
1589 ZOOM_API(ZOOM_record)
1590 ZOOM_resultset_record (ZOOM_resultset r, size_t pos)
1591 {
1592     ZOOM_record rec = ZOOM_resultset_record_immediate(r, pos);
1593
1594     if (!rec)
1595     {
1596         ZOOM_resultset_retrieve (r, 1, pos, 1);
1597         rec = ZOOM_resultset_record_immediate (r, pos);
1598     }
1599     return rec;
1600 }
1601
1602 ZOOM_API(void)
1603 ZOOM_record_destroy (ZOOM_record rec)
1604 {
1605     if (!rec)
1606         return;
1607     if (rec->wrbuf_marc)
1608         wrbuf_free (rec->wrbuf_marc, 1);
1609     if (rec->wrbuf_iconv)
1610         wrbuf_free (rec->wrbuf_iconv, 1);
1611     if (rec->wrbuf_opac)
1612         wrbuf_free (rec->wrbuf_opac, 1);
1613     odr_destroy (rec->odr);
1614     xfree (rec);
1615 }
1616
1617 static const char *marc_iconv_return(ZOOM_record rec, int marc_type,
1618                                      int *len,
1619                                      const char *buf, int sz,
1620                                      const char *record_charset)
1621 {
1622     char to[40];
1623     char from[40];
1624     yaz_iconv_t cd = 0;
1625     yaz_marc_t mt = yaz_marc_create();
1626
1627     *from = '\0';
1628     strcpy(to, "UTF-8");
1629     if (record_charset && *record_charset)
1630     {
1631         /* Use "from,to" or just "from" */
1632         const char *cp = strchr(record_charset, ',');
1633         int clen = strlen(record_charset);
1634         if (cp && cp[1])
1635         {
1636             strncpy( to, cp+1, sizeof(to)-1);
1637             to[sizeof(to)-1] = '\0';
1638             clen = cp - record_charset;
1639         }
1640         if (clen > sizeof(from)-1)
1641             clen = sizeof(from)-1;
1642         
1643         if (clen)
1644             strncpy(from, record_charset, clen);
1645         from[clen] = '\0';
1646     }
1647
1648     if (*from && *to)
1649     {
1650         cd = yaz_iconv_open(to, from);
1651         yaz_marc_iconv(mt, cd);
1652     }
1653
1654     yaz_marc_xml(mt, marc_type);
1655     if (!rec->wrbuf_marc)
1656         rec->wrbuf_marc = wrbuf_alloc();
1657     wrbuf_rewind (rec->wrbuf_marc);
1658     if (yaz_marc_decode_wrbuf (mt, buf, sz, rec->wrbuf_marc) > 0)
1659     {
1660         yaz_marc_destroy(mt);
1661         if (cd)
1662             yaz_iconv_close(cd);
1663         if (len)
1664             *len = wrbuf_len(rec->wrbuf_marc);
1665         return wrbuf_buf(rec->wrbuf_marc);
1666     }
1667     yaz_marc_destroy(mt);
1668     if (cd)
1669         yaz_iconv_close(cd);
1670     return 0;
1671 }
1672
1673 static const char *record_iconv_return(ZOOM_record rec, int *len,
1674                                        const char *buf, int sz,
1675                                        const char *record_charset)
1676 {
1677     char to[40];
1678     char from[40];
1679     yaz_iconv_t cd = 0;
1680
1681     *from = '\0';
1682     strcpy(to, "UTF-8");
1683     if (record_charset && *record_charset)
1684     {
1685         /* Use "from,to" or just "from" */
1686         const char *cp = strchr(record_charset, ',');
1687         int clen = strlen(record_charset);
1688         if (cp && cp[1])
1689         {
1690             strncpy( to, cp+1, sizeof(to)-1);
1691             to[sizeof(to)-1] = '\0';
1692             clen = cp - record_charset;
1693         }
1694         if (clen > sizeof(from)-1)
1695             clen = sizeof(from)-1;
1696         
1697         if (clen)
1698             strncpy(from, record_charset, clen);
1699         from[clen] = '\0';
1700     }
1701
1702     if (*from && *to && (cd = yaz_iconv_open(to, from)))
1703     {
1704         char outbuf[12];
1705         size_t inbytesleft = sz;
1706         const char *inp = buf;
1707         
1708         if (!rec->wrbuf_iconv)
1709             rec->wrbuf_iconv = wrbuf_alloc();
1710
1711         wrbuf_rewind(rec->wrbuf_iconv);
1712
1713         while (inbytesleft)
1714         {
1715             size_t outbytesleft = sizeof(outbuf);
1716             char *outp = outbuf;
1717             size_t r = yaz_iconv (cd, (char**) &inp,
1718                                   &inbytesleft, 
1719                                   &outp, &outbytesleft);
1720             if (r == (size_t) (-1))
1721             {
1722                 int e = yaz_iconv_error(cd);
1723                 if (e != YAZ_ICONV_E2BIG)
1724                     break;
1725             }
1726             wrbuf_write(rec->wrbuf_iconv, outbuf, outp - outbuf);
1727         }
1728         wrbuf_puts(rec->wrbuf_iconv, "");
1729         buf = wrbuf_buf(rec->wrbuf_iconv);
1730         sz = wrbuf_len(rec->wrbuf_iconv);
1731         yaz_iconv_close(cd);
1732     }
1733     if (len)
1734         *len = sz;
1735     return buf;
1736 }
1737
1738 ZOOM_API(const char *)
1739 ZOOM_record_get (ZOOM_record rec, const char *type_spec, int *len)
1740 {
1741     char type[40];
1742     char charset[40];
1743     char xpath[512];
1744     const char *cp;
1745     int i;
1746     Z_NamePlusRecord *npr;
1747     
1748     if (len)
1749         *len = 0; /* default return */
1750         
1751     if (!rec)
1752         return 0;
1753     npr = rec->npr;
1754     if (!npr)
1755         return 0;
1756
1757     cp = type_spec;
1758     for (i = 0; cp[i] && i < sizeof(type)-1; i++)
1759     {
1760         if (cp[i] == ';' || cp[i] == ' ')
1761             break;
1762         type[i] = cp[i];
1763     }
1764     type[i] = '\0';
1765     charset[0] = '\0';
1766     while (type_spec[i] == ';')
1767     {
1768         i++;
1769         while (type_spec[i] == ' ')
1770             i++;
1771         if (!strncmp(type_spec+i, "charset=", 8))
1772         {
1773             int j = 0;
1774             i = i + 8; /* skip charset= */
1775             for (j = 0; type_spec[i]  && j < sizeof(charset)-1; i++, j++)
1776             {
1777                 if (type_spec[i] == ';' || type_spec[i] == ' ')
1778                     break;
1779                 charset[j] = cp[i];
1780             }
1781             charset[j] = '\0';
1782         }
1783         else if (!strncmp(type_spec+i, "xpath=", 6))
1784         {
1785             int j = 0; 
1786             i = i + 6;
1787             for (j = 0; type_spec[i] && j < sizeof(xpath)-1; i++, j++)
1788                 xpath[j] = cp[i];
1789             xpath[j] = '\0';
1790         } 
1791         while (type_spec[i] == ' ')
1792             i++;
1793     }
1794     if (!strcmp (type, "database"))
1795     {
1796         if (len)
1797             *len = (npr->databaseName ? strlen(npr->databaseName) : 0);
1798         return npr->databaseName;
1799     }
1800     else if (!strcmp (type, "syntax"))
1801     {
1802         const char *desc = 0;   
1803         if (npr->which == Z_NamePlusRecord_databaseRecord)
1804         {
1805             Z_External *r = (Z_External *) npr->u.databaseRecord;
1806             oident *ent = oid_getentbyoid(r->direct_reference);
1807             if (ent)
1808                 desc = ent->desc;
1809         }
1810         if (!desc)
1811             desc = "none";
1812         if (len)
1813             *len = strlen(desc);
1814         return desc;
1815     }
1816     if (npr->which != Z_NamePlusRecord_databaseRecord)
1817         return 0;
1818
1819     /* from now on - we have a database record .. */
1820     if (!strcmp (type, "render"))
1821     {
1822         Z_External *r = (Z_External *) npr->u.databaseRecord;
1823         oident *ent = oid_getentbyoid(r->direct_reference);
1824
1825         /* render bibliographic record .. */
1826         if (r->which == Z_External_OPAC)
1827         {
1828             r = r->u.opac->bibliographicRecord;
1829             if (!r)
1830                 return 0;
1831             ent = oid_getentbyoid(r->direct_reference);
1832         }
1833         if (r->which == Z_External_sutrs)
1834             return record_iconv_return(rec, len,
1835                                        (char*) r->u.sutrs->buf,
1836                                        r->u.sutrs->len,
1837                                        charset);
1838         else if (r->which == Z_External_octet)
1839         {
1840             const char *ret_buf;
1841             switch (ent->value)
1842             {
1843             case VAL_SOIF:
1844             case VAL_HTML:
1845             case VAL_SUTRS:
1846                 break;
1847             case VAL_TEXT_XML:
1848             case VAL_APPLICATION_XML:
1849                 break;
1850             default:
1851                 ret_buf = marc_iconv_return(
1852                     rec, YAZ_MARC_LINE, len,
1853                     (const char *) r->u.octet_aligned->buf,
1854                     r->u.octet_aligned->len,
1855                     charset);
1856                 if (ret_buf)
1857                     return ret_buf;
1858             }
1859             return record_iconv_return(rec, len,
1860                                        (const char *) r->u.octet_aligned->buf,
1861                                        r->u.octet_aligned->len,
1862                                        charset);
1863         }
1864         else if (r->which == Z_External_grs1)
1865         {
1866             if (!rec->wrbuf_marc)
1867                 rec->wrbuf_marc = wrbuf_alloc();
1868             wrbuf_rewind (rec->wrbuf_marc);
1869             yaz_display_grs1(rec->wrbuf_marc, r->u.grs1, 0);
1870             return record_iconv_return(rec, len,
1871                                        wrbuf_buf(rec->wrbuf_marc),
1872                                        wrbuf_len(rec->wrbuf_marc),
1873                                        charset);
1874         }
1875         return 0;
1876     }
1877     else if (!strcmp (type, "xml") || !strcmp(type, "oai"))
1878     {
1879         Z_External *r = (Z_External *) npr->u.databaseRecord;
1880         oident *ent = oid_getentbyoid(r->direct_reference);
1881
1882         /* render bibliographic record .. */
1883         if (r->which == Z_External_OPAC)
1884         {
1885             r = r->u.opac->bibliographicRecord;
1886             if (!r)
1887                 return 0;
1888             ent = oid_getentbyoid(r->direct_reference);
1889         }
1890         
1891         if (r->which == Z_External_sutrs)
1892             return record_iconv_return(rec, len,
1893                                        (const char *) r->u.sutrs->buf,
1894                                        r->u.sutrs->len,
1895                                        charset);
1896         else if (r->which == Z_External_octet)
1897         {
1898             const char *ret_buf;
1899             int marc_decode_type = YAZ_MARC_MARCXML;
1900
1901             if (!strcmp(type, "oai"))
1902                 marc_decode_type = YAZ_MARC_OAIMARC;
1903             switch (ent->value)
1904             {
1905             case VAL_SOIF:
1906             case VAL_HTML:
1907             case VAL_SUTRS:
1908                 break;
1909             case VAL_TEXT_XML:
1910             case VAL_APPLICATION_XML:
1911                 break;
1912             default:
1913                 ret_buf = marc_iconv_return(
1914                     rec, marc_decode_type, len,
1915                     (const char *) r->u.octet_aligned->buf,
1916                     r->u.octet_aligned->len,
1917                     charset);
1918                 if (ret_buf)
1919                     return ret_buf;
1920             }
1921             return record_iconv_return(rec, len,
1922                                        (const char *) r->u.octet_aligned->buf,
1923                                        r->u.octet_aligned->len,
1924                                        charset);
1925         }
1926         else if (r->which == Z_External_grs1)
1927         {
1928             if (len) *len = 5;
1929             return "GRS-1";
1930         }
1931         return 0;
1932     }
1933     else if (!strcmp (type, "raw"))
1934     {
1935         Z_External *r = (Z_External *) npr->u.databaseRecord;
1936         
1937         if (r->which == Z_External_sutrs)
1938         {
1939             if (len) *len = r->u.sutrs->len;
1940             return (const char *) r->u.sutrs->buf;
1941         }
1942         else if (r->which == Z_External_octet)
1943         {
1944             if (len) *len = r->u.octet_aligned->len;
1945             return (const char *) r->u.octet_aligned->buf;
1946         }
1947         else /* grs-1, explain, OPAC, ... */
1948         {
1949             if (len) *len = -1;
1950             return (const char *) npr->u.databaseRecord;
1951         }
1952         return 0;
1953     }
1954     else if (!strcmp (type, "ext"))
1955     {
1956         if (len) *len = -1;
1957         return (const char *) npr->u.databaseRecord;
1958     }
1959     else if (!strcmp (type, "opac"))
1960              
1961     {
1962         Z_External *r = (Z_External *) npr->u.databaseRecord;
1963         if (r->which == Z_External_OPAC)
1964         {
1965             if (!rec->wrbuf_opac)
1966                 rec->wrbuf_opac = wrbuf_alloc();
1967             wrbuf_rewind (rec->wrbuf_opac);
1968             yaz_display_OPAC(rec->wrbuf_opac, r->u.opac, 0);
1969             return record_iconv_return(rec, len,
1970                                        wrbuf_buf(rec->wrbuf_opac),
1971                                        wrbuf_len(rec->wrbuf_opac),
1972                                        charset);
1973         }
1974     }
1975     return 0;
1976 }
1977
1978 static int strcmp_null(const char *v1, const char *v2)
1979 {
1980     if (!v1 && !v2)
1981         return 0;
1982     if (!v1 || !v2)
1983         return -1;
1984     return strcmp(v1, v2);
1985 }
1986
1987 static void record_cache_add (ZOOM_resultset r, Z_NamePlusRecord *npr, 
1988                               int pos)
1989 {
1990     ZOOM_record_cache rc;
1991     const char *elementSetName =
1992         ZOOM_resultset_option_get (r, "elementSetName");
1993     const char *syntax = 
1994         ZOOM_resultset_option_get (r, "preferredRecordSyntax");
1995     
1996     ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_RECV_RECORD);
1997     ZOOM_connection_put_event(r->connection, event);
1998
1999     for (rc = r->record_cache; rc; rc = rc->next)
2000     {
2001         if (pos == rc->pos)
2002         {
2003             if (strcmp_null(r->schema, rc->schema))
2004                 continue;
2005             if (strcmp_null(elementSetName,rc->elementSetName))
2006                 continue;
2007             if (strcmp_null(syntax, rc->syntax))
2008                 continue;
2009             /* not destroying rc->npr (it's handled by nmem )*/
2010             rc->rec.npr = npr;
2011             /* keeping wrbuf_marc too */
2012             return;
2013         }
2014     }
2015     rc = (ZOOM_record_cache) odr_malloc (r->odr, sizeof(*rc));
2016     rc->rec.npr = npr; 
2017     rc->rec.odr = 0;
2018     rc->rec.wrbuf_marc = 0;
2019     rc->rec.wrbuf_iconv = 0;
2020     rc->rec.wrbuf_opac = 0;
2021     if (elementSetName)
2022         rc->elementSetName = odr_strdup (r->odr, elementSetName);
2023     else
2024         rc->elementSetName = 0;
2025
2026     if (syntax)
2027         rc->syntax = odr_strdup (r->odr, syntax);
2028     else
2029         rc->syntax = 0;
2030
2031     if (r->schema)
2032         rc->schema = odr_strdup (r->odr, r->schema);
2033     else
2034         rc->schema = 0;
2035
2036     rc->pos = pos;
2037     rc->next = r->record_cache;
2038     r->record_cache = rc;
2039 }
2040
2041 static ZOOM_record record_cache_lookup (ZOOM_resultset r, int pos)
2042 {
2043     ZOOM_record_cache rc;
2044     const char *elementSetName =
2045         ZOOM_resultset_option_get (r, "elementSetName");
2046     const char *syntax = 
2047         ZOOM_resultset_option_get (r, "preferredRecordSyntax");
2048     
2049     for (rc = r->record_cache; rc; rc = rc->next)
2050     {
2051         if (pos == rc->pos)
2052         {
2053             if (strcmp_null(r->schema, rc->schema))
2054                 continue;
2055             if (strcmp_null(elementSetName,rc->elementSetName))
2056                 continue;
2057             if (strcmp_null(syntax, rc->syntax))
2058                 continue;
2059             return &rc->rec;
2060         }
2061     }
2062     return 0;
2063 }
2064                                              
2065 static void handle_records (ZOOM_connection c, Z_Records *sr,
2066                             int present_phase)
2067 {
2068     ZOOM_resultset resultset;
2069
2070     if (!c->tasks)
2071         return ;
2072     switch (c->tasks->which)
2073     {
2074     case ZOOM_TASK_SEARCH:
2075         resultset = c->tasks->u.search.resultset;
2076         break;
2077     case ZOOM_TASK_RETRIEVE:
2078         resultset = c->tasks->u.retrieve.resultset;        
2079         break;
2080     default:
2081         return;
2082     }
2083     if (sr && sr->which == Z_Records_NSD)
2084         response_default_diag(c, sr->u.nonSurrogateDiagnostic);
2085     else if (sr && sr->which == Z_Records_multipleNSD)
2086     {
2087         if (sr->u.multipleNonSurDiagnostics->num_diagRecs >= 1)
2088             response_diag(c, sr->u.multipleNonSurDiagnostics->diagRecs[0]);
2089         else
2090             set_ZOOM_error(c, ZOOM_ERROR_DECODE, 0);
2091     }
2092     else 
2093     {
2094         if (resultset->count + resultset->start > resultset->size)
2095             resultset->count = resultset->size - resultset->start;
2096         if (resultset->count < 0)
2097             resultset->count = 0;
2098         if (sr && sr->which == Z_Records_DBOSD)
2099         {
2100             int i;
2101             NMEM nmem = odr_extract_mem (c->odr_in);
2102             Z_NamePlusRecordList *p =
2103                 sr->u.databaseOrSurDiagnostics;
2104             for (i = 0; i<p->num_records; i++)
2105             {
2106                 record_cache_add (resultset, p->records[i],
2107                                   i+ resultset->start);
2108             }
2109             /* transfer our response to search_nmem .. we need it later */
2110             nmem_transfer (resultset->odr->mem, nmem);
2111             nmem_destroy (nmem);
2112             if (present_phase && p->num_records == 0)
2113             {
2114                 /* present response and we didn't get any records! */
2115                 Z_NamePlusRecord *myrec = 
2116                     zget_surrogateDiagRec(resultset->odr, 0, 14, 0);
2117                 record_cache_add(resultset, myrec, resultset->start);
2118             }
2119         }
2120         else if (present_phase)
2121         {
2122             /* present response and we didn't get any records! */
2123             Z_NamePlusRecord *myrec = 
2124                 zget_surrogateDiagRec(resultset->odr, 0, 14, 0);
2125             record_cache_add(resultset, myrec, resultset->start);
2126         }
2127     }
2128 }
2129
2130 static void handle_present_response (ZOOM_connection c, Z_PresentResponse *pr)
2131 {
2132     handle_records (c, pr->records, 1);
2133 }
2134
2135 static void handle_queryExpressionTerm(ZOOM_options opt, const char *name,
2136                                        Z_Term *term)
2137 {
2138     switch (term->which)
2139     {
2140     case Z_Term_general:
2141         ZOOM_options_setl(opt, name,
2142                           (const char *)(term->u.general->buf), 
2143                           term->u.general->len);
2144         break;
2145     case Z_Term_characterString:
2146         ZOOM_options_set(opt, name, term->u.characterString);
2147         break;
2148     case Z_Term_numeric:
2149         ZOOM_options_set_int(opt, name, *term->u.numeric);
2150         break;
2151     }
2152 }
2153
2154 static void handle_queryExpression(ZOOM_options opt, const char *name,
2155                                    Z_QueryExpression *exp)
2156 {
2157     char opt_name[80];
2158     
2159     switch (exp->which)
2160     {
2161     case Z_QueryExpression_term:
2162         if (exp->u.term && exp->u.term->queryTerm)
2163         {
2164             sprintf(opt_name, "%s.term", name);
2165             handle_queryExpressionTerm(opt, opt_name, exp->u.term->queryTerm);
2166         }
2167         break;
2168     case Z_QueryExpression_query:
2169         break;
2170     }
2171 }
2172
2173 static void handle_searchResult(ZOOM_connection c, ZOOM_resultset resultset,
2174                                 Z_OtherInformation *o)
2175 {
2176     int i;
2177     for (i = 0; o && i < o->num_elements; i++)
2178     {
2179         if (o->list[i]->which == Z_OtherInfo_externallyDefinedInfo)
2180         {
2181             Z_External *ext = o->list[i]->information.externallyDefinedInfo;
2182             
2183             if (ext->which == Z_External_searchResult1)
2184             {
2185                 int j;
2186                 Z_SearchInfoReport *sr = ext->u.searchResult1;
2187                 
2188                 if (sr->num)
2189                     ZOOM_options_set_int(
2190                         resultset->options, "searchresult.size", sr->num);
2191
2192                 for (j = 0; j < sr->num; j++)
2193                 {
2194                     Z_SearchInfoReport_s *ent =
2195                         ext->u.searchResult1->elements[j];
2196                     char pref[80];
2197                     
2198                     sprintf(pref, "searchresult.%d", j);
2199
2200                     if (ent->subqueryId)
2201                     {
2202                         char opt_name[80];
2203                         sprintf(opt_name, "%s.id", pref);
2204                         ZOOM_options_set(resultset->options, opt_name,
2205                                          ent->subqueryId);
2206                     }
2207                     if (ent->subqueryExpression)
2208                     {
2209                         char opt_name[80];
2210                         sprintf(opt_name, "%s.subquery", pref);
2211                         handle_queryExpression(resultset->options, opt_name,
2212                                                ent->subqueryExpression);
2213                     }
2214                     if (ent->subqueryInterpretation)
2215                     {
2216                         char opt_name[80];
2217                         sprintf(opt_name, "%s.interpretation", pref);
2218                         handle_queryExpression(resultset->options, opt_name,
2219                                                ent->subqueryInterpretation);
2220                     }
2221                     if (ent->subqueryRecommendation)
2222                     {
2223                         char opt_name[80];
2224                         sprintf(opt_name, "%s.recommendation", pref);
2225                         handle_queryExpression(resultset->options, opt_name,
2226                                                ent->subqueryRecommendation);
2227                     }
2228                     if (ent->subqueryCount)
2229                     {
2230                         char opt_name[80];
2231                         sprintf(opt_name, "%s.count", pref);
2232                         ZOOM_options_set_int(resultset->options, opt_name,
2233                                              *ent->subqueryCount);
2234                     }                                             
2235                 }
2236             }
2237         }
2238     }
2239 }
2240
2241 static void handle_search_response (ZOOM_connection c, Z_SearchResponse *sr)
2242 {
2243     ZOOM_resultset resultset;
2244     ZOOM_Event event;
2245
2246     if (!c->tasks || c->tasks->which != ZOOM_TASK_SEARCH)
2247         return ;
2248
2249     event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH);
2250     ZOOM_connection_put_event(c, event);
2251
2252     resultset = c->tasks->u.search.resultset;
2253
2254     handle_searchResult(c, resultset, sr->additionalSearchInfo);
2255
2256     resultset->size = *sr->resultCount;
2257     handle_records (c, sr->records, 0);
2258 }
2259
2260 static void sort_response (ZOOM_connection c, Z_SortResponse *res)
2261 {
2262     if (res->diagnostics && res->num_diagnostics > 0)
2263         response_diag (c, res->diagnostics[0]);
2264 }
2265
2266 static int scan_response (ZOOM_connection c, Z_ScanResponse *res)
2267 {
2268     NMEM nmem = odr_extract_mem (c->odr_in);
2269     ZOOM_scanset scan;
2270
2271     if (!c->tasks || c->tasks->which != ZOOM_TASK_SCAN)
2272         return 0;
2273     scan = c->tasks->u.scan.scan;
2274
2275     if (res->entries && res->entries->nonsurrogateDiagnostics)
2276         response_diag(c, res->entries->nonsurrogateDiagnostics[0]);
2277     scan->scan_response = res;
2278     nmem_transfer (scan->odr->mem, nmem);
2279     if (res->stepSize)
2280         ZOOM_options_set_int (scan->options, "stepSize", *res->stepSize);
2281     if (res->positionOfTerm)
2282         ZOOM_options_set_int (scan->options, "position", *res->positionOfTerm);
2283     if (res->scanStatus)
2284         ZOOM_options_set_int (scan->options, "scanStatus", *res->scanStatus);
2285     if (res->numberOfEntriesReturned)
2286         ZOOM_options_set_int (scan->options, "number",
2287                               *res->numberOfEntriesReturned);
2288     nmem_destroy (nmem);
2289     return 1;
2290 }
2291
2292 static zoom_ret send_sort (ZOOM_connection c,
2293                            ZOOM_resultset resultset)
2294 {
2295     if (c->error)
2296         resultset->r_sort_spec = 0;
2297     if (resultset->r_sort_spec)
2298     {
2299         Z_APDU *apdu = zget_APDU(c->odr_out, Z_APDU_sortRequest);
2300         Z_SortRequest *req = apdu->u.sortRequest;
2301         
2302         req->num_inputResultSetNames = 1;
2303         req->inputResultSetNames = (Z_InternationalString **)
2304             odr_malloc (c->odr_out, sizeof(*req->inputResultSetNames));
2305         req->inputResultSetNames[0] =
2306             odr_strdup (c->odr_out, resultset->setname);
2307         req->sortedResultSetName = odr_strdup (c->odr_out, resultset->setname);
2308         req->sortSequence = resultset->r_sort_spec;
2309         resultset->r_sort_spec = 0;
2310         return send_APDU (c, apdu);
2311     }
2312     return zoom_complete;
2313 }
2314
2315 static zoom_ret send_present(ZOOM_connection c)
2316 {
2317     Z_APDU *apdu = 0;
2318     Z_PresentRequest *req = 0;
2319     int i = 0;
2320     const char *syntax = 0;
2321     const char *elementSetName = 0;
2322     ZOOM_resultset  resultset;
2323
2324     if (!c->tasks)
2325     {
2326         yaz_log(log_details, "%p send_present no tasks", c);
2327         return zoom_complete;
2328     }
2329     switch (c->tasks->which)
2330     {
2331     case ZOOM_TASK_SEARCH:
2332         resultset = c->tasks->u.search.resultset;
2333         break;
2334     case ZOOM_TASK_RETRIEVE:
2335         resultset = c->tasks->u.retrieve.resultset;
2336         resultset->start = c->tasks->u.retrieve.start;
2337         resultset->count = c->tasks->u.retrieve.count;
2338
2339         if (resultset->start >= resultset->size)
2340         {
2341             yaz_log(log_details, "%p send_present start=%d >= size=%d",
2342                     c, resultset->start, resultset->size);
2343             return zoom_complete;
2344         }
2345         if (resultset->start + resultset->count > resultset->size)
2346             resultset->count = resultset->size - resultset->start;
2347         break;
2348     default:
2349         return zoom_complete;
2350     }
2351     yaz_log(log_details, "%p send_present start=%d count=%d",
2352             c, resultset->start, resultset->count);
2353
2354     syntax = ZOOM_resultset_option_get (resultset, "preferredRecordSyntax");
2355     elementSetName = ZOOM_resultset_option_get (resultset, "elementSetName");
2356
2357     if (c->error)                  /* don't continue on error */
2358         return zoom_complete;
2359     if (resultset->start < 0)
2360         return zoom_complete;
2361     for (i = 0; i<resultset->count; i++)
2362     {
2363         ZOOM_record rec =
2364             record_cache_lookup (resultset, i + resultset->start);
2365         if (!rec)
2366             break;
2367     }
2368     if (i == resultset->count)
2369     {
2370         yaz_log(log_details, "%p send_present skip=%d no more to fetch", c, i);
2371         return zoom_complete;
2372     }
2373
2374     apdu = zget_APDU(c->odr_out, Z_APDU_presentRequest);
2375     req = apdu->u.presentRequest;
2376
2377     resultset->start += i;
2378     resultset->count -= i;
2379
2380     if (i)
2381         yaz_log(log_details, "%p send_present skip=%d", c, i);
2382
2383     *req->resultSetStartPoint = resultset->start + 1;
2384     *req->numberOfRecordsRequested = resultset->step>0 ?
2385         resultset->step : resultset->count;
2386     assert (*req->numberOfRecordsRequested > 0);
2387
2388
2389     if (syntax && *syntax)
2390         req->preferredRecordSyntax =
2391             yaz_str_to_z3950oid (c->odr_out, CLASS_RECSYN, syntax);
2392
2393     if (resultset->schema && *resultset->schema)
2394     {
2395         Z_RecordComposition *compo = (Z_RecordComposition *)
2396             odr_malloc (c->odr_out, sizeof(*compo));
2397
2398         req->recordComposition = compo;
2399         compo->which = Z_RecordComp_complex;
2400         compo->u.complex = (Z_CompSpec *)
2401             odr_malloc(c->odr_out, sizeof(*compo->u.complex));
2402         compo->u.complex->selectAlternativeSyntax = (bool_t *) 
2403             odr_malloc(c->odr_out, sizeof(bool_t));
2404         *compo->u.complex->selectAlternativeSyntax = 0;
2405
2406         compo->u.complex->generic = (Z_Specification *)
2407             odr_malloc(c->odr_out, sizeof(*compo->u.complex->generic));
2408
2409         compo->u.complex->generic->which = Z_Schema_oid;
2410         compo->u.complex->generic->schema.oid = (Odr_oid *)
2411             yaz_str_to_z3950oid (c->odr_out, CLASS_SCHEMA, resultset->schema);
2412
2413         if (!compo->u.complex->generic->schema.oid)
2414         {
2415             /* OID wasn't a schema! Try record syntax instead. */
2416
2417             compo->u.complex->generic->schema.oid = (Odr_oid *)
2418                 yaz_str_to_z3950oid (c->odr_out, CLASS_RECSYN, resultset->schema);
2419         }
2420         if (elementSetName && *elementSetName)
2421         {
2422             compo->u.complex->generic->elementSpec = (Z_ElementSpec *)
2423                 odr_malloc(c->odr_out, sizeof(Z_ElementSpec));
2424             compo->u.complex->generic->elementSpec->which =
2425                 Z_ElementSpec_elementSetName;
2426             compo->u.complex->generic->elementSpec->u.elementSetName =
2427                 odr_strdup (c->odr_out, elementSetName);
2428         }
2429         else
2430             compo->u.complex->generic->elementSpec = 0;
2431         compo->u.complex->num_dbSpecific = 0;
2432         compo->u.complex->dbSpecific = 0;
2433         compo->u.complex->num_recordSyntax = 0;
2434         compo->u.complex->recordSyntax = 0;
2435     }
2436     else if (elementSetName && *elementSetName)
2437     {
2438         Z_ElementSetNames *esn = (Z_ElementSetNames *)
2439             odr_malloc (c->odr_out, sizeof(*esn));
2440         Z_RecordComposition *compo = (Z_RecordComposition *)
2441             odr_malloc (c->odr_out, sizeof(*compo));
2442         
2443         esn->which = Z_ElementSetNames_generic;
2444         esn->u.generic = odr_strdup (c->odr_out, elementSetName);
2445         compo->which = Z_RecordComp_simple;
2446         compo->u.simple = esn;
2447         req->recordComposition = compo;
2448     }
2449     req->resultSetId = odr_strdup(c->odr_out, resultset->setname);
2450     return send_APDU (c, apdu);
2451 }
2452
2453 ZOOM_API(ZOOM_scanset)
2454 ZOOM_connection_scan (ZOOM_connection c, const char *start)
2455 {
2456     ZOOM_scanset s;
2457     ZOOM_query q = ZOOM_query_create();
2458
2459     ZOOM_query_prefix (q, start);
2460
2461     s = ZOOM_connection_scan1(c, q);
2462     ZOOM_query_destroy (q);
2463     return s;
2464
2465 }
2466
2467 ZOOM_API(ZOOM_scanset)
2468 ZOOM_connection_scan1 (ZOOM_connection c, ZOOM_query q)
2469 {
2470     ZOOM_scanset scan = (ZOOM_scanset) xmalloc (sizeof(*scan));
2471     char *start;
2472     char *freeme = 0;
2473
2474     scan->connection = c;
2475     scan->odr = odr_createmem (ODR_DECODE);
2476     scan->options = ZOOM_options_create_with_parent (c->options);
2477     scan->refcount = 1;
2478     scan->scan_response = 0;
2479
2480     /*
2481      * We need to check the query-type, so we can recognise CQL and
2482      * compile it into a form that we can use here.  The ZOOM_query
2483      * structure has no explicit `type' member, but inspection of the
2484      * ZOOM_query_prefix() and ZOOM_query_cql() functions shows how
2485      * the structure is set up in each case.
2486      */
2487
2488     if (q->z_query->which == Z_Query_type_1) {
2489         yaz_log(log_api, "%p ZOOM_connection_scan1 q=%p PQF '%s'",
2490                 c, q, q->query_string);
2491         start = q->query_string;
2492     } else if (q->z_query->which == Z_Query_type_104) {
2493         yaz_log(log_api, "%p ZOOM_connection_scan1 q=%p CQL '%s'",
2494                 c, q, q->query_string);
2495         start = freeme = cql2pqf(c, q->query_string);
2496         if (start == 0)
2497             return 0;
2498     } else {
2499         yaz_log(YLOG_FATAL, "%p ZOOM_connection_scan1 q=%p unknown type '%s'",
2500                 c, q, q->query_string);
2501         abort();
2502     }
2503
2504     scan->termListAndStartPoint =
2505         p_query_scan(scan->odr, PROTO_Z3950, &scan->attributeSet, start);
2506     xfree (freeme);
2507     if (scan->termListAndStartPoint != 0)
2508     {
2509         ZOOM_task task = ZOOM_connection_add_task (c, ZOOM_TASK_SCAN);
2510         task->u.scan.scan = scan;
2511         
2512         (scan->refcount)++;
2513         if (!c->async)
2514         {
2515             while (ZOOM_event (1, &c))
2516                 ;
2517         }
2518     }
2519     return scan;
2520 }
2521
2522 ZOOM_API(void)
2523 ZOOM_scanset_destroy (ZOOM_scanset scan)
2524 {
2525     if (!scan)
2526         return;
2527     (scan->refcount)--;
2528     if (scan->refcount == 0)
2529     {
2530         odr_destroy (scan->odr);
2531         
2532         ZOOM_options_destroy (scan->options);
2533         xfree (scan);
2534     }
2535 }
2536
2537 static zoom_ret send_package (ZOOM_connection c)
2538 {
2539     ZOOM_Event event;
2540
2541     yaz_log(log_details, "%p send_package", c);
2542     if (!c->tasks)
2543         return zoom_complete;
2544     assert (c->tasks->which == ZOOM_TASK_PACKAGE);
2545     
2546     event = ZOOM_Event_create (ZOOM_EVENT_SEND_APDU);
2547     ZOOM_connection_put_event (c, event);
2548     
2549     return do_write_ex (c, c->tasks->u.package->buf_out,
2550                         c->tasks->u.package->len_out);
2551 }
2552
2553 static zoom_ret send_scan (ZOOM_connection c)
2554 {
2555     ZOOM_scanset scan;
2556     Z_APDU *apdu = zget_APDU(c->odr_out, Z_APDU_scanRequest);
2557     Z_ScanRequest *req = apdu->u.scanRequest;
2558
2559     yaz_log(log_details, "%p send_scan", c);
2560     if (!c->tasks)
2561         return zoom_complete;
2562     assert (c->tasks->which == ZOOM_TASK_SCAN);
2563     scan = c->tasks->u.scan.scan;
2564
2565     req->termListAndStartPoint = scan->termListAndStartPoint;
2566     req->attributeSet = scan->attributeSet;
2567
2568     *req->numberOfTermsRequested =
2569         ZOOM_options_get_int(scan->options, "number", 10);
2570
2571     req->preferredPositionInResponse =
2572         odr_intdup (c->odr_out,
2573                     ZOOM_options_get_int(scan->options, "position", 1));
2574
2575     req->stepSize =
2576         odr_intdup (c->odr_out,
2577                     ZOOM_options_get_int(scan->options, "stepSize", 0));
2578     
2579     req->databaseNames = set_DatabaseNames (c, scan->options, 
2580                                             &req->num_databaseNames);
2581
2582     return send_APDU (c, apdu);
2583 }
2584
2585 ZOOM_API(size_t)
2586 ZOOM_scanset_size (ZOOM_scanset scan)
2587 {
2588     if (!scan || !scan->scan_response || !scan->scan_response->entries)
2589         return 0;
2590     return scan->scan_response->entries->num_entries;
2591 }
2592
2593 ZOOM_API(const char *)
2594 ZOOM_scanset_term (ZOOM_scanset scan, size_t pos,
2595                    int *occ, int *len)
2596 {
2597     const char *term = 0;
2598     size_t noent = ZOOM_scanset_size (scan);
2599     Z_ScanResponse *res = scan->scan_response;
2600     
2601     *len = 0;
2602     *occ = 0;
2603     if (pos >= noent)
2604         return 0;
2605     if (res->entries->entries[pos]->which == Z_Entry_termInfo)
2606     {
2607         Z_TermInfo *t = res->entries->entries[pos]->u.termInfo;
2608         
2609         if (t->term->which == Z_Term_general)
2610         {
2611             term = (const char *) t->term->u.general->buf;
2612             *len = t->term->u.general->len;
2613         }
2614         *occ = t->globalOccurrences ? *t->globalOccurrences : 0;
2615     }
2616     return term;
2617 }
2618
2619 ZOOM_API(const char *)
2620 ZOOM_scanset_display_term (ZOOM_scanset scan, size_t pos,
2621                            int *occ, int *len)
2622 {
2623     const char *term = 0;
2624     size_t noent = ZOOM_scanset_size (scan);
2625     Z_ScanResponse *res = scan->scan_response;
2626     
2627     *len = 0;
2628     *occ = 0;
2629     if (pos >= noent)
2630         return 0;
2631     if (res->entries->entries[pos]->which == Z_Entry_termInfo)
2632     {
2633         Z_TermInfo *t = res->entries->entries[pos]->u.termInfo;
2634
2635         if (t->displayTerm)
2636         {
2637             term = t->displayTerm;
2638             *len = strlen(term);
2639         }
2640         else if (t->term->which == Z_Term_general)
2641         {
2642             term = (const char *) t->term->u.general->buf;
2643             *len = t->term->u.general->len;
2644         }
2645         *occ = t->globalOccurrences ? *t->globalOccurrences : 0;
2646     }
2647     return term;
2648 }
2649
2650 ZOOM_API(const char *)
2651 ZOOM_scanset_option_get (ZOOM_scanset scan, const char *key)
2652 {
2653     return ZOOM_options_get (scan->options, key);
2654 }
2655
2656 ZOOM_API(void)
2657 ZOOM_scanset_option_set (ZOOM_scanset scan, const char *key,
2658                               const char *val)
2659 {
2660     ZOOM_options_set (scan->options, key, val);
2661 }
2662
2663 static Z_APDU *create_es_package (ZOOM_package p, int type)
2664 {
2665     const char *str;
2666     Z_APDU *apdu = zget_APDU(p->odr_out, Z_APDU_extendedServicesRequest);
2667     Z_ExtendedServicesRequest *req = apdu->u.extendedServicesRequest;
2668     
2669     *req->function = Z_ExtendedServicesRequest_create;
2670     
2671     str = ZOOM_options_get(p->options, "package-name");
2672     if (str && *str)
2673         req->packageName = nmem_strdup (p->odr_out->mem, str);
2674     
2675     str = ZOOM_options_get(p->options, "user-id");
2676     if (str)
2677         req->userId = nmem_strdup (p->odr_out->mem, str);
2678     
2679     req->packageType = yaz_oidval_to_z3950oid(p->odr_out, CLASS_EXTSERV,
2680                                               type);
2681
2682     str = ZOOM_options_get(p->options, "function");
2683     if (str)
2684     {
2685         if (!strcmp (str, "create"))
2686             *req->function = 1;
2687         if (!strcmp (str, "delete"))
2688             *req->function = 2;
2689         if (!strcmp (str, "modify"))
2690             *req->function = 3;
2691     }
2692     return apdu;
2693 }
2694
2695 static const char *ill_array_lookup (void *clientData, const char *idx)
2696 {
2697     ZOOM_package p = (ZOOM_package) clientData;
2698     return ZOOM_options_get (p->options, idx+4);
2699 }
2700
2701 static Z_External *encode_ill_request (ZOOM_package p)
2702 {
2703     ODR out = p->odr_out;
2704     ILL_Request *req;
2705     Z_External *r = 0;
2706     struct ill_get_ctl ctl;
2707         
2708     ctl.odr = p->odr_out;
2709     ctl.clientData = p;
2710     ctl.f = ill_array_lookup;
2711         
2712     req = ill_get_ILLRequest(&ctl, "ill", 0);
2713         
2714     if (!ill_Request (out, &req, 0, 0))
2715     {
2716         int ill_request_size;
2717         char *ill_request_buf = odr_getbuf (out, &ill_request_size, 0);
2718         if (ill_request_buf)
2719             odr_setbuf (out, ill_request_buf, ill_request_size, 1);
2720         return 0;
2721     }
2722     else
2723     {
2724         oident oid;
2725         int illRequest_size = 0;
2726         char *illRequest_buf = odr_getbuf (out, &illRequest_size, 0);
2727                 
2728         oid.proto = PROTO_GENERAL;
2729         oid.oclass = CLASS_GENERAL;
2730         oid.value = VAL_ISO_ILL_1;
2731                 
2732         r = (Z_External *) odr_malloc (out, sizeof(*r));
2733         r->direct_reference = odr_oiddup(out,oid_getoidbyent(&oid)); 
2734         r->indirect_reference = 0;
2735         r->descriptor = 0;
2736         r->which = Z_External_single;
2737                 
2738         r->u.single_ASN1_type =
2739             odr_create_Odr_oct(out,
2740                                (unsigned char *)illRequest_buf,
2741                                illRequest_size);
2742     }
2743     return r;
2744 }
2745
2746 static Z_ItemOrder *encode_item_order(ZOOM_package p)
2747 {
2748     Z_ItemOrder *req = (Z_ItemOrder *) odr_malloc (p->odr_out, sizeof(*req));
2749     const char *str;
2750     
2751     req->which = Z_IOItemOrder_esRequest;
2752     req->u.esRequest = (Z_IORequest *) 
2753         odr_malloc(p->odr_out,sizeof(Z_IORequest));
2754
2755     /* to keep part ... */
2756     req->u.esRequest->toKeep = (Z_IOOriginPartToKeep *)
2757         odr_malloc(p->odr_out,sizeof(Z_IOOriginPartToKeep));
2758     req->u.esRequest->toKeep->supplDescription = 0;
2759     req->u.esRequest->toKeep->contact = (Z_IOContact *)
2760         odr_malloc (p->odr_out, sizeof(*req->u.esRequest->toKeep->contact));
2761         
2762     str = ZOOM_options_get(p->options, "contact-name");
2763     req->u.esRequest->toKeep->contact->name = str ?
2764         nmem_strdup (p->odr_out->mem, str) : 0;
2765         
2766     str = ZOOM_options_get(p->options, "contact-phone");
2767     req->u.esRequest->toKeep->contact->phone = str ?
2768         nmem_strdup (p->odr_out->mem, str) : 0;
2769         
2770     str = ZOOM_options_get(p->options, "contact-email");
2771     req->u.esRequest->toKeep->contact->email = str ?
2772         nmem_strdup (p->odr_out->mem, str) : 0;
2773         
2774     req->u.esRequest->toKeep->addlBilling = 0;
2775         
2776     /* not to keep part ... */
2777     req->u.esRequest->notToKeep = (Z_IOOriginPartNotToKeep *)
2778         odr_malloc(p->odr_out,sizeof(Z_IOOriginPartNotToKeep));
2779         
2780     str = ZOOM_options_get(p->options, "itemorder-setname");
2781     if (!str)
2782         str = "default";
2783
2784     if (!*str) 
2785         req->u.esRequest->notToKeep->resultSetItem = 0;
2786     else
2787     {
2788         req->u.esRequest->notToKeep->resultSetItem = (Z_IOResultSetItem *)
2789            odr_malloc(p->odr_out, sizeof(Z_IOResultSetItem));
2790
2791         req->u.esRequest->notToKeep->resultSetItem->resultSetId =
2792            nmem_strdup (p->odr_out->mem, str);
2793         req->u.esRequest->notToKeep->resultSetItem->item =
2794             (int *) odr_malloc(p->odr_out, sizeof(int));
2795         
2796         str = ZOOM_options_get(p->options, "itemorder-item");
2797         *req->u.esRequest->notToKeep->resultSetItem->item =
2798             (str ? atoi(str) : 1);
2799     }
2800     req->u.esRequest->notToKeep->itemRequest = encode_ill_request(p);
2801     
2802     return req;
2803 }
2804
2805 Z_APDU *create_admin_package(ZOOM_package p, int type, 
2806                              Z_ESAdminOriginPartToKeep **toKeepP,
2807                              Z_ESAdminOriginPartNotToKeep **notToKeepP)
2808 {
2809     Z_APDU *apdu = create_es_package (p, VAL_ADMINSERVICE);
2810     if (apdu)
2811     {
2812         Z_ESAdminOriginPartToKeep  *toKeep;
2813         Z_ESAdminOriginPartNotToKeep  *notToKeep;
2814         Z_External *r = (Z_External *) odr_malloc (p->odr_out, sizeof(*r));
2815         const char *first_db = "Default";
2816         int num_db;
2817         char **db = set_DatabaseNames(p->connection, p->options, &num_db);
2818         if (num_db > 0)
2819             first_db = db[0];
2820             
2821         r->direct_reference =
2822             yaz_oidval_to_z3950oid(p->odr_out, CLASS_EXTSERV,
2823                                    VAL_ADMINSERVICE);
2824         r->descriptor = 0;
2825         r->indirect_reference = 0;
2826         r->which = Z_External_ESAdmin;
2827         
2828         r->u.adminService = (Z_Admin *)
2829             odr_malloc(p->odr_out, sizeof(*r->u.adminService));
2830         r->u.adminService->which = Z_Admin_esRequest;
2831         r->u.adminService->u.esRequest = (Z_AdminEsRequest *)
2832             odr_malloc(p->odr_out, sizeof(*r->u.adminService->u.esRequest));
2833         
2834         toKeep = r->u.adminService->u.esRequest->toKeep =
2835             (Z_ESAdminOriginPartToKeep *) 
2836             odr_malloc(p->odr_out, sizeof(*r->u.adminService->u.esRequest->toKeep));
2837         toKeep->which = type;
2838         toKeep->databaseName = odr_strdup(p->odr_out, first_db);
2839         toKeep->u.create = odr_nullval();
2840         apdu->u.extendedServicesRequest->taskSpecificParameters = r;
2841         
2842         r->u.adminService->u.esRequest->notToKeep = notToKeep =
2843             (Z_ESAdminOriginPartNotToKeep *)
2844             odr_malloc(p->odr_out,
2845                        sizeof(*r->u.adminService->u.esRequest->notToKeep));
2846         notToKeep->which = Z_ESAdminOriginPartNotToKeep_recordsWillFollow;
2847         notToKeep->u.recordsWillFollow = odr_nullval();
2848         if (toKeepP)
2849             *toKeepP = toKeep;
2850         if (notToKeepP)
2851             *notToKeepP = notToKeep;
2852     }
2853     return apdu;
2854 }
2855
2856 static Z_APDU *create_xmlupdate_package(ZOOM_package p)
2857 {
2858     Z_APDU *apdu = create_es_package(p, VAL_XMLES);
2859     Z_ExtendedServicesRequest *req = apdu->u.extendedServicesRequest;
2860     Z_External *ext = (Z_External *) odr_malloc(p->odr_out, sizeof(*ext));
2861     const char *doc = ZOOM_options_get(p->options, "doc");
2862
2863     if (!doc)
2864         doc = "";
2865
2866     req->taskSpecificParameters = ext;
2867     ext->direct_reference = req->packageType;
2868     ext->descriptor = 0;
2869     ext->indirect_reference = 0;
2870     
2871     ext->which = Z_External_octet;
2872     ext->u.single_ASN1_type =
2873         odr_create_Odr_oct(p->odr_out, (const unsigned char *) doc,
2874                            strlen(doc));
2875     return apdu;
2876 }
2877
2878 static Z_APDU *create_update_package(ZOOM_package p)
2879 {
2880     Z_APDU *apdu = 0;
2881     const char *first_db = "Default";
2882     int num_db;
2883     char **db = set_DatabaseNames(p->connection, p->options, &num_db);
2884     const char *action = ZOOM_options_get(p->options, "action");
2885     const char *recordIdOpaque = ZOOM_options_get(p->options, "recordIdOpaque");
2886     const char *recordIdNumber = ZOOM_options_get(p->options, "recordIdNumber");
2887     const char *record_buf = ZOOM_options_get(p->options, "record");
2888     const char *syntax_str = ZOOM_options_get(p->options, "syntax");
2889     int syntax_oid = VAL_NONE;
2890     int action_no = -1;
2891     
2892     if (syntax_str)
2893         syntax_oid = oid_getvalbyname(syntax_str);
2894     if (!record_buf)
2895     {
2896         record_buf = "void";
2897         syntax_oid = VAL_SUTRS;
2898     }
2899     if (syntax_oid == VAL_NONE)
2900         syntax_oid = VAL_TEXT_XML;
2901     
2902     if (num_db > 0)
2903         first_db = db[0];
2904     
2905     if (!action)
2906         action = "specialUpdate";
2907     
2908     if (!strcmp(action, "recordInsert"))
2909         action_no = Z_IUOriginPartToKeep_recordInsert;
2910     else if (!strcmp(action, "recordReplace"))
2911         action_no = Z_IUOriginPartToKeep_recordReplace;
2912     else if (!strcmp(action, "recordDelete"))
2913         action_no = Z_IUOriginPartToKeep_recordDelete;
2914     else if (!strcmp(action, "elementUpdate"))
2915         action_no = Z_IUOriginPartToKeep_elementUpdate;
2916     else if (!strcmp(action, "specialUpdate"))
2917         action_no = Z_IUOriginPartToKeep_specialUpdate;
2918     else
2919         return 0;
2920
2921     apdu = create_es_package (p, VAL_DBUPDATE);
2922     if (apdu)
2923     {
2924         Z_IUOriginPartToKeep *toKeep;
2925         Z_IUSuppliedRecords *notToKeep;
2926         Z_External *r = (Z_External *)
2927             odr_malloc (p->odr_out, sizeof(*r));
2928         
2929         apdu->u.extendedServicesRequest->taskSpecificParameters = r;
2930         
2931         r->direct_reference =
2932             yaz_oidval_to_z3950oid(p->odr_out, CLASS_EXTSERV,
2933                                    VAL_DBUPDATE);
2934         r->descriptor = 0;
2935         r->which = Z_External_update;
2936         r->indirect_reference = 0;
2937         r->u.update = (Z_IUUpdate *)
2938             odr_malloc(p->odr_out, sizeof(*r->u.update));
2939         
2940         r->u.update->which = Z_IUUpdate_esRequest;
2941         r->u.update->u.esRequest = (Z_IUUpdateEsRequest *)
2942             odr_malloc(p->odr_out, sizeof(*r->u.update->u.esRequest));
2943         toKeep = r->u.update->u.esRequest->toKeep = 
2944             (Z_IUOriginPartToKeep *)
2945             odr_malloc(p->odr_out, sizeof(*toKeep));
2946         
2947         toKeep->databaseName = odr_strdup(p->odr_out, first_db);
2948         toKeep->schema = 0;
2949         toKeep->elementSetName = 0;
2950         toKeep->actionQualifier = 0;
2951         toKeep->action = odr_intdup(p->odr_out, action_no);
2952         
2953         notToKeep = r->u.update->u.esRequest->notToKeep = 
2954             (Z_IUSuppliedRecords *)
2955             odr_malloc(p->odr_out, sizeof(*notToKeep));
2956         notToKeep->num = 1;
2957         notToKeep->elements = (Z_IUSuppliedRecords_elem **)
2958             odr_malloc(p->odr_out, sizeof(*notToKeep->elements));
2959         notToKeep->elements[0] = (Z_IUSuppliedRecords_elem *)
2960             odr_malloc(p->odr_out, sizeof(**notToKeep->elements));
2961         notToKeep->elements[0]->which = Z_IUSuppliedRecords_elem_opaque;
2962         if (recordIdOpaque)
2963         {
2964             notToKeep->elements[0]->u.opaque = 
2965                 odr_create_Odr_oct(p->odr_out,
2966                                    (const unsigned char *) recordIdOpaque,
2967                                    strlen(recordIdOpaque));
2968         }
2969         else if (recordIdNumber)
2970         {
2971             notToKeep->elements[0]->which = Z_IUSuppliedRecords_elem_number;
2972             
2973             notToKeep->elements[0]->u.number =
2974                 odr_intdup(p->odr_out, atoi(recordIdNumber));
2975         }
2976         else
2977             notToKeep->elements[0]->u.opaque = 0;
2978         notToKeep->elements[0]->supplementalId = 0;
2979         notToKeep->elements[0]->correlationInfo = 0;
2980         notToKeep->elements[0]->record =
2981             z_ext_record(p->odr_out, syntax_oid,
2982                          record_buf, strlen(record_buf));
2983     }
2984     if (0 && apdu)
2985     {
2986        ODR print = odr_createmem(ODR_PRINT);
2987
2988        z_APDU(print, &apdu, 0, 0);
2989        odr_destroy(print);
2990     }
2991     return apdu;
2992 }
2993
2994 ZOOM_API(void)
2995     ZOOM_package_send (ZOOM_package p, const char *type)
2996 {
2997     Z_APDU *apdu = 0;
2998     ZOOM_connection c;
2999     if (!p)
3000         return;
3001     c = p->connection;
3002     odr_reset (p->odr_out);
3003     xfree (p->buf_out);
3004     p->buf_out = 0;
3005     if (!strcmp(type, "itemorder"))
3006     {
3007         apdu = create_es_package (p, VAL_ITEMORDER);
3008         if (apdu)
3009         {
3010             Z_External *r = (Z_External *) odr_malloc (p->odr_out, sizeof(*r));
3011             
3012             r->direct_reference =
3013                 yaz_oidval_to_z3950oid(p->odr_out, CLASS_EXTSERV,
3014                                        VAL_ITEMORDER);
3015             r->descriptor = 0;
3016             r->which = Z_External_itemOrder;
3017             r->indirect_reference = 0;
3018             r->u.itemOrder = encode_item_order (p);
3019
3020             apdu->u.extendedServicesRequest->taskSpecificParameters = r;
3021         }
3022     }
3023     else if (!strcmp(type, "create"))  /* create database */
3024     {
3025         apdu = create_admin_package(p, Z_ESAdminOriginPartToKeep_create,
3026                                     0, 0);
3027     }   
3028     else if (!strcmp(type, "drop"))  /* drop database */
3029     {
3030         apdu = create_admin_package(p, Z_ESAdminOriginPartToKeep_drop,
3031                                     0, 0);
3032     }
3033     else if (!strcmp(type, "commit"))  /* commit changes */
3034     {
3035         apdu = create_admin_package(p, Z_ESAdminOriginPartToKeep_commit,
3036                                     0, 0);
3037     }
3038     else if (!strcmp(type, "update")) /* update record(s) */
3039     {
3040         apdu = create_update_package(p);
3041     }
3042     else if (!strcmp(type, "xmlupdate"))
3043     {
3044         apdu = create_xmlupdate_package(p);
3045     }
3046     if (apdu)
3047     {
3048         if (encode_APDU(p->connection, apdu, p->odr_out) == 0)
3049         {
3050             char *buf;
3051
3052             ZOOM_task task = ZOOM_connection_add_task (c, ZOOM_TASK_PACKAGE);
3053             task->u.package = p;
3054             buf = odr_getbuf(p->odr_out, &p->len_out, 0);
3055             p->buf_out = (char *) xmalloc (p->len_out);
3056             memcpy (p->buf_out, buf, p->len_out);
3057             
3058             (p->refcount)++;
3059             if (!c->async)
3060             {
3061                 while (ZOOM_event (1, &c))
3062                     ;
3063             }
3064         }
3065     }
3066 }
3067
3068 ZOOM_API(ZOOM_package)
3069     ZOOM_connection_package (ZOOM_connection c, ZOOM_options options)
3070 {
3071     ZOOM_package p = (ZOOM_package) xmalloc (sizeof(*p));
3072
3073     p->connection = c;
3074     p->odr_out = odr_createmem (ODR_ENCODE);
3075     p->options = ZOOM_options_create_with_parent2 (options, c->options);
3076     p->refcount = 1;
3077     p->buf_out = 0;
3078     p->len_out = 0;
3079     return p;
3080 }
3081
3082 ZOOM_API(void)
3083     ZOOM_package_destroy(ZOOM_package p)
3084 {
3085     if (!p)
3086         return;
3087     (p->refcount)--;
3088     if (p->refcount == 0)
3089     {
3090         odr_destroy (p->odr_out);
3091         xfree (p->buf_out);
3092         
3093         ZOOM_options_destroy (p->options);
3094         xfree (p);
3095     }
3096 }
3097
3098 ZOOM_API(const char *)
3099 ZOOM_package_option_get (ZOOM_package p, const char *key)
3100 {
3101     return ZOOM_options_get (p->options, key);
3102 }
3103
3104
3105 ZOOM_API(void)
3106 ZOOM_package_option_set (ZOOM_package p, const char *key,
3107                               const char *val)
3108 {
3109     ZOOM_options_set (p->options, key, val);
3110 }
3111
3112 static int ZOOM_connection_exec_task (ZOOM_connection c)
3113 {
3114     ZOOM_task task = c->tasks;
3115     zoom_ret ret = zoom_complete;
3116
3117     if (!task)
3118         return 0;
3119     yaz_log(log_details, "%p ZOOM_connection_exec_task type=%d run=%d",
3120              c, task->which, task->running);
3121     if (c->error != ZOOM_ERROR_NONE)
3122     {
3123         yaz_log(log_details, "%p ZOOM_connection_exec_task "
3124                 "removing tasks because of error = %d", c, c->error);
3125         ZOOM_connection_remove_tasks (c);
3126         return 0;
3127     }
3128     if (task->running)
3129     {
3130         yaz_log(log_details, "%p ZOOM_connection_exec_task "
3131                  "task already running", c);
3132         return 0;
3133     }
3134     task->running = 1;
3135     ret = zoom_complete;
3136     if (c->cs || task->which == ZOOM_TASK_CONNECT)
3137     {
3138         switch (task->which)
3139         {
3140         case ZOOM_TASK_SEARCH:
3141             if (c->proto == PROTO_HTTP)
3142                 ret = ZOOM_connection_srw_send_search(c);
3143             else
3144                 ret = ZOOM_connection_send_search(c);
3145             break;
3146         case ZOOM_TASK_RETRIEVE:
3147             if (c->proto == PROTO_HTTP)
3148                 ret = ZOOM_connection_srw_send_search(c);
3149             else
3150                 ret = send_present (c);
3151             break;
3152         case ZOOM_TASK_CONNECT:
3153             ret = do_connect(c);
3154             break;
3155         case ZOOM_TASK_SCAN:
3156             ret = send_scan(c);
3157             break;
3158         case ZOOM_TASK_PACKAGE:
3159             ret = send_package(c);
3160             break;
3161         case ZOOM_TASK_SORT:
3162             c->tasks->u.sort.resultset->r_sort_spec = 
3163                 c->tasks->u.sort.q->sort_spec;
3164             ret = send_sort(c, c->tasks->u.sort.resultset);
3165             break;
3166         }
3167     }
3168     else
3169     {
3170         yaz_log(log_details, "%p ZOOM_connection_exec_task "
3171                 "remove tasks because no connection exist", c);
3172         ZOOM_connection_remove_tasks (c);
3173     }
3174     if (ret == zoom_complete)
3175     {
3176         yaz_log(log_details, "%p ZOOM_connection_exec_task "
3177                  "task removed (complete)", c);
3178         ZOOM_connection_remove_task (c);
3179         return 0;
3180     }
3181     yaz_log(log_details, "%p ZOOM_connection_exec_task "
3182             "task pending", c);
3183     return 1;
3184 }
3185
3186 static zoom_ret send_sort_present (ZOOM_connection c)
3187 {
3188     zoom_ret r = zoom_complete;
3189
3190     if (c->tasks && c->tasks->which == ZOOM_TASK_SEARCH)
3191         r = send_sort (c, c->tasks->u.search.resultset);
3192     if (r == zoom_complete)
3193         r = send_present (c);
3194     return r;
3195 }
3196
3197 static int es_response (ZOOM_connection c,
3198                         Z_ExtendedServicesResponse *res)
3199 {
3200     if (!c->tasks || c->tasks->which != ZOOM_TASK_PACKAGE)
3201         return 0;
3202     if (res->diagnostics && res->num_diagnostics > 0)
3203         response_diag(c, res->diagnostics[0]);
3204     if (res->taskPackage &&
3205         res->taskPackage->which == Z_External_extendedService)
3206     {
3207         Z_TaskPackage *taskPackage = res->taskPackage->u.extendedService;
3208         Odr_oct *id = taskPackage->targetReference;
3209         
3210         if (id)
3211             ZOOM_options_setl (c->tasks->u.package->options,
3212                                "targetReference", (char*) id->buf, id->len);
3213     }
3214     if (res->taskPackage && 
3215         res->taskPackage->which == Z_External_octet)
3216     {
3217         Odr_oct *doc = res->taskPackage->u.octet_aligned;
3218         ZOOM_options_setl (c->tasks->u.package->options,
3219                            "xmlUpdateDoc", (char*) doc->buf, doc->len);
3220     }
3221     return 1;
3222 }
3223
3224 static void interpret_init_diag(ZOOM_connection c,
3225                                 Z_DiagnosticFormat *diag)
3226 {
3227     if (diag->num > 0)
3228     {
3229         Z_DiagnosticFormat_s *ds = diag->elements[0];
3230         if (ds->which == Z_DiagnosticFormat_s_defaultDiagRec)
3231             response_default_diag(c, ds->u.defaultDiagRec);
3232     }
3233 }
3234
3235
3236 static void interpret_otherinformation_field(ZOOM_connection c,
3237                                              Z_OtherInformation *ui)
3238 {
3239     int i;
3240     for (i = 0; i < ui->num_elements; i++)
3241     {
3242         Z_OtherInformationUnit *unit = ui->list[i];
3243         if (unit->which == Z_OtherInfo_externallyDefinedInfo &&
3244             unit->information.externallyDefinedInfo &&
3245             unit->information.externallyDefinedInfo->which ==
3246             Z_External_diag1) 
3247         {
3248             interpret_init_diag(c, unit->information.externallyDefinedInfo->u.diag1);
3249         } 
3250     }
3251 }
3252
3253 static void recv_apdu (ZOOM_connection c, Z_APDU *apdu)
3254 {
3255     Z_InitResponse *initrs;
3256     
3257     c->mask = 0;
3258     yaz_log(log_details, "%p recv_apdu apdu->which=%d", c, apdu->which);
3259     switch(apdu->which)
3260     {
3261     case Z_APDU_initResponse:
3262         yaz_log(log_api, "%p recv_apd: Received Init response", c);
3263         initrs = apdu->u.initResponse;
3264         ZOOM_connection_option_set(c, "serverImplementationId",
3265                                    initrs->implementationId ?
3266                                    initrs->implementationId : "");
3267         ZOOM_connection_option_set(c, "serverImplementationName",
3268                                    initrs->implementationName ?
3269                                    initrs->implementationName : "");
3270         ZOOM_connection_option_set(c, "serverImplementationVersion",
3271                                    initrs->implementationVersion ?
3272                                    initrs->implementationVersion : "");
3273         /* Set the three old options too, for old applications */
3274         ZOOM_connection_option_set(c, "targetImplementationId",
3275                                    initrs->implementationId ?
3276                                    initrs->implementationId : "");
3277         ZOOM_connection_option_set(c, "targetImplementationName",
3278                                    initrs->implementationName ?
3279                                    initrs->implementationName : "");
3280         ZOOM_connection_option_set(c, "targetImplementationVersion",
3281                                    initrs->implementationVersion ?
3282                                    initrs->implementationVersion : "");
3283         if (!*initrs->result)
3284         {
3285             Z_External *uif = initrs->userInformationField;
3286
3287             set_ZOOM_error(c, ZOOM_ERROR_INIT, 0); /* default error */
3288
3289             if (uif && uif->which == Z_External_userInfo1)
3290                 interpret_otherinformation_field(c, uif->u.userInfo1);
3291         }
3292         else
3293         {
3294             char *cookie =
3295                 yaz_oi_get_string_oidval (&apdu->u.initResponse->otherInfo,
3296                                           VAL_COOKIE, 1, 0);
3297             xfree (c->cookie_in);
3298             c->cookie_in = 0;
3299             if (cookie)
3300                 c->cookie_in = xstrdup(cookie);
3301             if (ODR_MASK_GET(initrs->options, Z_Options_namedResultSets) &&
3302                 ODR_MASK_GET(initrs->protocolVersion, Z_ProtocolVersion_3))
3303                 c->support_named_resultsets = 1;
3304             if (c->tasks)
3305             {
3306                 assert (c->tasks->which == ZOOM_TASK_CONNECT);
3307                 ZOOM_connection_remove_task (c);
3308             }
3309             ZOOM_connection_exec_task (c);
3310         }
3311         if (ODR_MASK_GET(initrs->options, Z_Options_negotiationModel))
3312         {
3313             NMEM tmpmem = nmem_create();
3314             Z_CharSetandLanguageNegotiation *p =
3315                 yaz_get_charneg_record(initrs->otherInfo);
3316             
3317             if (p)
3318             {
3319                 char *charset = NULL, *lang = NULL;
3320                 int sel;
3321                 
3322                 yaz_get_response_charneg(tmpmem, p, &charset, &lang, &sel);
3323                 yaz_log(log_details, "%p recv_apdu target accepted: "
3324                         "charset %s, language %s, select %d",
3325                         c,
3326                         charset ? charset : "none", lang ? lang : "none", sel);
3327                 if (charset)
3328                     ZOOM_connection_option_set (c, "negotiation-charset",
3329                                                 charset);
3330                 if (lang)
3331                     ZOOM_connection_option_set (c, "negotiation-lang",
3332                                                 lang);
3333
3334                 ZOOM_connection_option_set (
3335                     c,  "negotiation-charset-in-effect-for-records",
3336                     (sel != 0) ? "1" : "0");
3337                 nmem_destroy(tmpmem);
3338             }
3339         }       
3340         break;
3341     case Z_APDU_searchResponse:
3342         yaz_log(log_api, "%p recv_apdu Search response", c);
3343         handle_search_response (c, apdu->u.searchResponse);
3344         if (send_sort_present (c) == zoom_complete)
3345             ZOOM_connection_remove_task (c);
3346         break;
3347     case Z_APDU_presentResponse:
3348         yaz_log(log_api, "%p recv_apdu Present response", c);
3349         handle_present_response (c, apdu->u.presentResponse);
3350         if (send_present (c) == zoom_complete)
3351             ZOOM_connection_remove_task (c);
3352         break;
3353     case Z_APDU_sortResponse:
3354         yaz_log(log_api, "%p recv_apdu Sort response", c);
3355         sort_response (c, apdu->u.sortResponse);
3356         if (send_present (c) == zoom_complete)
3357             ZOOM_connection_remove_task (c);
3358         break;
3359     case Z_APDU_scanResponse:
3360         yaz_log(log_api, "%p recv_apdu Scan response", c);
3361         scan_response (c, apdu->u.scanResponse);
3362         ZOOM_connection_remove_task (c);
3363         break;
3364     case Z_APDU_extendedServicesResponse:
3365         yaz_log(log_api, "%p recv_apdu Extended Services response", c);
3366         es_response (c, apdu->u.extendedServicesResponse);
3367         ZOOM_connection_remove_task (c);
3368         break;
3369     case Z_APDU_close:
3370         yaz_log(log_api, "%p recv_apdu Close PDU", c);
3371         if (c->reconnect_ok)
3372         {
3373             do_close(c);
3374             c->tasks->running = 0;
3375             ZOOM_connection_insert_task (c, ZOOM_TASK_CONNECT);
3376         }
3377         else
3378         {
3379             set_ZOOM_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
3380             do_close(c);
3381         }
3382         break;
3383     default:
3384         yaz_log(log_api, "%p Received unknown PDU", c);
3385         set_ZOOM_error(c, ZOOM_ERROR_DECODE, 0);
3386         do_close(c);
3387     }
3388 }
3389
3390 #if HAVE_XML2
3391 static void handle_srw_response(ZOOM_connection c,
3392                                 Z_SRW_searchRetrieveResponse *res)
3393 {
3394     ZOOM_resultset resultset = 0;
3395     int i;
3396     NMEM nmem;
3397     ZOOM_Event event;
3398
3399     if (!c->tasks)
3400         return;
3401
3402     if (c->tasks->which == ZOOM_TASK_SEARCH)
3403         resultset = c->tasks->u.search.resultset;
3404     else if (c->tasks->which == ZOOM_TASK_RETRIEVE)
3405         resultset = c->tasks->u.retrieve.resultset;
3406     else
3407         return ;
3408
3409     event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH);
3410     ZOOM_connection_put_event(c, event);
3411
3412     resultset->size = 0;
3413
3414     yaz_log(log_details, "%p handle_srw_response got SRW response OK", c);
3415     
3416     if (res->numberOfRecords)
3417         resultset->size = *res->numberOfRecords;
3418
3419     for (i = 0; i<res->num_records; i++)
3420     {
3421         int pos;
3422
3423         Z_NamePlusRecord *npr = (Z_NamePlusRecord *)
3424             odr_malloc(c->odr_in, sizeof(Z_NamePlusRecord));
3425
3426         if (res->records[i].recordPosition && 
3427             *res->records[i].recordPosition > 0)
3428             pos = *res->records[i].recordPosition - 1;
3429         else
3430             pos = resultset->start + i;
3431         
3432         npr->databaseName = 0;
3433         npr->which = Z_NamePlusRecord_databaseRecord;
3434         npr->u.databaseRecord = (Z_External *)
3435             odr_malloc(c->odr_in, sizeof(Z_External));
3436         npr->u.databaseRecord->descriptor = 0;
3437         npr->u.databaseRecord->direct_reference =
3438             yaz_oidval_to_z3950oid(c->odr_in, CLASS_RECSYN, VAL_TEXT_XML);
3439         npr->u.databaseRecord->which = Z_External_octet;
3440
3441         npr->u.databaseRecord->u.octet_aligned = (Odr_oct *)
3442             odr_malloc(c->odr_in, sizeof(Odr_oct));
3443         npr->u.databaseRecord->u.octet_aligned->buf = (unsigned char*)
3444             res->records[i].recordData_buf;
3445         npr->u.databaseRecord->u.octet_aligned->len = 
3446             npr->u.databaseRecord->u.octet_aligned->size = 
3447             res->records[i].recordData_len;
3448         record_cache_add (resultset, npr, pos);
3449     }
3450     if (res->num_diagnostics > 0)
3451     {
3452         const char *uri = res->diagnostics[0].uri;
3453         if (uri)
3454         {
3455             int code = 0;       
3456             const char *cp;
3457             if ((cp = strrchr(uri, '/')))
3458                 code = atoi(cp+1);
3459             set_dset_error(c, code, uri,
3460                            res->diagnostics[0].details, 0);
3461         }
3462     }
3463     nmem = odr_extract_mem(c->odr_in);
3464     nmem_transfer(resultset->odr->mem, nmem);
3465     nmem_destroy(nmem);
3466 }
3467 #endif
3468
3469 #if HAVE_XML2
3470 static void handle_http(ZOOM_connection c, Z_HTTP_Response *hres)
3471 {
3472     int ret = -1;
3473     const char *content_type = z_HTTP_header_lookup(hres->headers,
3474                                                     "Content-Type");
3475     const char *connection_head = z_HTTP_header_lookup(hres->headers,
3476                                                        "Connection");
3477     c->mask = 0;
3478     yaz_log(log_details, "%p handle_http", c);
3479
3480     if (content_type && !yaz_strcmp_del("text/xml", content_type, "; "))
3481     {
3482         Z_SOAP *soap_package = 0;
3483         ODR o = c->odr_in;
3484         Z_SOAP_Handler soap_handlers[2] = {
3485             {"http://www.loc.gov/zing/srw/", 0,
3486              (Z_SOAP_fun) yaz_srw_codec},
3487             {0, 0, 0}
3488         };
3489         ret = z_soap_codec(o, &soap_package,
3490                            &hres->content_buf, &hres->content_len,
3491                            soap_handlers);
3492         if (!ret && soap_package->which == Z_SOAP_generic &&
3493             soap_package->u.generic->no == 0)
3494         {
3495             Z_SRW_PDU *sr = (Z_SRW_PDU*) soap_package->u.generic->p;
3496             if (sr->which == Z_SRW_searchRetrieve_response)
3497                 handle_srw_response(c, sr->u.response);
3498             else
3499                 ret = -1;
3500         }
3501         else if (!ret && (soap_package->which == Z_SOAP_fault
3502                           || soap_package->which == Z_SOAP_error))
3503         {
3504             set_HTTP_error(c, hres->code,
3505                            soap_package->u.fault->fault_code,
3506                            soap_package->u.fault->fault_string);
3507         }
3508         else
3509             ret = -1;
3510     }
3511     if (ret)
3512     {
3513         if (hres->code != 200)
3514             set_HTTP_error(c, hres->code, 0, 0);
3515         else
3516             set_ZOOM_error(c, ZOOM_ERROR_DECODE, 0);
3517         do_close (c);
3518     }
3519     ZOOM_connection_remove_task(c);
3520     if (!strcmp(hres->version, "1.0"))
3521     {
3522         /* HTTP 1.0: only if Keep-Alive we stay alive.. */
3523         if (!connection_head || strcmp(connection_head, "Keep-Alive"))
3524             do_close(c);
3525     }
3526     else 
3527     {
3528         /* HTTP 1.1: only if no close we stay alive .. */
3529         if (connection_head && !strcmp(connection_head, "close"))
3530             do_close(c);
3531     }
3532 }
3533 #endif
3534
3535 static int do_read (ZOOM_connection c)
3536 {
3537     int r, more;
3538     ZOOM_Event event;
3539     
3540     event = ZOOM_Event_create (ZOOM_EVENT_RECV_DATA);
3541     ZOOM_connection_put_event (c, event);
3542     
3543     r = cs_get (c->cs, &c->buf_in, &c->len_in);
3544     more = cs_more(c->cs);
3545     yaz_log(log_details, "%p do_read len=%d more=%d", c, r, more);
3546     if (r == 1)
3547         return 0;
3548     if (r <= 0)
3549     {
3550         if (c->reconnect_ok)
3551         {
3552             do_close (c);
3553             c->reconnect_ok = 0;
3554             yaz_log(log_details, "%p do_read reconnect read", c);
3555             c->tasks->running = 0;
3556             ZOOM_connection_insert_task (c, ZOOM_TASK_CONNECT);
3557         }
3558         else
3559         {
3560             set_ZOOM_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
3561             do_close (c);
3562         }
3563     }
3564     else
3565     {
3566         Z_GDU *gdu;
3567         ZOOM_Event event;
3568
3569         odr_reset (c->odr_in);
3570         odr_setbuf (c->odr_in, c->buf_in, r, 0);
3571         event = ZOOM_Event_create (ZOOM_EVENT_RECV_APDU);
3572         ZOOM_connection_put_event (c, event);
3573
3574         if (!z_GDU (c->odr_in, &gdu, 0, 0))
3575         {
3576             int x;
3577             int err = odr_geterrorx(c->odr_in, &x);
3578             char msg[60];
3579             const char *element = odr_getelement(c->odr_in);
3580             sprintf (msg, "ODR code %d:%d element=%-20s",
3581                      err, x, element ? element : "<unknown>");
3582             set_ZOOM_error(c, ZOOM_ERROR_DECODE, msg);
3583             do_close (c);
3584         }
3585         else if (gdu->which == Z_GDU_Z3950)
3586             recv_apdu (c, gdu->u.z3950);
3587         else if (gdu->which == Z_GDU_HTTP_Response)
3588         {
3589 #if HAVE_XML2
3590             handle_http (c, gdu->u.HTTP_Response);
3591 #else
3592             set_ZOOM_error(c, ZOOM_ERROR_DECODE, 0);
3593             do_close (c);
3594 #endif
3595         }
3596         c->reconnect_ok = 0;
3597     }
3598     return 1;
3599 }
3600
3601 static zoom_ret do_write_ex (ZOOM_connection c, char *buf_out, int len_out)
3602 {
3603     int r;
3604     ZOOM_Event event;
3605     
3606     event = ZOOM_Event_create(ZOOM_EVENT_SEND_DATA);
3607     ZOOM_connection_put_event (c, event);
3608
3609     yaz_log(log_details, "%p do_write_ex len=%d", c, len_out);
3610     if ((r = cs_put(c->cs, buf_out, len_out)) < 0)
3611     {
3612         yaz_log(log_details, "%p do_write_ex write failed", c);
3613         if (c->reconnect_ok)
3614         {
3615             do_close (c);
3616             c->reconnect_ok = 0;
3617             yaz_log(log_details, "%p do_write_ex reconnect write", c);
3618             c->tasks->running = 0;
3619             ZOOM_connection_insert_task (c, ZOOM_TASK_CONNECT);
3620             return zoom_pending;
3621         }
3622         if (c->state == STATE_CONNECTING)
3623             set_ZOOM_error(c, ZOOM_ERROR_CONNECT, c->host_port);
3624         else
3625             set_ZOOM_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
3626         do_close (c);
3627         return zoom_complete;
3628     }
3629     else if (r == 1)
3630     {    
3631         c->mask = ZOOM_SELECT_EXCEPT;
3632         if (c->cs->io_pending & CS_WANT_WRITE)
3633             c->mask += ZOOM_SELECT_WRITE;
3634         if (c->cs->io_pending & CS_WANT_READ)
3635             c->mask += ZOOM_SELECT_READ;
3636         yaz_log(log_details, "%p do_write_ex write incomplete mask=%d",
3637                 c, c->mask);
3638     }
3639     else
3640     {
3641         c->mask = ZOOM_SELECT_READ|ZOOM_SELECT_EXCEPT;
3642         yaz_log(log_details, "%p do_write_ex write complete mask=%d",
3643                 c, c->mask);
3644     }
3645     return zoom_pending;
3646 }