src/zoom-c.c

   1 /*
   2  * Copyright (C) 1995-2005, Index Data ApS
   3  * See the file LICENSE for details.
   4  *
   5  * $Id: zoom-c.c,v 1.53 2005-12-07 16:15:15 mike Exp $
   6  */
   7 /**
   8  * \file zoom-c.c
   9  * \brief Implements ZOOM C interface.
  10  */
  11
  12 #include <assert.h>
  13 #include <string.h>
  14 #include "zoom-p.h"
  15
  16 #include <yaz/yaz-util.h>
  17 #include <yaz/xmalloc.h>
  18 #include <yaz/otherinfo.h>
  19 #include <yaz/log.h>
  20 #include <yaz/pquery.h>
  21 #include <yaz/marcdisp.h>
  22 #include <yaz/diagbib1.h>
  23 #include <yaz/charneg.h>
  24 #include <yaz/ill.h>
  25 #include <yaz/srw.h>
  26
  27 #if HAVE_SYS_TYPES_H
  28 #include <sys/types.h>
  29 #endif
  30 #if HAVE_SYS_TIME_H
  31 #include <sys/time.h>
  32 #endif
  33 #if HAVE_SYS_POLL_H
  34 #include <sys/poll.h>
  35 #endif
  36 #if HAVE_SYS_SELECT_H
  37 #include <sys/select.h>
  38 #endif
  39 #ifdef WIN32
  40 #include <winsock.h>
  41 #endif
  42
  43 static int log_api = 0;
  44 static int log_details = 0;
  45
  46 typedef enum {
  47     zoom_pending,
  48     zoom_complete
  49 } zoom_ret;
  50
  51 static zoom_ret ZOOM_connection_send_init (ZOOM_connection c);
  52 static zoom_ret do_write_ex (ZOOM_connection c, char *buf_out, int len_out);
  53
  54 static void initlog()
  55 {
  56     static int log_level_initialized = 0;
  57     if (!log_level_initialized)
  58     {
  59         log_api = yaz_log_module_level("zoom");
  60         log_details = yaz_log_module_level("zoomdetails");
  61         log_level_initialized = 1;
  62     }
  63 }
  64
  65 static ZOOM_Event ZOOM_Event_create (int kind)
  66 {
  67     ZOOM_Event event = (ZOOM_Event) xmalloc (sizeof(*event));
  68     event->kind = kind;
  69     event->next = 0;
  70     event->prev = 0;
  71     return event;
  72 }
  73
  74 static void ZOOM_Event_destroy (ZOOM_Event event)
  75 {
  76     xfree (event);
  77 }
  78
  79 static void ZOOM_connection_put_event (ZOOM_connection c, ZOOM_Event event)
  80 {
  81     if (c->m_queue_back)
  82     {
  83         c->m_queue_back->prev = event;
  84         assert (c->m_queue_front);
  85     }
  86     else
  87     {
  88         assert (!c->m_queue_front);
  89         c->m_queue_front = event;
  90     }
  91     event->next = c->m_queue_back;
  92     event->prev = 0;
  93     c->m_queue_back = event;
  94 }
  95
  96 static ZOOM_Event ZOOM_connection_get_event(ZOOM_connection c)
  97 {
  98     ZOOM_Event event = c->m_queue_front;
  99     if (!event)
 100     {
 101         c->last_event = ZOOM_EVENT_NONE;
 102         return 0;
 103     }
 104     assert (c->m_queue_back);
 105     c->m_queue_front = event->prev;
 106     if (c->m_queue_front)
 107     {
 108         assert (c->m_queue_back);
 109         c->m_queue_front->next = 0;
 110     }
 111     else
 112         c->m_queue_back = 0;
 113     c->last_event = event->kind;
 114     return event;
 115 }
 116
 117
 118 static void set_dset_error (ZOOM_connection c, int error,
 119                             const char *dset,
 120                             const char *addinfo, const char *addinfo2)
 121 {
 122     char *cp;
 123
 124     xfree (c->addinfo);
 125     c->addinfo = 0;
 126     c->error = error;
 127     if (!c->diagset || strcmp(dset, c->diagset))
 128     {
 129         xfree(c->diagset);
 130         c->diagset = xstrdup(dset);
 131         /* remove integer part from SRW diagset .. */
 132         if ((cp = strrchr(c->diagset, '/')))
 133            *cp = '\0';
 134     }
 135     if (addinfo && addinfo2)
 136     {
 137         c->addinfo = (char*) xmalloc(strlen(addinfo) + strlen(addinfo2) + 2);
 138         strcpy(c->addinfo, addinfo);
 139         strcat(c->addinfo, addinfo2);
 140     }
 141     else if (addinfo)
 142         c->addinfo = xstrdup(addinfo);
 143     if (error)
 144         yaz_log(log_api, "%p set_dset_error %s %s:%d %s %s",
 145                 c, c->host_port ? c->host_port : "<>", dset, error,
 146                 addinfo ? addinfo : "",
 147                 addinfo2 ? addinfo2 : "");
 148 }
 149
 150 #if HAVE_XML2
 151 static void set_HTTP_error (ZOOM_connection c, int error,
 152                             const char *addinfo, const char *addinfo2)
 153 {
 154     set_dset_error(c, error, "HTTP", addinfo, addinfo2);
 155 }
 156 #endif
 157
 158 static void set_ZOOM_error (ZOOM_connection c, int error,
 159                             const char *addinfo)
 160 {
 161     set_dset_error(c, error, "ZOOM", addinfo, 0);
 162 }
 163
 164 static void clear_error (ZOOM_connection c)
 165 {
 166
 167     switch (c->error)
 168     {
 169     case ZOOM_ERROR_CONNECT:
 170     case ZOOM_ERROR_MEMORY:
 171     case ZOOM_ERROR_DECODE:
 172     case ZOOM_ERROR_CONNECTION_LOST:
 173     case ZOOM_ERROR_INIT:
 174     case ZOOM_ERROR_INTERNAL:
 175     case ZOOM_ERROR_UNSUPPORTED_PROTOCOL:
 176         break;
 177     default:
 178         set_ZOOM_error(c, ZOOM_ERROR_NONE, 0);
 179     }
 180 }
 181
 182 ZOOM_task ZOOM_connection_add_task (ZOOM_connection c, int which)
 183 {
 184     ZOOM_task *taskp = &c->tasks;
 185     while (*taskp)
 186         taskp = &(*taskp)->next;
 187     *taskp = (ZOOM_task) xmalloc (sizeof(**taskp));
 188     (*taskp)->running = 0;
 189     (*taskp)->which = which;
 190     (*taskp)->next = 0;
 191     clear_error (c);
 192     return *taskp;
 193 }
 194
 195 ZOOM_task ZOOM_connection_insert_task (ZOOM_connection c, int which)
 196 {
 197     ZOOM_task task = (ZOOM_task) xmalloc (sizeof(*task));
 198
 199     task->next = c->tasks;
 200     c->tasks = task;
 201
 202     task->running = 0;
 203     task->which = which;
 204     clear_error (c);
 205     return task;
 206 }
 207
 208 void ZOOM_connection_remove_task (ZOOM_connection c)
 209 {
 210     ZOOM_task task = c->tasks;
 211
 212     if (task)
 213     {
 214         c->tasks = task->next;
 215         switch (task->which)
 216         {
 217         case ZOOM_TASK_SEARCH:
 218             ZOOM_resultset_destroy (task->u.search.resultset);
 219             break;
 220         case ZOOM_TASK_RETRIEVE:
 221             ZOOM_resultset_destroy (task->u.retrieve.resultset);
 222             break;
 223         case ZOOM_TASK_CONNECT:
 224             break;
 225         case ZOOM_TASK_SCAN:
 226             ZOOM_scanset_destroy (task->u.scan.scan);
 227             break;
 228         case ZOOM_TASK_PACKAGE:
 229             ZOOM_package_destroy (task->u.package);
 230             break;
 231         case ZOOM_TASK_SORT:
 232             ZOOM_resultset_destroy (task->u.sort.resultset);
 233             ZOOM_query_destroy(task->u.sort.q);
 234             break;
 235         default:
 236             assert (0);
 237         }
 238         xfree (task);
 239     }
 240 }
 241
 242 static int ZOOM_connection_exec_task (ZOOM_connection c);
 243
 244 void ZOOM_connection_remove_tasks (ZOOM_connection c)
 245 {
 246     while (c->tasks)
 247         ZOOM_connection_remove_task(c);
 248 }
 249
 250 static ZOOM_record record_cache_lookup (ZOOM_resultset r, int pos);
 251
 252 ZOOM_API(ZOOM_connection)
 253 ZOOM_connection_create (ZOOM_options options)
 254 {
 255     ZOOM_connection c = (ZOOM_connection) xmalloc (sizeof(*c));
 256
 257     initlog();
 258
 259     yaz_log(log_api, "%p ZOOM_connection_create", c);
 260
 261     c->proto = PROTO_Z3950;
 262     c->cs = 0;
 263     c->mask = 0;
 264     c->reconnect_ok = 0;
 265     c->state = STATE_IDLE;
 266     c->addinfo = 0;
 267     c->diagset = 0;
 268     set_ZOOM_error(c, ZOOM_ERROR_NONE, 0);
 269     c->buf_in = 0;
 270     c->len_in = 0;
 271     c->buf_out = 0;
 272     c->len_out = 0;
 273     c->resultsets = 0;
 274
 275     c->options = ZOOM_options_create_with_parent(options);
 276
 277     c->host_port = 0;
 278     c->path = 0;
 279     c->proxy = 0;
 280
 281     c->charset = c->lang = 0;
 282
 283     c->cookie_out = 0;
 284     c->cookie_in = 0;
 285     c->client_IP = 0;
 286     c->tasks = 0;
 287
 288     c->odr_in = odr_createmem (ODR_DECODE);
 289     c->odr_out = odr_createmem (ODR_ENCODE);
 290
 291     c->async = 0;
 292     c->support_named_resultsets = 0;
 293     c->last_event = ZOOM_EVENT_NONE;
 294
 295     c->m_queue_front = 0;
 296     c->m_queue_back = 0;
 297     return c;
 298 }
 299
 300 /* set database names. Take local databases (if set); otherwise
 301    take databases given in ZURL (if set); otherwise use Default */
 302 static char **set_DatabaseNames (ZOOM_connection con, ZOOM_options options,
 303                                  int *num)
 304 {
 305     char **databaseNames;
 306     const char *cp = ZOOM_options_get (options, "databaseName");
 307
 308     if (!cp || !*cp)
 309     {
 310         if (strncmp (con->host_port, "unix:", 5) == 0)
 311             cp = strchr(con->host_port+5, ':');
 312         else
 313             cp = strchr(con->host_port, '/');
 314         if (cp)
 315             cp++;
 316     }
 317     if (!cp)
 318         cp = "Default";
 319     nmem_strsplit(con->odr_out->mem, "+", cp,  &databaseNames, num);
 320     return databaseNames;
 321 }
 322
 323 ZOOM_API(ZOOM_connection)
 324 ZOOM_connection_new (const char *host, int portnum)
 325 {
 326     ZOOM_connection c = ZOOM_connection_create (0);
 327
 328     ZOOM_connection_connect (c, host, portnum);
 329     return c;
 330 }
 331
 332 ZOOM_API(void)
 333 ZOOM_connection_connect(ZOOM_connection c,
 334                         const char *host, int portnum)
 335 {
 336     const char *val;
 337     ZOOM_task task;
 338
 339     initlog();
 340
 341     yaz_log(log_api, "%p ZOOM_connection_connect host=%s portnum=%d",
 342             c, host, portnum);
 343
 344     if (c->cs)
 345     {
 346         yaz_log(log_details, "%p ZOOM_connection_connect reconnect ok", c);
 347         c->reconnect_ok = 1;
 348         return;
 349     }
 350     yaz_log(log_details, "%p ZOOM_connection_connect connect", c);
 351     xfree (c->proxy);
 352     val = ZOOM_options_get (c->options, "proxy");
 353     if (val && *val)
 354     {
 355         yaz_log(log_details, "%p ZOOM_connection_connect proxy=%s", c, val);
 356         c->proxy = xstrdup (val);
 357     }
 358     else
 359         c->proxy = 0;
 360
 361     xfree (c->charset);
 362     val = ZOOM_options_get (c->options, "charset");
 363     if (val && *val)
 364     {
 365         yaz_log(log_details, "%p ZOOM_connection_connect charset=%s", c, val);
 366         c->charset = xstrdup (val);
 367     }
 368     else
 369         c->charset = 0;
 370
 371     xfree (c->lang);
 372     val = ZOOM_options_get (c->options, "lang");
 373     if (val && *val)
 374     {
 375         yaz_log(log_details, "%p ZOOM_connection_connect lang=%s", c, val);
 376         c->lang = xstrdup (val);
 377     }
 378     else
 379         c->lang = 0;
 380
 381     xfree (c->host_port);
 382     if (portnum)
 383     {
 384         char hostn[128];
 385         sprintf (hostn, "%.80s:%d", host, portnum);
 386         c->host_port = xstrdup(hostn);
 387     }
 388     else
 389         c->host_port = xstrdup(host);
 390
 391     if ((val = strchr(c->host_port, '%')) != 0) {
 392         /* We recognise <username>:<password>%<string> for embedded
 393          * authentication.  This is slightly hacky syntax, but it's
 394          * hard to get into the comstack code in a
 395          * protocol-independent way.
 396          */
 397         *(char*)val = '\0';
 398         char *remainder = xstrdup(val+1);
 399         char *pass = strchr(c->host_port, ':');
 400         if (pass != 0) {
 401             *pass++ = '\0';
 402             ZOOM_connection_option_set(c, "user", c->host_port);
 403             ZOOM_connection_option_set(c, "password", pass);
 404         }
 405         xfree(c->host_port);
 406         c->host_port = remainder;
 407     }
 408
 409     ZOOM_options_set(c->options, "host", c->host_port);
 410
 411     val = ZOOM_options_get (c->options, "cookie");
 412     if (val && *val)
 413     {
 414         yaz_log(log_details, "%p ZOOM_connection_connect cookie=%s", c, val);
 415         c->cookie_out = xstrdup (val);
 416     }
 417
 418     val = ZOOM_options_get (c->options, "clientIP");
 419     if (val && *val)
 420     {
 421         yaz_log(log_details, "%p ZOOM_connection_connect clientIP=%s",
 422                 c, val);
 423         c->client_IP = xstrdup (val);
 424     }
 425
 426     c->async = ZOOM_options_get_bool (c->options, "async", 0);
 427     yaz_log(log_details, "%p ZOOM_connection_connect async=%d", c, c->async);
 428
 429     set_ZOOM_error(c, ZOOM_ERROR_NONE, 0);
 430
 431     task = ZOOM_connection_add_task (c, ZOOM_TASK_CONNECT);
 432
 433     if (!c->async)
 434     {
 435         while (ZOOM_event (1, &c))
 436             ;
 437     }
 438 }
 439
 440 ZOOM_API(ZOOM_query)
 441 ZOOM_query_create(void)
 442 {
 443     ZOOM_query s = (ZOOM_query) xmalloc (sizeof(*s));
 444
 445     yaz_log(log_details, "%p ZOOM_query_create", s);
 446     s->refcount = 1;
 447     s->z_query = 0;
 448     s->sort_spec = 0;
 449     s->odr = odr_createmem (ODR_ENCODE);
 450     s->query_string = 0;
 451
 452     return s;
 453 }
 454
 455 ZOOM_API(void)
 456 ZOOM_query_destroy(ZOOM_query s)
 457 {
 458     if (!s)
 459         return;
 460
 461     (s->refcount)--;
 462     yaz_log(log_details, "%p ZOOM_query_destroy count=%d", s, s->refcount);
 463     if (s->refcount == 0)
 464     {
 465         odr_destroy (s->odr);
 466         xfree (s);
 467     }
 468 }
 469
 470 ZOOM_API(int)
 471 ZOOM_query_prefix(ZOOM_query s, const char *str)
 472 {
 473     s->query_string = odr_strdup(s->odr, str);
 474     s->z_query = (Z_Query *) odr_malloc (s->odr, sizeof(*s->z_query));
 475     s->z_query->which = Z_Query_type_1;
 476     s->z_query->u.type_1 =  p_query_rpn(s->odr, PROTO_Z3950, str);
 477     if (!s->z_query->u.type_1)
 478     {
 479         yaz_log(log_details, "%p ZOOM_query_prefix str=%s failed", s, str);
 480         s->z_query = 0;
 481         return -1;
 482     }
 483     yaz_log(log_details, "%p ZOOM_query_prefix str=%s", s, str);
 484     return 0;
 485 }
 486
 487 ZOOM_API(int)
 488 ZOOM_query_cql(ZOOM_query s, const char *str)
 489 {
 490     Z_External *ext;
 491
 492     s->query_string = odr_strdup(s->odr, str);
 493
 494     ext = (Z_External *) odr_malloc(s->odr, sizeof(*ext));
 495     ext->direct_reference = odr_getoidbystr(s->odr, "1.2.840.10003.16.2");
 496     ext->indirect_reference = 0;
 497     ext->descriptor = 0;
 498     ext->which = Z_External_CQL;
 499     ext->u.cql = s->query_string;
 500
 501     s->z_query = (Z_Query *) odr_malloc (s->odr, sizeof(*s->z_query));
 502     s->z_query->which = Z_Query_type_104;
 503     s->z_query->u.type_104 =  ext;
 504
 505     yaz_log(log_details, "%p ZOOM_query_cql str=%s", s, str);
 506
 507     return 0;
 508 }
 509
 510 ZOOM_API(int)
 511 ZOOM_query_sortby(ZOOM_query s, const char *criteria)
 512 {
 513     s->sort_spec = yaz_sort_spec (s->odr, criteria);
 514     if (!s->sort_spec)
 515     {
 516         yaz_log(log_details, "%p ZOOM_query_sortby criteria=%s failed",
 517                 s, criteria);
 518         return -1;
 519     }
 520     yaz_log(log_details, "%p ZOOM_query_sortby criteria=%s", s, criteria);
 521     return 0;
 522 }
 523
 524 static zoom_ret do_write(ZOOM_connection c);
 525
 526 ZOOM_API(void)
 527 ZOOM_connection_destroy(ZOOM_connection c)
 528 {
 529     ZOOM_resultset r;
 530     if (!c)
 531         return;
 532     yaz_log(log_api, "%p ZOOM_connection_destroy", c);
 533     if (c->cs)
 534         cs_close (c->cs);
 535     for (r = c->resultsets; r; r = r->next)
 536         r->connection = 0;
 537
 538     xfree (c->buf_in);
 539     xfree (c->addinfo);
 540     xfree (c->diagset);
 541     odr_destroy (c->odr_in);
 542     odr_destroy (c->odr_out);
 543     ZOOM_options_destroy (c->options);
 544     ZOOM_connection_remove_tasks (c);
 545     xfree (c->host_port);
 546     xfree (c->path);
 547     xfree (c->proxy);
 548     xfree (c->charset);
 549     xfree (c->lang);
 550     xfree (c->cookie_out);
 551     xfree (c->cookie_in);
 552     xfree (c->client_IP);
 553     xfree (c);
 554 }
 555
 556 void ZOOM_resultset_addref (ZOOM_resultset r)
 557 {
 558     if (r)
 559     {
 560         (r->refcount)++;
 561         yaz_log(log_details, "%p ZOOM_resultset_addref count=%d",
 562                 r, r->refcount);
 563     }
 564 }
 565
 566 ZOOM_resultset ZOOM_resultset_create ()
 567 {
 568     ZOOM_resultset r = (ZOOM_resultset) xmalloc (sizeof(*r));
 569
 570     initlog();
 571
 572     yaz_log(log_details, "%p ZOOM_resultset_create", r);
 573     r->refcount = 1;
 574     r->size = 0;
 575     r->odr = odr_createmem (ODR_ENCODE);
 576     r->start = 0;
 577     r->piggyback = 1;
 578     r->setname = 0;
 579     r->schema = 0;
 580     r->count = 0;
 581     r->step = 0;
 582     r->record_cache = 0;
 583     r->r_sort_spec = 0;
 584     r->query = 0;
 585     r->connection = 0;
 586     r->next = 0;
 587     return r;
 588 }
 589
 590 ZOOM_API(ZOOM_resultset)
 591 ZOOM_connection_search_pqf(ZOOM_connection c, const char *q)
 592 {
 593     ZOOM_resultset r;
 594     ZOOM_query s = ZOOM_query_create();
 595
 596     ZOOM_query_prefix (s, q);
 597
 598     r = ZOOM_connection_search (c, s);
 599     ZOOM_query_destroy (s);
 600     return r;
 601 }
 602
 603 ZOOM_API(ZOOM_resultset)
 604 ZOOM_connection_search(ZOOM_connection c, ZOOM_query q)
 605 {
 606     ZOOM_resultset r = ZOOM_resultset_create ();
 607     ZOOM_task task;
 608     const char *cp;
 609
 610     yaz_log(log_api, "%p ZOOM_connection_search set %p query %p", c, r, q);
 611     r->r_sort_spec = q->sort_spec;
 612     r->query = q;
 613
 614     r->options = ZOOM_options_create_with_parent(c->options);
 615
 616     r->start = ZOOM_options_get_int(r->options, "start", 0);
 617     r->count = ZOOM_options_get_int(r->options, "count", 0);
 618     {
 619         /* If "presentChunk" is defined use that; otherwise "step" */
 620         const char *cp = ZOOM_options_get (r->options, "presentChunk");
 621         r->step = ZOOM_options_get_int(r->options,
 622                                        (cp != 0 ? "presentChunk": "step"), 0);
 623     }
 624     r->piggyback = ZOOM_options_get_bool (r->options, "piggyback", 1);
 625     cp = ZOOM_options_get (r->options, "setname");
 626     if (cp)
 627         r->setname = xstrdup(cp);
 628     cp = ZOOM_options_get (r->options, "schema");
 629     if (cp)
 630         r->schema = xstrdup(cp);
 631
 632     r->connection = c;
 633
 634     r->next = c->resultsets;
 635     c->resultsets = r;
 636
 637     if (c->host_port && c->proto == PROTO_HTTP)
 638     {
 639         if (!c->cs)
 640         {
 641             yaz_log(log_details, "ZOOM_connection_search: no comstack");
 642             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
 643         }
 644         else
 645         {
 646             yaz_log(log_details, "ZOOM_connection_search: reconnect");
 647             c->reconnect_ok = 1;
 648         }
 649     }
 650
 651     task = ZOOM_connection_add_task (c, ZOOM_TASK_SEARCH);
 652     task->u.search.resultset = r;
 653     ZOOM_resultset_addref (r);
 654
 655     (q->refcount)++;
 656
 657     if (!c->async)
 658     {
 659         while (ZOOM_event (1, &c))
 660             ;
 661     }
 662     return r;
 663 }
 664
 665 /*
 666  * This is the old result-set sorting API, which is maintained only
 667  * for the sake of binary compatibility.  There is no reason ever to
 668  * use this rather than ZOOM_resultset_sort1().
 669  */
 670 ZOOM_API(void)
 671 ZOOM_resultset_sort(ZOOM_resultset r,
 672                     const char *sort_type, const char *sort_spec)
 673 {
 674     (void) ZOOM_resultset_sort1(r, sort_type, sort_spec);
 675 }
 676
 677 ZOOM_API(int)
 678 ZOOM_resultset_sort1(ZOOM_resultset r,
 679                      const char *sort_type, const char *sort_spec)
 680 {
 681     ZOOM_connection c = r->connection;
 682     ZOOM_task task;
 683     ZOOM_query newq;
 684
 685     newq = ZOOM_query_create();
 686     if (ZOOM_query_sortby(newq, sort_spec) < 0)
 687         return -1;
 688
 689     yaz_log(log_api, "%p ZOOM_resultset_sort r=%p sort_type=%s sort_spec=%s",
 690             r, r, sort_type, sort_spec);
 691     if (!c)
 692         return 0;
 693
 694     if (c->host_port && c->proto == PROTO_HTTP)
 695     {
 696         if (!c->cs)
 697         {
 698             yaz_log(log_details, "%p ZOOM_resultset_sort: no comstack", r);
 699             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
 700         }
 701         else
 702         {
 703             yaz_log(log_details, "%p ZOOM_resultset_sort: prepare reconnect",
 704                     r);
 705             c->reconnect_ok = 1;
 706         }
 707     }
 708
 709     ZOOM_resultset_cache_reset(r);
 710     task = ZOOM_connection_add_task (c, ZOOM_TASK_SORT);
 711     task->u.sort.resultset = r;
 712     task->u.sort.q = newq;
 713
 714     ZOOM_resultset_addref (r);
 715
 716     if (!c->async)
 717     {
 718         while (ZOOM_event (1, &c))
 719             ;
 720     }
 721
 722     return 0;
 723 }
 724
 725 ZOOM_API(void)
 726     ZOOM_resultset_cache_reset(ZOOM_resultset r)
 727 {
 728     ZOOM_record_cache rc;
 729
 730     for (rc = r->record_cache; rc; rc = rc->next)
 731     {
 732         if (rc->rec.wrbuf_marc)
 733             wrbuf_free (rc->rec.wrbuf_marc, 1);
 734         if (rc->rec.wrbuf_iconv)
 735             wrbuf_free (rc->rec.wrbuf_iconv, 1);
 736         if (rc->rec.wrbuf_opac)
 737             wrbuf_free (rc->rec.wrbuf_opac, 1);
 738     }
 739     r->record_cache = 0;
 740 }
 741
 742 ZOOM_API(void)
 743 ZOOM_resultset_destroy(ZOOM_resultset r)
 744 {
 745     if (!r)
 746         return;
 747     (r->refcount)--;
 748     yaz_log(log_details, "%p ZOOM_resultset_destroy r=%p count=%d",
 749             r, r, r->refcount);
 750     if (r->refcount == 0)
 751     {
 752         ZOOM_resultset_cache_reset(r);
 753
 754         if (r->connection)
 755         {
 756             /* remove ourselves from the resultsets in connection */
 757             ZOOM_resultset *rp = &r->connection->resultsets;
 758             while (1)
 759             {
 760                 assert (*rp);   /* we must be in this list!! */
 761                 if (*rp == r)
 762                 {   /* OK, we're here - take us out of it */
 763                     *rp = (*rp)->next;
 764                     break;
 765                 }
 766                 rp = &(*rp)->next;
 767             }
 768         }
 769         ZOOM_query_destroy (r->query);
 770         ZOOM_options_destroy (r->options);
 771         odr_destroy (r->odr);
 772         xfree (r->setname);
 773         xfree (r->schema);
 774         xfree (r);
 775     }
 776 }
 777
 778 ZOOM_API(size_t)
 779 ZOOM_resultset_size (ZOOM_resultset r)
 780 {
 781     return r->size;
 782 }
 783
 784 static void do_close (ZOOM_connection c)
 785 {
 786     if (c->cs)
 787         cs_close(c->cs);
 788     c->cs = 0;
 789     c->mask = 0;
 790     c->state = STATE_IDLE;
 791 }
 792
 793 static void ZOOM_resultset_retrieve (ZOOM_resultset r,
 794                                      int force_sync, int start, int count)
 795 {
 796     ZOOM_task task;
 797     ZOOM_connection c;
 798     const char *cp;
 799
 800     if (!r)
 801         return;
 802     yaz_log(log_details, "%p ZOOM_resultset_retrieve force_sync=%d start=%d"
 803             " count=%d", r, force_sync, start, count);
 804     c = r->connection;
 805     if (!c)
 806         return;
 807
 808     if (c->host_port && c->proto == PROTO_HTTP)
 809     {
 810         if (!c->cs)
 811         {
 812             yaz_log(log_details, "%p ZOOM_resultset_retrieve: no comstack", r);
 813             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
 814         }
 815         else
 816         {
 817             yaz_log(log_details, "%p ZOOM_resultset_retrieve: prepare "
 818                     "reconnect", r);
 819             c->reconnect_ok = 1;
 820         }
 821     }
 822     task = ZOOM_connection_add_task (c, ZOOM_TASK_RETRIEVE);
 823     task->u.retrieve.resultset = r;
 824     task->u.retrieve.start = start;
 825     task->u.retrieve.count = count;
 826
 827     cp = ZOOM_options_get (r->options, "schema");
 828     if (cp)
 829     {
 830         if (!r->schema || strcmp(r->schema, cp))
 831         {
 832             xfree(r->schema);
 833             r->schema = xstrdup(cp);
 834         }
 835     }
 836
 837     ZOOM_resultset_addref (r);
 838
 839     if (!r->connection->async || force_sync)
 840         while (r->connection && ZOOM_event (1, &r->connection))
 841             ;
 842 }
 843
 844 ZOOM_API(void)
 845 ZOOM_resultset_records (ZOOM_resultset r, ZOOM_record *recs,
 846                         size_t start, size_t count)
 847 {
 848     int force_present = 0;
 849
 850     if (!r)
 851         return ;
 852     yaz_log(log_api, "%p ZOOM_resultset_records r=%p start=%d count=%d",
 853             r, r, start, count);
 854     if (count && recs)
 855         force_present = 1;
 856     ZOOM_resultset_retrieve (r, force_present, start, count);
 857     if (force_present)
 858     {
 859         size_t i;
 860         for (i = 0; i< count; i++)
 861             recs[i] = ZOOM_resultset_record_immediate (r, i+start);
 862     }
 863 }
 864
 865 static void get_cert(ZOOM_connection c)
 866 {
 867     char *cert_buf;
 868     int cert_len;
 869
 870     if (cs_get_peer_certificate_x509(c->cs, &cert_buf, &cert_len))
 871     {
 872         ZOOM_connection_option_setl(c, "sslPeerCert",
 873                                     cert_buf, cert_len);
 874         xfree(cert_buf);
 875     }
 876 }
 877
 878 static zoom_ret do_connect (ZOOM_connection c)
 879 {
 880     void *add;
 881     const char *effective_host;
 882
 883     if (c->proxy)
 884         effective_host = c->proxy;
 885     else
 886         effective_host = c->host_port;
 887
 888     yaz_log(log_details, "%p do_connect effective_host=%s", c, effective_host);
 889
 890     if (c->cs)
 891         cs_close(c->cs);
 892     c->cs = cs_create_host (effective_host, 0, &add);
 893
 894     if (c->cs && c->cs->protocol == PROTO_HTTP)
 895     {
 896 #if HAVE_XML2
 897         const char *path = 0;
 898
 899         c->proto = PROTO_HTTP;
 900         cs_get_host_args(c->host_port, &path);
 901         xfree(c->path);
 902         c->path = (char*) xmalloc(strlen(path)+2);
 903         c->path[0] = '/';
 904         strcpy (c->path+1, path);
 905 #else
 906         set_ZOOM_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, "SRW");
 907         do_close(c);
 908         return zoom_complete;
 909 #endif
 910     }
 911     if (c->cs)
 912     {
 913         int ret = cs_connect (c->cs, add);
 914         if (ret == 0)
 915         {
 916             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
 917             ZOOM_connection_put_event(c, event);
 918             get_cert(c);
 919             if (c->proto == PROTO_Z3950)
 920                 ZOOM_connection_send_init(c);
 921             else
 922             {
 923                 /* no init request for SRW .. */
 924                 assert (c->tasks->which == ZOOM_TASK_CONNECT);
 925                 ZOOM_connection_remove_task (c);
 926                 c->mask = 0;
 927                 ZOOM_connection_exec_task (c);
 928             }
 929             c->state = STATE_ESTABLISHED;
 930             return zoom_pending;
 931         }
 932         else if (ret > 0)
 933         {
 934             c->state = STATE_CONNECTING;
 935             c->mask = ZOOM_SELECT_EXCEPT;
 936             if (c->cs->io_pending & CS_WANT_WRITE)
 937                 c->mask += ZOOM_SELECT_WRITE;
 938             if (c->cs->io_pending & CS_WANT_READ)
 939                 c->mask += ZOOM_SELECT_READ;
 940             return zoom_pending;
 941         }
 942     }
 943     c->state = STATE_IDLE;
 944     set_ZOOM_error(c, ZOOM_ERROR_CONNECT, effective_host);
 945     return zoom_complete;
 946 }
 947
 948 int z3950_connection_socket(ZOOM_connection c)
 949 {
 950     if (c->cs)
 951         return cs_fileno(c->cs);
 952     return -1;
 953 }
 954
 955 int z3950_connection_mask(ZOOM_connection c)
 956 {
 957     if (c->cs)
 958         return c->mask;
 959     return 0;
 960 }
 961
 962 static void otherInfo_attach (ZOOM_connection c, Z_APDU *a, ODR out)
 963 {
 964     int i;
 965     for (i = 0; i<200; i++)
 966     {
 967         size_t len;
 968         Z_OtherInformation **oi;
 969         char buf[80];
 970         const char *val;
 971         const char *cp;
 972         int oidval;
 973
 974         sprintf (buf, "otherInfo%d", i);
 975         val = ZOOM_options_get (c->options, buf);
 976         if (!val)
 977             break;
 978         cp = strchr(val, ':');
 979         if (!cp)
 980             continue;
 981         len = cp - val;
 982         if (len >= sizeof(buf))
 983             len = sizeof(buf)-1;
 984         memcpy (buf, val, len);
 985         buf[len] = '\0';
 986         oidval = oid_getvalbyname (buf);
 987         if (oidval == VAL_NONE)
 988             continue;
 989
 990         yaz_oi_APDU(a, &oi);
 991         yaz_oi_set_string_oidval(oi, out, oidval, 1, cp+1);
 992     }
 993 }
 994
 995 static int encode_APDU(ZOOM_connection c, Z_APDU *a, ODR out)
 996 {
 997     assert (a);
 998     if (c->cookie_out)
 999     {
1000         Z_OtherInformation **oi;
1001         yaz_oi_APDU(a, &oi);
1002         yaz_oi_set_string_oidval(oi, out, VAL_COOKIE, 1, c->cookie_out);
1003     }
1004     if (c->client_IP)
1005     {
1006         Z_OtherInformation **oi;
1007         yaz_oi_APDU(a, &oi);
1008         yaz_oi_set_string_oidval(oi, out, VAL_CLIENT_IP, 1, c->client_IP);
1009     }
1010     otherInfo_attach (c, a, out);
1011     if (!z_APDU(out, &a, 0, 0))
1012     {
1013         FILE *outf = fopen("/tmp/apdu.txt", "a");
1014         if (a && outf)
1015         {
1016             ODR odr_pr = odr_createmem(ODR_PRINT);
1017             fprintf (outf, "a=%p\n", a);
1018             odr_setprint(odr_pr, outf);
1019             z_APDU(odr_pr, &a, 0, 0);
1020             odr_destroy(odr_pr);
1021         }
1022         yaz_log(log_api, "%p encoding_APDU: encoding failed", c);
1023         set_ZOOM_error(c, ZOOM_ERROR_ENCODE, 0);
1024         odr_reset(out);
1025         return -1;
1026     }
1027     yaz_log(log_details, "%p encoding_APDU encoding OK", c);
1028     return 0;
1029 }
1030
1031 static zoom_ret send_APDU (ZOOM_connection c, Z_APDU *a)
1032 {
1033     ZOOM_Event event;
1034     assert (a);
1035     if (encode_APDU(c, a, c->odr_out))
1036         return zoom_complete;
1037     yaz_log(log_details, "%p send APDU type=%d", c, a->which);
1038     c->buf_out = odr_getbuf(c->odr_out, &c->len_out, 0);
1039     event = ZOOM_Event_create (ZOOM_EVENT_SEND_APDU);
1040     ZOOM_connection_put_event (c, event);
1041     odr_reset(c->odr_out);
1042     return do_write (c);
1043 }
1044
1045 /* returns 1 if PDU was sent OK (still pending )
1046            0 if PDU was not sent OK (nothing to wait for)
1047 */
1048
1049 static zoom_ret ZOOM_connection_send_init (ZOOM_connection c)
1050 {
1051     Z_APDU *apdu = zget_APDU(c->odr_out, Z_APDU_initRequest);
1052     Z_InitRequest *ireq = apdu->u.initRequest;
1053     Z_IdAuthentication *auth = (Z_IdAuthentication *)
1054         odr_malloc(c->odr_out, sizeof(*auth));
1055     const char *auth_groupId = ZOOM_options_get (c->options, "group");
1056     const char *auth_userId = ZOOM_options_get (c->options, "user");
1057     const char *auth_password = ZOOM_options_get (c->options, "password");
1058     char *version;
1059
1060     /* support the pass for backwards compatibility */
1061     if (!auth_password)
1062         auth_password = ZOOM_options_get (c->options, "pass");
1063
1064     ODR_MASK_SET(ireq->options, Z_Options_search);
1065     ODR_MASK_SET(ireq->options, Z_Options_present);
1066     ODR_MASK_SET(ireq->options, Z_Options_scan);
1067     ODR_MASK_SET(ireq->options, Z_Options_sort);
1068     ODR_MASK_SET(ireq->options, Z_Options_extendedServices);
1069     ODR_MASK_SET(ireq->options, Z_Options_namedResultSets);
1070
1071     ODR_MASK_SET(ireq->protocolVersion, Z_ProtocolVersion_1);
1072     ODR_MASK_SET(ireq->protocolVersion, Z_ProtocolVersion_2);
1073     ODR_MASK_SET(ireq->protocolVersion, Z_ProtocolVersion_3);
1074
1075     /* Index Data's Z39.50 Implementor Id is 81 */
1076     ireq->implementationId = odr_prepend(c->odr_out,
1077         ZOOM_options_get(c->options, "implementationId"),
1078         odr_prepend(c->odr_out, "81", ireq->implementationId));
1079
1080     ireq->implementationName = odr_prepend(c->odr_out,
1081         ZOOM_options_get(c->options, "implementationName"),
1082         odr_prepend(c->odr_out, "ZOOM-C", ireq->implementationName));
1083
1084     version = odr_strdup(c->odr_out, "$Revision: 1.53 $");
1085     if (strlen(version) > 10)   /* check for unexpanded CVS strings */
1086         version[strlen(version)-2] = '\0';
1087     ireq->implementationVersion = odr_prepend(c->odr_out,
1088         ZOOM_options_get(c->options, "implementationVersion"),
1089         odr_prepend(c->odr_out, &version[11], ireq->implementationVersion));
1090
1091     *ireq->maximumRecordSize =
1092         ZOOM_options_get_int (c->options, "maximumRecordSize", 1024*1024);
1093     *ireq->preferredMessageSize =
1094         ZOOM_options_get_int (c->options, "preferredMessageSize", 1024*1024);
1095
1096     if (auth_groupId || auth_password)
1097     {
1098         Z_IdPass *pass = (Z_IdPass *) odr_malloc(c->odr_out, sizeof(*pass));
1099         int i = 0;
1100         pass->groupId = 0;
1101         if (auth_groupId && *auth_groupId)
1102         {
1103             pass->groupId = (char *)
1104                 odr_malloc(c->odr_out, strlen(auth_groupId)+1);
1105             strcpy(pass->groupId, auth_groupId);
1106             i++;
1107         }
1108         pass->userId = 0;
1109         if (auth_userId && *auth_userId)
1110         {
1111             pass->userId = (char *)
1112                 odr_malloc(c->odr_out, strlen(auth_userId)+1);
1113             strcpy(pass->userId, auth_userId);
1114             i++;
1115         }
1116         pass->password = 0;
1117         if (auth_password && *auth_password)
1118         {
1119             pass->password = (char *)
1120                 odr_malloc(c->odr_out, strlen(auth_password)+1);
1121             strcpy(pass->password, auth_password);
1122             i++;
1123         }
1124         if (i)
1125         {
1126             auth->which = Z_IdAuthentication_idPass;
1127             auth->u.idPass = pass;
1128             ireq->idAuthentication = auth;
1129         }
1130     }
1131     else if (auth_userId)
1132     {
1133         auth->which = Z_IdAuthentication_open;
1134         auth->u.open = (char *)
1135             odr_malloc(c->odr_out, strlen(auth_userId)+1);
1136         strcpy(auth->u.open, auth_userId);
1137         ireq->idAuthentication = auth;
1138     }
1139     if (c->proxy)
1140         yaz_oi_set_string_oidval(&ireq->otherInfo, c->odr_out,
1141                                  VAL_PROXY, 1, c->host_port);
1142     if (c->charset || c->lang)
1143     {
1144         Z_OtherInformation **oi;
1145         Z_OtherInformationUnit *oi_unit;
1146
1147         yaz_oi_APDU(apdu, &oi);
1148
1149         if ((oi_unit = yaz_oi_update(oi, c->odr_out, NULL, 0, 0)))
1150         {
1151             char **charsets_addresses = 0;
1152             char **langs_addresses = 0;
1153             int charsets_count = 0;
1154             int langs_count = 0;
1155
1156             if (c->charset)
1157                 nmem_strsplit_blank(c->odr_out->mem, c->charset,
1158                                     &charsets_addresses, &charsets_count);
1159             if (c->lang)
1160                 nmem_strsplit_blank(c->odr_out->mem, c->lang,
1161                                     &langs_addresses, &langs_count);
1162             ODR_MASK_SET(ireq->options, Z_Options_negotiationModel);
1163             oi_unit->which = Z_OtherInfo_externallyDefinedInfo;
1164             oi_unit->information.externallyDefinedInfo =
1165                 yaz_set_proposal_charneg(c->odr_out,
1166                                          (const char **) charsets_addresses,
1167                                          charsets_count,
1168                                          (const char **) langs_addresses,
1169                                          langs_count,
1170                                          1);
1171         }
1172     }
1173     assert (apdu);
1174     return send_APDU (c, apdu);
1175 }
1176
1177 #if HAVE_XML2
1178 static zoom_ret send_srw (ZOOM_connection c, Z_SRW_PDU *sr)
1179 {
1180     char ctype[50];
1181     Z_SOAP_Handler h[2] = {
1182         {"http://www.loc.gov/zing/srw/", 0, (Z_SOAP_fun) yaz_srw_codec},
1183         {0, 0, 0}
1184     };
1185     ODR o = odr_createmem(ODR_ENCODE);
1186     int ret;
1187     Z_SOAP *p = (Z_SOAP*) odr_malloc(o, sizeof(*p));
1188     Z_GDU *gdu;
1189     ZOOM_Event event;
1190
1191     gdu = z_get_HTTP_Request(c->odr_out);
1192     gdu->u.HTTP_Request->path = c->path;
1193
1194     if (c->host_port)
1195     {
1196         const char *cp0 = strstr(c->host_port, "://");
1197         const char *cp1 = 0;
1198         if (cp0)
1199             cp0 = cp0+3;
1200         else
1201             cp0 = c->host_port;
1202
1203         cp1 = strchr(cp0, '/');
1204         if (!cp1)
1205             cp1 = cp0+strlen(cp0);
1206
1207         if (cp0 && cp1)
1208         {
1209             char *h = (char*) odr_malloc(c->odr_out, cp1 - cp0 + 1);
1210             memcpy (h, cp0, cp1 - cp0);
1211             h[cp1-cp0] = '\0';
1212             z_HTTP_header_add(c->odr_out, &gdu->u.HTTP_Request->headers,
1213                               "Host", h);
1214         }
1215     }
1216
1217     strcpy(ctype, "text/xml");
1218     if (c->charset && strlen(c->charset) < 20)
1219     {
1220         strcat(ctype, "; charset=");
1221         strcat(ctype, c->charset);
1222     }
1223     z_HTTP_header_add(c->odr_out, &gdu->u.HTTP_Request->headers,
1224                       "Content-Type", ctype);
1225     z_HTTP_header_add(c->odr_out, &gdu->u.HTTP_Request->headers,
1226                       "SOAPAction", "\"\"");
1227     p->which = Z_SOAP_generic;
1228     p->u.generic = (Z_SOAP_Generic *) odr_malloc(o, sizeof(*p->u.generic));
1229     p->u.generic->no = 0;
1230     p->u.generic->ns = 0;
1231     p->u.generic->p = sr;
1232     p->ns = "http://schemas.xmlsoap.org/soap/envelope/";
1233
1234     ret = z_soap_codec_enc(o, &p,
1235                            &gdu->u.HTTP_Request->content_buf,
1236                            &gdu->u.HTTP_Request->content_len, h,
1237                            c->charset);
1238
1239     if (!z_GDU(c->odr_out, &gdu, 0, 0))
1240         return zoom_complete;
1241     c->buf_out = odr_getbuf(c->odr_out, &c->len_out, 0);
1242
1243     odr_destroy(o);
1244
1245     event = ZOOM_Event_create (ZOOM_EVENT_SEND_APDU);
1246     ZOOM_connection_put_event (c, event);
1247     odr_reset(c->odr_out);
1248     return do_write (c);
1249 }
1250 #endif
1251
1252 #if HAVE_XML2
1253 static zoom_ret ZOOM_connection_srw_send_search(ZOOM_connection c)
1254 {
1255     int i;
1256     ZOOM_resultset resultset = 0;
1257     Z_SRW_PDU *sr = 0;
1258     const char *recordPacking = 0;
1259
1260     if (c->error)                  /* don't continue on error */
1261         return zoom_complete;
1262     assert (c->tasks);
1263     if (c->tasks->which == ZOOM_TASK_SEARCH)
1264     {
1265         resultset = c->tasks->u.search.resultset;
1266         resultset->setname = xstrdup ("default");
1267         ZOOM_options_set (resultset->options, "setname", resultset->setname);
1268     }
1269     else if(c->tasks->which == ZOOM_TASK_RETRIEVE)
1270     {
1271         resultset = c->tasks->u.retrieve.resultset;
1272
1273         resultset->start = c->tasks->u.retrieve.start;
1274         resultset->count = c->tasks->u.retrieve.count;
1275
1276         if (resultset->start >= resultset->size)
1277             return zoom_complete;
1278         if (resultset->start + resultset->count > resultset->size)
1279             resultset->count = resultset->size - resultset->start;
1280
1281         for (i = 0; i<resultset->count; i++)
1282         {
1283             ZOOM_record rec =
1284                 record_cache_lookup (resultset, i + resultset->start);
1285             if (!rec)
1286                 break;
1287         }
1288         if (i == resultset->count)
1289             return zoom_complete;
1290     }
1291     assert(resultset->query);
1292
1293     sr = yaz_srw_get(c->odr_out, Z_SRW_searchRetrieve_request);
1294
1295     if (resultset->query->z_query->which == Z_Query_type_104
1296         && resultset->query->z_query->u.type_104->which == Z_External_CQL)
1297     {
1298
1299         sr->u.request->query_type = Z_SRW_query_type_cql;
1300         sr->u.request->query.cql =resultset->query->z_query->u.type_104->u.cql;
1301     }
1302     else if (resultset->query->z_query->which == Z_Query_type_1 &&
1303              resultset->query->z_query->u.type_1)
1304     {
1305         sr->u.request->query_type = Z_SRW_query_type_pqf;
1306         sr->u.request->query.pqf = resultset->query->query_string;
1307     }
1308     else
1309     {
1310         set_ZOOM_error(c, ZOOM_ERROR_UNSUPPORTED_QUERY, 0);
1311         return zoom_complete;
1312     }
1313     sr->u.request->startRecord = odr_intdup (c->odr_out, resultset->start + 1);
1314     sr->u.request->maximumRecords = odr_intdup (
1315         c->odr_out, resultset->step>0 ? resultset->step : resultset->count);
1316     sr->u.request->recordSchema = resultset->schema;
1317
1318     recordPacking = ZOOM_resultset_option_get (resultset, "recordPacking");
1319
1320     if (recordPacking)
1321         sr->u.request->recordPacking = odr_strdup(c->odr_out, recordPacking);
1322
1323     return send_srw(c, sr);
1324 }
1325 #else
1326 static zoom_ret ZOOM_connection_srw_send_search(ZOOM_connection c)
1327 {
1328     return zoom_complete;
1329 }
1330 #endif
1331
1332 static zoom_ret ZOOM_connection_send_search (ZOOM_connection c)
1333 {
1334     ZOOM_resultset r;
1335     int lslb, ssub, mspn;
1336     const char *syntax;
1337     Z_APDU *apdu = zget_APDU(c->odr_out, Z_APDU_searchRequest);
1338     Z_SearchRequest *search_req = apdu->u.searchRequest;
1339     const char *elementSetName;
1340     const char *smallSetElementSetName;
1341     const char *mediumSetElementSetName;
1342
1343     assert (c->tasks);
1344     assert (c->tasks->which == ZOOM_TASK_SEARCH);
1345
1346     r = c->tasks->u.search.resultset;
1347
1348     yaz_log(log_details, "%p ZOOM_connection_send_search set=%p", c, r);
1349
1350     elementSetName =
1351         ZOOM_options_get (r->options, "elementSetName");
1352     smallSetElementSetName  =
1353         ZOOM_options_get (r->options, "smallSetElementSetName");
1354     mediumSetElementSetName =
1355         ZOOM_options_get (r->options, "mediumSetElementSetName");
1356
1357     if (!smallSetElementSetName)
1358         smallSetElementSetName = elementSetName;
1359
1360     if (!mediumSetElementSetName)
1361         mediumSetElementSetName = elementSetName;
1362
1363     assert (r);
1364     assert (r->query);
1365
1366     /* prepare query for the search request */
1367     search_req->query = r->query->z_query;
1368     if (!search_req->query)
1369     {
1370         set_ZOOM_error(c, ZOOM_ERROR_INVALID_QUERY, 0);
1371         return zoom_complete;
1372     }
1373
1374     search_req->databaseNames =
1375         set_DatabaseNames (c, r->options, &search_req->num_databaseNames);
1376
1377     /* get syntax (no need to provide unless piggyback is in effect) */
1378     syntax = ZOOM_options_get (r->options, "preferredRecordSyntax");
1379
1380     lslb = ZOOM_options_get_int (r->options, "largeSetLowerBound", -1);
1381     ssub = ZOOM_options_get_int (r->options, "smallSetUpperBound", -1);
1382     mspn = ZOOM_options_get_int (r->options, "mediumSetPresentNumber", -1);
1383     if (lslb != -1 && ssub != -1 && mspn != -1)
1384     {
1385         /* So're a Z39.50 expert? Let's hope you don't do sort */
1386         *search_req->largeSetLowerBound = lslb;
1387         *search_req->smallSetUpperBound = ssub;
1388         *search_req->mediumSetPresentNumber = mspn;
1389     }
1390     else if (r->start == 0 && r->count > 0
1391              && r->piggyback && !r->r_sort_spec && !r->schema)
1392     {
1393         /* Regular piggyback - do it unless we're going to do sort */
1394         *search_req->largeSetLowerBound = 2000000000;
1395         *search_req->smallSetUpperBound = 1;
1396         *search_req->mediumSetPresentNumber = r->step>0 ? r->step : r->count;
1397     }
1398     else
1399     {
1400         /* non-piggyback. Need not provide elementsets or syntaxes .. */
1401         smallSetElementSetName = 0;
1402         mediumSetElementSetName = 0;
1403         syntax = 0;
1404     }
1405     if (smallSetElementSetName && *smallSetElementSetName)
1406     {
1407         Z_ElementSetNames *esn = (Z_ElementSetNames *)
1408             odr_malloc (c->odr_out, sizeof(*esn));
1409
1410         esn->which = Z_ElementSetNames_generic;
1411         esn->u.generic = odr_strdup (c->odr_out, smallSetElementSetName);
1412         search_req->smallSetElementSetNames = esn;
1413     }
1414     if (mediumSetElementSetName && *mediumSetElementSetName)
1415     {
1416         Z_ElementSetNames *esn = (Z_ElementSetNames *)
1417             odr_malloc (c->odr_out, sizeof(*esn));
1418
1419         esn->which = Z_ElementSetNames_generic;
1420         esn->u.generic = odr_strdup (c->odr_out, mediumSetElementSetName);
1421         search_req->mediumSetElementSetNames = esn;
1422     }
1423     if (syntax)
1424         search_req->preferredRecordSyntax =
1425             yaz_str_to_z3950oid (c->odr_out, CLASS_RECSYN, syntax);
1426
1427     if (!r->setname)
1428     {
1429         if (c->support_named_resultsets)
1430         {
1431             char setname[14];
1432             int ord;
1433             /* find the lowest unused ordinal so that we re-use
1434                result sets on the server. */
1435             for (ord = 1; ; ord++)
1436             {
1437                 ZOOM_resultset rp;
1438                 sprintf (setname, "%d", ord);
1439                 for (rp = c->resultsets; rp; rp = rp->next)
1440                     if (rp->setname && !strcmp (rp->setname, setname))
1441                         break;
1442                 if (!rp)
1443                     break;
1444             }
1445             r->setname = xstrdup (setname);
1446             yaz_log(log_details, "%p ZOOM_connection_send_search: allocating "
1447                     "set %s", c, r->setname);
1448         }
1449         else
1450         {
1451             yaz_log(log_details, "%p ZOOM_connection_send_search: using "
1452                     "default set", c);
1453             r->setname = xstrdup ("default");
1454         }
1455         ZOOM_options_set (r->options, "setname", r->setname);
1456     }
1457     search_req->resultSetName = odr_strdup(c->odr_out, r->setname);
1458     return send_APDU (c, apdu);
1459 }
1460
1461 static void response_default_diag(ZOOM_connection c, Z_DefaultDiagFormat *r)
1462 {
1463     int oclass;
1464     char *addinfo = 0;
1465
1466     switch (r->which)
1467     {
1468     case Z_DefaultDiagFormat_v2Addinfo:
1469         addinfo = r->u.v2Addinfo;
1470         break;
1471     case Z_DefaultDiagFormat_v3Addinfo:
1472         addinfo = r->u.v3Addinfo;
1473         break;
1474     }
1475     xfree (c->addinfo);
1476     c->addinfo = 0;
1477     set_dset_error(c, *r->condition,
1478                    yaz_z3950oid_to_str(r->diagnosticSetId, &oclass),
1479                    addinfo, 0);
1480 }
1481
1482 static void response_diag(ZOOM_connection c, Z_DiagRec *p)
1483 {
1484     if (p->which != Z_DiagRec_defaultFormat)
1485         set_ZOOM_error(c, ZOOM_ERROR_DECODE, 0);
1486     else
1487         response_default_diag(c, p->u.defaultFormat);
1488 }
1489
1490 ZOOM_API(ZOOM_record)
1491 ZOOM_record_clone (ZOOM_record srec)
1492 {
1493     char *buf;
1494     int size;
1495     ODR odr_enc;
1496     ZOOM_record nrec;
1497
1498     odr_enc = odr_createmem(ODR_ENCODE);
1499     if (!z_NamePlusRecord (odr_enc, &srec->npr, 0, 0))
1500         return 0;
1501     buf = odr_getbuf (odr_enc, &size, 0);
1502
1503     nrec = (ZOOM_record) xmalloc (sizeof(*nrec));
1504     nrec->odr = odr_createmem(ODR_DECODE);
1505     nrec->wrbuf_marc = 0;
1506     nrec->wrbuf_iconv = 0;
1507     nrec->wrbuf_opac = 0;
1508     odr_setbuf (nrec->odr, buf, size, 0);
1509     z_NamePlusRecord (nrec->odr, &nrec->npr, 0, 0);
1510
1511     odr_destroy (odr_enc);
1512     return nrec;
1513 }
1514
1515 ZOOM_API(ZOOM_record)
1516 ZOOM_resultset_record_immediate (ZOOM_resultset s,size_t pos)
1517 {
1518     return record_cache_lookup (s, pos);
1519 }
1520
1521 ZOOM_API(ZOOM_record)
1522 ZOOM_resultset_record (ZOOM_resultset r, size_t pos)
1523 {
1524     ZOOM_record rec = ZOOM_resultset_record_immediate(r, pos);
1525
1526     if (!rec)
1527     {
1528         ZOOM_resultset_retrieve (r, 1, pos, 1);
1529         rec = ZOOM_resultset_record_immediate (r, pos);
1530     }
1531     return rec;
1532 }
1533
1534 ZOOM_API(void)
1535 ZOOM_record_destroy (ZOOM_record rec)
1536 {
1537     if (!rec)
1538         return;
1539     if (rec->wrbuf_marc)
1540         wrbuf_free (rec->wrbuf_marc, 1);
1541     if (rec->wrbuf_iconv)
1542         wrbuf_free (rec->wrbuf_iconv, 1);
1543     if (rec->wrbuf_opac)
1544         wrbuf_free (rec->wrbuf_opac, 1);
1545     odr_destroy (rec->odr);
1546     xfree (rec);
1547 }
1548
1549 static const char *marc_iconv_return(ZOOM_record rec, int marc_type,
1550                                      int *len,
1551                                      const char *buf, int sz,
1552                                      const char *record_charset)
1553 {
1554     char to[40];
1555     char from[40];
1556     yaz_iconv_t cd = 0;
1557     yaz_marc_t mt = yaz_marc_create();
1558
1559     *from = '\0';
1560     strcpy(to, "UTF-8");
1561     if (record_charset && *record_charset)
1562     {
1563         /* Use "from,to" or just "from" */
1564         const char *cp = strchr(record_charset, ',');
1565         int clen = strlen(record_charset);
1566         if (cp && cp[1])
1567         {
1568             strncpy( to, cp+1, sizeof(to)-1);
1569             to[sizeof(to)-1] = '\0';
1570             clen = cp - record_charset;
1571         }
1572         if (clen > sizeof(from)-1)
1573             clen = sizeof(from)-1;
1574
1575         if (clen)
1576             strncpy(from, record_charset, clen);
1577         from[clen] = '\0';
1578     }
1579
1580     if (*from && *to)
1581     {
1582         cd = yaz_iconv_open(to, from);
1583         yaz_marc_iconv(mt, cd);
1584     }
1585
1586     yaz_marc_xml(mt, marc_type);
1587     if (!rec->wrbuf_marc)
1588         rec->wrbuf_marc = wrbuf_alloc();
1589     wrbuf_rewind (rec->wrbuf_marc);
1590     if (yaz_marc_decode_wrbuf (mt, buf, sz, rec->wrbuf_marc) > 0)
1591     {
1592         yaz_marc_destroy(mt);
1593         if (cd)
1594             yaz_iconv_close(cd);
1595         if (len)
1596             *len = wrbuf_len(rec->wrbuf_marc);
1597         return wrbuf_buf(rec->wrbuf_marc);
1598     }
1599     yaz_marc_destroy(mt);
1600     if (cd)
1601         yaz_iconv_close(cd);
1602     return 0;
1603 }
1604
1605 static const char *record_iconv_return(ZOOM_record rec, int *len,
1606                                        const char *buf, int sz,
1607                                        const char *record_charset)
1608 {
1609     char to[40];
1610     char from[40];
1611     yaz_iconv_t cd = 0;
1612
1613     *from = '\0';
1614     strcpy(to, "UTF-8");
1615     if (record_charset && *record_charset)
1616     {
1617         /* Use "from,to" or just "from" */
1618         const char *cp = strchr(record_charset, ',');
1619         int clen = strlen(record_charset);
1620         if (cp && cp[1])
1621         {
1622             strncpy( to, cp+1, sizeof(to)-1);
1623             to[sizeof(to)-1] = '\0';
1624             clen = cp - record_charset;
1625         }
1626         if (clen > sizeof(from)-1)
1627             clen = sizeof(from)-1;
1628
1629         if (clen)
1630             strncpy(from, record_charset, clen);
1631         from[clen] = '\0';
1632     }
1633
1634     if (*from && *to && (cd = yaz_iconv_open(to, from)))
1635     {
1636         char outbuf[12];
1637         size_t inbytesleft = sz;
1638         const char *inp = buf;
1639
1640         if (!rec->wrbuf_iconv)
1641             rec->wrbuf_iconv = wrbuf_alloc();
1642
1643         wrbuf_rewind(rec->wrbuf_iconv);
1644
1645         while (inbytesleft)
1646         {
1647             size_t outbytesleft = sizeof(outbuf);
1648             char *outp = outbuf;
1649             size_t r = yaz_iconv (cd, (char**) &inp,
1650                                   &inbytesleft,
1651                                   &outp, &outbytesleft);
1652             if (r == (size_t) (-1))
1653             {
1654                 int e = yaz_iconv_error(cd);
1655                 if (e != YAZ_ICONV_E2BIG)
1656                     break;
1657             }
1658             wrbuf_write(rec->wrbuf_iconv, outbuf, outp - outbuf);
1659         }
1660         wrbuf_puts(rec->wrbuf_iconv, "");
1661         buf = wrbuf_buf(rec->wrbuf_iconv);
1662         sz = wrbuf_len(rec->wrbuf_iconv);
1663         yaz_iconv_close(cd);
1664     }
1665     if (len)
1666         *len = sz;
1667     return buf;
1668 }
1669
1670 ZOOM_API(const char *)
1671 ZOOM_record_get (ZOOM_record rec, const char *type_spec, int *len)
1672 {
1673     char type[40];
1674     char charset[40];
1675     char xpath[512];
1676     const char *cp;
1677     int i;
1678     Z_NamePlusRecord *npr;
1679
1680     if (len)
1681         *len = 0; /* default return */
1682
1683     if (!rec)
1684         return 0;
1685     npr = rec->npr;
1686     if (!npr)
1687         return 0;
1688
1689     cp = type_spec;
1690     for (i = 0; cp[i] && i < sizeof(type)-1; i++)
1691     {
1692         if (cp[i] == ';' || cp[i] == ' ')
1693             break;
1694         type[i] = cp[i];
1695     }
1696     type[i] = '\0';
1697     charset[0] = '\0';
1698     while (type_spec[i] == ';')
1699     {
1700         i++;
1701         while (type_spec[i] == ' ')
1702             i++;
1703         if (!strncmp(type_spec+i, "charset=", 8))
1704         {
1705             int j = 0;
1706             i = i + 8; /* skip charset= */
1707             for (j = 0; type_spec[i]  && j < sizeof(charset)-1; i++, j++)
1708             {
1709                 if (type_spec[i] == ';' || type_spec[i] == ' ')
1710                     break;
1711                 charset[j] = cp[i];
1712             }
1713             charset[j] = '\0';
1714         }
1715         else if (!strncmp(type_spec+i, "xpath=", 6))
1716         {
1717             int j = 0;
1718             i = i + 6;
1719             for (j = 0; type_spec[i] && j < sizeof(xpath)-1; i++, j++)
1720                 xpath[j] = cp[i];
1721             xpath[j] = '\0';
1722         }
1723         while (type_spec[i] == ' ')
1724             i++;
1725     }
1726     if (!strcmp (type, "database"))
1727     {
1728         if (len)
1729             *len = (npr->databaseName ? strlen(npr->databaseName) : 0);
1730         return npr->databaseName;
1731     }
1732     else if (!strcmp (type, "syntax"))
1733     {
1734         const char *desc = 0;
1735         if (npr->which == Z_NamePlusRecord_databaseRecord)
1736         {
1737             Z_External *r = (Z_External *) npr->u.databaseRecord;
1738             oident *ent = oid_getentbyoid(r->direct_reference);
1739             if (ent)
1740                 desc = ent->desc;
1741         }
1742         if (!desc)
1743             desc = "none";
1744         if (len)
1745             *len = strlen(desc);
1746         return desc;
1747     }
1748     if (npr->which != Z_NamePlusRecord_databaseRecord)
1749         return 0;
1750
1751     /* from now on - we have a database record .. */
1752     if (!strcmp (type, "render"))
1753     {
1754         Z_External *r = (Z_External *) npr->u.databaseRecord;
1755         oident *ent = oid_getentbyoid(r->direct_reference);
1756
1757         /* render bibliographic record .. */
1758         if (r->which == Z_External_OPAC)
1759         {
1760             r = r->u.opac->bibliographicRecord;
1761             if (!r)
1762                 return 0;
1763             ent = oid_getentbyoid(r->direct_reference);
1764         }
1765         if (r->which == Z_External_sutrs)
1766             return record_iconv_return(rec, len,
1767                                        (char*) r->u.sutrs->buf,
1768                                        r->u.sutrs->len,
1769                                        charset);
1770         else if (r->which == Z_External_octet)
1771         {
1772             const char *ret_buf;
1773             switch (ent->value)
1774             {
1775             case VAL_SOIF:
1776             case VAL_HTML:
1777             case VAL_SUTRS:
1778                 break;
1779             case VAL_TEXT_XML:
1780             case VAL_APPLICATION_XML:
1781                 break;
1782             default:
1783                 ret_buf = marc_iconv_return(
1784                     rec, YAZ_MARC_LINE, len,
1785                     (const char *) r->u.octet_aligned->buf,
1786                     r->u.octet_aligned->len,
1787                     charset);
1788                 if (ret_buf)
1789                     return ret_buf;
1790             }
1791             return record_iconv_return(rec, len,
1792                                        (const char *) r->u.octet_aligned->buf,
1793                                        r->u.octet_aligned->len,
1794                                        charset);
1795         }
1796         else if (r->which == Z_External_grs1)
1797         {
1798             if (!rec->wrbuf_marc)
1799                 rec->wrbuf_marc = wrbuf_alloc();
1800             wrbuf_rewind (rec->wrbuf_marc);
1801             yaz_display_grs1(rec->wrbuf_marc, r->u.grs1, 0);
1802             return record_iconv_return(rec, len,
1803                                        wrbuf_buf(rec->wrbuf_marc),
1804                                        wrbuf_len(rec->wrbuf_marc),
1805                                        charset);
1806         }
1807         return 0;
1808     }
1809     else if (!strcmp (type, "xml") || !strcmp(type, "oai"))
1810     {
1811         Z_External *r = (Z_External *) npr->u.databaseRecord;
1812         oident *ent = oid_getentbyoid(r->direct_reference);
1813
1814         /* render bibliographic record .. */
1815         if (r->which == Z_External_OPAC)
1816         {
1817             r = r->u.opac->bibliographicRecord;
1818             if (!r)
1819                 return 0;
1820             ent = oid_getentbyoid(r->direct_reference);
1821         }
1822
1823         if (r->which == Z_External_sutrs)
1824             return record_iconv_return(rec, len,
1825                                        (const char *) r->u.sutrs->buf,
1826                                        r->u.sutrs->len,
1827                                        charset);
1828         else if (r->which == Z_External_octet)
1829         {
1830             const char *ret_buf;
1831             int marc_decode_type = YAZ_MARC_MARCXML;
1832
1833             if (!strcmp(type, "oai"))
1834                 marc_decode_type = YAZ_MARC_OAIMARC;
1835             switch (ent->value)
1836             {
1837             case VAL_SOIF:
1838             case VAL_HTML:
1839             case VAL_SUTRS:
1840                 break;
1841             case VAL_TEXT_XML:
1842             case VAL_APPLICATION_XML:
1843                 break;
1844             default:
1845                 ret_buf = marc_iconv_return(
1846                     rec, marc_decode_type, len,
1847                     (const char *) r->u.octet_aligned->buf,
1848                     r->u.octet_aligned->len,
1849                     charset);
1850                 if (ret_buf)
1851                     return ret_buf;
1852             }
1853             return record_iconv_return(rec, len,
1854                                        (const char *) r->u.octet_aligned->buf,
1855                                        r->u.octet_aligned->len,
1856                                        charset);
1857         }
1858         else if (r->which == Z_External_grs1)
1859         {
1860             if (len) *len = 5;
1861             return "GRS-1";
1862         }
1863         return 0;
1864     }
1865     else if (!strcmp (type, "raw"))
1866     {
1867         Z_External *r = (Z_External *) npr->u.databaseRecord;
1868
1869         if (r->which == Z_External_sutrs)
1870         {
1871             if (len) *len = r->u.sutrs->len;
1872             return (const char *) r->u.sutrs->buf;
1873         }
1874         else if (r->which == Z_External_octet)
1875         {
1876             if (len) *len = r->u.octet_aligned->len;
1877             return (const char *) r->u.octet_aligned->buf;
1878         }
1879         else /* grs-1, explain, OPAC, ... */
1880         {
1881             if (len) *len = -1;
1882             return (const char *) npr->u.databaseRecord;
1883         }
1884         return 0;
1885     }
1886     else if (!strcmp (type, "ext"))
1887     {
1888         if (len) *len = -1;
1889         return (const char *) npr->u.databaseRecord;
1890     }
1891     else if (!strcmp (type, "opac"))
1892
1893     {
1894         Z_External *r = (Z_External *) npr->u.databaseRecord;
1895         if (r->which == Z_External_OPAC)
1896         {
1897             if (!rec->wrbuf_opac)
1898                 rec->wrbuf_opac = wrbuf_alloc();
1899             wrbuf_rewind (rec->wrbuf_opac);
1900             yaz_display_OPAC(rec->wrbuf_opac, r->u.opac, 0);
1901             return record_iconv_return(rec, len,
1902                                        wrbuf_buf(rec->wrbuf_opac),
1903                                        wrbuf_len(rec->wrbuf_opac),
1904                                        charset);
1905         }
1906     }
1907     return 0;
1908 }
1909
1910 static int strcmp_null(const char *v1, const char *v2)
1911 {
1912     if (!v1 && !v2)
1913         return 0;
1914     if (!v1 || !v2)
1915         return -1;
1916     return strcmp(v1, v2);
1917 }
1918
1919 static void record_cache_add (ZOOM_resultset r, Z_NamePlusRecord *npr,
1920                               int pos)
1921 {
1922     ZOOM_record_cache rc;
1923     const char *elementSetName =
1924         ZOOM_resultset_option_get (r, "elementSetName");
1925     const char *syntax =
1926         ZOOM_resultset_option_get (r, "preferredRecordSyntax");
1927
1928     ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_RECV_RECORD);
1929     ZOOM_connection_put_event(r->connection, event);
1930
1931     for (rc = r->record_cache; rc; rc = rc->next)
1932     {
1933         if (pos == rc->pos)
1934         {
1935             if (strcmp_null(r->schema, rc->schema))
1936                 continue;
1937             if (strcmp_null(elementSetName,rc->elementSetName))
1938                 continue;
1939             if (strcmp_null(syntax, rc->syntax))
1940                 continue;
1941             /* not destroying rc->npr (it's handled by nmem )*/
1942             rc->rec.npr = npr;
1943             /* keeping wrbuf_marc too */
1944             return;
1945         }
1946     }
1947     rc = (ZOOM_record_cache) odr_malloc (r->odr, sizeof(*rc));
1948     rc->rec.npr = npr;
1949     rc->rec.odr = 0;
1950     rc->rec.wrbuf_marc = 0;
1951     rc->rec.wrbuf_iconv = 0;
1952     rc->rec.wrbuf_opac = 0;
1953     if (elementSetName)
1954         rc->elementSetName = odr_strdup (r->odr, elementSetName);
1955     else
1956         rc->elementSetName = 0;
1957
1958     if (syntax)
1959         rc->syntax = odr_strdup (r->odr, syntax);
1960     else
1961         rc->syntax = 0;
1962
1963     if (r->schema)
1964         rc->schema = odr_strdup (r->odr, r->schema);
1965     else
1966         rc->schema = 0;
1967
1968     rc->pos = pos;
1969     rc->next = r->record_cache;
1970     r->record_cache = rc;
1971 }
1972
1973 static ZOOM_record record_cache_lookup (ZOOM_resultset r, int pos)
1974 {
1975     ZOOM_record_cache rc;
1976     const char *elementSetName =
1977         ZOOM_resultset_option_get (r, "elementSetName");
1978     const char *syntax =
1979         ZOOM_resultset_option_get (r, "preferredRecordSyntax");
1980
1981     for (rc = r->record_cache; rc; rc = rc->next)
1982     {
1983         if (pos == rc->pos)
1984         {
1985             if (strcmp_null(r->schema, rc->schema))
1986                 continue;
1987             if (strcmp_null(elementSetName,rc->elementSetName))
1988                 continue;
1989             if (strcmp_null(syntax, rc->syntax))
1990                 continue;
1991             return &rc->rec;
1992         }
1993     }
1994     return 0;
1995 }
1996
1997 static void handle_records (ZOOM_connection c, Z_Records *sr,
1998                             int present_phase)
1999 {
2000     ZOOM_resultset resultset;
2001
2002     if (!c->tasks)
2003         return ;
2004     switch (c->tasks->which)
2005     {
2006     case ZOOM_TASK_SEARCH:
2007         resultset = c->tasks->u.search.resultset;
2008         break;
2009     case ZOOM_TASK_RETRIEVE:
2010         resultset = c->tasks->u.retrieve.resultset;
2011         break;
2012     default:
2013         return;
2014     }
2015     if (sr && sr->which == Z_Records_NSD)
2016         response_default_diag(c, sr->u.nonSurrogateDiagnostic);
2017     else if (sr && sr->which == Z_Records_multipleNSD)
2018     {
2019         if (sr->u.multipleNonSurDiagnostics->num_diagRecs >= 1)
2020             response_diag(c, sr->u.multipleNonSurDiagnostics->diagRecs[0]);
2021         else
2022             set_ZOOM_error(c, ZOOM_ERROR_DECODE, 0);
2023     }
2024     else
2025     {
2026         if (resultset->count + resultset->start > resultset->size)
2027             resultset->count = resultset->size - resultset->start;
2028         if (resultset->count < 0)
2029             resultset->count = 0;
2030         if (sr && sr->which == Z_Records_DBOSD)
2031         {
2032             int i;
2033             NMEM nmem = odr_extract_mem (c->odr_in);
2034             Z_NamePlusRecordList *p =
2035                 sr->u.databaseOrSurDiagnostics;
2036             for (i = 0; i<p->num_records; i++)
2037             {
2038                 record_cache_add (resultset, p->records[i],
2039                                   i+ resultset->start);
2040             }
2041             /* transfer our response to search_nmem .. we need it later */
2042             nmem_transfer (resultset->odr->mem, nmem);
2043             nmem_destroy (nmem);
2044             if (present_phase && p->num_records == 0)
2045             {
2046                 /* present response and we didn't get any records! */
2047                 Z_NamePlusRecord *myrec =
2048                     zget_surrogateDiagRec(resultset->odr, 0, 14, 0);
2049                 record_cache_add(resultset, myrec, resultset->start);
2050             }
2051         }
2052         else if (present_phase)
2053         {
2054             /* present response and we didn't get any records! */
2055             Z_NamePlusRecord *myrec =
2056                 zget_surrogateDiagRec(resultset->odr, 0, 14, 0);
2057             record_cache_add(resultset, myrec, resultset->start);
2058         }
2059     }
2060 }
2061
2062 static void handle_present_response (ZOOM_connection c, Z_PresentResponse *pr)
2063 {
2064     handle_records (c, pr->records, 1);
2065 }
2066
2067 static void handle_queryExpressionTerm(ZOOM_options opt, const char *name,
2068                                        Z_Term *term)
2069 {
2070     switch (term->which)
2071     {
2072     case Z_Term_general:
2073         ZOOM_options_setl(opt, name,
2074                           term->u.general->buf, term->u.general->len);
2075         break;
2076     case Z_Term_characterString:
2077         ZOOM_options_set(opt, name, term->u.characterString);
2078         break;
2079     case Z_Term_numeric:
2080         ZOOM_options_set_int(opt, name, *term->u.numeric);
2081         break;
2082     }
2083 }
2084
2085 static void handle_queryExpression(ZOOM_options opt, const char *name,
2086                                    Z_QueryExpression *exp)
2087 {
2088     char opt_name[80];
2089
2090     switch (exp->which)
2091     {
2092     case Z_QueryExpression_term:
2093         if (exp->u.term && exp->u.term->queryTerm)
2094         {
2095             sprintf(opt_name, "%s.term", name);
2096             handle_queryExpressionTerm(opt, opt_name, exp->u.term->queryTerm);
2097         }
2098         break;
2099     case Z_QueryExpression_query:
2100         break;
2101     }
2102 }
2103
2104 static void handle_searchResult(ZOOM_connection c, ZOOM_resultset resultset,
2105                                 Z_OtherInformation *o)
2106 {
2107     int i;
2108     for (i = 0; o && i < o->num_elements; i++)
2109     {
2110         if (o->list[i]->which == Z_OtherInfo_externallyDefinedInfo)
2111         {
2112             Z_External *ext = o->list[i]->information.externallyDefinedInfo;
2113
2114             if (ext->which == Z_External_searchResult1)
2115             {
2116                 int j;
2117                 Z_SearchInfoReport *sr = ext->u.searchResult1;
2118
2119                 if (sr->num)
2120                     ZOOM_options_set_int(
2121                         resultset->options, "searchresult.size", sr->num);
2122
2123                 for (j = 0; j < sr->num; j++)
2124                 {
2125                     Z_SearchInfoReport_s *ent =
2126                         ext->u.searchResult1->elements[j];
2127                     char pref[80];
2128
2129                     sprintf(pref, "searchresult.%d", j);
2130
2131                     if (ent->subqueryId)
2132                     {
2133                         char opt_name[80];
2134                         sprintf(opt_name, "%s.id", pref);
2135                         ZOOM_options_set(resultset->options, opt_name,
2136                                          ent->subqueryId);
2137                     }
2138                     if (ent->subqueryExpression)
2139                     {
2140                         char opt_name[80];
2141                         sprintf(opt_name, "%s.subquery", pref);
2142                         handle_queryExpression(resultset->options, opt_name,
2143                                                ent->subqueryExpression);
2144                     }
2145                     if (ent->subqueryInterpretation)
2146                     {
2147                         char opt_name[80];
2148                         sprintf(opt_name, "%s.interpretation", pref);
2149                         handle_queryExpression(resultset->options, opt_name,
2150                                                ent->subqueryInterpretation);
2151                     }
2152                     if (ent->subqueryRecommendation)
2153                     {
2154                         char opt_name[80];
2155                         sprintf(opt_name, "%s.recommendation", pref);
2156                         handle_queryExpression(resultset->options, opt_name,
2157                                                ent->subqueryRecommendation);
2158                     }
2159                     if (ent->subqueryCount)
2160                     {
2161                         char opt_name[80];
2162                         sprintf(opt_name, "%s.count", pref);
2163                         ZOOM_options_set_int(resultset->options, opt_name,
2164                                              *ent->subqueryCount);
2165                     }
2166                 }
2167             }
2168         }
2169     }
2170 }
2171
2172 static void handle_search_response (ZOOM_connection c, Z_SearchResponse *sr)
2173 {
2174     ZOOM_resultset resultset;
2175     ZOOM_Event event;
2176
2177     if (!c->tasks || c->tasks->which != ZOOM_TASK_SEARCH)
2178         return ;
2179
2180     event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH);
2181     ZOOM_connection_put_event(c, event);
2182
2183     resultset = c->tasks->u.search.resultset;
2184
2185     handle_searchResult(c, resultset, sr->additionalSearchInfo);
2186
2187     resultset->size = *sr->resultCount;
2188     handle_records (c, sr->records, 0);
2189 }
2190
2191 static void sort_response (ZOOM_connection c, Z_SortResponse *res)
2192 {
2193     if (res->diagnostics && res->num_diagnostics > 0)
2194         response_diag (c, res->diagnostics[0]);
2195 }
2196
2197 static int scan_response (ZOOM_connection c, Z_ScanResponse *res)
2198 {
2199     NMEM nmem = odr_extract_mem (c->odr_in);
2200     ZOOM_scanset scan;
2201
2202     if (!c->tasks || c->tasks->which != ZOOM_TASK_SCAN)
2203         return 0;
2204     scan = c->tasks->u.scan.scan;
2205
2206     if (res->entries && res->entries->nonsurrogateDiagnostics)
2207         response_diag(c, res->entries->nonsurrogateDiagnostics[0]);
2208     scan->scan_response = res;
2209     nmem_transfer (scan->odr->mem, nmem);
2210     if (res->stepSize)
2211         ZOOM_options_set_int (scan->options, "stepSize", *res->stepSize);
2212     if (res->positionOfTerm)
2213         ZOOM_options_set_int (scan->options, "position", *res->positionOfTerm);
2214     if (res->scanStatus)
2215         ZOOM_options_set_int (scan->options, "scanStatus", *res->scanStatus);
2216     if (res->numberOfEntriesReturned)
2217         ZOOM_options_set_int (scan->options, "number",
2218                               *res->numberOfEntriesReturned);
2219     nmem_destroy (nmem);
2220     return 1;
2221 }
2222
2223 static zoom_ret send_sort (ZOOM_connection c,
2224                            ZOOM_resultset resultset)
2225 {
2226     if (c->error)
2227         resultset->r_sort_spec = 0;
2228     if (resultset->r_sort_spec)
2229     {
2230         Z_APDU *apdu = zget_APDU(c->odr_out, Z_APDU_sortRequest);
2231         Z_SortRequest *req = apdu->u.sortRequest;
2232
2233         req->num_inputResultSetNames = 1;
2234         req->inputResultSetNames = (Z_InternationalString **)
2235             odr_malloc (c->odr_out, sizeof(*req->inputResultSetNames));
2236         req->inputResultSetNames[0] =
2237             odr_strdup (c->odr_out, resultset->setname);
2238         req->sortedResultSetName = odr_strdup (c->odr_out, resultset->setname);
2239         req->sortSequence = resultset->r_sort_spec;
2240         resultset->r_sort_spec = 0;
2241         return send_APDU (c, apdu);
2242     }
2243     return zoom_complete;
2244 }
2245
2246 static zoom_ret send_present(ZOOM_connection c)
2247 {
2248     Z_APDU *apdu = 0;
2249     Z_PresentRequest *req = 0;
2250     int i = 0;
2251     const char *syntax = 0;
2252     const char *elementSetName = 0;
2253     ZOOM_resultset  resultset;
2254
2255     if (!c->tasks)
2256     {
2257         yaz_log(log_details, "%p send_present no tasks", c);
2258         return zoom_complete;
2259     }
2260     switch (c->tasks->which)
2261     {
2262     case ZOOM_TASK_SEARCH:
2263         resultset = c->tasks->u.search.resultset;
2264         break;
2265     case ZOOM_TASK_RETRIEVE:
2266         resultset = c->tasks->u.retrieve.resultset;
2267         resultset->start = c->tasks->u.retrieve.start;
2268         resultset->count = c->tasks->u.retrieve.count;
2269
2270         if (resultset->start >= resultset->size)
2271         {
2272             yaz_log(log_details, "%p send_present start=%d >= size=%d",
2273                     c, resultset->start, resultset->size);
2274             return zoom_complete;
2275         }
2276         if (resultset->start + resultset->count > resultset->size)
2277             resultset->count = resultset->size - resultset->start;
2278         break;
2279     default:
2280         return zoom_complete;
2281     }
2282     yaz_log(log_details, "%p send_present start=%d count=%d",
2283             c, resultset->start, resultset->count);
2284
2285     syntax = ZOOM_resultset_option_get (resultset, "preferredRecordSyntax");
2286     elementSetName = ZOOM_resultset_option_get (resultset, "elementSetName");
2287
2288     if (c->error)                  /* don't continue on error */
2289         return zoom_complete;
2290     if (resultset->start < 0)
2291         return zoom_complete;
2292     for (i = 0; i<resultset->count; i++)
2293     {
2294         ZOOM_record rec =
2295             record_cache_lookup (resultset, i + resultset->start);
2296         if (!rec)
2297             break;
2298     }
2299     if (i == resultset->count)
2300     {
2301         yaz_log(log_details, "%p send_present skip=%d no more to fetch", c, i);
2302         return zoom_complete;
2303     }
2304
2305     apdu = zget_APDU(c->odr_out, Z_APDU_presentRequest);
2306     req = apdu->u.presentRequest;
2307
2308     resultset->start += i;
2309     resultset->count -= i;
2310
2311     if (i)
2312         yaz_log(log_details, "%p send_present skip=%d", c, i);
2313
2314     *req->resultSetStartPoint = resultset->start + 1;
2315     *req->numberOfRecordsRequested = resultset->step>0 ?
2316         resultset->step : resultset->count;
2317     assert (*req->numberOfRecordsRequested > 0);
2318
2319
2320     if (syntax && *syntax)
2321         req->preferredRecordSyntax =
2322             yaz_str_to_z3950oid (c->odr_out, CLASS_RECSYN, syntax);
2323
2324     if (resultset->schema && *resultset->schema)
2325     {
2326         Z_RecordComposition *compo = (Z_RecordComposition *)
2327             odr_malloc (c->odr_out, sizeof(*compo));
2328
2329         req->recordComposition = compo;
2330         compo->which = Z_RecordComp_complex;
2331         compo->u.complex = (Z_CompSpec *)
2332             odr_malloc(c->odr_out, sizeof(*compo->u.complex));
2333         compo->u.complex->selectAlternativeSyntax = (bool_t *)
2334             odr_malloc(c->odr_out, sizeof(bool_t));
2335         *compo->u.complex->selectAlternativeSyntax = 0;
2336
2337         compo->u.complex->generic = (Z_Specification *)
2338             odr_malloc(c->odr_out, sizeof(*compo->u.complex->generic));
2339
2340         compo->u.complex->generic->which = Z_Schema_oid;
2341         compo->u.complex->generic->schema.oid = (Odr_oid *)
2342             yaz_str_to_z3950oid (c->odr_out, CLASS_SCHEMA, resultset->schema);
2343
2344         if (!compo->u.complex->generic->schema.oid)
2345         {
2346             /* OID wasn't a schema! Try record syntax instead. */
2347
2348             compo->u.complex->generic->schema.oid = (Odr_oid *)
2349                 yaz_str_to_z3950oid (c->odr_out, CLASS_RECSYN, resultset->schema);
2350         }
2351         if (elementSetName && *elementSetName)
2352         {
2353             compo->u.complex->generic->elementSpec = (Z_ElementSpec *)
2354                 odr_malloc(c->odr_out, sizeof(Z_ElementSpec));
2355             compo->u.complex->generic->elementSpec->which =
2356                 Z_ElementSpec_elementSetName;
2357             compo->u.complex->generic->elementSpec->u.elementSetName =
2358                 odr_strdup (c->odr_out, elementSetName);
2359         }
2360         else
2361             compo->u.complex->generic->elementSpec = 0;
2362         compo->u.complex->num_dbSpecific = 0;
2363         compo->u.complex->dbSpecific = 0;
2364         compo->u.complex->num_recordSyntax = 0;
2365         compo->u.complex->recordSyntax = 0;
2366     }
2367     else if (elementSetName && *elementSetName)
2368     {
2369         Z_ElementSetNames *esn = (Z_ElementSetNames *)
2370             odr_malloc (c->odr_out, sizeof(*esn));
2371         Z_RecordComposition *compo = (Z_RecordComposition *)
2372             odr_malloc (c->odr_out, sizeof(*compo));
2373
2374         esn->which = Z_ElementSetNames_generic;
2375         esn->u.generic = odr_strdup (c->odr_out, elementSetName);
2376         compo->which = Z_RecordComp_simple;
2377         compo->u.simple = esn;
2378         req->recordComposition = compo;
2379     }
2380     req->resultSetId = odr_strdup(c->odr_out, resultset->setname);
2381     return send_APDU (c, apdu);
2382 }
2383
2384 ZOOM_API(ZOOM_scanset)
2385 ZOOM_connection_scan (ZOOM_connection c, const char *start)
2386 {
2387     ZOOM_scanset scan = (ZOOM_scanset) xmalloc (sizeof(*scan));
2388
2389     scan->connection = c;
2390     scan->odr = odr_createmem (ODR_DECODE);
2391     scan->options = ZOOM_options_create_with_parent (c->options);
2392     scan->refcount = 1;
2393     scan->scan_response = 0;
2394
2395     if ((scan->termListAndStartPoint =
2396          p_query_scan(scan->odr, PROTO_Z3950, &scan->attributeSet,
2397                       start)))
2398     {
2399         ZOOM_task task = ZOOM_connection_add_task (c, ZOOM_TASK_SCAN);
2400         task->u.scan.scan = scan;
2401
2402         (scan->refcount)++;
2403         if (!c->async)
2404         {
2405             while (ZOOM_event (1, &c))
2406                 ;
2407         }
2408     }
2409     return scan;
2410 }
2411
2412 ZOOM_API(void)
2413 ZOOM_scanset_destroy (ZOOM_scanset scan)
2414 {
2415     if (!scan)
2416         return;
2417     (scan->refcount)--;
2418     if (scan->refcount == 0)
2419     {
2420         odr_destroy (scan->odr);
2421
2422         ZOOM_options_destroy (scan->options);
2423         xfree (scan);
2424     }
2425 }
2426
2427 static zoom_ret send_package (ZOOM_connection c)
2428 {
2429     ZOOM_Event event;
2430
2431     yaz_log(log_details, "%p send_package", c);
2432     if (!c->tasks)
2433         return zoom_complete;
2434     assert (c->tasks->which == ZOOM_TASK_PACKAGE);
2435
2436     event = ZOOM_Event_create (ZOOM_EVENT_SEND_APDU);
2437     ZOOM_connection_put_event (c, event);
2438
2439     return do_write_ex (c, c->tasks->u.package->buf_out,
2440                         c->tasks->u.package->len_out);
2441 }
2442
2443 static zoom_ret send_scan (ZOOM_connection c)
2444 {
2445     ZOOM_scanset scan;
2446     Z_APDU *apdu = zget_APDU(c->odr_out, Z_APDU_scanRequest);
2447     Z_ScanRequest *req = apdu->u.scanRequest;
2448
2449     yaz_log(log_details, "%p send_scan", c);
2450     if (!c->tasks)
2451         return zoom_complete;
2452     assert (c->tasks->which == ZOOM_TASK_SCAN);
2453     scan = c->tasks->u.scan.scan;
2454
2455     req->termListAndStartPoint = scan->termListAndStartPoint;
2456     req->attributeSet = scan->attributeSet;
2457
2458     *req->numberOfTermsRequested =
2459         ZOOM_options_get_int(scan->options, "number", 10);
2460
2461     req->preferredPositionInResponse =
2462         odr_intdup (c->odr_out,
2463                     ZOOM_options_get_int(scan->options, "position", 1));
2464
2465     req->stepSize =
2466         odr_intdup (c->odr_out,
2467                     ZOOM_options_get_int(scan->options, "stepSize", 0));
2468
2469     req->databaseNames = set_DatabaseNames (c, scan->options,
2470                                             &req->num_databaseNames);
2471
2472     return send_APDU (c, apdu);
2473 }
2474
2475 ZOOM_API(size_t)
2476 ZOOM_scanset_size (ZOOM_scanset scan)
2477 {
2478     if (!scan || !scan->scan_response || !scan->scan_response->entries)
2479         return 0;
2480     return scan->scan_response->entries->num_entries;
2481 }
2482
2483 ZOOM_API(const char *)
2484 ZOOM_scanset_term (ZOOM_scanset scan, size_t pos,
2485                    int *occ, int *len)
2486 {
2487     const char *term = 0;
2488     size_t noent = ZOOM_scanset_size (scan);
2489     Z_ScanResponse *res = scan->scan_response;
2490
2491     *len = 0;
2492     *occ = 0;
2493     if (pos >= noent)
2494         return 0;
2495     if (res->entries->entries[pos]->which == Z_Entry_termInfo)
2496     {
2497         Z_TermInfo *t = res->entries->entries[pos]->u.termInfo;
2498
2499         if (t->term->which == Z_Term_general)
2500         {
2501             term = (const char *) t->term->u.general->buf;
2502             *len = t->term->u.general->len;
2503         }
2504         *occ = t->globalOccurrences ? *t->globalOccurrences : 0;
2505     }
2506     return term;
2507 }
2508
2509 ZOOM_API(const char *)
2510 ZOOM_scanset_display_term (ZOOM_scanset scan, size_t pos,
2511                            int *occ, int *len)
2512 {
2513     const char *term = 0;
2514     size_t noent = ZOOM_scanset_size (scan);
2515     Z_ScanResponse *res = scan->scan_response;
2516
2517     *len = 0;
2518     *occ = 0;
2519     if (pos >= noent)
2520         return 0;
2521     if (res->entries->entries[pos]->which == Z_Entry_termInfo)
2522     {
2523         Z_TermInfo *t = res->entries->entries[pos]->u.termInfo;
2524
2525         if (t->displayTerm)
2526         {
2527             term = t->displayTerm;
2528             *len = strlen(term);
2529         }
2530         else if (t->term->which == Z_Term_general)
2531         {
2532             term = (const char *) t->term->u.general->buf;
2533             *len = t->term->u.general->len;
2534         }
2535         *occ = t->globalOccurrences ? *t->globalOccurrences : 0;
2536     }
2537     return term;
2538 }
2539
2540 ZOOM_API(const char *)
2541 ZOOM_scanset_option_get (ZOOM_scanset scan, const char *key)
2542 {
2543     return ZOOM_options_get (scan->options, key);
2544 }
2545
2546 ZOOM_API(void)
2547 ZOOM_scanset_option_set (ZOOM_scanset scan, const char *key,
2548                               const char *val)
2549 {
2550     ZOOM_options_set (scan->options, key, val);
2551 }
2552
2553 static Z_APDU *create_es_package (ZOOM_package p, int type)
2554 {
2555     const char *str;
2556     Z_APDU *apdu = zget_APDU(p->odr_out, Z_APDU_extendedServicesRequest);
2557     Z_ExtendedServicesRequest *req = apdu->u.extendedServicesRequest;
2558
2559     *req->function = Z_ExtendedServicesRequest_create;
2560
2561     str = ZOOM_options_get(p->options, "package-name");
2562     if (str && *str)
2563         req->packageName = nmem_strdup (p->odr_out->mem, str);
2564
2565     str = ZOOM_options_get(p->options, "user-id");
2566     if (str)
2567         req->userId = nmem_strdup (p->odr_out->mem, str);
2568
2569     req->packageType = yaz_oidval_to_z3950oid(p->odr_out, CLASS_EXTSERV,
2570                                               type);
2571
2572     str = ZOOM_options_get(p->options, "function");
2573     if (str)
2574     {
2575         if (!strcmp (str, "create"))
2576             *req->function = 1;
2577         if (!strcmp (str, "delete"))
2578             *req->function = 2;
2579         if (!strcmp (str, "modify"))
2580             *req->function = 3;
2581     }
2582     return apdu;
2583 }
2584
2585 static const char *ill_array_lookup (void *clientData, const char *idx)
2586 {
2587     ZOOM_package p = (ZOOM_package) clientData;
2588     return ZOOM_options_get (p->options, idx+4);
2589 }
2590
2591 static Z_External *encode_ill_request (ZOOM_package p)
2592 {
2593     ODR out = p->odr_out;
2594     ILL_Request *req;
2595     Z_External *r = 0;
2596     struct ill_get_ctl ctl;
2597
2598     ctl.odr = p->odr_out;
2599     ctl.clientData = p;
2600     ctl.f = ill_array_lookup;
2601
2602     req = ill_get_ILLRequest(&ctl, "ill", 0);
2603
2604     if (!ill_Request (out, &req, 0, 0))
2605     {
2606         int ill_request_size;
2607         char *ill_request_buf = odr_getbuf (out, &ill_request_size, 0);
2608         if (ill_request_buf)
2609             odr_setbuf (out, ill_request_buf, ill_request_size, 1);
2610         return 0;
2611     }
2612     else
2613     {
2614         oident oid;
2615         int illRequest_size = 0;
2616         char *illRequest_buf = odr_getbuf (out, &illRequest_size, 0);
2617
2618         oid.proto = PROTO_GENERAL;
2619         oid.oclass = CLASS_GENERAL;
2620         oid.value = VAL_ISO_ILL_1;
2621
2622         r = (Z_External *) odr_malloc (out, sizeof(*r));
2623         r->direct_reference = odr_oiddup(out,oid_getoidbyent(&oid));
2624         r->indirect_reference = 0;
2625         r->descriptor = 0;
2626         r->which = Z_External_single;
2627
2628         r->u.single_ASN1_type = (Odr_oct *)
2629             odr_malloc (out, sizeof(*r->u.single_ASN1_type));
2630         r->u.single_ASN1_type->buf = (unsigned char*)
2631             odr_malloc (out, illRequest_size);
2632         r->u.single_ASN1_type->len = illRequest_size;
2633         r->u.single_ASN1_type->size = illRequest_size;
2634         memcpy (r->u.single_ASN1_type->buf, illRequest_buf, illRequest_size);
2635     }
2636     return r;
2637 }
2638
2639 static Z_ItemOrder *encode_item_order(ZOOM_package p)
2640 {
2641     Z_ItemOrder *req = (Z_ItemOrder *) odr_malloc (p->odr_out, sizeof(*req));
2642     const char *str;
2643
2644     req->which = Z_IOItemOrder_esRequest;
2645     req->u.esRequest = (Z_IORequest *)
2646         odr_malloc(p->odr_out,sizeof(Z_IORequest));
2647
2648     /* to keep part ... */
2649     req->u.esRequest->toKeep = (Z_IOOriginPartToKeep *)
2650         odr_malloc(p->odr_out,sizeof(Z_IOOriginPartToKeep));
2651     req->u.esRequest->toKeep->supplDescription = 0;
2652     req->u.esRequest->toKeep->contact = (Z_IOContact *)
2653         odr_malloc (p->odr_out, sizeof(*req->u.esRequest->toKeep->contact));
2654
2655     str = ZOOM_options_get(p->options, "contact-name");
2656     req->u.esRequest->toKeep->contact->name = str ?
2657         nmem_strdup (p->odr_out->mem, str) : 0;
2658
2659     str = ZOOM_options_get(p->options, "contact-phone");
2660     req->u.esRequest->toKeep->contact->phone = str ?
2661         nmem_strdup (p->odr_out->mem, str) : 0;
2662
2663     str = ZOOM_options_get(p->options, "contact-email");
2664     req->u.esRequest->toKeep->contact->email = str ?
2665         nmem_strdup (p->odr_out->mem, str) : 0;
2666
2667     req->u.esRequest->toKeep->addlBilling = 0;
2668
2669     /* not to keep part ... */
2670     req->u.esRequest->notToKeep = (Z_IOOriginPartNotToKeep *)
2671         odr_malloc(p->odr_out,sizeof(Z_IOOriginPartNotToKeep));
2672
2673     str = ZOOM_options_get(p->options, "itemorder-setname");
2674     if (!str)
2675         str = "default";
2676
2677     if (!*str)
2678         req->u.esRequest->notToKeep->resultSetItem = 0;
2679     else
2680     {
2681         req->u.esRequest->notToKeep->resultSetItem = (Z_IOResultSetItem *)
2682            odr_malloc(p->odr_out, sizeof(Z_IOResultSetItem));
2683
2684         req->u.esRequest->notToKeep->resultSetItem->resultSetId =
2685            nmem_strdup (p->odr_out->mem, str);
2686         req->u.esRequest->notToKeep->resultSetItem->item =
2687             (int *) odr_malloc(p->odr_out, sizeof(int));
2688
2689         str = ZOOM_options_get(p->options, "itemorder-item");
2690         *req->u.esRequest->notToKeep->resultSetItem->item =
2691             (str ? atoi(str) : 1);
2692     }
2693     req->u.esRequest->notToKeep->itemRequest = encode_ill_request(p);
2694
2695     return req;
2696 }
2697
2698 Z_APDU *create_admin_package(ZOOM_package p, int type,
2699                              Z_ESAdminOriginPartToKeep **toKeepP,
2700                              Z_ESAdminOriginPartNotToKeep **notToKeepP)
2701 {
2702     Z_APDU *apdu = create_es_package (p, VAL_ADMINSERVICE);
2703     if (apdu)
2704     {
2705         Z_ESAdminOriginPartToKeep  *toKeep;
2706         Z_ESAdminOriginPartNotToKeep  *notToKeep;
2707         Z_External *r = (Z_External *) odr_malloc (p->odr_out, sizeof(*r));
2708         const char *first_db = "Default";
2709         int num_db;
2710         char **db = set_DatabaseNames(p->connection, p->options, &num_db);
2711         if (num_db > 0)
2712             first_db = db[0];
2713
2714         r->direct_reference =
2715             yaz_oidval_to_z3950oid(p->odr_out, CLASS_EXTSERV,
2716                                    VAL_ADMINSERVICE);
2717         r->descriptor = 0;
2718         r->indirect_reference = 0;
2719         r->which = Z_External_ESAdmin;
2720
2721         r->u.adminService = (Z_Admin *)
2722             odr_malloc(p->odr_out, sizeof(*r->u.adminService));
2723         r->u.adminService->which = Z_Admin_esRequest;
2724         r->u.adminService->u.esRequest = (Z_AdminEsRequest *)
2725             odr_malloc(p->odr_out, sizeof(*r->u.adminService->u.esRequest));
2726
2727         toKeep = r->u.adminService->u.esRequest->toKeep =
2728             (Z_ESAdminOriginPartToKeep *)
2729             odr_malloc(p->odr_out, sizeof(*r->u.adminService->u.esRequest->toKeep));
2730         toKeep->which = type;
2731         toKeep->databaseName = odr_strdup(p->odr_out, first_db);
2732         toKeep->u.create = odr_nullval();
2733         apdu->u.extendedServicesRequest->taskSpecificParameters = r;
2734
2735         r->u.adminService->u.esRequest->notToKeep = notToKeep =
2736             (Z_ESAdminOriginPartNotToKeep *)
2737             odr_malloc(p->odr_out,
2738                        sizeof(*r->u.adminService->u.esRequest->notToKeep));
2739         notToKeep->which = Z_ESAdminOriginPartNotToKeep_recordsWillFollow;
2740         notToKeep->u.recordsWillFollow = odr_nullval();
2741         if (toKeepP)
2742             *toKeepP = toKeep;
2743         if (notToKeepP)
2744             *notToKeepP = notToKeep;
2745     }
2746     return apdu;
2747 }
2748
2749 static Z_APDU *create_xmlupdate_package(ZOOM_package p)
2750 {
2751     Z_APDU *apdu = create_es_package(p, VAL_XMLES);
2752     Z_ExtendedServicesRequest *req = apdu->u.extendedServicesRequest;
2753     Z_External *ext = (Z_External *) odr_malloc(p->odr_out, sizeof(*ext));
2754     const char *doc = ZOOM_options_get(p->options, "doc");
2755
2756     req->taskSpecificParameters = ext;
2757     ext->direct_reference = req->packageType;
2758     ext->descriptor = 0;
2759     ext->indirect_reference = 0;
2760
2761     ext->which = Z_External_octet;
2762     ext->u.single_ASN1_type = (Odr_oct *)
2763         odr_malloc (p->odr_out, sizeof(Odr_oct));
2764
2765     if (!doc)
2766         doc = "";
2767     ext->u.single_ASN1_type->buf = (unsigned char*)
2768         odr_strdup(p->odr_out, doc);
2769     ext->u.single_ASN1_type->size = ext->u.single_ASN1_type->len = strlen(doc);
2770     return apdu;
2771 }
2772
2773 static Z_APDU *create_update_package(ZOOM_package p)
2774 {
2775     Z_APDU *apdu = 0;
2776     const char *first_db = "Default";
2777     int num_db;
2778     char **db = set_DatabaseNames(p->connection, p->options, &num_db);
2779     const char *action = ZOOM_options_get(p->options, "action");
2780     const char *recordIdOpaque = ZOOM_options_get(p->options, "recordIdOpaque");
2781     const char *recordIdNumber = ZOOM_options_get(p->options, "recordIdNumber");
2782     const char *record_buf = ZOOM_options_get(p->options, "record");
2783     const char *syntax_str = ZOOM_options_get(p->options, "syntax");
2784     int syntax_oid = VAL_NONE;
2785     int action_no = -1;
2786
2787     if (syntax_str)
2788         syntax_oid = oid_getvalbyname(syntax_str);
2789     if (!record_buf)
2790     {
2791         record_buf = "void";
2792         syntax_oid = VAL_SUTRS;
2793     }
2794     if (syntax_oid == VAL_NONE)
2795         syntax_oid = VAL_TEXT_XML;
2796
2797     if (num_db > 0)
2798         first_db = db[0];
2799
2800     if (!action)
2801         action = "specialUpdate";
2802
2803     if (!strcmp(action, "recordInsert"))
2804         action_no = Z_IUOriginPartToKeep_recordInsert;
2805     else if (!strcmp(action, "recordReplace"))
2806         action_no = Z_IUOriginPartToKeep_recordReplace;
2807     else if (!strcmp(action, "recordDelete"))
2808         action_no = Z_IUOriginPartToKeep_recordDelete;
2809     else if (!strcmp(action, "elementUpdate"))
2810         action_no = Z_IUOriginPartToKeep_elementUpdate;
2811     else if (!strcmp(action, "specialUpdate"))
2812         action_no = Z_IUOriginPartToKeep_specialUpdate;
2813     else
2814         return 0;
2815
2816     apdu = create_es_package (p, VAL_DBUPDATE);
2817     if (apdu)
2818     {
2819         Z_IUOriginPartToKeep *toKeep;
2820         Z_IUSuppliedRecords *notToKeep;
2821         Z_External *r = (Z_External *)
2822             odr_malloc (p->odr_out, sizeof(*r));
2823
2824         apdu->u.extendedServicesRequest->taskSpecificParameters = r;
2825
2826         r->direct_reference =
2827             yaz_oidval_to_z3950oid(p->odr_out, CLASS_EXTSERV,
2828                                    VAL_DBUPDATE);
2829         r->descriptor = 0;
2830         r->which = Z_External_update;
2831         r->indirect_reference = 0;
2832         r->u.update = (Z_IUUpdate *)
2833             odr_malloc(p->odr_out, sizeof(*r->u.update));
2834
2835         r->u.update->which = Z_IUUpdate_esRequest;
2836         r->u.update->u.esRequest = (Z_IUUpdateEsRequest *)
2837             odr_malloc(p->odr_out, sizeof(*r->u.update->u.esRequest));
2838         toKeep = r->u.update->u.esRequest->toKeep =
2839             (Z_IUOriginPartToKeep *)
2840             odr_malloc(p->odr_out, sizeof(*toKeep));
2841
2842         toKeep->databaseName = odr_strdup(p->odr_out, first_db);
2843         toKeep->schema = 0;
2844         toKeep->elementSetName = 0;
2845         toKeep->actionQualifier = 0;
2846         toKeep->action = odr_intdup(p->odr_out, action_no);
2847
2848         notToKeep = r->u.update->u.esRequest->notToKeep =
2849             (Z_IUSuppliedRecords *)
2850             odr_malloc(p->odr_out, sizeof(*notToKeep));
2851         notToKeep->num = 1;
2852         notToKeep->elements = (Z_IUSuppliedRecords_elem **)
2853             odr_malloc(p->odr_out, sizeof(*notToKeep->elements));
2854         notToKeep->elements[0] = (Z_IUSuppliedRecords_elem *)
2855             odr_malloc(p->odr_out, sizeof(**notToKeep->elements));
2856         notToKeep->elements[0]->which = Z_IUSuppliedRecords_elem_opaque;
2857         if (recordIdOpaque)
2858         {
2859             notToKeep->elements[0]->u.opaque = (Odr_oct *)
2860                 odr_malloc (p->odr_out, sizeof(Odr_oct));
2861             notToKeep->elements[0]->u.opaque->size =
2862                 notToKeep->elements[0]->u.opaque->len = strlen(recordIdOpaque);
2863             notToKeep->elements[0]->u.opaque->buf = (unsigned char*)
2864                 odr_strdup(p->odr_out, recordIdOpaque);
2865         }
2866         else if (recordIdNumber)
2867         {
2868             notToKeep->elements[0]->which = Z_IUSuppliedRecords_elem_number;
2869
2870             notToKeep->elements[0]->u.number =
2871                 odr_intdup(p->odr_out, atoi(recordIdNumber));
2872         }
2873         else
2874             notToKeep->elements[0]->u.opaque = 0;
2875         notToKeep->elements[0]->supplementalId = 0;
2876         notToKeep->elements[0]->correlationInfo = 0;
2877         notToKeep->elements[0]->record =
2878             z_ext_record(p->odr_out, syntax_oid,
2879                          record_buf, strlen(record_buf));
2880     }
2881     if (0 && apdu)
2882     {
2883        ODR print = odr_createmem(ODR_PRINT);
2884
2885        z_APDU(print, &apdu, 0, 0);
2886        odr_destroy(print);
2887     }
2888     return apdu;
2889 }
2890
2891 ZOOM_API(void)
2892     ZOOM_package_send (ZOOM_package p, const char *type)
2893 {
2894     Z_APDU *apdu = 0;
2895     ZOOM_connection c;
2896     if (!p)
2897         return;
2898     c = p->connection;
2899     odr_reset (p->odr_out);
2900     xfree (p->buf_out);
2901     p->buf_out = 0;
2902     if (!strcmp(type, "itemorder"))
2903     {
2904         apdu = create_es_package (p, VAL_ITEMORDER);
2905         if (apdu)
2906         {
2907             Z_External *r = (Z_External *) odr_malloc (p->odr_out, sizeof(*r));
2908
2909             r->direct_reference =
2910                 yaz_oidval_to_z3950oid(p->odr_out, CLASS_EXTSERV,
2911                                        VAL_ITEMORDER);
2912             r->descriptor = 0;
2913             r->which = Z_External_itemOrder;
2914             r->indirect_reference = 0;
2915             r->u.itemOrder = encode_item_order (p);
2916
2917             apdu->u.extendedServicesRequest->taskSpecificParameters = r;
2918         }
2919     }
2920     else if (!strcmp(type, "create"))  /* create database */
2921     {
2922         apdu = create_admin_package(p, Z_ESAdminOriginPartToKeep_create,
2923                                     0, 0);
2924     }
2925     else if (!strcmp(type, "drop"))  /* drop database */
2926     {
2927         apdu = create_admin_package(p, Z_ESAdminOriginPartToKeep_drop,
2928                                     0, 0);
2929     }
2930     else if (!strcmp(type, "commit"))  /* commit changes */
2931     {
2932         apdu = create_admin_package(p, Z_ESAdminOriginPartToKeep_commit,
2933                                     0, 0);
2934     }
2935     else if (!strcmp(type, "update")) /* update record(s) */
2936     {
2937         apdu = create_update_package(p);
2938     }
2939     else if (!strcmp(type, "xmlupdate"))
2940     {
2941         apdu = create_xmlupdate_package(p);
2942     }
2943     if (apdu)
2944     {
2945         if (encode_APDU(p->connection, apdu, p->odr_out) == 0)
2946         {
2947             char *buf;
2948
2949             ZOOM_task task = ZOOM_connection_add_task (c, ZOOM_TASK_PACKAGE);
2950             task->u.package = p;
2951             buf = odr_getbuf(p->odr_out, &p->len_out, 0);
2952             p->buf_out = (char *) xmalloc (p->len_out);
2953             memcpy (p->buf_out, buf, p->len_out);
2954
2955             (p->refcount)++;
2956             if (!c->async)
2957             {
2958                 while (ZOOM_event (1, &c))
2959                     ;
2960             }
2961         }
2962     }
2963 }
2964
2965 ZOOM_API(ZOOM_package)
2966     ZOOM_connection_package (ZOOM_connection c, ZOOM_options options)
2967 {
2968     ZOOM_package p = (ZOOM_package) xmalloc (sizeof(*p));
2969
2970     p->connection = c;
2971     p->odr_out = odr_createmem (ODR_ENCODE);
2972     p->options = ZOOM_options_create_with_parent2 (options, c->options);
2973     p->refcount = 1;
2974     p->buf_out = 0;
2975     p->len_out = 0;
2976     return p;
2977 }
2978
2979 ZOOM_API(void)
2980     ZOOM_package_destroy(ZOOM_package p)
2981 {
2982     if (!p)
2983         return;
2984     (p->refcount)--;
2985     if (p->refcount == 0)
2986     {
2987         odr_destroy (p->odr_out);
2988         xfree (p->buf_out);
2989
2990         ZOOM_options_destroy (p->options);
2991         xfree (p);
2992     }
2993 }
2994
2995 ZOOM_API(const char *)
2996 ZOOM_package_option_get (ZOOM_package p, const char *key)
2997 {
2998     return ZOOM_options_get (p->options, key);
2999 }
3000
3001
3002 ZOOM_API(void)
3003 ZOOM_package_option_set (ZOOM_package p, const char *key,
3004                               const char *val)
3005 {
3006     ZOOM_options_set (p->options, key, val);
3007 }
3008
3009 static int ZOOM_connection_exec_task (ZOOM_connection c)
3010 {
3011     ZOOM_task task = c->tasks;
3012     zoom_ret ret = zoom_complete;
3013
3014     if (!task)
3015         return 0;
3016     yaz_log(log_details, "%p ZOOM_connection_exec_task type=%d run=%d",
3017              c, task->which, task->running);
3018     if (c->error != ZOOM_ERROR_NONE)
3019     {
3020         yaz_log(log_details, "%p ZOOM_connection_exec_task "
3021                 "removing tasks because of error = %d", c, c->error);
3022         ZOOM_connection_remove_tasks (c);
3023         return 0;
3024     }
3025     if (task->running)
3026     {
3027         yaz_log(log_details, "%p ZOOM_connection_exec_task "
3028                  "task already running", c);
3029         return 0;
3030     }
3031     task->running = 1;
3032     ret = zoom_complete;
3033     if (c->cs || task->which == ZOOM_TASK_CONNECT)
3034     {
3035         switch (task->which)
3036         {
3037         case ZOOM_TASK_SEARCH:
3038             if (c->proto == PROTO_HTTP)
3039                 ret = ZOOM_connection_srw_send_search(c);
3040             else
3041                 ret = ZOOM_connection_send_search(c);
3042             break;
3043         case ZOOM_TASK_RETRIEVE:
3044             if (c->proto == PROTO_HTTP)
3045                 ret = ZOOM_connection_srw_send_search(c);
3046             else
3047                 ret = send_present (c);
3048             break;
3049         case ZOOM_TASK_CONNECT:
3050             ret = do_connect(c);
3051             break;
3052         case ZOOM_TASK_SCAN:
3053             ret = send_scan(c);
3054             break;
3055         case ZOOM_TASK_PACKAGE:
3056             ret = send_package(c);
3057             break;
3058         case ZOOM_TASK_SORT:
3059             c->tasks->u.sort.resultset->r_sort_spec =
3060                 c->tasks->u.sort.q->sort_spec;
3061             ret = send_sort(c, c->tasks->u.sort.resultset);
3062             break;
3063         }
3064     }
3065     else
3066     {
3067         yaz_log(log_details, "%p ZOOM_connection_exec_task "
3068                 "remove tasks because no connection exist", c);
3069         ZOOM_connection_remove_tasks (c);
3070     }
3071     if (ret == zoom_complete)
3072     {
3073         yaz_log(log_details, "%p ZOOM_connection_exec_task "
3074                  "task removed (complete)", c);
3075         ZOOM_connection_remove_task (c);
3076         return 0;
3077     }
3078     yaz_log(log_details, "%p ZOOM_connection_exec_task "
3079             "task pending", c);
3080     return 1;
3081 }
3082
3083 static zoom_ret send_sort_present (ZOOM_connection c)
3084 {
3085     zoom_ret r = zoom_complete;
3086
3087     if (c->tasks && c->tasks->which == ZOOM_TASK_SEARCH)
3088         r = send_sort (c, c->tasks->u.search.resultset);
3089     if (r == zoom_complete)
3090         r = send_present (c);
3091     return r;
3092 }
3093
3094 static int es_response (ZOOM_connection c,
3095                         Z_ExtendedServicesResponse *res)
3096 {
3097     if (!c->tasks || c->tasks->which != ZOOM_TASK_PACKAGE)
3098         return 0;
3099     if (res->diagnostics && res->num_diagnostics > 0)
3100         response_diag(c, res->diagnostics[0]);
3101     if (res->taskPackage &&
3102         res->taskPackage->which == Z_External_extendedService)
3103     {
3104         Z_TaskPackage *taskPackage = res->taskPackage->u.extendedService;
3105         Odr_oct *id = taskPackage->targetReference;
3106
3107         if (id)
3108             ZOOM_options_setl (c->tasks->u.package->options,
3109                                "targetReference", (char*) id->buf, id->len);
3110     }
3111     if (res->taskPackage &&
3112         res->taskPackage->which == Z_External_octet)
3113     {
3114         Odr_oct *doc = res->taskPackage->u.octet_aligned;
3115         ZOOM_options_setl (c->tasks->u.package->options,
3116                            "xmlUpdateDoc", (char*) doc->buf, doc->len);
3117     }
3118     return 1;
3119 }
3120
3121 static void interpret_init_diag(ZOOM_connection c,
3122                                 Z_DiagnosticFormat *diag)
3123 {
3124     if (diag->num > 0)
3125     {
3126         Z_DiagnosticFormat_s *ds = diag->elements[0];
3127         if (ds->which == Z_DiagnosticFormat_s_defaultDiagRec)
3128             response_default_diag(c, ds->u.defaultDiagRec);
3129     }
3130 }
3131
3132
3133 static void interpret_otherinformation_field(ZOOM_connection c,
3134                                              Z_OtherInformation *ui)
3135 {
3136     int i;
3137     for (i = 0; i < ui->num_elements; i++)
3138     {
3139         Z_OtherInformationUnit *unit = ui->list[i];
3140         if (unit->which == Z_OtherInfo_externallyDefinedInfo &&
3141             unit->information.externallyDefinedInfo &&
3142             unit->information.externallyDefinedInfo->which ==
3143             Z_External_diag1)
3144         {
3145             interpret_init_diag(c, unit->information.externallyDefinedInfo->u.diag1);
3146         }
3147     }
3148 }
3149
3150 static void recv_apdu (ZOOM_connection c, Z_APDU *apdu)
3151 {
3152     Z_InitResponse *initrs;
3153
3154     c->mask = 0;
3155     yaz_log(log_details, "%p recv_apdu apdu->which=%d", c, apdu->which);
3156     switch(apdu->which)
3157     {
3158     case Z_APDU_initResponse:
3159         yaz_log(log_api, "%p recv_apd: Received Init response", c);
3160         initrs = apdu->u.initResponse;
3161         ZOOM_connection_option_set(c, "serverImplementationId",
3162                                    initrs->implementationId ?
3163                                    initrs->implementationId : "");
3164         ZOOM_connection_option_set(c, "serverImplementationName",
3165                                    initrs->implementationName ?
3166                                    initrs->implementationName : "");
3167         ZOOM_connection_option_set(c, "serverImplementationVersion",
3168                                    initrs->implementationVersion ?
3169                                    initrs->implementationVersion : "");
3170         /* Set the three old options too, for old applications */
3171         ZOOM_connection_option_set(c, "targetImplementationId",
3172                                    initrs->implementationId ?
3173                                    initrs->implementationId : "");
3174         ZOOM_connection_option_set(c, "targetImplementationName",
3175                                    initrs->implementationName ?
3176                                    initrs->implementationName : "");
3177         ZOOM_connection_option_set(c, "targetImplementationVersion",
3178                                    initrs->implementationVersion ?
3179                                    initrs->implementationVersion : "");
3180         if (!*initrs->result)
3181         {
3182             Z_External *uif = initrs->userInformationField;
3183
3184             set_ZOOM_error(c, ZOOM_ERROR_INIT, 0); /* default error */
3185
3186             if (uif && uif->which == Z_External_userInfo1)
3187                 interpret_otherinformation_field(c, uif->u.userInfo1);
3188         }
3189         else
3190         {
3191             char *cookie =
3192                 yaz_oi_get_string_oidval (&apdu->u.initResponse->otherInfo,
3193                                           VAL_COOKIE, 1, 0);
3194             xfree (c->cookie_in);
3195             c->cookie_in = 0;
3196             if (cookie)
3197                 c->cookie_in = xstrdup(cookie);
3198             if (ODR_MASK_GET(initrs->options, Z_Options_namedResultSets) &&
3199                 ODR_MASK_GET(initrs->protocolVersion, Z_ProtocolVersion_3))
3200                 c->support_named_resultsets = 1;
3201             if (c->tasks)
3202             {
3203                 assert (c->tasks->which == ZOOM_TASK_CONNECT);
3204                 ZOOM_connection_remove_task (c);
3205             }
3206             ZOOM_connection_exec_task (c);
3207         }
3208         if (ODR_MASK_GET(initrs->options, Z_Options_negotiationModel))
3209         {
3210             NMEM tmpmem = nmem_create();
3211             Z_CharSetandLanguageNegotiation *p =
3212                 yaz_get_charneg_record(initrs->otherInfo);
3213
3214             if (p)
3215             {
3216                 char *charset = NULL, *lang = NULL;
3217                 int sel;
3218
3219                 yaz_get_response_charneg(tmpmem, p, &charset, &lang, &sel);
3220                 yaz_log(log_details, "%p recv_apdu target accepted: "
3221                         "charset %s, language %s, select %d",
3222                         c,
3223                         charset ? charset : "none", lang ? lang : "none", sel);
3224                 if (charset)
3225                     ZOOM_connection_option_set (c, "negotiation-charset",
3226                                                 charset);
3227                 if (lang)
3228                     ZOOM_connection_option_set (c, "negotiation-lang",
3229                                                 lang);
3230
3231                 ZOOM_connection_option_set (
3232                     c,  "negotiation-charset-in-effect-for-records",
3233                     (sel != 0) ? "1" : "0");
3234                 nmem_destroy(tmpmem);
3235             }
3236         }
3237         break;
3238     case Z_APDU_searchResponse:
3239         yaz_log(log_api, "%p recv_apdu Search response", c);
3240         handle_search_response (c, apdu->u.searchResponse);
3241         if (send_sort_present (c) == zoom_complete)
3242             ZOOM_connection_remove_task (c);
3243         break;
3244     case Z_APDU_presentResponse:
3245         yaz_log(log_api, "%p recv_apdu Present response", c);
3246         handle_present_response (c, apdu->u.presentResponse);
3247         if (send_present (c) == zoom_complete)
3248             ZOOM_connection_remove_task (c);
3249         break;
3250     case Z_APDU_sortResponse:
3251         yaz_log(log_api, "%p recv_apdu Sort response", c);
3252         sort_response (c, apdu->u.sortResponse);
3253         if (send_present (c) == zoom_complete)
3254             ZOOM_connection_remove_task (c);
3255         break;
3256     case Z_APDU_scanResponse:
3257         yaz_log(log_api, "%p recv_apdu Scan response", c);
3258         scan_response (c, apdu->u.scanResponse);
3259         ZOOM_connection_remove_task (c);
3260         break;
3261     case Z_APDU_extendedServicesResponse:
3262         yaz_log(log_api, "%p recv_apdu Extended Services response", c);
3263         es_response (c, apdu->u.extendedServicesResponse);
3264         ZOOM_connection_remove_task (c);
3265         break;
3266     case Z_APDU_close:
3267         yaz_log(log_api, "%p recv_apdu Close PDU", c);
3268         if (c->reconnect_ok)
3269         {
3270             do_close(c);
3271             c->tasks->running = 0;
3272             ZOOM_connection_insert_task (c, ZOOM_TASK_CONNECT);
3273         }
3274         else
3275         {
3276             set_ZOOM_error(c, ZOOM_ERROR_CONNECTION_LOST, 0);
3277             do_close(c);
3278         }
3279         break;
3280     default:
3281         yaz_log(log_api, "%p Received unknown PDU", c);
3282         set_ZOOM_error(c, ZOOM_ERROR_DECODE, 0);
3283         do_close(c);
3284     }
3285 }
3286
3287 #if HAVE_XML2
3288 static void handle_srw_response(ZOOM_connection c,
3289                                 Z_SRW_searchRetrieveResponse *res)
3290 {
3291     ZOOM_resultset resultset = 0;
3292     int i;
3293     NMEM nmem;
3294     ZOOM_Event event;
3295
3296     if (!c->tasks)
3297         return;
3298
3299     if (c->tasks->which == ZOOM_TASK_SEARCH)
3300         resultset = c->tasks->u.search.resultset;
3301     else if (c->tasks->which == ZOOM_TASK_RETRIEVE)
3302         resultset = c->tasks->u.retrieve.resultset;
3303     else
3304         return ;
3305
3306     event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH);
3307     ZOOM_connection_put_event(c, event);
3308
3309     resultset->size = 0;
3310
3311     yaz_log(log_details, "%p handle_srw_response got SRW response OK", c);
3312
3313     if (res->numberOfRecords)
3314         resultset->size = *res->numberOfRecords;
3315
3316     for (i = 0; i<res->num_records; i++)
3317     {
3318         int pos;
3319
3320         Z_NamePlusRecord *npr = (Z_NamePlusRecord *)
3321             odr_malloc(c->odr_in, sizeof(Z_NamePlusRecord));
3322
3323         if (res->records[i].recordPosition &&
3324             *res->records[i].recordPosition > 0)
3325             pos = *res->records[i].recordPosition - 1;
3326         else
3327             pos = resultset->start + i;
3328
3329         npr->databaseName = 0;
3330         npr->which = Z_NamePlusRecord_databaseRecord;
3331         npr->u.databaseRecord = (Z_External *)
3332             odr_malloc(c->odr_in, sizeof(Z_External));
3333         npr->u.databaseRecord->descriptor = 0;
3334         npr->u.databaseRecord->direct_reference =
3335             yaz_oidval_to_z3950oid(c->odr_in, CLASS_RECSYN, VAL_TEXT_XML);
3336         npr->u.databaseRecord->which = Z_External_octet;
3337         npr->u.databaseRecord->u.octet_aligned = (Odr_oct *)
3338             odr_malloc(c->odr_in, sizeof(Odr_oct));
3339         npr->u.databaseRecord->u.octet_aligned->buf = (unsigned char*)
3340             res->records[i].recordData_buf;
3341         npr->u.databaseRecord->u.octet_aligned->len =
3342             npr->u.databaseRecord->u.octet_aligned->size =
3343             res->records[i].recordData_len;
3344         record_cache_add (resultset, npr, pos);
3345     }
3346     if (res->num_diagnostics > 0)
3347     {
3348         const char *uri = res->diagnostics[0].uri;
3349         if (uri)
3350         {
3351             int code = 0;
3352             const char *cp;
3353             if ((cp = strrchr(uri, '/')))
3354                 code = atoi(cp+1);
3355             set_dset_error(c, code, uri,
3356                            res->diagnostics[0].details, 0);
3357         }
3358     }
3359     nmem = odr_extract_mem(c->odr_in);
3360     nmem_transfer(resultset->odr->mem, nmem);
3361     nmem_destroy(nmem);
3362 }
3363 #endif
3364
3365 #if HAVE_XML2
3366 static void handle_http(ZOOM_connection c, Z_HTTP_Response *hres)
3367 {
3368     int ret = -1;
3369     const char *content_type = z_HTTP_header_lookup(hres->headers,
3370                                                     "Content-Type");
3371     const char *connection_head = z_HTTP_header_lookup(hres->headers,
3372                                                        "Connection");
3373     c->mask = 0;
3374     yaz_log(log_details, "%p handle_http", c);
3375
3376     if (content_type && !yaz_strcmp_del("text/xml", content_type, "; "))
3377     {
3378         Z_SOAP *soap_package = 0;
3379         ODR o = c->odr_in;
3380         Z_SOAP_Handler soap_handlers[2] = {
3381             {"http://www.loc.gov/zing/srw/", 0,
3382              (Z_SOAP_fun) yaz_srw_codec},
3383             {0, 0, 0}
3384         };
3385         ret = z_soap_codec(o, &soap_package,
3386                            &hres->content_buf, &hres->content_len,
3387                            soap_handlers);
3388         if (!ret && soap_package->which == Z_SOAP_generic &&
3389             soap_package->u.generic->no == 0)
3390         {
3391             Z_SRW_PDU *sr = (Z_SRW_PDU*) soap_package->u.generic->p;
3392             if (sr->which == Z_SRW_searchRetrieve_response)
3393                 handle_srw_response(c, sr->u.response);
3394             else
3395                 ret = -1;
3396         }
3397         else if (!ret && (soap_package->which == Z_SOAP_fault
3398                           || soap_package->which == Z_SOAP_error))
3399         {
3400             set_HTTP_error(c, hres->code,
3401                            soap_package->u.fault->fault_code,
3402                            soap_package->u.fault->fault_string);
3403         }
3404         else
3405             ret = -1;
3406     }
3407     if (ret)
3408     {
3409         if (hres->code != 200)
3410             set_HTTP_error(c, hres->code, 0, 0);
3411         else
3412             set_ZOOM_error(c, ZOOM_ERROR_DECODE, 0);
3413         do_close (c);
3414     }
3415     ZOOM_connection_remove_task(c);
3416     if (!strcmp(hres->version, "1.0"))
3417     {
3418         /* HTTP 1.0: only if Keep-Alive we stay alive.. */
3419         if (!connection_head || strcmp(connection_head, "Keep-Alive"))
3420             do_close(c);
3421     }
3422     else
3423     {
3424         /* HTTP 1.1: only if no close we stay alive .. */
3425         if (connection_head && !strcmp(connection_head, "close"))
3426             do_close(c);
3427     }
3428 }
3429 #endif
3430
3431 static int do_read (ZOOM_connection c)
3432 {
3433     int r, more;
3434     ZOOM_Event event;
3435
3436     event = ZOOM_Event_create (ZOOM_EVENT_RECV_DATA);
3437     ZOOM_connection_put_event (c, event);
3438
3439     r = cs_get (c->cs, &c->buf_in, &c->len_in);
3440     more = cs_more(c->cs);
3441     yaz_log(log_details, "%p do_read len=%d more=%d", c, r, more);
3442     if (r == 1)
3443         return 0;
3444     if (r <= 0)
3445     {
3446         if (c->reconnect_ok)
3447         {
3448             do_close (c);
3449             c->reconnect_ok = 0;
3450             yaz_log(log_details, "%p do_read reconnect read", c);
3451             c->tasks->running = 0;
3452             ZOOM_connection_insert_task (c, ZOOM_TASK_CONNECT);
3453         }
3454         else
3455         {
3456             set_ZOOM_error(c, ZOOM_ERROR_CONNECTION_LOST, 0);
3457             do_close (c);
3458         }
3459     }
3460     else
3461     {
3462         Z_GDU *gdu;
3463         ZOOM_Event event;
3464
3465         odr_reset (c->odr_in);
3466         odr_setbuf (c->odr_in, c->buf_in, r, 0);
3467         event = ZOOM_Event_create (ZOOM_EVENT_RECV_APDU);
3468         ZOOM_connection_put_event (c, event);
3469
3470         if (!z_GDU (c->odr_in, &gdu, 0, 0))
3471         {
3472             int x;
3473             int err = odr_geterrorx(c->odr_in, &x);
3474             char msg[60];
3475             const char *element = odr_getelement(c->odr_in);
3476             sprintf (msg, "ODR code %d:%d element=%-20s",
3477                      err, x, element ? element : "<unknown>");
3478             set_ZOOM_error(c, ZOOM_ERROR_DECODE, msg);
3479             do_close (c);
3480         }
3481         else if (gdu->which == Z_GDU_Z3950)
3482             recv_apdu (c, gdu->u.z3950);
3483         else if (gdu->which == Z_GDU_HTTP_Response)
3484         {
3485 #if HAVE_XML2
3486             handle_http (c, gdu->u.HTTP_Response);
3487 #else
3488             set_ZOOM_error(c, ZOOM_ERROR_DECODE, 0);
3489             do_close (c);
3490 #endif
3491         }
3492         c->reconnect_ok = 0;
3493     }
3494     return 1;
3495 }
3496
3497 static zoom_ret do_write_ex (ZOOM_connection c, char *buf_out, int len_out)
3498 {
3499     int r;
3500     ZOOM_Event event;
3501
3502     event = ZOOM_Event_create(ZOOM_EVENT_SEND_DATA);
3503     ZOOM_connection_put_event (c, event);
3504
3505     yaz_log(log_details, "%p do_write_ex len=%d", c, len_out);
3506     if ((r = cs_put(c->cs, buf_out, len_out)) < 0)
3507     {
3508         yaz_log(log_details, "%p do_write_ex write failed", c);
3509         if (c->reconnect_ok)
3510         {
3511             do_close (c);
3512             c->reconnect_ok = 0;
3513             yaz_log(log_details, "%p do_write_ex reconnect write", c);
3514             c->tasks->running = 0;
3515             ZOOM_connection_insert_task (c, ZOOM_TASK_CONNECT);
3516             return zoom_pending;
3517         }
3518         if (c->state == STATE_CONNECTING)
3519             set_ZOOM_error(c, ZOOM_ERROR_CONNECT, 0);
3520         else
3521             set_ZOOM_error(c, ZOOM_ERROR_CONNECTION_LOST, 0);
3522         do_close (c);
3523         return zoom_complete;
3524     }
3525     else if (r == 1)
3526     {
3527         c->mask = ZOOM_SELECT_EXCEPT;
3528         if (c->cs->io_pending & CS_WANT_WRITE)
3529             c->mask += ZOOM_SELECT_WRITE;
3530         if (c->cs->io_pending & CS_WANT_READ)
3531             c->mask += ZOOM_SELECT_READ;
3532         yaz_log(log_details, "%p do_write_ex write incomplete mask=%d",
3533                 c, c->mask);
3534     }
3535     else
3536     {
3537         c->mask = ZOOM_SELECT_READ|ZOOM_SELECT_EXCEPT;
3538         yaz_log(log_details, "%p do_write_ex write complete mask=%d",
3539                 c, c->mask);
3540     }
3541     return zoom_pending;
3542 }
3543
3544 static zoom_ret do_write(ZOOM_connection c)
3545 {
3546     return do_write_ex (c, c->buf_out, c->len_out);
3547 }
3548
3549
3550 ZOOM_API(const char *)
3551 ZOOM_connection_option_get (ZOOM_connection c, const char *key)
3552 {
3553     return ZOOM_options_get (c->options, key);
3554 }
3555
3556 ZOOM_API(const char *)
3557 ZOOM_connection_option_getl (ZOOM_connection c, const char *key, int *lenp)
3558 {
3559     return ZOOM_options_getl (c->options, key, lenp);
3560 }
3561
3562 ZOOM_API(void)
3563 ZOOM_connection_option_set (ZOOM_connection c, const char *key,
3564                             const char *val)
3565 {
3566     ZOOM_options_set (c->options, key, val);
3567 }
3568
3569 ZOOM_API(void)
3570 ZOOM_connection_option_setl (ZOOM_connection c, const char *key,
3571                              const char *val, int len)
3572 {
3573     ZOOM_options_setl (c->options, key, val, len);
3574 }
3575
3576 ZOOM_API(const char *)
3577 ZOOM_resultset_option_get (ZOOM_resultset r, const char *key)
3578 {
3579     return ZOOM_options_get (r->options, key);
3580 }
3581
3582 ZOOM_API(void)
3583 ZOOM_resultset_option_set (ZOOM_resultset r, const char *key,
3584                                   const char *val)
3585 {
3586     ZOOM_options_set (r->options, key, val);
3587 }
3588
3589
3590 ZOOM_API(int)
3591 ZOOM_connection_errcode (ZOOM_connection c)
3592 {
3593     return ZOOM_connection_error (c, 0, 0);
3594 }
3595
3596 ZOOM_API(const char *)
3597 ZOOM_connection_errmsg (ZOOM_connection c)
3598 {
3599     const char *msg;
3600     ZOOM_connection_error (c, &msg, 0);
3601     return msg;
3602 }
3603
3604 ZOOM_API(const char *)
3605 ZOOM_connection_addinfo (ZOOM_connection c)
3606 {
3607     const char *addinfo;
3608     ZOOM_connection_error (c, 0, &addinfo);
3609     return addinfo;
3610 }
3611
3612 ZOOM_API(const char *)
3613 ZOOM_connection_diagset (ZOOM_connection c)
3614 {
3615     const char *diagset;
3616     ZOOM_connection_error_x (c, 0, 0, &diagset);
3617     return diagset;
3618 }
3619
3620 ZOOM_API(const char *)
3621 ZOOM_diag_str (int error)
3622 {
3623     switch (error)
3624     {
3625     case ZOOM_ERROR_NONE:
3626         return "No error";
3627     case ZOOM_ERROR_CONNECT:
3628         return "Connect failed";
3629     case ZOOM_ERROR_MEMORY:
3630         return "Out of memory";
3631     case ZOOM_ERROR_ENCODE:
3632         return "Encoding failed";
3633     case ZOOM_ERROR_DECODE:
3634         return "Decoding failed";
3635     case ZOOM_ERROR_CONNECTION_LOST:
3636         return "Connection lost";
3637     case ZOOM_ERROR_INIT:
3638         return "Init rejected";
3639     case ZOOM_ERROR_INTERNAL:
3640         return "Internal failure";
3641     case ZOOM_ERROR_TIMEOUT:
3642         return "Timeout";
3643     case ZOOM_ERROR_UNSUPPORTED_PROTOCOL:
3644         return "Unsupported protocol";
3645     case ZOOM_ERROR_UNSUPPORTED_QUERY:
3646         return "Unsupported query type";
3647     case ZOOM_ERROR_INVALID_QUERY:
3648         return "Invalid query";
3649     default:
3650         return diagbib1_str (error);
3651     }
3652 }
3653
3654 ZOOM_API(int)
3655 ZOOM_connection_error_x (ZOOM_connection c, const char **cp,
3656                          const char **addinfo, const char **diagset)
3657 {
3658     int error = c->error;
3659     if (cp)
3660     {
3661         if (!c->diagset || !strcmp(c->diagset, "ZOOM"))
3662             *cp = ZOOM_diag_str(error);
3663         else if (!strcmp(c->diagset, "HTTP"))
3664             *cp = z_HTTP_errmsg(c->error);
3665         else if (!strcmp(c->diagset, "Bib-1"))
3666             *cp = ZOOM_diag_str(error);
3667         else if (!strcmp(c->diagset, "info:srw/diagnostic/1"))
3668             *cp = yaz_diag_srw_str(c->error);
3669         else
3670             *cp = "Unknown error and diagnostic set";
3671     }
3672     if (addinfo)
3673         *addinfo = c->addinfo ? c->addinfo : "";
3674     if (diagset)
3675         *diagset = c->diagset ? c->diagset : "";
3676     return c->error;
3677 }
3678
3679 ZOOM_API(int)
3680 ZOOM_connection_error (ZOOM_connection c, const char **cp,
3681                        const char **addinfo)
3682 {
3683     return ZOOM_connection_error_x(c, cp, addinfo, 0);
3684 }
3685
3686 static int ZOOM_connection_do_io(ZOOM_connection c, int mask)
3687 {
3688     ZOOM_Event event = 0;
3689     int r = cs_look(c->cs);
3690     yaz_log(log_details, "%p ZOOM_connection_do_io mask=%d cs_look=%d",
3691              c, mask, r);
3692
3693     if (r == CS_NONE)
3694     {
3695         event = ZOOM_Event_create (ZOOM_EVENT_CONNECT);
3696         set_ZOOM_error(c, ZOOM_ERROR_CONNECT, 0);
3697         do_close (c);
3698         ZOOM_connection_put_event (c, event);
3699     }
3700     else if (r == CS_CONNECT)
3701     {
3702         int ret;
3703         event = ZOOM_Event_create (ZOOM_EVENT_CONNECT);
3704
3705         ret = cs_rcvconnect (c->cs);
3706         yaz_log(log_details, "%p ZOOM_connection_do_io "
3707                 "cs_rcvconnect returned %d", c, ret);
3708         if (ret == 1)
3709         {
3710             c->mask = ZOOM_SELECT_EXCEPT;
3711             if (c->cs->io_pending & CS_WANT_WRITE)
3712                 c->mask += ZOOM_SELECT_WRITE;
3713             if (c->cs->io_pending & CS_WANT_READ)
3714                 c->mask += ZOOM_SELECT_READ;
3715             ZOOM_connection_put_event (c, event);
3716         }
3717         else if (ret == 0)
3718         {
3719             ZOOM_connection_put_event (c, event);
3720             get_cert(c);
3721             if (c->proto == PROTO_Z3950)
3722                 ZOOM_connection_send_init(c);
3723             else
3724             {
3725                 /* no init request for SRW .. */
3726                 assert (c->tasks->which == ZOOM_TASK_CONNECT);
3727                 ZOOM_connection_remove_task (c);
3728                 c->mask = 0;
3729                 ZOOM_connection_exec_task (c);
3730             }
3731             c->state = STATE_ESTABLISHED;
3732         }
3733         else
3734         {
3735             set_ZOOM_error(c, ZOOM_ERROR_CONNECT, 0);
3736             do_close (c);
3737             ZOOM_connection_put_event (c, event);
3738         }
3739     }
3740     else
3741     {
3742         if (mask & ZOOM_SELECT_READ)
3743             do_read (c);
3744         if (c->cs && (mask & ZOOM_SELECT_WRITE))
3745             do_write (c);
3746     }
3747     return 1;
3748 }
3749
3750 ZOOM_API(int)
3751 ZOOM_connection_last_event(ZOOM_connection cs)
3752 {
3753     if (!cs)
3754         return ZOOM_EVENT_NONE;
3755     return cs->last_event;
3756 }
3757
3758 ZOOM_API(int)
3759 ZOOM_event (int no, ZOOM_connection *cs)
3760 {
3761     int timeout = 30;      /* default timeout in seconds */
3762     int timeout_set = 0;   /* whether it was overriden at all */
3763 #if HAVE_SYS_POLL_H
3764     struct pollfd pollfds[1024];
3765     ZOOM_connection poll_cs[1024];
3766 #else
3767     struct timeval tv;
3768     fd_set input, output, except;
3769 #endif
3770     int i, r, nfds;
3771     int max_fd = 0;
3772
3773     for (i = 0; i<no; i++)
3774     {
3775         ZOOM_connection c = cs[i];
3776         ZOOM_Event event;
3777         if (c && (event = ZOOM_connection_get_event(c)))
3778         {
3779             ZOOM_Event_destroy (event);
3780             return i+1;
3781         }
3782     }
3783     for (i = 0; i<no; i++)
3784     {
3785         ZOOM_connection c = cs[i];
3786         ZOOM_Event event;
3787         if (c && ZOOM_connection_exec_task (c))
3788         {
3789             if ((event = ZOOM_connection_get_event(c)))
3790             {
3791                 ZOOM_Event_destroy (event);
3792                 return i+1;
3793             }
3794         }
3795     }
3796 #if HAVE_SYS_POLL_H
3797
3798 #else
3799     FD_ZERO (&input);
3800     FD_ZERO (&output);
3801     FD_ZERO (&except);
3802 #endif
3803     nfds = 0;
3804     for (i = 0; i<no; i++)
3805     {
3806         ZOOM_connection c = cs[i];
3807         int fd, mask;
3808         int this_timeout;
3809
3810         if (!c)
3811             continue;
3812         fd = z3950_connection_socket(c);
3813         mask = z3950_connection_mask(c);
3814
3815         if (fd == -1)
3816             continue;
3817         if (max_fd < fd)
3818             max_fd = fd;
3819
3820         /* -1 is used for indefinite timeout (no timeout), so -2 here. */
3821         this_timeout = ZOOM_options_get_int (c->options, "timeout", -2);
3822         if (this_timeout != -2)
3823         {
3824             /* ensure the minimum timeout is used */
3825             if (!timeout_set)
3826                 timeout = this_timeout;
3827             else if (this_timeout != -1 && this_timeout < timeout)
3828                 timeout = this_timeout;
3829             timeout_set = 1;
3830         }
3831 #if HAVE_SYS_POLL_H
3832         if (mask)
3833         {
3834             short poll_events = 0;
3835
3836             if (mask & ZOOM_SELECT_READ)
3837                 poll_events += POLLIN;
3838             if (mask & ZOOM_SELECT_WRITE)
3839                 poll_events += POLLOUT;
3840             if (mask & ZOOM_SELECT_EXCEPT)
3841                 poll_events += POLLERR;
3842             pollfds[nfds].fd = fd;
3843             pollfds[nfds].events = poll_events;
3844             pollfds[nfds].revents = 0;
3845             poll_cs[nfds] = c;
3846             nfds++;
3847         }
3848 #else
3849         if (mask & ZOOM_SELECT_READ)
3850         {
3851             FD_SET (fd, &input);
3852             nfds++;
3853         }
3854         if (mask & ZOOM_SELECT_WRITE)
3855         {
3856             FD_SET (fd, &output);
3857             nfds++;
3858         }
3859         if (mask & ZOOM_SELECT_EXCEPT)
3860         {
3861             FD_SET (fd, &except);
3862             nfds++;
3863         }
3864 #endif
3865     }
3866     if (!nfds)
3867         return 0;
3868
3869 #if HAVE_SYS_POLL_H
3870     r = poll (pollfds, nfds, (timeout == -1 ? -1 : timeout * 1000));
3871     for (i = 0; i<nfds; i++)
3872     {
3873         ZOOM_connection c = poll_cs[i];
3874         if (r && c->mask)
3875         {
3876             int mask = 0;
3877             if (pollfds[i].revents & POLLIN)
3878                 mask += ZOOM_SELECT_READ;
3879             if (pollfds[i].revents & POLLOUT)
3880                 mask += ZOOM_SELECT_WRITE;
3881             if (pollfds[i].revents & POLLERR)
3882                 mask += ZOOM_SELECT_EXCEPT;
3883             if (mask)
3884                 ZOOM_connection_do_io(c, mask);
3885         }
3886         else if (r == 0 && c->mask)
3887         {
3888             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_TIMEOUT);
3889             /* timeout and this connection was waiting */
3890             set_ZOOM_error(c, ZOOM_ERROR_TIMEOUT, 0);
3891             do_close (c);
3892             ZOOM_connection_put_event(c, event);
3893         }
3894     }
3895 #else
3896     tv.tv_sec = timeout;
3897     tv.tv_usec = 0;
3898     r = select (max_fd+1, &input, &output, &except, (timeout == -1 ? 0 : &tv));
3899     for (i = 0; i<no; i++)
3900     {
3901         ZOOM_connection c = cs[i];
3902         int fd, mask;
3903
3904         if (!c)
3905             continue;
3906         fd = z3950_connection_socket(c);
3907         mask = 0;
3908         if (r && c->mask)
3909         {
3910             /* no timeout and real socket */
3911             if (FD_ISSET(fd, &input))
3912                 mask += ZOOM_SELECT_READ;
3913             if (FD_ISSET(fd, &output))
3914                 mask += ZOOM_SELECT_WRITE;
3915             if (FD_ISSET(fd, &except))
3916                 mask += ZOOM_SELECT_EXCEPT;
3917             if (mask)
3918                 ZOOM_connection_do_io(c, mask);
3919         }
3920         if (r == 0 && c->mask)
3921         {
3922             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_TIMEOUT);
3923             /* timeout and this connection was waiting */
3924             set_ZOOM_error(c, ZOOM_ERROR_TIMEOUT, 0);
3925             do_close (c);
3926             ZOOM_connection_put_event(c, event);
3927         }
3928     }
3929 #endif
3930     for (i = 0; i<no; i++)
3931     {
3932         ZOOM_connection c = cs[i];
3933         ZOOM_Event event;
3934         if (c && (event = ZOOM_connection_get_event(c)))
3935         {
3936             ZOOM_Event_destroy (event);
3937             return i+1;
3938         }
3939     }
3940     return 0;
3941 }
3942 /*
3943  * Local variables:
3944  * c-basic-offset: 4
3945  * indent-tabs-mode: nil
3946  * End:
3947  * vim: shiftwidth=4 tabstop=8 expandtab
3948  */
3949