src/zoom-c.c

   1 /* This file is part of the YAZ toolkit.
   2  * Copyright (C) 1995-2010 Index Data
   3  * See the file LICENSE for details.
   4  */
   5 /**
   6  * \file zoom-c.c
   7  * \brief Implements ZOOM C interface.
   8  */
   9
  10 #include <assert.h>
  11 #include <string.h>
  12 #include <errno.h>
  13 #include "zoom-p.h"
  14
  15 #include <yaz/yaz-util.h>
  16 #include <yaz/xmalloc.h>
  17 #include <yaz/otherinfo.h>
  18 #include <yaz/log.h>
  19 #include <yaz/pquery.h>
  20 #include <yaz/marcdisp.h>
  21 #include <yaz/diagbib1.h>
  22 #include <yaz/charneg.h>
  23 #include <yaz/ill.h>
  24 #include <yaz/srw.h>
  25 #include <yaz/cql.h>
  26 #include <yaz/ccl.h>
  27 #include <yaz/query-charset.h>
  28 #include <yaz/copy_types.h>
  29 #include <yaz/snprintf.h>
  30
  31 #include <yaz/shptr.h>
  32
  33 #if SHPTR
  34 YAZ_SHPTR_TYPE(WRBUF)
  35 #endif
  36
  37 static int log_api = 0;
  38 static int log_details = 0;
  39
  40 typedef enum {
  41     zoom_pending,
  42     zoom_complete
  43 } zoom_ret;
  44
  45 static void resultset_destroy(ZOOM_resultset r);
  46 static zoom_ret ZOOM_connection_send_init(ZOOM_connection c);
  47 static zoom_ret do_write_ex(ZOOM_connection c, char *buf_out, int len_out);
  48 static char *cql2pqf(ZOOM_connection c, const char *cql);
  49
  50 ZOOM_API(const char *) ZOOM_get_event_str(int event)
  51 {
  52     static const char *ar[] = {
  53         "NONE",
  54         "CONNECT",
  55         "SEND_DATA",
  56         "RECV_DATA",
  57         "TIMEOUT",
  58         "UNKNOWN",
  59         "SEND_APDU",
  60         "RECV_APDU",
  61         "RECV_RECORD",
  62         "RECV_SEARCH",
  63         "END"
  64     };
  65     return ar[event];
  66 }
  67
  68 /*
  69  * This wrapper is just for logging failed lookups.  It would be nicer
  70  * if it could cause failure when a lookup fails, but that's hard.
  71  */
  72 static Odr_oid *zoom_yaz_str_to_z3950oid(ZOOM_connection c,
  73                                      oid_class oid_class, const char *str) {
  74     Odr_oid *res = yaz_string_to_oid_odr(yaz_oid_std(), oid_class, str,
  75                                      c->odr_out);
  76     if (res == 0)
  77         yaz_log(YLOG_WARN, "%p OID lookup (%d, '%s') failed",
  78                 c, (int) oid_class, str);
  79     return res;
  80 }
  81
  82
  83 static void initlog(void)
  84 {
  85     static int log_level_initialized = 0;
  86     if (!log_level_initialized)
  87     {
  88         log_api = yaz_log_module_level("zoom");
  89         log_details = yaz_log_module_level("zoomdetails");
  90         log_level_initialized = 1;
  91     }
  92 }
  93
  94 static ZOOM_Event ZOOM_Event_create(int kind)
  95 {
  96     ZOOM_Event event = (ZOOM_Event) xmalloc(sizeof(*event));
  97     event->kind = kind;
  98     event->next = 0;
  99     event->prev = 0;
 100     yaz_log(log_details, "ZOOM_Event_create(kind=%d)", kind);
 101     return event;
 102 }
 103
 104 static void ZOOM_Event_destroy(ZOOM_Event event)
 105 {
 106     xfree(event);
 107 }
 108
 109 static void ZOOM_connection_put_event(ZOOM_connection c, ZOOM_Event event)
 110 {
 111     if (c->m_queue_back)
 112     {
 113         c->m_queue_back->prev = event;
 114         assert(c->m_queue_front);
 115     }
 116     else
 117     {
 118         assert(!c->m_queue_front);
 119         c->m_queue_front = event;
 120     }
 121     event->next = c->m_queue_back;
 122     event->prev = 0;
 123     c->m_queue_back = event;
 124 }
 125
 126 static ZOOM_Event ZOOM_connection_get_event(ZOOM_connection c)
 127 {
 128     ZOOM_Event event = c->m_queue_front;
 129     if (!event)
 130     {
 131         c->last_event = ZOOM_EVENT_NONE;
 132         return 0;
 133     }
 134     assert(c->m_queue_back);
 135     c->m_queue_front = event->prev;
 136     if (c->m_queue_front)
 137     {
 138         assert(c->m_queue_back);
 139         c->m_queue_front->next = 0;
 140     }
 141     else
 142         c->m_queue_back = 0;
 143     c->last_event = event->kind;
 144     return event;
 145 }
 146
 147 static void ZOOM_connection_remove_events(ZOOM_connection c)
 148 {
 149     ZOOM_Event event;
 150     while ((event = ZOOM_connection_get_event(c)))
 151         ZOOM_Event_destroy(event);
 152 }
 153
 154 ZOOM_API(int) ZOOM_connection_peek_event(ZOOM_connection c)
 155 {
 156     ZOOM_Event event = c->m_queue_front;
 157
 158     return event ? event->kind : ZOOM_EVENT_NONE;
 159 }
 160
 161 void ZOOM_connection_remove_tasks(ZOOM_connection c);
 162
 163 static void set_dset_error(ZOOM_connection c, int error,
 164                            const char *dset,
 165                            const char *addinfo, const char *addinfo2)
 166 {
 167     char *cp;
 168
 169     xfree(c->addinfo);
 170     c->addinfo = 0;
 171     c->error = error;
 172     if (!c->diagset || strcmp(dset, c->diagset))
 173     {
 174         xfree(c->diagset);
 175         c->diagset = xstrdup(dset);
 176         /* remove integer part from SRW diagset .. */
 177         if ((cp = strrchr(c->diagset, '/')))
 178             *cp = '\0';
 179     }
 180     if (addinfo && addinfo2)
 181     {
 182         c->addinfo = (char*) xmalloc(strlen(addinfo) + strlen(addinfo2) + 2);
 183         strcpy(c->addinfo, addinfo);
 184         strcat(c->addinfo, addinfo2);
 185     }
 186     else if (addinfo)
 187         c->addinfo = xstrdup(addinfo);
 188     if (error != ZOOM_ERROR_NONE)
 189     {
 190         yaz_log(log_api, "%p set_dset_error %s %s:%d %s %s",
 191                 c, c->host_port ? c->host_port : "<>", dset, error,
 192                 addinfo ? addinfo : "",
 193                 addinfo2 ? addinfo2 : "");
 194         ZOOM_connection_remove_tasks(c);
 195     }
 196 }
 197
 198 static int uri_to_code(const char *uri)
 199 {
 200     int code = 0;
 201     const char *cp;
 202     if ((cp = strrchr(uri, '/')))
 203         code = atoi(cp+1);
 204     return code;
 205 }
 206
 207 #if YAZ_HAVE_XML2
 208 static void set_HTTP_error(ZOOM_connection c, int error,
 209                            const char *addinfo, const char *addinfo2)
 210 {
 211     set_dset_error(c, error, "HTTP", addinfo, addinfo2);
 212 }
 213
 214 static void set_SRU_error(ZOOM_connection c, Z_SRW_diagnostic *d)
 215 {
 216     const char *uri = d->uri;
 217     if (uri)
 218         set_dset_error(c, uri_to_code(uri), uri, d->details, 0);
 219 }
 220
 221 #endif
 222
 223
 224 static void set_ZOOM_error(ZOOM_connection c, int error,
 225                            const char *addinfo)
 226 {
 227     set_dset_error(c, error, "ZOOM", addinfo, 0);
 228 }
 229
 230 static void clear_error(ZOOM_connection c)
 231 {
 232     /*
 233      * If an error is tied to an operation then it's ok to clear: for
 234      * example, a diagnostic returned from a search is cleared by a
 235      * subsequent search.  However, problems such as Connection Lost
 236      * or Init Refused are not cleared, because they are not
 237      * recoverable: doing another search doesn't help.
 238      */
 239
 240     ZOOM_connection_remove_events(c);
 241     switch (c->error)
 242     {
 243     case ZOOM_ERROR_CONNECT:
 244     case ZOOM_ERROR_MEMORY:
 245     case ZOOM_ERROR_DECODE:
 246     case ZOOM_ERROR_CONNECTION_LOST:
 247     case ZOOM_ERROR_INIT:
 248     case ZOOM_ERROR_INTERNAL:
 249     case ZOOM_ERROR_UNSUPPORTED_PROTOCOL:
 250         break;
 251     default:
 252         set_ZOOM_error(c, ZOOM_ERROR_NONE, 0);
 253     }
 254 }
 255
 256 void ZOOM_connection_show_task(ZOOM_task task)
 257 {
 258     switch(task->which)
 259     {
 260     case ZOOM_TASK_SEARCH:
 261         yaz_log(YLOG_LOG, "search p=%p", task);
 262         break;
 263     case ZOOM_TASK_RETRIEVE:
 264         yaz_log(YLOG_LOG, "retrieve p=%p", task);
 265         break;
 266     case ZOOM_TASK_CONNECT:
 267         yaz_log(YLOG_LOG, "connect p=%p", task);
 268         break;
 269     case ZOOM_TASK_SCAN:
 270         yaz_log(YLOG_LOG, "scan p=%p", task);
 271         break;
 272     }
 273 }
 274
 275 void ZOOM_connection_show_tasks(ZOOM_connection c)
 276 {
 277     ZOOM_task task;
 278     yaz_log(YLOG_LOG, "connection p=%p tasks", c);
 279     for (task = c->tasks; task; task = task->next)
 280         ZOOM_connection_show_task(task);
 281 }
 282
 283 ZOOM_task ZOOM_connection_add_task(ZOOM_connection c, int which)
 284 {
 285     ZOOM_task *taskp = &c->tasks;
 286     while (*taskp)
 287         taskp = &(*taskp)->next;
 288     *taskp = (ZOOM_task) xmalloc(sizeof(**taskp));
 289     (*taskp)->running = 0;
 290     (*taskp)->which = which;
 291     (*taskp)->next = 0;
 292     clear_error(c);
 293     return *taskp;
 294 }
 295
 296 ZOOM_API(int) ZOOM_connection_is_idle(ZOOM_connection c)
 297 {
 298     return c->tasks ? 0 : 1;
 299 }
 300
 301 ZOOM_task ZOOM_connection_insert_task(ZOOM_connection c, int which)
 302 {
 303     ZOOM_task task = (ZOOM_task) xmalloc(sizeof(*task));
 304
 305     task->next = c->tasks;
 306     c->tasks = task;
 307
 308     task->running = 0;
 309     task->which = which;
 310     clear_error(c);
 311     return task;
 312 }
 313
 314 void ZOOM_connection_remove_task(ZOOM_connection c)
 315 {
 316     ZOOM_task task = c->tasks;
 317
 318     if (task)
 319     {
 320         c->tasks = task->next;
 321         switch (task->which)
 322         {
 323         case ZOOM_TASK_SEARCH:
 324             resultset_destroy(task->u.search.resultset);
 325             xfree(task->u.search.syntax);
 326             xfree(task->u.search.elementSetName);
 327             break;
 328         case ZOOM_TASK_RETRIEVE:
 329             resultset_destroy(task->u.retrieve.resultset);
 330             xfree(task->u.retrieve.syntax);
 331             xfree(task->u.retrieve.elementSetName);
 332             break;
 333         case ZOOM_TASK_CONNECT:
 334             break;
 335         case ZOOM_TASK_SCAN:
 336             ZOOM_scanset_destroy(task->u.scan.scan);
 337             break;
 338         case ZOOM_TASK_PACKAGE:
 339             ZOOM_package_destroy(task->u.package);
 340             break;
 341         case ZOOM_TASK_SORT:
 342             resultset_destroy(task->u.sort.resultset);
 343             ZOOM_query_destroy(task->u.sort.q);
 344             break;
 345         default:
 346             assert(0);
 347         }
 348         xfree(task);
 349
 350         if (!c->tasks)
 351         {
 352             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_END);
 353             ZOOM_connection_put_event(c, event);
 354         }
 355     }
 356 }
 357
 358 void ZOOM_connection_remove_tasks(ZOOM_connection c)
 359 {
 360     while (c->tasks)
 361         ZOOM_connection_remove_task(c);
 362 }
 363
 364 static ZOOM_record record_cache_lookup(ZOOM_resultset r, int pos,
 365                                        const char *syntax,
 366                                        const char *elementSetName);
 367
 368 ZOOM_API(ZOOM_connection)
 369     ZOOM_connection_create(ZOOM_options options)
 370 {
 371     ZOOM_connection c = (ZOOM_connection) xmalloc(sizeof(*c));
 372
 373     initlog();
 374
 375     yaz_log(log_api, "%p ZOOM_connection_create", c);
 376
 377     c->proto = PROTO_Z3950;
 378     c->cs = 0;
 379     ZOOM_connection_set_mask(c, 0);
 380     c->reconnect_ok = 0;
 381     c->state = STATE_IDLE;
 382     c->addinfo = 0;
 383     c->diagset = 0;
 384     set_ZOOM_error(c, ZOOM_ERROR_NONE, 0);
 385     c->buf_in = 0;
 386     c->len_in = 0;
 387     c->buf_out = 0;
 388     c->len_out = 0;
 389     c->resultsets = 0;
 390
 391     c->options = ZOOM_options_create_with_parent(options);
 392
 393     c->host_port = 0;
 394     c->path = 0;
 395     c->proxy = 0;
 396
 397     c->charset = c->lang = 0;
 398
 399     c->cookie_out = 0;
 400     c->cookie_in = 0;
 401     c->client_IP = 0;
 402     c->tasks = 0;
 403
 404     c->user = 0;
 405     c->group = 0;
 406     c->password = 0;
 407
 408     c->maximum_record_size = 0;
 409     c->preferred_message_size = 0;
 410
 411     c->odr_in = odr_createmem(ODR_DECODE);
 412     c->odr_out = odr_createmem(ODR_ENCODE);
 413     c->odr_print = 0;
 414
 415     c->async = 0;
 416     c->support_named_resultsets = 0;
 417     c->last_event = ZOOM_EVENT_NONE;
 418
 419     c->m_queue_front = 0;
 420     c->m_queue_back = 0;
 421
 422     c->sru_version = 0;
 423     c->no_redirects = 0;
 424     return c;
 425 }
 426
 427
 428 /* set database names. Take local databases (if set); otherwise
 429    take databases given in ZURL (if set); otherwise use Default */
 430 static char **set_DatabaseNames(ZOOM_connection con, ZOOM_options options,
 431                                 int *num, ODR odr)
 432 {
 433     char **databaseNames;
 434     const char *cp = ZOOM_options_get(options, "databaseName");
 435
 436     if ((!cp || !*cp) && con->host_port)
 437     {
 438         if (strncmp(con->host_port, "unix:", 5) == 0)
 439             cp = strchr(con->host_port+5, ':');
 440         else
 441             cp = strchr(con->host_port, '/');
 442         if (cp)
 443             cp++;
 444     }
 445     if (!cp)
 446         cp = "Default";
 447     nmem_strsplit(odr_getmem(odr), "+", cp,  &databaseNames, num);
 448     return databaseNames;
 449 }
 450
 451 ZOOM_API(ZOOM_connection)
 452     ZOOM_connection_new(const char *host, int portnum)
 453 {
 454     ZOOM_connection c = ZOOM_connection_create(0);
 455
 456     ZOOM_connection_connect(c, host, portnum);
 457     return c;
 458 }
 459
 460 static zoom_sru_mode get_sru_mode_from_string(const char *s)
 461 {
 462     if (!s || !*s)
 463         return zoom_sru_soap;
 464     if (!yaz_matchstr(s, "soap"))
 465         return zoom_sru_soap;
 466     else if (!yaz_matchstr(s, "get"))
 467         return zoom_sru_get;
 468     else if (!yaz_matchstr(s, "post"))
 469         return zoom_sru_post;
 470     return zoom_sru_error;
 471 }
 472
 473 ZOOM_API(void)
 474     ZOOM_connection_connect(ZOOM_connection c,
 475                             const char *host, int portnum)
 476 {
 477     const char *val;
 478     ZOOM_task task;
 479
 480     initlog();
 481
 482     yaz_log(log_api, "%p ZOOM_connection_connect host=%s portnum=%d",
 483             c, host ? host : "null", portnum);
 484
 485     set_ZOOM_error(c, ZOOM_ERROR_NONE, 0);
 486     ZOOM_connection_remove_tasks(c);
 487
 488     if (c->odr_print)
 489     {
 490         odr_setprint(c->odr_print, 0); /* prevent destroy from fclose'ing */
 491         odr_destroy(c->odr_print);
 492     }
 493     if (ZOOM_options_get_bool(c->options, "apdulog", 0))
 494     {
 495         c->odr_print = odr_createmem(ODR_PRINT);
 496         odr_setprint(c->odr_print, yaz_log_file());
 497     }
 498     else
 499         c->odr_print = 0;
 500
 501     if (c->cs)
 502     {
 503         yaz_log(log_details, "%p ZOOM_connection_connect reconnect ok", c);
 504         c->reconnect_ok = 1;
 505         return;
 506     }
 507     yaz_log(log_details, "%p ZOOM_connection_connect connect", c);
 508     xfree(c->proxy);
 509     c->proxy = 0;
 510     val = ZOOM_options_get(c->options, "proxy");
 511     if (val && *val)
 512     {
 513         yaz_log(log_details, "%p ZOOM_connection_connect proxy=%s", c, val);
 514         c->proxy = xstrdup(val);
 515     }
 516
 517     xfree(c->charset);
 518     c->charset = 0;
 519     val = ZOOM_options_get(c->options, "charset");
 520     if (val && *val)
 521     {
 522         yaz_log(log_details, "%p ZOOM_connection_connect charset=%s", c, val);
 523         c->charset = xstrdup(val);
 524     }
 525
 526     xfree(c->lang);
 527     val = ZOOM_options_get(c->options, "lang");
 528     if (val && *val)
 529     {
 530         yaz_log(log_details, "%p ZOOM_connection_connect lang=%s", c, val);
 531         c->lang = xstrdup(val);
 532     }
 533     else
 534         c->lang = 0;
 535
 536     if (host)
 537     {
 538         xfree(c->host_port);
 539         if (portnum)
 540         {
 541             char hostn[128];
 542             sprintf(hostn, "%.80s:%d", host, portnum);
 543             c->host_port = xstrdup(hostn);
 544         }
 545         else
 546             c->host_port = xstrdup(host);
 547     }
 548
 549     {
 550         /*
 551          * If the "<scheme>:" part of the host string is preceded by one
 552          * or more comma-separated <name>=<value> pairs, these are taken
 553          * to be options to be set on the connection object.  Among other
 554          * applications, this facility can be used to embed authentication
 555          * in a host string:
 556          *          user=admin,password=secret,tcp:localhost:9999
 557          */
 558         char *remainder = c->host_port;
 559         char *pcolon = strchr(remainder, ':');
 560         char *pcomma;
 561         char *pequals;
 562         while ((pcomma = strchr(remainder, ',')) != 0 &&
 563                (pcolon == 0 || pcomma < pcolon)) {
 564             *pcomma = '\0';
 565             if ((pequals = strchr(remainder, '=')) != 0) {
 566                 *pequals = '\0';
 567                 /*printf("# setting '%s'='%s'\n", remainder, pequals+1);*/
 568                 ZOOM_connection_option_set(c, remainder, pequals+1);
 569             }
 570             remainder = pcomma+1;
 571         }
 572
 573         if (remainder != c->host_port) {
 574             xfree(c->host_port);
 575             c->host_port = xstrdup(remainder);
 576             /*printf("# reset hp='%s'\n", remainder);*/
 577         }
 578     }
 579
 580     val = ZOOM_options_get(c->options, "sru");
 581     c->sru_mode = get_sru_mode_from_string(val);
 582
 583     xfree(c->sru_version);
 584     val = ZOOM_options_get(c->options, "sru_version");
 585     c->sru_version = xstrdup(val ? val : "1.2");
 586
 587     ZOOM_options_set(c->options, "host", c->host_port);
 588
 589     xfree(c->cookie_out);
 590     c->cookie_out = 0;
 591     val = ZOOM_options_get(c->options, "cookie");
 592     if (val && *val)
 593     {
 594         yaz_log(log_details, "%p ZOOM_connection_connect cookie=%s", c, val);
 595         c->cookie_out = xstrdup(val);
 596     }
 597
 598     xfree(c->client_IP);
 599     c->client_IP = 0;
 600     val = ZOOM_options_get(c->options, "clientIP");
 601     if (val && *val)
 602     {
 603         yaz_log(log_details, "%p ZOOM_connection_connect clientIP=%s",
 604                 c, val);
 605         c->client_IP = xstrdup(val);
 606     }
 607
 608     xfree(c->group);
 609     c->group = 0;
 610     val = ZOOM_options_get(c->options, "group");
 611     if (val && *val)
 612         c->group = xstrdup(val);
 613
 614     xfree(c->user);
 615     c->user = 0;
 616     val = ZOOM_options_get(c->options, "user");
 617     if (val && *val)
 618         c->user = xstrdup(val);
 619
 620     xfree(c->password);
 621     c->password = 0;
 622     val = ZOOM_options_get(c->options, "password");
 623     if (!val)
 624         val = ZOOM_options_get(c->options, "pass");
 625
 626     if (val && *val)
 627         c->password = xstrdup(val);
 628
 629     c->maximum_record_size =
 630         ZOOM_options_get_int(c->options, "maximumRecordSize", 1024*1024);
 631     c->preferred_message_size =
 632         ZOOM_options_get_int(c->options, "preferredMessageSize", 1024*1024);
 633
 634     c->async = ZOOM_options_get_bool(c->options, "async", 0);
 635     yaz_log(log_details, "%p ZOOM_connection_connect async=%d", c, c->async);
 636
 637     task = ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
 638
 639     if (!c->async)
 640     {
 641         while (ZOOM_event(1, &c))
 642             ;
 643     }
 644 }
 645
 646 ZOOM_API(ZOOM_query)
 647     ZOOM_query_create(void)
 648 {
 649     ZOOM_query s = (ZOOM_query) xmalloc(sizeof(*s));
 650
 651     yaz_log(log_details, "%p ZOOM_query_create", s);
 652     s->refcount = 1;
 653     s->z_query = 0;
 654     s->sort_spec = 0;
 655     s->odr = odr_createmem(ODR_ENCODE);
 656     s->query_string = 0;
 657
 658     return s;
 659 }
 660
 661 ZOOM_API(void)
 662     ZOOM_query_destroy(ZOOM_query s)
 663 {
 664     if (!s)
 665         return;
 666
 667     (s->refcount)--;
 668     yaz_log(log_details, "%p ZOOM_query_destroy count=%d", s, s->refcount);
 669     if (s->refcount == 0)
 670     {
 671         odr_destroy(s->odr);
 672         xfree(s);
 673     }
 674 }
 675
 676 ZOOM_API(int)
 677     ZOOM_query_prefix(ZOOM_query s, const char *str)
 678 {
 679     s->query_string = odr_strdup(s->odr, str);
 680     s->z_query = (Z_Query *) odr_malloc(s->odr, sizeof(*s->z_query));
 681     s->z_query->which = Z_Query_type_1;
 682     s->z_query->u.type_1 =  p_query_rpn(s->odr, str);
 683     if (!s->z_query->u.type_1)
 684     {
 685         yaz_log(log_details, "%p ZOOM_query_prefix str=%s failed", s, str);
 686         s->z_query = 0;
 687         return -1;
 688     }
 689     yaz_log(log_details, "%p ZOOM_query_prefix str=%s", s, str);
 690     return 0;
 691 }
 692
 693 ZOOM_API(int)
 694     ZOOM_query_cql(ZOOM_query s, const char *str)
 695 {
 696     Z_External *ext;
 697
 698     s->query_string = odr_strdup(s->odr, str);
 699
 700     ext = (Z_External *) odr_malloc(s->odr, sizeof(*ext));
 701     ext->direct_reference = odr_oiddup(s->odr, yaz_oid_userinfo_cql);
 702     ext->indirect_reference = 0;
 703     ext->descriptor = 0;
 704     ext->which = Z_External_CQL;
 705     ext->u.cql = s->query_string;
 706
 707     s->z_query = (Z_Query *) odr_malloc(s->odr, sizeof(*s->z_query));
 708     s->z_query->which = Z_Query_type_104;
 709     s->z_query->u.type_104 =  ext;
 710
 711     yaz_log(log_details, "%p ZOOM_query_cql str=%s", s, str);
 712
 713     return 0;
 714 }
 715
 716 /*
 717  * Translate the CQL string client-side into RPN which is passed to
 718  * the server.  This is useful for server's that don't themselves
 719  * support CQL, for which ZOOM_query_cql() is useless.  `conn' is used
 720  * only as a place to stash diagnostics if compilation fails; if this
 721  * information is not needed, a null pointer may be used.
 722  */
 723 ZOOM_API(int)
 724     ZOOM_query_cql2rpn(ZOOM_query s, const char *str, ZOOM_connection conn)
 725 {
 726     char *rpn;
 727     int ret;
 728     ZOOM_connection freeme = 0;
 729
 730     yaz_log(log_details, "%p ZOOM_query_cql2rpn str=%s conn=%p", s, str, conn);
 731     if (conn == 0)
 732         conn = freeme = ZOOM_connection_create(0);
 733
 734     rpn = cql2pqf(conn, str);
 735     if (freeme != 0)
 736         ZOOM_connection_destroy(freeme);
 737     if (rpn == 0)
 738         return -1;
 739
 740     ret = ZOOM_query_prefix(s, rpn);
 741     xfree(rpn);
 742     return ret;
 743 }
 744
 745 /*
 746  * Analogous in every way to ZOOM_query_cql2rpn(), except that there
 747  * is no analogous ZOOM_query_ccl() that just sends uninterpreted CCL
 748  * to the server, as the YAZ GFS doesn't know how to handle this.
 749  */
 750 ZOOM_API(int)
 751     ZOOM_query_ccl2rpn(ZOOM_query s, const char *str, const char *config,
 752                        int *ccl_error, const char **error_string,
 753                        int *error_pos)
 754 {
 755     int ret;
 756     struct ccl_rpn_node *rpn;
 757     CCL_bibset bibset = ccl_qual_mk();
 758
 759     if (config)
 760         ccl_qual_buf(bibset, config);
 761
 762     rpn = ccl_find_str(bibset, str, ccl_error, error_pos);
 763     if (!rpn)
 764     {
 765         *error_string = ccl_err_msg(*ccl_error);
 766         ret = -1;
 767     }
 768     else
 769     {
 770         WRBUF wr = wrbuf_alloc();
 771         ccl_pquery(wr, rpn);
 772         ccl_rpn_delete(rpn);
 773         ret = ZOOM_query_prefix(s, wrbuf_cstr(wr));
 774         wrbuf_destroy(wr);
 775     }
 776     ccl_qual_rm(&bibset);
 777     return ret;
 778 }
 779
 780 ZOOM_API(int)
 781     ZOOM_query_sortby(ZOOM_query s, const char *criteria)
 782 {
 783     s->sort_spec = yaz_sort_spec(s->odr, criteria);
 784     if (!s->sort_spec)
 785     {
 786         yaz_log(log_details, "%p ZOOM_query_sortby criteria=%s failed",
 787                 s, criteria);
 788         return -1;
 789     }
 790     yaz_log(log_details, "%p ZOOM_query_sortby criteria=%s", s, criteria);
 791     return 0;
 792 }
 793
 794 static zoom_ret do_write(ZOOM_connection c);
 795
 796 ZOOM_API(void)
 797     ZOOM_connection_destroy(ZOOM_connection c)
 798 {
 799     ZOOM_resultsets list;
 800     if (!c)
 801         return;
 802     yaz_log(log_api, "%p ZOOM_connection_destroy", c);
 803     if (c->cs)
 804         cs_close(c->cs);
 805
 806     // Remove the connection's usage of resultsets
 807     list = c->resultsets;
 808     while (list) {
 809         ZOOM_resultsets removed = list;
 810         ZOOM_resultset_destroy(list->resultset);
 811         list = list->next;
 812         xfree(removed);
 813     }
 814
 815     xfree(c->buf_in);
 816     xfree(c->addinfo);
 817     xfree(c->diagset);
 818     odr_destroy(c->odr_in);
 819     odr_destroy(c->odr_out);
 820     if (c->odr_print)
 821     {
 822         odr_setprint(c->odr_print, 0); /* prevent destroy from fclose'ing */
 823         odr_destroy(c->odr_print);
 824     }
 825     ZOOM_options_destroy(c->options);
 826     ZOOM_connection_remove_tasks(c);
 827     ZOOM_connection_remove_events(c);
 828     xfree(c->host_port);
 829     xfree(c->path);
 830     xfree(c->proxy);
 831     xfree(c->charset);
 832     xfree(c->lang);
 833     xfree(c->cookie_out);
 834     xfree(c->cookie_in);
 835     xfree(c->client_IP);
 836     xfree(c->user);
 837     xfree(c->group);
 838     xfree(c->password);
 839     xfree(c->sru_version);
 840     xfree(c);
 841 }
 842
 843 void ZOOM_resultset_addref(ZOOM_resultset r)
 844 {
 845     if (r)
 846     {
 847         yaz_mutex_enter(r->mutex);
 848         (r->refcount)++;
 849         yaz_log(log_details, "%p ZOOM_resultset_addref count=%d",
 850                 r, r->refcount);
 851         yaz_mutex_leave(r->mutex);
 852     }
 853 }
 854
 855 ZOOM_resultset ZOOM_resultset_create(void)
 856 {
 857     int i;
 858     ZOOM_resultset r = (ZOOM_resultset) xmalloc(sizeof(*r));
 859
 860     initlog();
 861
 862     yaz_log(log_details, "%p ZOOM_resultset_create", r);
 863     r->refcount = 1;
 864     r->size = 0;
 865     r->odr = odr_createmem(ODR_ENCODE);
 866     r->piggyback = 1;
 867     r->setname = 0;
 868     r->schema = 0;
 869     r->step = 0;
 870     for (i = 0; i<RECORD_HASH_SIZE; i++)
 871         r->record_hash[i] = 0;
 872     r->r_sort_spec = 0;
 873     r->query = 0;
 874     r->connection = 0;
 875     r->databaseNames = 0;
 876     r->num_databaseNames = 0;
 877     r->mutex = 0;
 878     yaz_mutex_create(&r->mutex);
 879 #if SHPTR
 880     {
 881         WRBUF w = wrbuf_alloc();
 882         YAZ_SHPTR_INIT(r->record_wrbuf, w);
 883     }
 884 #endif
 885     return r;
 886 }
 887
 888 ZOOM_API(ZOOM_resultset)
 889     ZOOM_connection_search_pqf(ZOOM_connection c, const char *q)
 890 {
 891     ZOOM_resultset r;
 892     ZOOM_query s = ZOOM_query_create();
 893
 894     ZOOM_query_prefix(s, q);
 895
 896     r = ZOOM_connection_search(c, s);
 897     ZOOM_query_destroy(s);
 898     return r;
 899 }
 900
 901 ZOOM_API(ZOOM_resultset)
 902     ZOOM_connection_search(ZOOM_connection c, ZOOM_query q)
 903 {
 904     ZOOM_resultset r = ZOOM_resultset_create();
 905     ZOOM_task task;
 906     const char *cp;
 907     int start, count;
 908     const char *syntax, *elementSetName;
 909     ZOOM_resultsets set;
 910
 911     yaz_log(log_api, "%p ZOOM_connection_search set %p query %p", c, r, q);
 912     r->r_sort_spec = q->sort_spec;
 913     r->query = q;
 914
 915     r->options = ZOOM_options_create_with_parent(c->options);
 916
 917     start = ZOOM_options_get_int(r->options, "start", 0);
 918     count = ZOOM_options_get_int(r->options, "count", 0);
 919     {
 920         /* If "presentChunk" is defined use that; otherwise "step" */
 921         const char *cp = ZOOM_options_get(r->options, "presentChunk");
 922         r->step = ZOOM_options_get_int(r->options,
 923                                        (cp != 0 ? "presentChunk": "step"), 0);
 924     }
 925     r->piggyback = ZOOM_options_get_bool(r->options, "piggyback", 1);
 926     cp = ZOOM_options_get(r->options, "setname");
 927     if (cp)
 928         r->setname = xstrdup(cp);
 929     cp = ZOOM_options_get(r->options, "schema");
 930     if (cp)
 931         r->schema = xstrdup(cp);
 932
 933     r->databaseNames = set_DatabaseNames(c, c->options, &r->num_databaseNames,
 934                                          r->odr);
 935
 936     r->connection = c;
 937
 938     yaz_log(log_details, "%p ZOOM_connection_search: Adding new resultset (%p) to resultsets (%p) ", c, r, c->resultsets);
 939     set = xmalloc(sizeof(*set));
 940     ZOOM_resultset_addref(r);
 941     set->resultset = r;
 942     set->next = c->resultsets;
 943     c->resultsets = set;
 944
 945     if (c->host_port && c->proto == PROTO_HTTP)
 946     {
 947         if (!c->cs)
 948         {
 949             yaz_log(log_details, "ZOOM_connection_search: no comstack");
 950             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
 951         }
 952         else
 953         {
 954             yaz_log(log_details, "ZOOM_connection_search: reconnect");
 955             c->reconnect_ok = 1;
 956         }
 957     }
 958
 959     task = ZOOM_connection_add_task(c, ZOOM_TASK_SEARCH);
 960     task->u.search.resultset = r;
 961     task->u.search.start = start;
 962     task->u.search.count = count;
 963     task->u.search.recv_search_fired = 0;
 964
 965     syntax = ZOOM_options_get(r->options, "preferredRecordSyntax");
 966     task->u.search.syntax = syntax ? xstrdup(syntax) : 0;
 967     elementSetName = ZOOM_options_get(r->options, "elementSetName");
 968     task->u.search.elementSetName = elementSetName
 969         ? xstrdup(elementSetName) : 0;
 970
 971     ZOOM_resultset_addref(r);
 972
 973     (q->refcount)++;
 974
 975     if (!c->async)
 976     {
 977         while (ZOOM_event(1, &c))
 978             ;
 979     }
 980     return r;
 981 }
 982
 983 ZOOM_API(void)
 984     ZOOM_resultset_sort(ZOOM_resultset r,
 985                          const char *sort_type, const char *sort_spec)
 986 {
 987     (void) ZOOM_resultset_sort1(r, sort_type, sort_spec);
 988 }
 989
 990 ZOOM_API(int)
 991     ZOOM_resultset_sort1(ZOOM_resultset r,
 992                          const char *sort_type, const char *sort_spec)
 993 {
 994     ZOOM_connection c = r->connection;
 995     ZOOM_task task;
 996     ZOOM_query newq;
 997
 998     newq = ZOOM_query_create();
 999     if (ZOOM_query_sortby(newq, sort_spec) < 0)
1000         return -1;
1001
1002     yaz_log(log_api, "%p ZOOM_resultset_sort r=%p sort_type=%s sort_spec=%s",
1003             r, r, sort_type, sort_spec);
1004     if (!c)
1005         return 0;
1006
1007     if (c->host_port && c->proto == PROTO_HTTP)
1008     {
1009         if (!c->cs)
1010         {
1011             yaz_log(log_details, "%p ZOOM_resultset_sort: no comstack", r);
1012             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
1013         }
1014         else
1015         {
1016             yaz_log(log_details, "%p ZOOM_resultset_sort: prepare reconnect",
1017                     r);
1018             c->reconnect_ok = 1;
1019         }
1020     }
1021
1022     ZOOM_resultset_cache_reset(r);
1023     task = ZOOM_connection_add_task(c, ZOOM_TASK_SORT);
1024     task->u.sort.resultset = r;
1025     task->u.sort.q = newq;
1026
1027     ZOOM_resultset_addref(r);
1028
1029     if (!c->async)
1030     {
1031         while (ZOOM_event(1, &c))
1032             ;
1033     }
1034
1035     return 0;
1036 }
1037
1038 static void ZOOM_record_release(ZOOM_record rec)
1039 {
1040     if (!rec)
1041         return;
1042
1043 #if SHPTR
1044     if (rec->record_wrbuf)
1045         YAZ_SHPTR_DEC(rec->record_wrbuf, wrbuf_destroy);
1046 #else
1047     if (rec->wrbuf)
1048         wrbuf_destroy(rec->wrbuf);
1049 #endif
1050
1051 #if YAZ_HAVE_XML2
1052     if (rec->xml_mem)
1053         xmlFree(rec->xml_mem);
1054 #endif
1055     if (rec->odr)
1056         odr_destroy(rec->odr);
1057 }
1058
1059 ZOOM_API(void)
1060     ZOOM_resultset_cache_reset(ZOOM_resultset r)
1061 {
1062     int i;
1063     for (i = 0; i<RECORD_HASH_SIZE; i++)
1064     {
1065         ZOOM_record_cache rc;
1066         for (rc = r->record_hash[i]; rc; rc = rc->next)
1067         {
1068             ZOOM_record_release(&rc->rec);
1069         }
1070         r->record_hash[i] = 0;
1071     }
1072 }
1073
1074 ZOOM_API(void)
1075     ZOOM_resultset_destroy(ZOOM_resultset r)
1076 {
1077     resultset_destroy(r);
1078 }
1079
1080 static void resultset_destroy(ZOOM_resultset r)
1081 {
1082     if (!r)
1083         return;
1084     yaz_mutex_enter(r->mutex);
1085     (r->refcount)--;
1086     yaz_log(log_details, "%p ZOOM_resultset_destroy r=%p count=%d",
1087             r, r, r->refcount);
1088     if (r->refcount == 0)
1089     {
1090         yaz_mutex_leave(r->mutex);
1091
1092         yaz_log(log_details, "%p ZOOM_connection resultset_destroy: Deleting resultset (%p) ", r->connection, r);
1093         ZOOM_resultset_cache_reset(r);
1094         ZOOM_query_destroy(r->query);
1095         ZOOM_options_destroy(r->options);
1096         odr_destroy(r->odr);
1097         xfree(r->setname);
1098         xfree(r->schema);
1099         yaz_mutex_destroy(&r->mutex);
1100 #if SHPTR
1101         YAZ_SHPTR_DEC(r->record_wrbuf, wrbuf_destroy);
1102 #endif
1103         xfree(r);
1104     }
1105     else
1106         yaz_mutex_leave(r->mutex);
1107 }
1108
1109 ZOOM_API(size_t)
1110     ZOOM_resultset_size(ZOOM_resultset r)
1111 {
1112     yaz_log(log_details, "ZOOM_resultset_size r=%p count=" ODR_INT_PRINTF,
1113             r, r->size);
1114     return r->size;
1115 }
1116
1117 static void do_close(ZOOM_connection c)
1118 {
1119     if (c->cs)
1120         cs_close(c->cs);
1121     c->cs = 0;
1122     ZOOM_connection_set_mask(c, 0);
1123     c->state = STATE_IDLE;
1124 }
1125
1126 static int ZOOM_test_reconnect(ZOOM_connection c)
1127 {
1128     ZOOM_Event event;
1129
1130     if (!c->reconnect_ok)
1131         return 0;
1132     do_close(c);
1133     c->reconnect_ok = 0;
1134     c->tasks->running = 0;
1135     ZOOM_connection_insert_task(c, ZOOM_TASK_CONNECT);
1136
1137     event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1138     ZOOM_connection_put_event(c, event);
1139
1140     return 1;
1141 }
1142
1143 static void ZOOM_resultset_retrieve(ZOOM_resultset r,
1144                                     int force_sync, int start, int count)
1145 {
1146     ZOOM_task task;
1147     ZOOM_connection c;
1148     const char *cp;
1149     const char *syntax, *elementSetName;
1150
1151     if (!r)
1152         return;
1153     yaz_log(log_details, "%p ZOOM_resultset_retrieve force_sync=%d start=%d"
1154             " count=%d", r, force_sync, start, count);
1155     c = r->connection;
1156     if (!c)
1157         return;
1158
1159     if (c->host_port && c->proto == PROTO_HTTP)
1160     {
1161         if (!c->cs)
1162         {
1163             yaz_log(log_details, "%p ZOOM_resultset_retrieve: no comstack", r);
1164             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
1165         }
1166         else
1167         {
1168             yaz_log(log_details, "%p ZOOM_resultset_retrieve: prepare "
1169                     "reconnect", r);
1170             c->reconnect_ok = 1;
1171         }
1172     }
1173     task = ZOOM_connection_add_task(c, ZOOM_TASK_RETRIEVE);
1174     task->u.retrieve.resultset = r;
1175     task->u.retrieve.start = start;
1176     task->u.retrieve.count = count;
1177
1178     syntax = ZOOM_options_get(r->options, "preferredRecordSyntax");
1179     task->u.retrieve.syntax = syntax ? xstrdup(syntax) : 0;
1180     elementSetName = ZOOM_options_get(r->options, "elementSetName");
1181     task->u.retrieve.elementSetName = elementSetName
1182         ? xstrdup(elementSetName) : 0;
1183
1184     cp = ZOOM_options_get(r->options, "schema");
1185     if (cp)
1186     {
1187         if (!r->schema || strcmp(r->schema, cp))
1188         {
1189             xfree(r->schema);
1190             r->schema = xstrdup(cp);
1191         }
1192     }
1193
1194     ZOOM_resultset_addref(r);
1195
1196     if (!r->connection->async || force_sync)
1197         while (r->connection && ZOOM_event(1, &r->connection))
1198             ;
1199 }
1200
1201 ZOOM_API(void)
1202     ZOOM_resultset_records(ZOOM_resultset r, ZOOM_record *recs,
1203                            size_t start, size_t count)
1204 {
1205     int force_present = 0;
1206
1207     if (!r)
1208         return ;
1209     yaz_log(log_api, "%p ZOOM_resultset_records r=%p start=%ld count=%ld",
1210             r, r, (long) start, (long) count);
1211     if (count && recs)
1212         force_present = 1;
1213     ZOOM_resultset_retrieve(r, force_present, start, count);
1214     if (force_present)
1215     {
1216         size_t i;
1217         for (i = 0; i< count; i++)
1218             recs[i] = ZOOM_resultset_record_immediate(r, i+start);
1219     }
1220 }
1221
1222 static void get_cert(ZOOM_connection c)
1223 {
1224     char *cert_buf;
1225     int cert_len;
1226
1227     if (cs_get_peer_certificate_x509(c->cs, &cert_buf, &cert_len))
1228     {
1229         ZOOM_connection_option_setl(c, "sslPeerCert",
1230                                     cert_buf, cert_len);
1231         xfree(cert_buf);
1232     }
1233 }
1234
1235 static zoom_ret do_connect_host(ZOOM_connection c,
1236                                 const char *effective_host,
1237                                 const char *logical_url);
1238
1239 static zoom_ret do_connect(ZOOM_connection c)
1240 {
1241     const char *effective_host;
1242
1243     if (c->proxy)
1244         effective_host = c->proxy;
1245     else
1246         effective_host = c->host_port;
1247     return do_connect_host(c, effective_host, c->host_port);
1248 }
1249
1250 static zoom_ret do_connect_host(ZOOM_connection c, const char *effective_host,
1251     const char *logical_url)
1252 {
1253     void *add;
1254
1255     yaz_log(log_details, "%p do_connect effective_host=%s", c, effective_host);
1256
1257     if (c->cs)
1258         cs_close(c->cs);
1259     c->cs = cs_create_host(effective_host, 0, &add);
1260
1261     if (c->cs && c->cs->protocol == PROTO_HTTP)
1262     {
1263 #if YAZ_HAVE_XML2
1264         if (logical_url)
1265         {
1266             const char *db = 0;
1267
1268             c->proto = PROTO_HTTP;
1269             cs_get_host_args(logical_url, &db);
1270             xfree(c->path);
1271
1272             c->path = xmalloc(strlen(db) * 3 + 2);
1273             yaz_encode_sru_dbpath_buf(c->path, db);
1274         }
1275 #else
1276         set_ZOOM_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, "SRW");
1277         do_close(c);
1278         return zoom_complete;
1279 #endif
1280     }
1281     if (c->cs)
1282     {
1283         int ret = cs_connect(c->cs, add);
1284         if (ret == 0)
1285         {
1286             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1287             ZOOM_connection_put_event(c, event);
1288             get_cert(c);
1289             if (c->proto == PROTO_Z3950)
1290                 ZOOM_connection_send_init(c);
1291             else
1292             {
1293                 /* no init request for SRW .. */
1294                 assert(c->tasks->which == ZOOM_TASK_CONNECT);
1295                 ZOOM_connection_remove_task(c);
1296                 ZOOM_connection_set_mask(c, 0);
1297                 ZOOM_connection_exec_task(c);
1298             }
1299             c->state = STATE_ESTABLISHED;
1300             return zoom_pending;
1301         }
1302         else if (ret > 0)
1303         {
1304             int mask = ZOOM_SELECT_EXCEPT;
1305             if (c->cs->io_pending & CS_WANT_WRITE)
1306                 mask += ZOOM_SELECT_WRITE;
1307             if (c->cs->io_pending & CS_WANT_READ)
1308                 mask += ZOOM_SELECT_READ;
1309             ZOOM_connection_set_mask(c, mask);
1310             c->state = STATE_CONNECTING;
1311             return zoom_pending;
1312         }
1313     }
1314     c->state = STATE_IDLE;
1315     set_ZOOM_error(c, ZOOM_ERROR_CONNECT, logical_url);
1316     return zoom_complete;
1317 }
1318
1319 static void otherInfo_attach(ZOOM_connection c, Z_APDU *a, ODR out)
1320 {
1321     int i;
1322     for (i = 0; i<200; i++)
1323     {
1324         size_t len;
1325         Odr_oid *oid;
1326         Z_OtherInformation **oi;
1327         char buf[80];
1328         const char *val;
1329         const char *cp;
1330
1331         sprintf(buf, "otherInfo%d", i);
1332         val = ZOOM_options_get(c->options, buf);
1333         if (!val)
1334             break;
1335         cp = strchr(val, ':');
1336         if (!cp)
1337             continue;
1338         len = cp - val;
1339         if (len >= sizeof(buf))
1340             len = sizeof(buf)-1;
1341         memcpy(buf, val, len);
1342         buf[len] = '\0';
1343
1344         oid = yaz_string_to_oid_odr(yaz_oid_std(), CLASS_USERINFO,
1345                                     buf, out);
1346         if (!oid)
1347             continue;
1348
1349         yaz_oi_APDU(a, &oi);
1350         yaz_oi_set_string_oid(oi, out, oid, 1, cp+1);
1351     }
1352 }
1353
1354 static int encode_APDU(ZOOM_connection c, Z_APDU *a, ODR out)
1355 {
1356     assert(a);
1357     if (c->cookie_out)
1358     {
1359         Z_OtherInformation **oi;
1360         yaz_oi_APDU(a, &oi);
1361         yaz_oi_set_string_oid(oi, out, yaz_oid_userinfo_cookie,
1362                               1, c->cookie_out);
1363     }
1364     if (c->client_IP)
1365     {
1366         Z_OtherInformation **oi;
1367         yaz_oi_APDU(a, &oi);
1368         yaz_oi_set_string_oid(oi, out, yaz_oid_userinfo_client_ip,
1369                               1, c->client_IP);
1370     }
1371     otherInfo_attach(c, a, out);
1372     if (!z_APDU(out, &a, 0, 0))
1373     {
1374         FILE *outf = fopen("/tmp/apdu.txt", "a");
1375         if (a && outf)
1376         {
1377             ODR odr_pr = odr_createmem(ODR_PRINT);
1378             fprintf(outf, "a=%p\n", a);
1379             odr_setprint(odr_pr, outf);
1380             z_APDU(odr_pr, &a, 0, 0);
1381             odr_destroy(odr_pr);
1382         }
1383         yaz_log(log_api, "%p encoding_APDU: encoding failed", c);
1384         set_ZOOM_error(c, ZOOM_ERROR_ENCODE, 0);
1385         odr_reset(out);
1386         return -1;
1387     }
1388     if (c->odr_print)
1389         z_APDU(c->odr_print, &a, 0, 0);
1390     yaz_log(log_details, "%p encoding_APDU encoding OK", c);
1391     return 0;
1392 }
1393
1394 static zoom_ret send_APDU(ZOOM_connection c, Z_APDU *a)
1395 {
1396     ZOOM_Event event;
1397     assert(a);
1398     if (encode_APDU(c, a, c->odr_out))
1399         return zoom_complete;
1400     yaz_log(log_details, "%p send APDU type=%d", c, a->which);
1401     c->buf_out = odr_getbuf(c->odr_out, &c->len_out, 0);
1402     event = ZOOM_Event_create(ZOOM_EVENT_SEND_APDU);
1403     ZOOM_connection_put_event(c, event);
1404     odr_reset(c->odr_out);
1405     return do_write(c);
1406 }
1407
1408 /* returns 1 if PDU was sent OK (still pending )
1409    0 if PDU was not sent OK (nothing to wait for)
1410 */
1411
1412 static zoom_ret ZOOM_connection_send_init(ZOOM_connection c)
1413 {
1414     Z_APDU *apdu = zget_APDU(c->odr_out, Z_APDU_initRequest);
1415     Z_InitRequest *ireq = apdu->u.initRequest;
1416     Z_IdAuthentication *auth = (Z_IdAuthentication *)
1417         odr_malloc(c->odr_out, sizeof(*auth));
1418
1419     ODR_MASK_SET(ireq->options, Z_Options_search);
1420     ODR_MASK_SET(ireq->options, Z_Options_present);
1421     ODR_MASK_SET(ireq->options, Z_Options_scan);
1422     ODR_MASK_SET(ireq->options, Z_Options_sort);
1423     ODR_MASK_SET(ireq->options, Z_Options_extendedServices);
1424     ODR_MASK_SET(ireq->options, Z_Options_namedResultSets);
1425
1426     ODR_MASK_SET(ireq->protocolVersion, Z_ProtocolVersion_1);
1427     ODR_MASK_SET(ireq->protocolVersion, Z_ProtocolVersion_2);
1428     ODR_MASK_SET(ireq->protocolVersion, Z_ProtocolVersion_3);
1429
1430     ireq->implementationId =
1431         odr_prepend(c->odr_out,
1432                     ZOOM_options_get(c->options, "implementationId"),
1433                     ireq->implementationId);
1434
1435     ireq->implementationName =
1436         odr_prepend(c->odr_out,
1437                     ZOOM_options_get(c->options, "implementationName"),
1438                     odr_prepend(c->odr_out, "ZOOM-C",
1439                                 ireq->implementationName));
1440
1441     ireq->implementationVersion =
1442         odr_prepend(c->odr_out,
1443                     ZOOM_options_get(c->options, "implementationVersion"),
1444                                 ireq->implementationVersion);
1445
1446     *ireq->maximumRecordSize = c->maximum_record_size;
1447     *ireq->preferredMessageSize = c->preferred_message_size;
1448
1449     if (c->group || c->password)
1450     {
1451         Z_IdPass *pass = (Z_IdPass *) odr_malloc(c->odr_out, sizeof(*pass));
1452         pass->groupId = odr_strdup_null(c->odr_out, c->group);
1453         pass->userId = odr_strdup_null(c->odr_out, c->user);
1454         pass->password = odr_strdup_null(c->odr_out, c->password);
1455         auth->which = Z_IdAuthentication_idPass;
1456         auth->u.idPass = pass;
1457         ireq->idAuthentication = auth;
1458     }
1459     else if (c->user)
1460     {
1461         auth->which = Z_IdAuthentication_open;
1462         auth->u.open = odr_strdup(c->odr_out, c->user);
1463         ireq->idAuthentication = auth;
1464     }
1465     if (c->proxy)
1466     {
1467         yaz_oi_set_string_oid(&ireq->otherInfo, c->odr_out,
1468                               yaz_oid_userinfo_proxy, 1, c->host_port);
1469     }
1470     if (c->charset || c->lang)
1471     {
1472         Z_OtherInformation **oi;
1473         Z_OtherInformationUnit *oi_unit;
1474
1475         yaz_oi_APDU(apdu, &oi);
1476
1477         if ((oi_unit = yaz_oi_update(oi, c->odr_out, NULL, 0, 0)))
1478         {
1479             ODR_MASK_SET(ireq->options, Z_Options_negotiationModel);
1480             oi_unit->which = Z_OtherInfo_externallyDefinedInfo;
1481             oi_unit->information.externallyDefinedInfo =
1482                 yaz_set_proposal_charneg_list(c->odr_out, " ",
1483                                               c->charset, c->lang, 1);
1484         }
1485     }
1486     assert(apdu);
1487     return send_APDU(c, apdu);
1488 }
1489
1490 #if YAZ_HAVE_XML2
1491 static zoom_ret send_srw(ZOOM_connection c, Z_SRW_PDU *sr)
1492 {
1493     Z_GDU *gdu;
1494     ZOOM_Event event;
1495     const char *database =  ZOOM_options_get(c->options, "databaseName");
1496     char *fdatabase = 0;
1497
1498     if (database)
1499         fdatabase = yaz_encode_sru_dbpath_odr(c->odr_out, database);
1500     gdu = z_get_HTTP_Request_host_path(c->odr_out, c->host_port,
1501                                        fdatabase ? fdatabase : c->path);
1502
1503     if (c->sru_mode == zoom_sru_get)
1504     {
1505         yaz_sru_get_encode(gdu->u.HTTP_Request, sr, c->odr_out, c->charset);
1506     }
1507     else if (c->sru_mode == zoom_sru_post)
1508     {
1509         yaz_sru_post_encode(gdu->u.HTTP_Request, sr, c->odr_out, c->charset);
1510     }
1511     else if (c->sru_mode == zoom_sru_soap)
1512     {
1513         yaz_sru_soap_encode(gdu->u.HTTP_Request, sr, c->odr_out, c->charset);
1514     }
1515     if (!z_GDU(c->odr_out, &gdu, 0, 0))
1516         return zoom_complete;
1517     if (c->odr_print)
1518         z_GDU(c->odr_print, &gdu, 0, 0);
1519     c->buf_out = odr_getbuf(c->odr_out, &c->len_out, 0);
1520
1521     event = ZOOM_Event_create(ZOOM_EVENT_SEND_APDU);
1522     ZOOM_connection_put_event(c, event);
1523     odr_reset(c->odr_out);
1524     return do_write(c);
1525 }
1526 #endif
1527
1528 #if YAZ_HAVE_XML2
1529 static Z_SRW_PDU *ZOOM_srw_get_pdu(ZOOM_connection c, int type)
1530 {
1531     Z_SRW_PDU *sr = yaz_srw_get_pdu(c->odr_out, type, c->sru_version);
1532     sr->username = c->user;
1533     sr->password = c->password;
1534     return sr;
1535 }
1536 #endif
1537
1538 #if YAZ_HAVE_XML2
1539 static zoom_ret ZOOM_connection_srw_send_search(ZOOM_connection c)
1540 {
1541     int i;
1542     int *start, *count;
1543     ZOOM_resultset resultset = 0;
1544     Z_SRW_PDU *sr = 0;
1545     const char *option_val = 0;
1546
1547     if (c->error)                  /* don't continue on error */
1548         return zoom_complete;
1549     assert(c->tasks);
1550     switch(c->tasks->which)
1551     {
1552     case ZOOM_TASK_SEARCH:
1553         resultset = c->tasks->u.search.resultset;
1554         if (!resultset->setname)
1555             resultset->setname = xstrdup("default");
1556         ZOOM_options_set(resultset->options, "setname", resultset->setname);
1557         start = &c->tasks->u.search.start;
1558         count = &c->tasks->u.search.count;
1559         break;
1560     case ZOOM_TASK_RETRIEVE:
1561         resultset = c->tasks->u.retrieve.resultset;
1562
1563         start = &c->tasks->u.retrieve.start;
1564         count = &c->tasks->u.retrieve.count;
1565
1566         if (*start >= resultset->size)
1567             return zoom_complete;
1568         if (*start + *count > resultset->size)
1569             *count = resultset->size - *start;
1570
1571         for (i = 0; i < *count; i++)
1572         {
1573             ZOOM_record rec =
1574                 record_cache_lookup(resultset, i + *start,
1575                                     c->tasks->u.retrieve.syntax,
1576                                     c->tasks->u.retrieve.elementSetName);
1577             if (!rec)
1578                 break;
1579             else
1580             {
1581                 ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_RECV_RECORD);
1582                 ZOOM_connection_put_event(c, event);
1583             }
1584         }
1585         *start += i;
1586         *count -= i;
1587
1588         if (*count == 0)
1589             return zoom_complete;
1590         break;
1591     default:
1592         return zoom_complete;
1593     }
1594     assert(resultset->query);
1595
1596     sr = ZOOM_srw_get_pdu(c, Z_SRW_searchRetrieve_request);
1597     if (resultset->query->z_query->which == Z_Query_type_104
1598         && resultset->query->z_query->u.type_104->which == Z_External_CQL)
1599     {
1600         sr->u.request->query_type = Z_SRW_query_type_cql;
1601         sr->u.request->query.cql =resultset->query->z_query->u.type_104->u.cql;
1602     }
1603     else if (resultset->query->z_query->which == Z_Query_type_1 &&
1604              resultset->query->z_query->u.type_1)
1605     {
1606         sr->u.request->query_type = Z_SRW_query_type_pqf;
1607         sr->u.request->query.pqf = resultset->query->query_string;
1608     }
1609     else
1610     {
1611         set_ZOOM_error(c, ZOOM_ERROR_UNSUPPORTED_QUERY, 0);
1612         return zoom_complete;
1613     }
1614     sr->u.request->startRecord = odr_intdup(c->odr_out, *start + 1);
1615     sr->u.request->maximumRecords = odr_intdup(
1616         c->odr_out, (resultset->step > 0 && resultset->step < *count) ?
1617         resultset->step : *count);
1618     sr->u.request->recordSchema = resultset->schema;
1619
1620     option_val = ZOOM_resultset_option_get(resultset, "recordPacking");
1621     if (option_val)
1622         sr->u.request->recordPacking = odr_strdup(c->odr_out, option_val);
1623
1624     option_val = ZOOM_resultset_option_get(resultset, "extraArgs");
1625     yaz_encode_sru_extra(sr, c->odr_out, option_val);
1626     return send_srw(c, sr);
1627 }
1628 #else
1629 static zoom_ret ZOOM_connection_srw_send_search(ZOOM_connection c)
1630 {
1631     return zoom_complete;
1632 }
1633 #endif
1634
1635 static zoom_ret ZOOM_connection_send_search(ZOOM_connection c)
1636 {
1637     ZOOM_resultset r;
1638     int lslb, ssub, mspn;
1639     const char *syntax;
1640     Z_APDU *apdu = zget_APDU(c->odr_out, Z_APDU_searchRequest);
1641     Z_SearchRequest *search_req = apdu->u.searchRequest;
1642     const char *elementSetName;
1643     const char *smallSetElementSetName;
1644     const char *mediumSetElementSetName;
1645
1646     assert(c->tasks);
1647     assert(c->tasks->which == ZOOM_TASK_SEARCH);
1648
1649     r = c->tasks->u.search.resultset;
1650
1651     yaz_log(log_details, "%p ZOOM_connection_send_search set=%p", c, r);
1652
1653     elementSetName =
1654         ZOOM_options_get(r->options, "elementSetName");
1655     smallSetElementSetName  =
1656         ZOOM_options_get(r->options, "smallSetElementSetName");
1657     mediumSetElementSetName =
1658         ZOOM_options_get(r->options, "mediumSetElementSetName");
1659
1660     if (!smallSetElementSetName)
1661         smallSetElementSetName = elementSetName;
1662
1663     if (!mediumSetElementSetName)
1664         mediumSetElementSetName = elementSetName;
1665
1666     assert(r);
1667     assert(r->query);
1668
1669     /* prepare query for the search request */
1670     search_req->query = r->query->z_query;
1671     if (!search_req->query)
1672     {
1673         set_ZOOM_error(c, ZOOM_ERROR_INVALID_QUERY, 0);
1674         return zoom_complete;
1675     }
1676     if (r->query->z_query->which == Z_Query_type_1 ||
1677         r->query->z_query->which == Z_Query_type_101)
1678     {
1679         const char *cp = ZOOM_options_get(r->options, "rpnCharset");
1680         if (cp)
1681         {
1682             yaz_iconv_t cd = yaz_iconv_open(cp, "UTF-8");
1683             if (cd)
1684             {
1685                 int r;
1686                 search_req->query = yaz_copy_Z_Query(search_req->query,
1687                                                      c->odr_out);
1688
1689                 r = yaz_query_charset_convert_rpnquery_check(
1690                     search_req->query->u.type_1,
1691                     c->odr_out, cd);
1692                 yaz_iconv_close(cd);
1693                 if (r)
1694                 {  /* query could not be char converted */
1695                     set_ZOOM_error(c, ZOOM_ERROR_INVALID_QUERY, 0);
1696                     return zoom_complete;
1697                 }
1698             }
1699         }
1700     }
1701     search_req->databaseNames = r->databaseNames;
1702     search_req->num_databaseNames = r->num_databaseNames;
1703
1704     /* get syntax (no need to provide unless piggyback is in effect) */
1705     syntax = c->tasks->u.search.syntax;
1706
1707     lslb = ZOOM_options_get_int(r->options, "largeSetLowerBound", -1);
1708     ssub = ZOOM_options_get_int(r->options, "smallSetUpperBound", -1);
1709     mspn = ZOOM_options_get_int(r->options, "mediumSetPresentNumber", -1);
1710     if (lslb != -1 && ssub != -1 && mspn != -1)
1711     {
1712         /* So're a Z39.50 expert? Let's hope you don't do sort */
1713         *search_req->largeSetLowerBound = lslb;
1714         *search_req->smallSetUpperBound = ssub;
1715         *search_req->mediumSetPresentNumber = mspn;
1716     }
1717     else if (c->tasks->u.search.start == 0 && c->tasks->u.search.count > 0
1718              && r->piggyback && !r->r_sort_spec && !r->schema)
1719     {
1720         /* Regular piggyback - do it unless we're going to do sort */
1721         *search_req->largeSetLowerBound = 2000000000;
1722         *search_req->smallSetUpperBound = 1;
1723         *search_req->mediumSetPresentNumber =
1724             r->step>0 ? r->step : c->tasks->u.search.count;
1725     }
1726     else
1727     {
1728         /* non-piggyback. Need not provide elementsets or syntaxes .. */
1729         smallSetElementSetName = 0;
1730         mediumSetElementSetName = 0;
1731         syntax = 0;
1732     }
1733     if (smallSetElementSetName && *smallSetElementSetName)
1734     {
1735         Z_ElementSetNames *esn = (Z_ElementSetNames *)
1736             odr_malloc(c->odr_out, sizeof(*esn));
1737
1738         esn->which = Z_ElementSetNames_generic;
1739         esn->u.generic = odr_strdup(c->odr_out, smallSetElementSetName);
1740         search_req->smallSetElementSetNames = esn;
1741     }
1742     if (mediumSetElementSetName && *mediumSetElementSetName)
1743     {
1744         Z_ElementSetNames *esn =(Z_ElementSetNames *)
1745             odr_malloc(c->odr_out, sizeof(*esn));
1746
1747         esn->which = Z_ElementSetNames_generic;
1748         esn->u.generic = odr_strdup(c->odr_out, mediumSetElementSetName);
1749         search_req->mediumSetElementSetNames = esn;
1750     }
1751     if (syntax)
1752         search_req->preferredRecordSyntax =
1753             zoom_yaz_str_to_z3950oid(c, CLASS_RECSYN, syntax);
1754
1755     if (!r->setname)
1756     {
1757         if (c->support_named_resultsets)
1758         {
1759             char setname[14];
1760             int ord;
1761             /* find the lowest unused ordinal so that we re-use
1762                result sets on the server. */
1763             for (ord = 1; ; ord++)
1764             {
1765                 ZOOM_resultsets rsp;
1766                 sprintf(setname, "%d", ord);
1767                 for (rsp = c->resultsets; rsp; rsp = rsp->next)
1768                     if (rsp->resultset->setname && !strcmp(rsp->resultset->setname, setname))
1769                         break;
1770                 if (!rsp)
1771                     break;
1772             }
1773             r->setname = xstrdup(setname);
1774             yaz_log(log_details, "%p ZOOM_connection_send_search: allocating "
1775                     "set %s", c, r->setname);
1776         }
1777         else
1778         {
1779             yaz_log(log_details, "%p ZOOM_connection_send_search: using "
1780                     "default set", c);
1781             r->setname = xstrdup("default");
1782         }
1783         ZOOM_options_set(r->options, "setname", r->setname);
1784     }
1785     search_req->resultSetName = odr_strdup(c->odr_out, r->setname);
1786     return send_APDU(c, apdu);
1787 }
1788
1789 static void response_default_diag(ZOOM_connection c, Z_DefaultDiagFormat *r)
1790 {
1791     char oid_name_buf[OID_STR_MAX];
1792     const char *oid_name;
1793     char *addinfo = 0;
1794
1795     oid_name = yaz_oid_to_string_buf(r->diagnosticSetId, 0, oid_name_buf);
1796     switch (r->which)
1797     {
1798     case Z_DefaultDiagFormat_v2Addinfo:
1799         addinfo = r->u.v2Addinfo;
1800         break;
1801     case Z_DefaultDiagFormat_v3Addinfo:
1802         addinfo = r->u.v3Addinfo;
1803         break;
1804     }
1805     xfree(c->addinfo);
1806     c->addinfo = 0;
1807     set_dset_error(c, *r->condition, oid_name, addinfo, 0);
1808 }
1809
1810 static void response_diag(ZOOM_connection c, Z_DiagRec *p)
1811 {
1812     if (p->which != Z_DiagRec_defaultFormat)
1813         set_ZOOM_error(c, ZOOM_ERROR_DECODE, 0);
1814     else
1815         response_default_diag(c, p->u.defaultFormat);
1816 }
1817
1818 ZOOM_API(ZOOM_record)
1819     ZOOM_record_clone(ZOOM_record srec)
1820 {
1821     char *buf;
1822     int size;
1823     ODR odr_enc;
1824     ZOOM_record nrec;
1825
1826     odr_enc = odr_createmem(ODR_ENCODE);
1827     if (!z_NamePlusRecord(odr_enc, &srec->npr, 0, 0))
1828         return 0;
1829     buf = odr_getbuf(odr_enc, &size, 0);
1830
1831     nrec = (ZOOM_record) xmalloc(sizeof(*nrec));
1832     nrec->odr = odr_createmem(ODR_DECODE);
1833 #if SHPTR
1834     nrec->record_wrbuf = 0;
1835 #else
1836     nrec->wrbuf = 0;
1837 #endif
1838 #if YAZ_HAVE_XML2
1839     nrec->xml_mem = 0;
1840     nrec->xml_size = 0;
1841 #endif
1842     odr_setbuf(nrec->odr, buf, size, 0);
1843     z_NamePlusRecord(nrec->odr, &nrec->npr, 0, 0);
1844
1845     nrec->schema = odr_strdup_null(nrec->odr, srec->schema);
1846     nrec->diag_uri = odr_strdup_null(nrec->odr, srec->diag_uri);
1847     nrec->diag_message = odr_strdup_null(nrec->odr, srec->diag_message);
1848     nrec->diag_details = odr_strdup_null(nrec->odr, srec->diag_details);
1849     nrec->diag_set = odr_strdup_null(nrec->odr, srec->diag_set);
1850     odr_destroy(odr_enc);
1851     return nrec;
1852 }
1853
1854 ZOOM_API(ZOOM_record)
1855     ZOOM_resultset_record_immediate(ZOOM_resultset s,size_t pos)
1856 {
1857     const char *syntax =
1858         ZOOM_options_get(s->options, "preferredRecordSyntax");
1859     const char *elementSetName =
1860         ZOOM_options_get(s->options, "elementSetName");
1861
1862     return record_cache_lookup(s, pos, syntax, elementSetName);
1863 }
1864
1865 ZOOM_API(ZOOM_record)
1866     ZOOM_resultset_record(ZOOM_resultset r, size_t pos)
1867 {
1868     ZOOM_record rec = ZOOM_resultset_record_immediate(r, pos);
1869
1870     if (!rec)
1871     {
1872         /*
1873          * MIKE: I think force_sync should always be zero, but I don't
1874          * want to make this change until I get the go-ahead from
1875          * Adam, in case something depends on the old synchronous
1876          * behaviour.
1877          */
1878         int force_sync = 1;
1879         if (getenv("ZOOM_RECORD_NO_FORCE_SYNC")) force_sync = 0;
1880         ZOOM_resultset_retrieve(r, force_sync, pos, 1);
1881         rec = ZOOM_resultset_record_immediate(r, pos);
1882     }
1883     return rec;
1884 }
1885
1886 ZOOM_API(void)
1887     ZOOM_record_destroy(ZOOM_record rec)
1888 {
1889     ZOOM_record_release(rec);
1890     xfree(rec);
1891 }
1892
1893
1894 static yaz_iconv_t iconv_create_charset(const char *record_charset)
1895 {
1896     char to[40];
1897     char from[40];
1898     yaz_iconv_t cd = 0;
1899
1900     *from = '\0';
1901     strcpy(to, "UTF-8");
1902     if (record_charset && *record_charset)
1903     {
1904         /* Use "from,to" or just "from" */
1905         const char *cp = strchr(record_charset, ',');
1906         size_t clen = strlen(record_charset);
1907         if (cp && cp[1])
1908         {
1909             strncpy( to, cp+1, sizeof(to)-1);
1910             to[sizeof(to)-1] = '\0';
1911             clen = cp - record_charset;
1912         }
1913         if (clen > sizeof(from)-1)
1914             clen = sizeof(from)-1;
1915
1916         if (clen)
1917             strncpy(from, record_charset, clen);
1918         from[clen] = '\0';
1919     }
1920     if (*from && *to)
1921         cd = yaz_iconv_open(to, from);
1922     return cd;
1923 }
1924
1925 static const char *return_marc_record(ZOOM_record rec, WRBUF wrbuf,
1926                                       int marc_type,
1927                                       int *len,
1928                                       const char *buf, int sz,
1929                                       const char *record_charset)
1930 {
1931     yaz_iconv_t cd = iconv_create_charset(record_charset);
1932     yaz_marc_t mt = yaz_marc_create();
1933     const char *ret_string = 0;
1934
1935     if (cd)
1936         yaz_marc_iconv(mt, cd);
1937     yaz_marc_xml(mt, marc_type);
1938     if (yaz_marc_decode_wrbuf(mt, buf, sz, wrbuf) > 0)
1939     {
1940         if (len)
1941             *len = wrbuf_len(wrbuf);
1942         ret_string = wrbuf_cstr(wrbuf);
1943     }
1944     yaz_marc_destroy(mt);
1945     if (cd)
1946         yaz_iconv_close(cd);
1947     return ret_string;
1948 }
1949
1950 static const char *return_opac_record(ZOOM_record rec, WRBUF wrbuf,
1951                                       int marc_type,
1952                                       int *len,
1953                                       Z_OPACRecord *opac_rec,
1954                                       const char *record_charset)
1955 {
1956     yaz_iconv_t cd = iconv_create_charset(record_charset);
1957     yaz_marc_t mt = yaz_marc_create();
1958
1959     if (cd)
1960         yaz_marc_iconv(mt, cd);
1961     yaz_marc_xml(mt, marc_type);
1962
1963     yaz_opac_decode_wrbuf(mt, opac_rec, wrbuf);
1964     yaz_marc_destroy(mt);
1965
1966     if (cd)
1967         yaz_iconv_close(cd);
1968     if (len)
1969         *len = wrbuf_len(wrbuf);
1970     return wrbuf_cstr(wrbuf);
1971 }
1972
1973 static const char *return_string_record(ZOOM_record rec, WRBUF wrbuf,
1974                                         int *len,
1975                                         const char *buf, int sz,
1976                                         const char *record_charset)
1977 {
1978     yaz_iconv_t cd = iconv_create_charset(record_charset);
1979
1980     if (cd)
1981     {
1982         wrbuf_iconv_write(wrbuf, cd, buf, sz);
1983         wrbuf_iconv_reset(wrbuf, cd);
1984
1985         buf = wrbuf_cstr(wrbuf);
1986         sz = wrbuf_len(wrbuf);
1987         yaz_iconv_close(cd);
1988     }
1989     if (len)
1990         *len = sz;
1991     return buf;
1992 }
1993
1994 static const char *return_record(ZOOM_record rec, int *len,
1995                                  Z_NamePlusRecord *npr,
1996                                  int marctype, const char *charset)
1997 {
1998     Z_External *r = (Z_External *) npr->u.databaseRecord;
1999     const Odr_oid *oid = r->direct_reference;
2000     WRBUF wrbuf;
2001
2002 #if SHPTR
2003     if (!rec->record_wrbuf)
2004     {
2005         WRBUF w = wrbuf_alloc();
2006         YAZ_SHPTR_INIT(rec->record_wrbuf, w);
2007     }
2008     wrbuf = rec->record_wrbuf->ptr;
2009 #else
2010     if (!rec->wrbuf)
2011         rec->wrbuf = wrbuf_alloc();
2012     wrbuf = rec->wrbuf;
2013 #endif
2014     wrbuf_rewind(wrbuf);
2015     /* render bibliographic record .. */
2016     if (r->which == Z_External_OPAC)
2017     {
2018         return return_opac_record(rec, wrbuf, marctype, len,
2019                                   r->u.opac, charset);
2020     }
2021     if (r->which == Z_External_sutrs)
2022         return return_string_record(rec, wrbuf, len,
2023                                     (char*) r->u.sutrs->buf,
2024                                     r->u.sutrs->len,
2025                                     charset);
2026     else if (r->which == Z_External_octet)
2027     {
2028         if (yaz_oid_is_iso2709(oid))
2029         {
2030             const char *ret_buf = return_marc_record(
2031                 rec, wrbuf, marctype, len,
2032                 (const char *) r->u.octet_aligned->buf,
2033                 r->u.octet_aligned->len,
2034                 charset);
2035             if (ret_buf)
2036                 return ret_buf;
2037             /* bad ISO2709. Return fail unless raw (ISO2709) is wanted */
2038             if (marctype != YAZ_MARC_ISO2709)
2039                 return 0;
2040         }
2041         return return_string_record(rec, wrbuf, len,
2042                                     (const char *) r->u.octet_aligned->buf,
2043                                     r->u.octet_aligned->len,
2044                                     charset);
2045     }
2046     else if (r->which == Z_External_grs1)
2047     {
2048         yaz_display_grs1(wrbuf, r->u.grs1, 0);
2049         return return_string_record(rec, wrbuf, len,
2050                                     wrbuf_buf(wrbuf),
2051                                     wrbuf_len(wrbuf),
2052                                     charset);
2053     }
2054     return 0;
2055 }
2056
2057
2058 ZOOM_API(int)
2059     ZOOM_record_error(ZOOM_record rec, const char **cp,
2060                       const char **addinfo, const char **diagset)
2061 {
2062     Z_NamePlusRecord *npr;
2063
2064     if (!rec)
2065         return 0;
2066
2067     npr = rec->npr;
2068     if (rec->diag_uri)
2069     {
2070         if (cp)
2071             *cp = rec->diag_message;
2072         if (addinfo)
2073             *addinfo = rec->diag_details;
2074         if (diagset)
2075             *diagset = rec->diag_set;
2076         return uri_to_code(rec->diag_uri);
2077     }
2078     if (npr && npr->which == Z_NamePlusRecord_surrogateDiagnostic)
2079     {
2080         Z_DiagRec *diag_rec = npr->u.surrogateDiagnostic;
2081         int error = YAZ_BIB1_UNSPECIFIED_ERROR;
2082         const char *add = 0;
2083
2084         if (diag_rec->which == Z_DiagRec_defaultFormat)
2085         {
2086             Z_DefaultDiagFormat *ddf = diag_rec->u.defaultFormat;
2087             oid_class oclass;
2088
2089             error = *ddf->condition;
2090             switch (ddf->which)
2091             {
2092             case Z_DefaultDiagFormat_v2Addinfo:
2093                 add = ddf->u.v2Addinfo;
2094                 break;
2095             case Z_DefaultDiagFormat_v3Addinfo:
2096                 add = ddf->u.v3Addinfo;
2097                 break;
2098             }
2099             if (diagset)
2100                 *diagset =
2101                     yaz_oid_to_string(yaz_oid_std(),
2102                                       ddf->diagnosticSetId, &oclass);
2103         }
2104         else
2105         {
2106             if (diagset)
2107                 *diagset = "Bib-1";
2108         }
2109         if (addinfo)
2110             *addinfo = add ? add : "";
2111         if (cp)
2112             *cp = diagbib1_str(error);
2113         return error;
2114     }
2115     return 0;
2116 }
2117
2118 static const char *get_record_format(ZOOM_record rec, int *len,
2119                                      Z_NamePlusRecord *npr,
2120                                      int marctype, const char *charset,
2121                                      const char *format)
2122 {
2123     const char *res = return_record(rec, len, npr, marctype, charset);
2124 #if YAZ_HAVE_XML2
2125     if (*format == '1' && len)
2126     {
2127         /* try to XML format res */
2128         xmlDocPtr doc;
2129         xmlKeepBlanksDefault(0); /* get get xmlDocFormatMemory to work! */
2130         doc = xmlParseMemory(res, *len);
2131         if (doc)
2132         {
2133             if (rec->xml_mem)
2134                 xmlFree(rec->xml_mem);
2135             xmlDocDumpFormatMemory(doc, &rec->xml_mem, &rec->xml_size, 1);
2136             xmlFreeDoc(doc);
2137             res = (char *) rec->xml_mem;
2138             *len = rec->xml_size;
2139         }
2140     }
2141 #endif
2142     return res;
2143 }
2144
2145
2146 ZOOM_API(const char *)
2147     ZOOM_record_get(ZOOM_record rec, const char *type_spec, int *len)
2148 {
2149     char type[40];
2150     char charset[40];
2151     char format[3];
2152     const char *cp;
2153     size_t i;
2154     Z_NamePlusRecord *npr;
2155
2156     if (len)
2157         *len = 0; /* default return */
2158
2159     if (!rec)
2160         return 0;
2161     npr = rec->npr;
2162     if (!npr)
2163         return 0;
2164
2165     cp = type_spec;
2166     for (i = 0; cp[i] && cp[i] != ';' && cp[i] != ' ' && i < sizeof(type)-1;
2167          i++)
2168         type[i] = cp[i];
2169     type[i] = '\0';
2170     charset[0] = '\0';
2171     format[0] = '\0';
2172     while (1)
2173     {
2174         while (cp[i] == ' ')
2175             i++;
2176         if (cp[i] != ';')
2177             break;
2178         i++;
2179         while (cp[i] == ' ')
2180             i++;
2181         if (!strncmp(cp + i, "charset=", 8))
2182         {
2183             size_t j = 0;
2184             i = i + 8; /* skip charset= */
2185             for (j = 0; cp[i] && cp[i] != ';' && cp[i] != ' '; i++)
2186             {
2187                 if (j < sizeof(charset)-1)
2188                     charset[j++] = cp[i];
2189             }
2190             charset[j] = '\0';
2191         }
2192         else if (!strncmp(cp + i, "format=", 7))
2193         {
2194             size_t j = 0;
2195             i = i + 7;
2196             for (j = 0; cp[i] && cp[i] != ';' && cp[i] != ' '; i++)
2197             {
2198                 if (j < sizeof(format)-1)
2199                     format[j++] = cp[i];
2200             }
2201             format[j] = '\0';
2202         }
2203     }
2204     if (!strcmp(type, "database"))
2205     {
2206         if (len)
2207             *len = (npr->databaseName ? strlen(npr->databaseName) : 0);
2208         return npr->databaseName;
2209     }
2210     else if (!strcmp(type, "schema"))
2211     {
2212         if (len)
2213             *len = rec->schema ? strlen(rec->schema) : 0;
2214         return rec->schema;
2215     }
2216     else if (!strcmp(type, "syntax"))
2217     {
2218         const char *desc = 0;
2219         if (npr->which == Z_NamePlusRecord_databaseRecord)
2220         {
2221             Z_External *r = (Z_External *) npr->u.databaseRecord;
2222             desc = yaz_oid_to_string(yaz_oid_std(), r->direct_reference, 0);
2223         }
2224         if (!desc)
2225             desc = "none";
2226         if (len)
2227             *len = strlen(desc);
2228         return desc;
2229     }
2230     if (npr->which != Z_NamePlusRecord_databaseRecord)
2231         return 0;
2232
2233     /* from now on - we have a database record .. */
2234     if (!strcmp(type, "render"))
2235     {
2236         return get_record_format(rec, len, npr, YAZ_MARC_LINE, charset, format);
2237     }
2238     else if (!strcmp(type, "xml"))
2239     {
2240         return get_record_format(rec, len, npr, YAZ_MARC_MARCXML, charset,
2241                                  format);
2242     }
2243     else if (!strcmp(type, "txml"))
2244     {
2245         return get_record_format(rec, len, npr, YAZ_MARC_TURBOMARC, charset,
2246                                  format);
2247     }
2248     else if (!strcmp(type, "raw"))
2249     {
2250         return get_record_format(rec, len, npr, YAZ_MARC_ISO2709, charset,
2251             format);
2252     }
2253     else if (!strcmp(type, "ext"))
2254     {
2255         if (len) *len = -1;
2256         return (const char *) npr->u.databaseRecord;
2257     }
2258     else if (!strcmp(type, "opac"))
2259     {
2260         if (npr->u.databaseRecord->which == Z_External_OPAC)
2261             return get_record_format(rec, len, npr, YAZ_MARC_MARCXML, charset,
2262                 format);
2263     }
2264     return 0;
2265 }
2266
2267 static int strcmp_null(const char *v1, const char *v2)
2268 {
2269     if (!v1 && !v2)
2270         return 0;
2271     if (!v1 || !v2)
2272         return -1;
2273     return strcmp(v1, v2);
2274 }
2275
2276 static size_t record_hash(int pos)
2277 {
2278     if (pos < 0)
2279         pos = 0;
2280     return pos % RECORD_HASH_SIZE;
2281 }
2282
2283 static void record_cache_add(ZOOM_resultset r, Z_NamePlusRecord *npr,
2284                              int pos,
2285                              const char *syntax, const char *elementSetName,
2286                              const char *schema,
2287                              Z_SRW_diagnostic *diag)
2288 {
2289     ZOOM_record_cache rc = 0;
2290
2291     ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_RECV_RECORD);
2292     ZOOM_connection_put_event(r->connection, event);
2293
2294     for (rc = r->record_hash[record_hash(pos)]; rc; rc = rc->next)
2295     {
2296         if (pos == rc->pos
2297             && strcmp_null(r->schema, rc->schema) == 0
2298             && strcmp_null(elementSetName,rc->elementSetName) == 0
2299             && strcmp_null(syntax, rc->syntax) == 0)
2300             break;
2301     }
2302     if (!rc)
2303     {
2304         rc = (ZOOM_record_cache) odr_malloc(r->odr, sizeof(*rc));
2305         rc->rec.odr = 0;
2306 #if SHPTR
2307         YAZ_SHPTR_INC(r->record_wrbuf);
2308         rc->rec.record_wrbuf = r->record_wrbuf;
2309 #else
2310         rc->rec.wrbuf = 0;
2311 #endif
2312 #if YAZ_HAVE_XML2
2313         rc->rec.xml_mem = 0;
2314 #endif
2315         rc->elementSetName = odr_strdup_null(r->odr, elementSetName);
2316
2317         rc->syntax = odr_strdup_null(r->odr, syntax);
2318
2319         rc->schema = odr_strdup_null(r->odr, r->schema);
2320
2321         rc->pos = pos;
2322         rc->next = r->record_hash[record_hash(pos)];
2323         r->record_hash[record_hash(pos)] = rc;
2324     }
2325     rc->rec.npr = npr;
2326     rc->rec.schema = odr_strdup_null(r->odr, schema);
2327     rc->rec.diag_set = 0;
2328     rc->rec.diag_uri = 0;
2329     rc->rec.diag_message = 0;
2330     rc->rec.diag_details = 0;
2331     if (diag)
2332     {
2333         if (diag->uri)
2334         {
2335             char *cp;
2336             rc->rec.diag_set = odr_strdup(r->odr, diag->uri);
2337             if ((cp = strrchr(rc->rec.diag_set, '/')))
2338                 *cp = '\0';
2339             rc->rec.diag_uri = odr_strdup(r->odr, diag->uri);
2340         }
2341         rc->rec.diag_message = odr_strdup_null(r->odr, diag->message);
2342         rc->rec.diag_details = odr_strdup_null(r->odr, diag->details);
2343     }
2344 }
2345
2346 static ZOOM_record record_cache_lookup(ZOOM_resultset r, int pos,
2347                                        const char *syntax,
2348                                        const char *elementSetName)
2349 {
2350     ZOOM_record_cache rc;
2351
2352     for (rc = r->record_hash[record_hash(pos)]; rc; rc = rc->next)
2353     {
2354         if (pos == rc->pos)
2355         {
2356             if (strcmp_null(r->schema, rc->schema))
2357                 continue;
2358             if (strcmp_null(elementSetName,rc->elementSetName))
2359                 continue;
2360             if (strcmp_null(syntax, rc->syntax))
2361                 continue;
2362             return &rc->rec;
2363         }
2364     }
2365     return 0;
2366 }
2367
2368 static void handle_records(ZOOM_connection c, Z_Records *sr,
2369                            int present_phase)
2370 {
2371     ZOOM_resultset resultset;
2372     int *start, *count;
2373     const char *syntax = 0, *elementSetName = 0;
2374
2375     if (!c->tasks)
2376         return ;
2377     switch (c->tasks->which)
2378     {
2379     case ZOOM_TASK_SEARCH:
2380         resultset = c->tasks->u.search.resultset;
2381         start = &c->tasks->u.search.start;
2382         count = &c->tasks->u.search.count;
2383         syntax = c->tasks->u.search.syntax;
2384         elementSetName = c->tasks->u.search.elementSetName;
2385         break;
2386     case ZOOM_TASK_RETRIEVE:
2387         resultset = c->tasks->u.retrieve.resultset;
2388         start = &c->tasks->u.retrieve.start;
2389         count = &c->tasks->u.retrieve.count;
2390         syntax = c->tasks->u.retrieve.syntax;
2391         elementSetName = c->tasks->u.retrieve.elementSetName;
2392         break;
2393     default:
2394         return;
2395     }
2396     if (sr && sr->which == Z_Records_NSD)
2397         response_default_diag(c, sr->u.nonSurrogateDiagnostic);
2398     else if (sr && sr->which == Z_Records_multipleNSD)
2399     {
2400         if (sr->u.multipleNonSurDiagnostics->num_diagRecs >= 1)
2401             response_diag(c, sr->u.multipleNonSurDiagnostics->diagRecs[0]);
2402         else
2403             set_ZOOM_error(c, ZOOM_ERROR_DECODE, 0);
2404     }
2405     else
2406     {
2407         if (*count + *start > resultset->size)
2408             *count = resultset->size - *start;
2409         if (*count < 0)
2410             *count = 0;
2411         if (sr && sr->which == Z_Records_DBOSD)
2412         {
2413             int i;
2414             NMEM nmem = odr_extract_mem(c->odr_in);
2415             Z_NamePlusRecordList *p =
2416                 sr->u.databaseOrSurDiagnostics;
2417             for (i = 0; i<p->num_records; i++)
2418             {
2419                 record_cache_add(resultset, p->records[i], i + *start,
2420                                  syntax, elementSetName,
2421                                  elementSetName, 0);
2422             }
2423             *count -= i;
2424             if (*count < 0)
2425                 *count = 0;
2426             *start += i;
2427             yaz_log(log_details,
2428                     "handle_records resultset=%p start=%d count=%d",
2429                     resultset, *start, *count);
2430
2431             /* transfer our response to search_nmem .. we need it later */
2432             nmem_transfer(odr_getmem(resultset->odr), nmem);
2433             nmem_destroy(nmem);
2434             if (present_phase && p->num_records == 0)
2435             {
2436                 /* present response and we didn't get any records! */
2437                 Z_NamePlusRecord *myrec =
2438                     zget_surrogateDiagRec(
2439                         resultset->odr, 0,
2440                         YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS,
2441                         "ZOOM C generated. Present phase and no records");
2442                 record_cache_add(resultset, myrec, *start,
2443                                  syntax, elementSetName, 0, 0);
2444             }
2445         }
2446         else if (present_phase)
2447         {
2448             /* present response and we didn't get any records! */
2449             Z_NamePlusRecord *myrec =
2450                 zget_surrogateDiagRec(
2451                     resultset->odr, 0,
2452                     YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS,
2453                     "ZOOM C generated: Present response and no records");
2454             record_cache_add(resultset, myrec, *start, syntax, elementSetName,
2455                              0, 0);
2456         }
2457     }
2458 }
2459
2460 static void handle_present_response(ZOOM_connection c, Z_PresentResponse *pr)
2461 {
2462     handle_records(c, pr->records, 1);
2463 }
2464
2465 static void handle_queryExpressionTerm(ZOOM_options opt, const char *name,
2466                                        Z_Term *term)
2467 {
2468     switch (term->which)
2469     {
2470     case Z_Term_general:
2471         ZOOM_options_setl(opt, name,
2472                           (const char *)(term->u.general->buf),
2473                           term->u.general->len);
2474         break;
2475     case Z_Term_characterString:
2476         ZOOM_options_set(opt, name, term->u.characterString);
2477         break;
2478     case Z_Term_numeric:
2479         ZOOM_options_set_int(opt, name, *term->u.numeric);
2480         break;
2481     }
2482 }
2483
2484 static void handle_queryExpression(ZOOM_options opt, const char *name,
2485                                    Z_QueryExpression *exp)
2486 {
2487     char opt_name[80];
2488
2489     switch (exp->which)
2490     {
2491     case Z_QueryExpression_term:
2492         if (exp->u.term && exp->u.term->queryTerm)
2493         {
2494             sprintf(opt_name, "%s.term", name);
2495             handle_queryExpressionTerm(opt, opt_name, exp->u.term->queryTerm);
2496         }
2497         break;
2498     case Z_QueryExpression_query:
2499         break;
2500     }
2501 }
2502
2503 static void handle_searchResult(ZOOM_connection c, ZOOM_resultset resultset,
2504                                 Z_OtherInformation *o)
2505 {
2506     int i;
2507     for (i = 0; o && i < o->num_elements; i++)
2508     {
2509         if (o->list[i]->which == Z_OtherInfo_externallyDefinedInfo)
2510         {
2511             Z_External *ext = o->list[i]->information.externallyDefinedInfo;
2512
2513             if (ext->which == Z_External_searchResult1)
2514             {
2515                 int j;
2516                 Z_SearchInfoReport *sr = ext->u.searchResult1;
2517
2518                 if (sr->num)
2519                     ZOOM_options_set_int(
2520                         resultset->options, "searchresult.size", sr->num);
2521
2522                 for (j = 0; j < sr->num; j++)
2523                 {
2524                     Z_SearchInfoReport_s *ent =
2525                         ext->u.searchResult1->elements[j];
2526                     char pref[80];
2527
2528                     sprintf(pref, "searchresult.%d", j);
2529
2530                     if (ent->subqueryId)
2531                     {
2532                         char opt_name[80];
2533                         sprintf(opt_name, "%s.id", pref);
2534                         ZOOM_options_set(resultset->options, opt_name,
2535                                          ent->subqueryId);
2536                     }
2537                     if (ent->subqueryExpression)
2538                     {
2539                         char opt_name[80];
2540                         sprintf(opt_name, "%s.subquery", pref);
2541                         handle_queryExpression(resultset->options, opt_name,
2542                                                ent->subqueryExpression);
2543                     }
2544                     if (ent->subqueryInterpretation)
2545                     {
2546                         char opt_name[80];
2547                         sprintf(opt_name, "%s.interpretation", pref);
2548                         handle_queryExpression(resultset->options, opt_name,
2549                                                ent->subqueryInterpretation);
2550                     }
2551                     if (ent->subqueryRecommendation)
2552                     {
2553                         char opt_name[80];
2554                         sprintf(opt_name, "%s.recommendation", pref);
2555                         handle_queryExpression(resultset->options, opt_name,
2556                                                ent->subqueryRecommendation);
2557                     }
2558                     if (ent->subqueryCount)
2559                     {
2560                         char opt_name[80];
2561                         sprintf(opt_name, "%s.count", pref);
2562                         ZOOM_options_set_int(resultset->options, opt_name,
2563                                              *ent->subqueryCount);
2564                     }
2565                 }
2566             }
2567         }
2568     }
2569 }
2570
2571 static void handle_search_response(ZOOM_connection c, Z_SearchResponse *sr)
2572 {
2573     ZOOM_resultset resultset;
2574     ZOOM_Event event;
2575
2576     if (!c->tasks || c->tasks->which != ZOOM_TASK_SEARCH)
2577         return ;
2578
2579     event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH);
2580     ZOOM_connection_put_event(c, event);
2581
2582     resultset = c->tasks->u.search.resultset;
2583
2584     if (sr->resultSetStatus)
2585     {
2586         ZOOM_options_set_int(resultset->options, "resultSetStatus",
2587                              *sr->resultSetStatus);
2588     }
2589     if (sr->presentStatus)
2590     {
2591         ZOOM_options_set_int(resultset->options, "presentStatus",
2592                              *sr->presentStatus);
2593     }
2594     handle_searchResult(c, resultset, sr->additionalSearchInfo);
2595
2596     resultset->size = *sr->resultCount;
2597     handle_records(c, sr->records, 0);
2598 }
2599
2600 static void sort_response(ZOOM_connection c, Z_SortResponse *res)
2601 {
2602     if (res->diagnostics && res->num_diagnostics > 0)
2603         response_diag(c, res->diagnostics[0]);
2604 }
2605
2606 static int scan_response(ZOOM_connection c, Z_ScanResponse *res)
2607 {
2608     NMEM nmem = odr_extract_mem(c->odr_in);
2609     ZOOM_scanset scan;
2610
2611     if (!c->tasks || c->tasks->which != ZOOM_TASK_SCAN)
2612         return 0;
2613     scan = c->tasks->u.scan.scan;
2614
2615     if (res->entries && res->entries->nonsurrogateDiagnostics)
2616         response_diag(c, res->entries->nonsurrogateDiagnostics[0]);
2617     scan->scan_response = res;
2618     scan->srw_scan_response = 0;
2619     nmem_transfer(odr_getmem(scan->odr), nmem);
2620     if (res->stepSize)
2621         ZOOM_options_set_int(scan->options, "stepSize", *res->stepSize);
2622     if (res->positionOfTerm)
2623         ZOOM_options_set_int(scan->options, "position", *res->positionOfTerm);
2624     if (res->scanStatus)
2625         ZOOM_options_set_int(scan->options, "scanStatus", *res->scanStatus);
2626     if (res->numberOfEntriesReturned)
2627         ZOOM_options_set_int(scan->options, "number",
2628                              *res->numberOfEntriesReturned);
2629     nmem_destroy(nmem);
2630     return 1;
2631 }
2632
2633 static zoom_ret send_sort(ZOOM_connection c,
2634                           ZOOM_resultset resultset)
2635 {
2636     if (c->error)
2637         resultset->r_sort_spec = 0;
2638     if (resultset->r_sort_spec)
2639     {
2640         Z_APDU *apdu = zget_APDU(c->odr_out, Z_APDU_sortRequest);
2641         Z_SortRequest *req = apdu->u.sortRequest;
2642
2643         req->num_inputResultSetNames = 1;
2644         req->inputResultSetNames = (Z_InternationalString **)
2645             odr_malloc(c->odr_out, sizeof(*req->inputResultSetNames));
2646         req->inputResultSetNames[0] =
2647             odr_strdup(c->odr_out, resultset->setname);
2648         req->sortedResultSetName = odr_strdup(c->odr_out, resultset->setname);
2649         req->sortSequence = resultset->r_sort_spec;
2650         resultset->r_sort_spec = 0;
2651         return send_APDU(c, apdu);
2652     }
2653     return zoom_complete;
2654 }
2655
2656 static zoom_ret send_present(ZOOM_connection c)
2657 {
2658     Z_APDU *apdu = 0;
2659     Z_PresentRequest *req = 0;
2660     int i = 0;
2661     const char *syntax = 0;
2662     const char *elementSetName = 0;
2663     ZOOM_resultset  resultset;
2664     int *start, *count;
2665
2666     if (!c->tasks)
2667     {
2668         yaz_log(log_details, "%p send_present no tasks", c);
2669         return zoom_complete;
2670     }
2671
2672     switch (c->tasks->which)
2673     {
2674     case ZOOM_TASK_SEARCH:
2675         resultset = c->tasks->u.search.resultset;
2676         start = &c->tasks->u.search.start;
2677         count = &c->tasks->u.search.count;
2678         syntax = c->tasks->u.search.syntax;
2679         elementSetName = c->tasks->u.search.elementSetName;
2680         break;
2681     case ZOOM_TASK_RETRIEVE:
2682         resultset = c->tasks->u.retrieve.resultset;
2683         start = &c->tasks->u.retrieve.start;
2684         count = &c->tasks->u.retrieve.count;
2685         syntax = c->tasks->u.retrieve.syntax;
2686         elementSetName = c->tasks->u.retrieve.elementSetName;
2687         break;
2688     default:
2689         return zoom_complete;
2690     }
2691     yaz_log(log_details, "%p send_present start=%d count=%d",
2692             c, *start, *count);
2693
2694     if (*start < 0 || *count < 0 || *start + *count > resultset->size)
2695     {
2696         set_dset_error(c, YAZ_BIB1_PRESENT_REQUEST_OUT_OF_RANGE, "Bib-1",
2697                        "", 0);
2698     }
2699     if (c->error)                  /* don't continue on error */
2700         return zoom_complete;
2701     yaz_log(log_details, "send_present resultset=%p start=%d count=%d",
2702             resultset, *start, *count);
2703
2704     for (i = 0; i < *count; i++)
2705     {
2706         ZOOM_record rec =
2707             record_cache_lookup(resultset, i + *start, syntax, elementSetName);
2708         if (!rec)
2709             break;
2710         else
2711         {
2712             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_RECV_RECORD);
2713             ZOOM_connection_put_event(c, event);
2714         }
2715     }
2716     *start += i;
2717     *count -= i;
2718
2719     if (*count == 0)
2720     {
2721         yaz_log(log_details, "%p send_present skip=%d no more to fetch", c, i);
2722         return zoom_complete;
2723     }
2724
2725     apdu = zget_APDU(c->odr_out, Z_APDU_presentRequest);
2726     req = apdu->u.presentRequest;
2727
2728     if (i)
2729         yaz_log(log_details, "%p send_present skip=%d", c, i);
2730
2731     *req->resultSetStartPoint = *start + 1;
2732
2733     if (resultset->step > 0 && resultset->step < *count)
2734         *req->numberOfRecordsRequested = resultset->step;
2735     else
2736         *req->numberOfRecordsRequested = *count;
2737
2738     if (*req->numberOfRecordsRequested + *start > resultset->size)
2739         *req->numberOfRecordsRequested = resultset->size - *start;
2740     assert(*req->numberOfRecordsRequested > 0);
2741
2742     if (syntax && *syntax)
2743         req->preferredRecordSyntax =
2744             zoom_yaz_str_to_z3950oid(c, CLASS_RECSYN, syntax);
2745
2746     if (resultset->schema && *resultset->schema)
2747     {
2748         Z_RecordComposition *compo = (Z_RecordComposition *)
2749             odr_malloc(c->odr_out, sizeof(*compo));
2750
2751         req->recordComposition = compo;
2752         compo->which = Z_RecordComp_complex;
2753         compo->u.complex = (Z_CompSpec *)
2754             odr_malloc(c->odr_out, sizeof(*compo->u.complex));
2755         compo->u.complex->selectAlternativeSyntax = (bool_t *)
2756             odr_malloc(c->odr_out, sizeof(bool_t));
2757         *compo->u.complex->selectAlternativeSyntax = 0;
2758
2759         compo->u.complex->generic = (Z_Specification *)
2760             odr_malloc(c->odr_out, sizeof(*compo->u.complex->generic));
2761
2762         compo->u.complex->generic->which = Z_Schema_oid;
2763         compo->u.complex->generic->schema.oid = (Odr_oid *)
2764             zoom_yaz_str_to_z3950oid (c, CLASS_SCHEMA, resultset->schema);
2765
2766         if (!compo->u.complex->generic->schema.oid)
2767         {
2768             /* OID wasn't a schema! Try record syntax instead. */
2769
2770             compo->u.complex->generic->schema.oid = (Odr_oid *)
2771                 zoom_yaz_str_to_z3950oid (c, CLASS_RECSYN, resultset->schema);
2772         }
2773         if (elementSetName && *elementSetName)
2774         {
2775             compo->u.complex->generic->elementSpec = (Z_ElementSpec *)
2776                 odr_malloc(c->odr_out, sizeof(Z_ElementSpec));
2777             compo->u.complex->generic->elementSpec->which =
2778                 Z_ElementSpec_elementSetName;
2779             compo->u.complex->generic->elementSpec->u.elementSetName =
2780                 odr_strdup(c->odr_out, elementSetName);
2781         }
2782         else
2783             compo->u.complex->generic->elementSpec = 0;
2784         compo->u.complex->num_dbSpecific = 0;
2785         compo->u.complex->dbSpecific = 0;
2786         compo->u.complex->num_recordSyntax = 0;
2787         compo->u.complex->recordSyntax = 0;
2788     }
2789     else if (elementSetName && *elementSetName)
2790     {
2791         Z_ElementSetNames *esn = (Z_ElementSetNames *)
2792             odr_malloc(c->odr_out, sizeof(*esn));
2793         Z_RecordComposition *compo = (Z_RecordComposition *)
2794             odr_malloc(c->odr_out, sizeof(*compo));
2795
2796         esn->which = Z_ElementSetNames_generic;
2797         esn->u.generic = odr_strdup(c->odr_out, elementSetName);
2798         compo->which = Z_RecordComp_simple;
2799         compo->u.simple = esn;
2800         req->recordComposition = compo;
2801     }
2802     req->resultSetId = odr_strdup(c->odr_out, resultset->setname);
2803     return send_APDU(c, apdu);
2804 }
2805
2806 ZOOM_API(ZOOM_scanset)
2807     ZOOM_connection_scan(ZOOM_connection c, const char *start)
2808 {
2809     ZOOM_scanset s;
2810     ZOOM_query q = ZOOM_query_create();
2811
2812     ZOOM_query_prefix(q, start);
2813
2814     s = ZOOM_connection_scan1(c, q);
2815     ZOOM_query_destroy(q);
2816     return s;
2817
2818 }
2819
2820 ZOOM_API(ZOOM_scanset)
2821     ZOOM_connection_scan1(ZOOM_connection c, ZOOM_query q)
2822 {
2823     ZOOM_scanset scan = 0;
2824
2825     if (!q->z_query)
2826         return 0;
2827     scan = (ZOOM_scanset) xmalloc(sizeof(*scan));
2828     scan->connection = c;
2829     scan->odr = odr_createmem(ODR_DECODE);
2830     scan->options = ZOOM_options_create_with_parent(c->options);
2831     scan->refcount = 1;
2832     scan->scan_response = 0;
2833     scan->srw_scan_response = 0;
2834
2835     scan->query = q;
2836     (q->refcount)++;
2837     scan->databaseNames = set_DatabaseNames(c, c->options,
2838                                             &scan->num_databaseNames,
2839                                             scan->odr);
2840
2841     if (1)
2842     {
2843         ZOOM_task task = ZOOM_connection_add_task(c, ZOOM_TASK_SCAN);
2844         task->u.scan.scan = scan;
2845
2846         (scan->refcount)++;
2847         if (!c->async)
2848         {
2849             while (ZOOM_event(1, &c))
2850                 ;
2851         }
2852     }
2853     return scan;
2854 }
2855
2856 ZOOM_API(void)
2857     ZOOM_scanset_destroy(ZOOM_scanset scan)
2858 {
2859     if (!scan)
2860         return;
2861     (scan->refcount)--;
2862     if (scan->refcount == 0)
2863     {
2864         ZOOM_query_destroy(scan->query);
2865
2866         odr_destroy(scan->odr);
2867
2868         ZOOM_options_destroy(scan->options);
2869         xfree(scan);
2870     }
2871 }
2872
2873 static zoom_ret send_package(ZOOM_connection c)
2874 {
2875     ZOOM_Event event;
2876
2877     yaz_log(log_details, "%p send_package", c);
2878     if (!c->tasks)
2879         return zoom_complete;
2880     assert (c->tasks->which == ZOOM_TASK_PACKAGE);
2881
2882     event = ZOOM_Event_create(ZOOM_EVENT_SEND_APDU);
2883     ZOOM_connection_put_event(c, event);
2884
2885     c->buf_out = c->tasks->u.package->buf_out;
2886     c->len_out = c->tasks->u.package->len_out;
2887
2888     return do_write(c);
2889 }
2890
2891 static zoom_ret ZOOM_connection_send_scan(ZOOM_connection c)
2892 {
2893     ZOOM_scanset scan;
2894     Z_APDU *apdu = zget_APDU(c->odr_out, Z_APDU_scanRequest);
2895     Z_ScanRequest *req = apdu->u.scanRequest;
2896
2897     yaz_log(log_details, "%p send_scan", c);
2898     if (!c->tasks)
2899         return zoom_complete;
2900     assert (c->tasks->which == ZOOM_TASK_SCAN);
2901     scan = c->tasks->u.scan.scan;
2902
2903     /* Z39.50 scan can only carry RPN */
2904     if (scan->query->z_query->which == Z_Query_type_1 ||
2905         scan->query->z_query->which == Z_Query_type_101)
2906     {
2907         Z_RPNQuery *rpn = scan->query->z_query->u.type_1;
2908         const char *cp = ZOOM_options_get(scan->options, "rpnCharset");
2909         if (cp)
2910         {
2911             yaz_iconv_t cd = yaz_iconv_open(cp, "UTF-8");
2912             if (cd)
2913             {
2914                 rpn = yaz_copy_z_RPNQuery(rpn, c->odr_out);
2915
2916                 yaz_query_charset_convert_rpnquery(
2917                     rpn, c->odr_out, cd);
2918                 yaz_iconv_close(cd);
2919             }
2920         }
2921         req->attributeSet = rpn->attributeSetId;
2922         if (!req->attributeSet)
2923             req->attributeSet = odr_oiddup(c->odr_out, yaz_oid_attset_bib_1);
2924         if (rpn->RPNStructure->which == Z_RPNStructure_simple &&
2925             rpn->RPNStructure->u.simple->which == Z_Operand_APT)
2926         {
2927             req->termListAndStartPoint =
2928                 rpn->RPNStructure->u.simple->u.attributesPlusTerm;
2929         }
2930         else
2931         {
2932             set_ZOOM_error(c, ZOOM_ERROR_INVALID_QUERY, 0);
2933             return zoom_complete;
2934         }
2935     }
2936     else
2937     {
2938         set_ZOOM_error(c, ZOOM_ERROR_UNSUPPORTED_QUERY, 0);
2939         return zoom_complete;
2940     }
2941
2942     *req->numberOfTermsRequested =
2943         ZOOM_options_get_int(scan->options, "number", 20);
2944
2945     req->preferredPositionInResponse =
2946         odr_intdup(c->odr_out,
2947                    ZOOM_options_get_int(scan->options, "position", 1));
2948
2949     req->stepSize =
2950         odr_intdup(c->odr_out,
2951                    ZOOM_options_get_int(scan->options, "stepSize", 0));
2952
2953     req->databaseNames = scan->databaseNames;
2954     req->num_databaseNames = scan->num_databaseNames;
2955
2956     return send_APDU(c, apdu);
2957 }
2958
2959 #if YAZ_HAVE_XML2
2960 static zoom_ret ZOOM_connection_srw_send_scan(ZOOM_connection c)
2961 {
2962     ZOOM_scanset scan;
2963     Z_SRW_PDU *sr = 0;
2964     const char *option_val = 0;
2965
2966     if (!c->tasks)
2967         return zoom_complete;
2968     assert (c->tasks->which == ZOOM_TASK_SCAN);
2969     scan = c->tasks->u.scan.scan;
2970
2971     sr = ZOOM_srw_get_pdu(c, Z_SRW_scan_request);
2972
2973     /* SRU scan can only carry CQL and PQF */
2974     if (scan->query->z_query->which == Z_Query_type_104)
2975     {
2976         sr->u.scan_request->query_type = Z_SRW_query_type_cql;
2977         sr->u.scan_request->scanClause.cql = scan->query->query_string;
2978     }
2979     else if (scan->query->z_query->which == Z_Query_type_1
2980              || scan->query->z_query->which == Z_Query_type_101)
2981     {
2982         sr->u.scan_request->query_type = Z_SRW_query_type_pqf;
2983         sr->u.scan_request->scanClause.pqf = scan->query->query_string;
2984     }
2985     else
2986     {
2987         set_ZOOM_error(c, ZOOM_ERROR_UNSUPPORTED_QUERY, 0);
2988         return zoom_complete;
2989     }
2990
2991     sr->u.scan_request->maximumTerms = odr_intdup(
2992         c->odr_out, ZOOM_options_get_int(scan->options, "number", 10));
2993
2994     sr->u.scan_request->responsePosition = odr_intdup(
2995         c->odr_out, ZOOM_options_get_int(scan->options, "position", 1));
2996
2997     option_val = ZOOM_options_get(scan->options, "extraArgs");
2998     yaz_encode_sru_extra(sr, c->odr_out, option_val);
2999     return send_srw(c, sr);
3000 }
3001 #else
3002 static zoom_ret ZOOM_connection_srw_send_scan(ZOOM_connection c)
3003 {
3004     return zoom_complete;
3005 }
3006 #endif
3007
3008
3009 ZOOM_API(size_t)
3010     ZOOM_scanset_size(ZOOM_scanset scan)
3011 {
3012     if (!scan)
3013         return 0;
3014
3015     if (scan->scan_response && scan->scan_response->entries)
3016         return scan->scan_response->entries->num_entries;
3017     else if (scan->srw_scan_response)
3018         return scan->srw_scan_response->num_terms;
3019     return 0;
3020 }
3021
3022 static void ZOOM_scanset_term_x(ZOOM_scanset scan, size_t pos,
3023                                 size_t *occ,
3024                                 const char **value_term, size_t *value_len,
3025                                 const char **disp_term, size_t *disp_len)
3026 {
3027     size_t noent = ZOOM_scanset_size(scan);
3028
3029     *value_term = 0;
3030     *value_len = 0;
3031
3032     *disp_term = 0;
3033     *disp_len = 0;
3034
3035     *occ = 0;
3036     if (pos >= noent)
3037         return;
3038     if (scan->scan_response)
3039     {
3040         Z_ScanResponse *res = scan->scan_response;
3041         if (res->entries->entries[pos]->which == Z_Entry_termInfo)
3042         {
3043             Z_TermInfo *t = res->entries->entries[pos]->u.termInfo;
3044
3045             *value_term = (const char *) t->term->u.general->buf;
3046             *value_len = t->term->u.general->len;
3047             if (t->displayTerm)
3048             {
3049                 *disp_term = t->displayTerm;
3050                 *disp_len = strlen(*disp_term);
3051             }
3052             else if (t->term->which == Z_Term_general)
3053             {
3054                 *disp_term = (const char *) t->term->u.general->buf;
3055                 *disp_len = t->term->u.general->len;
3056             }
3057             *occ = t->globalOccurrences ? *t->globalOccurrences : 0;
3058         }
3059     }
3060     if (scan->srw_scan_response)
3061     {
3062         Z_SRW_scanResponse *res = scan->srw_scan_response;
3063         Z_SRW_scanTerm *t = res->terms + pos;
3064         if (t)
3065         {
3066             *value_term = t->value;
3067             *value_len = strlen(*value_term);
3068
3069             if (t->displayTerm)
3070                 *disp_term = t->displayTerm;
3071             else
3072                 *disp_term = t->value;
3073             *disp_len = strlen(*disp_term);
3074             *occ = t->numberOfRecords ? *t->numberOfRecords : 0;
3075         }
3076     }
3077 }
3078
3079 ZOOM_API(const char *)
3080     ZOOM_scanset_term(ZOOM_scanset scan, size_t pos,
3081                       size_t *occ, size_t *len)
3082 {
3083     const char *value_term = 0;
3084     size_t value_len = 0;
3085     const char *disp_term = 0;
3086     size_t disp_len = 0;
3087
3088     ZOOM_scanset_term_x(scan, pos, occ, &value_term, &value_len,
3089                         &disp_term, &disp_len);
3090
3091     *len = value_len;
3092     return value_term;
3093 }
3094
3095 ZOOM_API(const char *)
3096     ZOOM_scanset_display_term(ZOOM_scanset scan, size_t pos,
3097                               size_t *occ, size_t *len)
3098 {
3099     const char *value_term = 0;
3100     size_t value_len = 0;
3101     const char *disp_term = 0;
3102     size_t disp_len = 0;
3103
3104     ZOOM_scanset_term_x(scan, pos, occ, &value_term, &value_len,
3105                         &disp_term, &disp_len);
3106
3107     *len = disp_len;
3108     return disp_term;
3109 }
3110
3111 ZOOM_API(const char *)
3112     ZOOM_scanset_option_get(ZOOM_scanset scan, const char *key)
3113 {
3114     return ZOOM_options_get(scan->options, key);
3115 }
3116
3117 ZOOM_API(void)
3118     ZOOM_scanset_option_set(ZOOM_scanset scan, const char *key,
3119                             const char *val)
3120 {
3121     ZOOM_options_set(scan->options, key, val);
3122 }
3123
3124 static Z_APDU *create_es_package(ZOOM_package p, const Odr_oid *oid)
3125 {
3126     const char *str;
3127     Z_APDU *apdu = zget_APDU(p->odr_out, Z_APDU_extendedServicesRequest);
3128     Z_ExtendedServicesRequest *req = apdu->u.extendedServicesRequest;
3129
3130     str = ZOOM_options_get(p->options, "package-name");
3131     if (str && *str)
3132         req->packageName = odr_strdup(p->odr_out, str);
3133
3134     str = ZOOM_options_get(p->options, "user-id");
3135     if (str)
3136         req->userId = odr_strdup_null(p->odr_out, str);
3137
3138     req->packageType = odr_oiddup(p->odr_out, oid);
3139
3140     str = ZOOM_options_get(p->options, "function");
3141     if (str)
3142     {
3143         if (!strcmp (str, "create"))
3144             *req->function = Z_ExtendedServicesRequest_create;
3145         if (!strcmp (str, "delete"))
3146             *req->function = Z_ExtendedServicesRequest_delete;
3147         if (!strcmp (str, "modify"))
3148             *req->function = Z_ExtendedServicesRequest_modify;
3149     }
3150
3151     str = ZOOM_options_get(p->options, "waitAction");
3152     if (str)
3153     {
3154         if (!strcmp (str, "wait"))
3155             *req->waitAction = Z_ExtendedServicesRequest_wait;
3156         if (!strcmp (str, "waitIfPossible"))
3157             *req->waitAction = Z_ExtendedServicesRequest_waitIfPossible;
3158         if (!strcmp (str, "dontWait"))
3159             *req->waitAction = Z_ExtendedServicesRequest_dontWait;
3160         if (!strcmp (str, "dontReturnPackage"))
3161             *req->waitAction = Z_ExtendedServicesRequest_dontReturnPackage;
3162     }
3163     return apdu;
3164 }
3165
3166 static const char *ill_array_lookup(void *clientData, const char *idx)
3167 {
3168     ZOOM_package p = (ZOOM_package) clientData;
3169     return ZOOM_options_get(p->options, idx+4);
3170 }
3171
3172 static Z_External *encode_ill_request(ZOOM_package p)
3173 {
3174     ODR out = p->odr_out;
3175     ILL_Request *req;
3176     Z_External *r = 0;
3177     struct ill_get_ctl ctl;
3178
3179     ctl.odr = p->odr_out;
3180     ctl.clientData = p;
3181     ctl.f = ill_array_lookup;
3182
3183     req = ill_get_ILLRequest(&ctl, "ill", 0);
3184
3185     if (!ill_Request(out, &req, 0, 0))
3186     {
3187         int ill_request_size;
3188         char *ill_request_buf = odr_getbuf(out, &ill_request_size, 0);
3189         if (ill_request_buf)
3190             odr_setbuf(out, ill_request_buf, ill_request_size, 1);
3191         return 0;
3192     }
3193     else
3194     {
3195         int illRequest_size = 0;
3196         char *illRequest_buf = odr_getbuf(out, &illRequest_size, 0);
3197
3198         r = (Z_External *) odr_malloc(out, sizeof(*r));
3199         r->direct_reference = odr_oiddup(out, yaz_oid_general_isoill_1);
3200         r->indirect_reference = 0;
3201         r->descriptor = 0;
3202         r->which = Z_External_single;
3203
3204         r->u.single_ASN1_type =
3205             odr_create_Odr_oct(out,
3206                                (unsigned char *)illRequest_buf,
3207                                illRequest_size);
3208     }
3209     return r;
3210 }
3211
3212 static Z_ItemOrder *encode_item_order(ZOOM_package p)
3213 {
3214     Z_ItemOrder *req = (Z_ItemOrder *) odr_malloc(p->odr_out, sizeof(*req));
3215     const char *str;
3216     int len;
3217
3218     req->which = Z_IOItemOrder_esRequest;
3219     req->u.esRequest = (Z_IORequest *)
3220         odr_malloc(p->odr_out,sizeof(Z_IORequest));
3221
3222     /* to keep part ... */
3223     req->u.esRequest->toKeep = (Z_IOOriginPartToKeep *)
3224         odr_malloc(p->odr_out,sizeof(Z_IOOriginPartToKeep));
3225     req->u.esRequest->toKeep->supplDescription = 0;
3226     req->u.esRequest->toKeep->contact = (Z_IOContact *)
3227         odr_malloc(p->odr_out, sizeof(*req->u.esRequest->toKeep->contact));
3228
3229     str = ZOOM_options_get(p->options, "contact-name");
3230     req->u.esRequest->toKeep->contact->name =
3231         odr_strdup_null(p->odr_out, str);
3232
3233     str = ZOOM_options_get(p->options, "contact-phone");
3234     req->u.esRequest->toKeep->contact->phone =
3235         odr_strdup_null(p->odr_out, str);
3236
3237     str = ZOOM_options_get(p->options, "contact-email");
3238     req->u.esRequest->toKeep->contact->email =
3239         odr_strdup_null(p->odr_out, str);
3240
3241     req->u.esRequest->toKeep->addlBilling = 0;
3242
3243     /* not to keep part ... */
3244     req->u.esRequest->notToKeep = (Z_IOOriginPartNotToKeep *)
3245         odr_malloc(p->odr_out,sizeof(Z_IOOriginPartNotToKeep));
3246
3247     str = ZOOM_options_get(p->options, "itemorder-setname");
3248     if (!str)
3249         str = "default";
3250
3251     if (!*str)
3252         req->u.esRequest->notToKeep->resultSetItem = 0;
3253     else
3254     {
3255         req->u.esRequest->notToKeep->resultSetItem = (Z_IOResultSetItem *)
3256             odr_malloc(p->odr_out, sizeof(Z_IOResultSetItem));
3257
3258         req->u.esRequest->notToKeep->resultSetItem->resultSetId =
3259             odr_strdup(p->odr_out, str);
3260         req->u.esRequest->notToKeep->resultSetItem->item =
3261             odr_intdup(p->odr_out, 0);
3262
3263         str = ZOOM_options_get(p->options, "itemorder-item");
3264         *req->u.esRequest->notToKeep->resultSetItem->item =
3265             (str ? atoi(str) : 1);
3266     }
3267
3268     str = ZOOM_options_getl(p->options, "doc", &len);
3269     if (str)
3270     {
3271         req->u.esRequest->notToKeep->itemRequest =
3272             z_ext_record_xml(p->odr_out, str, len);
3273     }
3274     else
3275         req->u.esRequest->notToKeep->itemRequest = encode_ill_request(p);
3276
3277     return req;
3278 }
3279
3280 Z_APDU *create_admin_package(ZOOM_package p, int type,
3281                              Z_ESAdminOriginPartToKeep **toKeepP,
3282                              Z_ESAdminOriginPartNotToKeep **notToKeepP)
3283 {
3284     Z_APDU *apdu = create_es_package(p, yaz_oid_extserv_admin);
3285     if (apdu)
3286     {
3287         Z_ESAdminOriginPartToKeep  *toKeep;
3288         Z_ESAdminOriginPartNotToKeep  *notToKeep;
3289         Z_External *r = (Z_External *) odr_malloc(p->odr_out, sizeof(*r));
3290         const char *first_db = "Default";
3291         int num_db;
3292         char **db = set_DatabaseNames(p->connection, p->options, &num_db,
3293                                       p->odr_out);
3294         if (num_db > 0)
3295             first_db = db[0];
3296
3297         r->direct_reference = odr_oiddup(p->odr_out, yaz_oid_extserv_admin);
3298         r->descriptor = 0;
3299         r->indirect_reference = 0;
3300         r->which = Z_External_ESAdmin;
3301
3302         r->u.adminService = (Z_Admin *)
3303             odr_malloc(p->odr_out, sizeof(*r->u.adminService));
3304         r->u.adminService->which = Z_Admin_esRequest;
3305         r->u.adminService->u.esRequest = (Z_AdminEsRequest *)
3306             odr_malloc(p->odr_out, sizeof(*r->u.adminService->u.esRequest));
3307
3308         toKeep = r->u.adminService->u.esRequest->toKeep =
3309             (Z_ESAdminOriginPartToKeep *)
3310             odr_malloc(p->odr_out, sizeof(*r->u.adminService->u.esRequest->toKeep));
3311         toKeep->which = type;
3312         toKeep->databaseName = odr_strdup(p->odr_out, first_db);
3313         toKeep->u.create = odr_nullval();
3314         apdu->u.extendedServicesRequest->taskSpecificParameters = r;
3315
3316         r->u.adminService->u.esRequest->notToKeep = notToKeep =
3317             (Z_ESAdminOriginPartNotToKeep *)
3318             odr_malloc(p->odr_out,
3319                        sizeof(*r->u.adminService->u.esRequest->notToKeep));
3320         notToKeep->which = Z_ESAdminOriginPartNotToKeep_recordsWillFollow;
3321         notToKeep->u.recordsWillFollow = odr_nullval();
3322         if (toKeepP)
3323             *toKeepP = toKeep;
3324         if (notToKeepP)
3325             *notToKeepP = notToKeep;
3326     }
3327     return apdu;
3328 }
3329
3330 static Z_APDU *create_xmlupdate_package(ZOOM_package p)
3331 {
3332     Z_APDU *apdu = create_es_package(p, yaz_oid_extserv_xml_es);
3333     Z_ExtendedServicesRequest *req = apdu->u.extendedServicesRequest;
3334     Z_External *ext = (Z_External *) odr_malloc(p->odr_out, sizeof(*ext));
3335     int len;
3336     const char *doc = ZOOM_options_getl(p->options, "doc", &len);
3337
3338     if (!doc)
3339     {
3340         doc = "";
3341         len = 0;
3342     }
3343
3344     req->taskSpecificParameters = ext;
3345     ext->direct_reference = req->packageType;
3346     ext->descriptor = 0;
3347     ext->indirect_reference = 0;
3348
3349     ext->which = Z_External_octet;
3350     ext->u.single_ASN1_type =
3351         odr_create_Odr_oct(p->odr_out, (const unsigned char *) doc, len);
3352     return apdu;
3353 }
3354
3355 static Z_APDU *create_update_package(ZOOM_package p)
3356 {
3357     Z_APDU *apdu = 0;
3358     const char *first_db = "Default";
3359     int num_db;
3360     char **db = set_DatabaseNames(p->connection, p->options, &num_db, p->odr_out);
3361     const char *action = ZOOM_options_get(p->options, "action");
3362     int recordIdOpaque_len;
3363     const char *recordIdOpaque = ZOOM_options_getl(p->options, "recordIdOpaque",
3364         &recordIdOpaque_len);
3365     const char *recordIdNumber = ZOOM_options_get(p->options, "recordIdNumber");
3366     int record_len;
3367     const char *record_buf = ZOOM_options_getl(p->options, "record",
3368         &record_len);
3369     int recordOpaque_len;
3370     const char *recordOpaque_buf = ZOOM_options_getl(p->options, "recordOpaque",
3371         &recordOpaque_len);
3372     const char *syntax_str = ZOOM_options_get(p->options, "syntax");
3373     const char *version = ZOOM_options_get(p->options, "updateVersion");
3374
3375     const char *correlationInfo_note =
3376         ZOOM_options_get(p->options, "correlationInfo.note");
3377     const char *correlationInfo_id =
3378         ZOOM_options_get(p->options, "correlationInfo.id");
3379     int action_no = -1;
3380     Odr_oid *syntax_oid = 0;
3381     const Odr_oid *package_oid = yaz_oid_extserv_database_update;
3382
3383     if (!version)
3384         version = "3";
3385     if (!syntax_str)
3386         syntax_str = "xml";
3387     if (!record_buf && !recordOpaque_buf)
3388     {
3389         record_buf = "void";
3390         record_len = 4;
3391         syntax_str = "SUTRS";
3392     }
3393
3394     if (syntax_str)
3395     {
3396         syntax_oid = yaz_string_to_oid_odr(yaz_oid_std(),
3397                                            CLASS_RECSYN, syntax_str,
3398                                            p->odr_out);
3399     }
3400     if (!syntax_oid)
3401         return 0;
3402
3403     if (num_db > 0)
3404         first_db = db[0];
3405
3406     switch(*version)
3407     {
3408     case '1':
3409         package_oid = yaz_oid_extserv_database_update_first_version;
3410         /* old update does not support specialUpdate */
3411         if (!action)
3412             action = "recordInsert";
3413         break;
3414     case '2':
3415         if (!action)
3416             action = "specialUpdate";
3417         package_oid = yaz_oid_extserv_database_update_second_version;
3418         break;
3419     case '3':
3420         if (!action)
3421             action = "specialUpdate";
3422         package_oid = yaz_oid_extserv_database_update;
3423         break;
3424     default:
3425         return 0;
3426     }
3427
3428     if (!strcmp(action, "recordInsert"))
3429         action_no = Z_IUOriginPartToKeep_recordInsert;
3430     else if (!strcmp(action, "recordReplace"))
3431         action_no = Z_IUOriginPartToKeep_recordReplace;
3432     else if (!strcmp(action, "recordDelete"))
3433         action_no = Z_IUOriginPartToKeep_recordDelete;
3434     else if (!strcmp(action, "elementUpdate"))
3435         action_no = Z_IUOriginPartToKeep_elementUpdate;
3436     else if (!strcmp(action, "specialUpdate"))
3437         action_no = Z_IUOriginPartToKeep_specialUpdate;
3438     else
3439         return 0;
3440
3441     apdu = create_es_package(p, package_oid);
3442     if (apdu)
3443     {
3444         Z_IUOriginPartToKeep *toKeep;
3445         Z_IUSuppliedRecords *notToKeep;
3446         Z_External *r = (Z_External *)
3447             odr_malloc(p->odr_out, sizeof(*r));
3448         const char *elementSetName =
3449             ZOOM_options_get(p->options, "elementSetName");
3450
3451         apdu->u.extendedServicesRequest->taskSpecificParameters = r;
3452
3453         r->direct_reference = odr_oiddup(p->odr_out, package_oid);
3454         r->descriptor = 0;
3455         r->which = Z_External_update;
3456         r->indirect_reference = 0;
3457         r->u.update = (Z_IUUpdate *)
3458             odr_malloc(p->odr_out, sizeof(*r->u.update));
3459
3460         r->u.update->which = Z_IUUpdate_esRequest;
3461         r->u.update->u.esRequest = (Z_IUUpdateEsRequest *)
3462             odr_malloc(p->odr_out, sizeof(*r->u.update->u.esRequest));
3463         toKeep = r->u.update->u.esRequest->toKeep =
3464             (Z_IUOriginPartToKeep *)
3465             odr_malloc(p->odr_out, sizeof(*toKeep));
3466
3467         toKeep->databaseName = odr_strdup(p->odr_out, first_db);
3468         toKeep->schema = 0;
3469
3470         toKeep->elementSetName = odr_strdup_null(p->odr_out, elementSetName);
3471
3472         toKeep->actionQualifier = 0;
3473         toKeep->action = odr_intdup(p->odr_out, action_no);
3474
3475         notToKeep = r->u.update->u.esRequest->notToKeep =
3476             (Z_IUSuppliedRecords *)
3477             odr_malloc(p->odr_out, sizeof(*notToKeep));
3478         notToKeep->num = 1;
3479         notToKeep->elements = (Z_IUSuppliedRecords_elem **)
3480             odr_malloc(p->odr_out, sizeof(*notToKeep->elements));
3481         notToKeep->elements[0] = (Z_IUSuppliedRecords_elem *)
3482             odr_malloc(p->odr_out, sizeof(**notToKeep->elements));
3483         notToKeep->elements[0]->which = Z_IUSuppliedRecords_elem_opaque;
3484         if (recordIdOpaque)
3485         {
3486             notToKeep->elements[0]->u.opaque =
3487                 odr_create_Odr_oct(p->odr_out,
3488                                    (const unsigned char *) recordIdOpaque,
3489                                    recordIdOpaque_len);
3490         }
3491         else if (recordIdNumber)
3492         {
3493             notToKeep->elements[0]->which = Z_IUSuppliedRecords_elem_number;
3494
3495             notToKeep->elements[0]->u.number =
3496                 odr_intdup(p->odr_out, atoi(recordIdNumber));
3497         }
3498         else
3499             notToKeep->elements[0]->u.opaque = 0;
3500         notToKeep->elements[0]->supplementalId = 0;
3501         if (correlationInfo_note || correlationInfo_id)
3502         {
3503             Z_IUCorrelationInfo *ci;
3504             ci = notToKeep->elements[0]->correlationInfo =
3505                 (Z_IUCorrelationInfo *) odr_malloc(p->odr_out, sizeof(*ci));
3506             ci->note = odr_strdup_null(p->odr_out, correlationInfo_note);
3507             ci->id = correlationInfo_id ?
3508                 odr_intdup(p->odr_out, atoi(correlationInfo_id)) : 0;
3509         }
3510         else
3511             notToKeep->elements[0]->correlationInfo = 0;
3512         if (recordOpaque_buf)
3513         {
3514             notToKeep->elements[0]->record =
3515                 z_ext_record_oid_any(p->odr_out, syntax_oid,
3516                                  recordOpaque_buf, recordOpaque_len);
3517         }
3518         else
3519         {
3520             notToKeep->elements[0]->record =
3521                 z_ext_record_oid(p->odr_out, syntax_oid,
3522                                  record_buf, record_len);
3523         }
3524     }
3525     if (0 && apdu)
3526     {
3527         ODR print = odr_createmem(ODR_PRINT);
3528
3529         z_APDU(print, &apdu, 0, 0);
3530         odr_destroy(print);
3531     }
3532     return apdu;
3533 }
3534
3535 ZOOM_API(void)
3536     ZOOM_package_send(ZOOM_package p, const char *type)
3537 {
3538     Z_APDU *apdu = 0;
3539     ZOOM_connection c;
3540     if (!p)
3541         return;
3542     c = p->connection;
3543     odr_reset(p->odr_out);
3544     xfree(p->buf_out);
3545     p->buf_out = 0;
3546     if (!strcmp(type, "itemorder"))
3547     {
3548         apdu = create_es_package(p, yaz_oid_extserv_item_order);
3549         if (apdu)
3550         {
3551             Z_External *r = (Z_External *) odr_malloc(p->odr_out, sizeof(*r));
3552
3553             r->direct_reference =
3554                 odr_oiddup(p->odr_out, yaz_oid_extserv_item_order);
3555             r->descriptor = 0;
3556             r->which = Z_External_itemOrder;
3557             r->indirect_reference = 0;
3558             r->u.itemOrder = encode_item_order(p);
3559
3560             apdu->u.extendedServicesRequest->taskSpecificParameters = r;
3561         }
3562     }
3563     else if (!strcmp(type, "create"))  /* create database */
3564     {
3565         apdu = create_admin_package(p, Z_ESAdminOriginPartToKeep_create,
3566                                     0, 0);
3567     }
3568     else if (!strcmp(type, "drop"))  /* drop database */
3569     {
3570         apdu = create_admin_package(p, Z_ESAdminOriginPartToKeep_drop,
3571                                     0, 0);
3572     }
3573     else if (!strcmp(type, "commit"))  /* commit changes */
3574     {
3575         apdu = create_admin_package(p, Z_ESAdminOriginPartToKeep_commit,
3576                                     0, 0);
3577     }
3578     else if (!strcmp(type, "update")) /* update record(s) */
3579     {
3580         apdu = create_update_package(p);
3581     }
3582     else if (!strcmp(type, "xmlupdate"))
3583     {
3584         apdu = create_xmlupdate_package(p);
3585     }
3586     if (apdu)
3587     {
3588         if (encode_APDU(p->connection, apdu, p->odr_out) == 0)
3589         {
3590             char *buf;
3591
3592             ZOOM_task task = ZOOM_connection_add_task(c, ZOOM_TASK_PACKAGE);
3593             task->u.package = p;
3594             buf = odr_getbuf(p->odr_out, &p->len_out, 0);
3595             p->buf_out = (char *) xmalloc(p->len_out);
3596             memcpy(p->buf_out, buf, p->len_out);
3597
3598             (p->refcount)++;
3599             if (!c->async)
3600             {
3601                 while (ZOOM_event(1, &c))
3602                     ;
3603             }
3604         }
3605     }
3606 }
3607
3608 ZOOM_API(ZOOM_package)
3609     ZOOM_connection_package(ZOOM_connection c, ZOOM_options options)
3610 {
3611     ZOOM_package p = (ZOOM_package) xmalloc(sizeof(*p));
3612
3613     p->connection = c;
3614     p->odr_out = odr_createmem(ODR_ENCODE);
3615     p->options = ZOOM_options_create_with_parent2(options, c->options);
3616     p->refcount = 1;
3617     p->buf_out = 0;
3618     p->len_out = 0;
3619     return p;
3620 }
3621
3622 ZOOM_API(void)
3623     ZOOM_package_destroy(ZOOM_package p)
3624 {
3625     if (!p)
3626         return;
3627     (p->refcount)--;
3628     if (p->refcount == 0)
3629     {
3630         odr_destroy(p->odr_out);
3631         xfree(p->buf_out);
3632
3633         ZOOM_options_destroy(p->options);
3634         xfree(p);
3635     }
3636 }
3637
3638 ZOOM_API(const char *)
3639     ZOOM_package_option_get(ZOOM_package p, const char *key)
3640 {
3641     return ZOOM_options_get(p->options, key);
3642 }
3643
3644 ZOOM_API(const char *)
3645     ZOOM_package_option_getl(ZOOM_package p, const char *key, int *lenp)
3646 {
3647     return ZOOM_options_getl(p->options, key, lenp);
3648 }
3649
3650 ZOOM_API(void)
3651     ZOOM_package_option_set(ZOOM_package p, const char *key,
3652                             const char *val)
3653 {
3654     ZOOM_options_set(p->options, key, val);
3655 }
3656
3657 ZOOM_API(void)
3658     ZOOM_package_option_setl(ZOOM_package p, const char *key,
3659                              const char *val, int len)
3660 {
3661     ZOOM_options_setl(p->options, key, val, len);
3662 }
3663
3664 ZOOM_API(int)
3665     ZOOM_connection_exec_task(ZOOM_connection c)
3666 {
3667     ZOOM_task task = c->tasks;
3668     zoom_ret ret = zoom_complete;
3669
3670     if (!task)
3671         return 0;
3672     yaz_log(log_details, "%p ZOOM_connection_exec_task type=%d run=%d",
3673             c, task->which, task->running);
3674     if (c->error != ZOOM_ERROR_NONE)
3675     {
3676         yaz_log(log_details, "%p ZOOM_connection_exec_task "
3677                 "removing tasks because of error = %d", c, c->error);
3678         ZOOM_connection_remove_tasks(c);
3679         return 0;
3680     }
3681     if (task->running)
3682     {
3683         yaz_log(log_details, "%p ZOOM_connection_exec_task "
3684                 "task already running", c);
3685         return 0;
3686     }
3687     task->running = 1;
3688     ret = zoom_complete;
3689     if (c->cs || task->which == ZOOM_TASK_CONNECT)
3690     {
3691         switch (task->which)
3692         {
3693         case ZOOM_TASK_SEARCH:
3694             if (c->proto == PROTO_HTTP)
3695                 ret = ZOOM_connection_srw_send_search(c);
3696             else
3697                 ret = ZOOM_connection_send_search(c);
3698             break;
3699         case ZOOM_TASK_RETRIEVE:
3700             if (c->proto == PROTO_HTTP)
3701                 ret = ZOOM_connection_srw_send_search(c);
3702             else
3703                 ret = send_present(c);
3704             break;
3705         case ZOOM_TASK_CONNECT:
3706             ret = do_connect(c);
3707             break;
3708         case ZOOM_TASK_SCAN:
3709             if (c->proto == PROTO_HTTP)
3710                 ret = ZOOM_connection_srw_send_scan(c);
3711             else
3712                 ret = ZOOM_connection_send_scan(c);
3713             break;
3714         case ZOOM_TASK_PACKAGE:
3715             ret = send_package(c);
3716             break;
3717         case ZOOM_TASK_SORT:
3718             c->tasks->u.sort.resultset->r_sort_spec =
3719                 c->tasks->u.sort.q->sort_spec;
3720             ret = send_sort(c, c->tasks->u.sort.resultset);
3721             break;
3722         }
3723     }
3724     else
3725     {
3726         yaz_log(log_details, "%p ZOOM_connection_exec_task "
3727                 "remove tasks because no connection exist", c);
3728         ZOOM_connection_remove_tasks(c);
3729     }
3730     if (ret == zoom_complete)
3731     {
3732         yaz_log(log_details, "%p ZOOM_connection_exec_task "
3733                 "task removed (complete)", c);
3734         ZOOM_connection_remove_task(c);
3735         return 0;
3736     }
3737     yaz_log(log_details, "%p ZOOM_connection_exec_task "
3738             "task pending", c);
3739     return 1;
3740 }
3741
3742 static zoom_ret send_sort_present(ZOOM_connection c)
3743 {
3744     zoom_ret r = zoom_complete;
3745
3746     if (c->tasks && c->tasks->which == ZOOM_TASK_SEARCH)
3747         r = send_sort(c, c->tasks->u.search.resultset);
3748     if (r == zoom_complete)
3749         r = send_present(c);
3750     return r;
3751 }
3752
3753 static int es_response_taskpackage_update(ZOOM_connection c,
3754                 Z_IUUpdateTaskPackage *utp)
3755 {
3756         if (utp && utp->targetPart)
3757         {
3758                 Z_IUTargetPart *targetPart = utp->targetPart;
3759                 switch ( *targetPart->updateStatus ) {
3760                         case Z_IUTargetPart_success:
3761                                 ZOOM_options_set(c->tasks->u.package->options,"updateStatus", "success");
3762                                 break;
3763                         case Z_IUTargetPart_partial:
3764                                 ZOOM_options_set(c->tasks->u.package->options,"updateStatus", "partial");
3765                                 break;
3766                         case Z_IUTargetPart_failure:
3767                                 ZOOM_options_set(c->tasks->u.package->options,"updateStatus", "failure");
3768                                 if (targetPart->globalDiagnostics && targetPart->num_globalDiagnostics > 0)
3769                                         response_diag(c, targetPart->globalDiagnostics[0]);
3770                                 break;
3771                 }
3772                 // NOTE: Individual record status, surrogate diagnostics, and supplemental diagnostics ARE NOT REPORTED.
3773         }
3774     return 1;
3775 }
3776
3777 static int es_response_taskpackage(ZOOM_connection c,
3778                                    Z_TaskPackage *taskPackage)
3779 {
3780         // targetReference
3781         Odr_oct *id = taskPackage->targetReference;
3782         if (id)
3783                 ZOOM_options_setl(c->tasks->u.package->options,
3784                                                         "targetReference", (char*) id->buf, id->len);
3785
3786         // taskStatus
3787         switch ( *taskPackage->taskStatus ) {
3788                 case Z_TaskPackage_pending:
3789                         ZOOM_options_set(c->tasks->u.package->options,"taskStatus", "pending");
3790                         break;
3791                 case Z_TaskPackage_active:
3792                         ZOOM_options_set(c->tasks->u.package->options,"taskStatus", "active");
3793                         break;
3794                 case Z_TaskPackage_complete:
3795                         ZOOM_options_set(c->tasks->u.package->options,"taskStatus", "complete");
3796                         break;
3797                 case Z_TaskPackage_aborted:
3798                         ZOOM_options_set(c->tasks->u.package->options,"taskStatus", "aborted");
3799                         if ( taskPackage->num_packageDiagnostics && taskPackage->packageDiagnostics )
3800                                 response_diag(c, taskPackage->packageDiagnostics[0]);
3801                         break;
3802         }
3803
3804         // taskSpecificParameters
3805         // NOTE: Only Update implemented, no others.
3806         if ( taskPackage->taskSpecificParameters->which == Z_External_update ) {
3807                         Z_IUUpdateTaskPackage *utp = taskPackage->taskSpecificParameters->u.update->u.taskPackage;
3808                         es_response_taskpackage_update(c, utp);
3809         }
3810         return 1;
3811 }
3812
3813
3814 static int es_response(ZOOM_connection c,
3815                        Z_ExtendedServicesResponse *res)
3816 {
3817     if (!c->tasks || c->tasks->which != ZOOM_TASK_PACKAGE)
3818         return 0;
3819     switch (*res->operationStatus) {
3820         case Z_ExtendedServicesResponse_done:
3821             ZOOM_options_set(c->tasks->u.package->options,"operationStatus", "done");
3822             break;
3823         case Z_ExtendedServicesResponse_accepted:
3824             ZOOM_options_set(c->tasks->u.package->options,"operationStatus", "accepted");
3825             break;
3826         case Z_ExtendedServicesResponse_failure:
3827             ZOOM_options_set(c->tasks->u.package->options,"operationStatus", "failure");
3828             if (res->diagnostics && res->num_diagnostics > 0)
3829                 response_diag(c, res->diagnostics[0]);
3830             break;
3831     }
3832     if (res->taskPackage &&
3833         res->taskPackage->which == Z_External_extendedService)
3834     {
3835         Z_TaskPackage *taskPackage = res->taskPackage->u.extendedService;
3836         es_response_taskpackage(c, taskPackage);
3837     }
3838     if (res->taskPackage &&
3839         res->taskPackage->which == Z_External_octet)
3840     {
3841         Odr_oct *doc = res->taskPackage->u.octet_aligned;
3842         ZOOM_options_setl(c->tasks->u.package->options,
3843                           "xmlUpdateDoc", (char*) doc->buf, doc->len);
3844     }
3845     return 1;
3846 }
3847
3848 static void interpret_init_diag(ZOOM_connection c,
3849                                 Z_DiagnosticFormat *diag)
3850 {
3851     if (diag->num > 0)
3852     {
3853         Z_DiagnosticFormat_s *ds = diag->elements[0];
3854         if (ds->which == Z_DiagnosticFormat_s_defaultDiagRec)
3855             response_default_diag(c, ds->u.defaultDiagRec);
3856     }
3857 }
3858
3859
3860 static void interpret_otherinformation_field(ZOOM_connection c,
3861                                              Z_OtherInformation *ui)
3862 {
3863     int i;
3864     for (i = 0; i < ui->num_elements; i++)
3865     {
3866         Z_OtherInformationUnit *unit = ui->list[i];
3867         if (unit->which == Z_OtherInfo_externallyDefinedInfo &&
3868             unit->information.externallyDefinedInfo &&
3869             unit->information.externallyDefinedInfo->which ==
3870             Z_External_diag1)
3871         {
3872             interpret_init_diag(c, unit->information.externallyDefinedInfo->u.diag1);
3873         }
3874     }
3875 }
3876
3877
3878 static void set_init_option(const char *name, void *clientData) {
3879     ZOOM_connection c = (ZOOM_connection) clientData;
3880     char buf[80];
3881
3882     sprintf(buf, "init_opt_%.70s", name);
3883     ZOOM_connection_option_set(c, buf, "1");
3884 }
3885
3886
3887 static void recv_apdu(ZOOM_connection c, Z_APDU *apdu)
3888 {
3889     Z_InitResponse *initrs;
3890
3891     ZOOM_connection_set_mask(c, 0);
3892     yaz_log(log_details, "%p recv_apdu apdu->which=%d", c, apdu->which);
3893     switch(apdu->which)
3894     {
3895     case Z_APDU_initResponse:
3896         yaz_log(log_api, "%p recv_apdu: Received Init response", c);
3897         initrs = apdu->u.initResponse;
3898         ZOOM_connection_option_set(c, "serverImplementationId",
3899                                    initrs->implementationId ?
3900                                    initrs->implementationId : "");
3901         ZOOM_connection_option_set(c, "serverImplementationName",
3902                                    initrs->implementationName ?
3903                                    initrs->implementationName : "");
3904         ZOOM_connection_option_set(c, "serverImplementationVersion",
3905                                    initrs->implementationVersion ?
3906                                    initrs->implementationVersion : "");
3907         /* Set the three old options too, for old applications */
3908         ZOOM_connection_option_set(c, "targetImplementationId",
3909                                    initrs->implementationId ?
3910                                    initrs->implementationId : "");
3911         ZOOM_connection_option_set(c, "targetImplementationName",
3912                                    initrs->implementationName ?
3913                                    initrs->implementationName : "");
3914         ZOOM_connection_option_set(c, "targetImplementationVersion",
3915                                    initrs->implementationVersion ?
3916                                    initrs->implementationVersion : "");
3917
3918         /* Make initrs->options available as ZOOM-level options */
3919         yaz_init_opt_decode(initrs->options, set_init_option, (void*) c);
3920
3921         if (!*initrs->result)
3922         {
3923             Z_External *uif = initrs->userInformationField;
3924
3925             set_ZOOM_error(c, ZOOM_ERROR_INIT, 0); /* default error */
3926
3927             if (uif && uif->which == Z_External_userInfo1)
3928                 interpret_otherinformation_field(c, uif->u.userInfo1);
3929         }
3930         else
3931         {
3932             char *cookie =
3933                 yaz_oi_get_string_oid(&apdu->u.initResponse->otherInfo,
3934                                       yaz_oid_userinfo_cookie, 1, 0);
3935             xfree(c->cookie_in);
3936             c->cookie_in = 0;
3937             if (cookie)
3938                 c->cookie_in = xstrdup(cookie);
3939             if (ODR_MASK_GET(initrs->options, Z_Options_namedResultSets) &&
3940                 ODR_MASK_GET(initrs->protocolVersion, Z_ProtocolVersion_3))
3941                 c->support_named_resultsets = 1;
3942             if (c->tasks)
3943             {
3944                 assert(c->tasks->which == ZOOM_TASK_CONNECT);
3945                 ZOOM_connection_remove_task(c);
3946             }
3947             ZOOM_connection_exec_task(c);
3948         }
3949         if (ODR_MASK_GET(initrs->options, Z_Options_negotiationModel))
3950         {
3951             NMEM tmpmem = nmem_create();
3952             Z_CharSetandLanguageNegotiation *p =
3953                 yaz_get_charneg_record(initrs->otherInfo);
3954
3955             if (p)
3956             {
3957                 char *charset = NULL, *lang = NULL;
3958                 int sel;
3959
3960                 yaz_get_response_charneg(tmpmem, p, &charset, &lang, &sel);
3961                 yaz_log(log_details, "%p recv_apdu target accepted: "
3962                         "charset %s, language %s, select %d",
3963                         c,
3964                         charset ? charset : "none", lang ? lang : "none", sel);
3965                 if (charset)
3966                     ZOOM_connection_option_set(c, "negotiation-charset",
3967                                                charset);
3968                 if (lang)
3969                     ZOOM_connection_option_set(c, "negotiation-lang",
3970                                                lang);
3971
3972                 ZOOM_connection_option_set(
3973                     c,  "negotiation-charset-in-effect-for-records",
3974                     (sel != 0) ? "1" : "0");
3975                 nmem_destroy(tmpmem);
3976             }
3977         }
3978         break;
3979     case Z_APDU_searchResponse:
3980         yaz_log(log_api, "%p recv_apdu Search response", c);
3981         handle_search_response(c, apdu->u.searchResponse);
3982         if (send_sort_present(c) == zoom_complete)
3983             ZOOM_connection_remove_task(c);
3984         break;
3985     case Z_APDU_presentResponse:
3986         yaz_log(log_api, "%p recv_apdu Present response", c);
3987         handle_present_response(c, apdu->u.presentResponse);
3988         if (send_present(c) == zoom_complete)
3989             ZOOM_connection_remove_task(c);
3990         break;
3991     case Z_APDU_sortResponse:
3992         yaz_log(log_api, "%p recv_apdu Sort response", c);
3993         sort_response(c, apdu->u.sortResponse);
3994         if (send_present(c) == zoom_complete)
3995             ZOOM_connection_remove_task(c);
3996         break;
3997     case Z_APDU_scanResponse:
3998         yaz_log(log_api, "%p recv_apdu Scan response", c);
3999         scan_response(c, apdu->u.scanResponse);
4000         ZOOM_connection_remove_task(c);
4001         break;
4002     case Z_APDU_extendedServicesResponse:
4003         yaz_log(log_api, "%p recv_apdu Extended Services response", c);
4004         es_response(c, apdu->u.extendedServicesResponse);
4005         ZOOM_connection_remove_task(c);
4006         break;
4007     case Z_APDU_close:
4008         yaz_log(log_api, "%p recv_apdu Close PDU", c);
4009         if (!ZOOM_test_reconnect(c))
4010         {
4011             set_ZOOM_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
4012             do_close(c);
4013         }
4014         break;
4015     default:
4016         yaz_log(log_api, "%p Received unknown PDU", c);
4017         set_ZOOM_error(c, ZOOM_ERROR_DECODE, 0);
4018         do_close(c);
4019     }
4020 }
4021
4022 #if YAZ_HAVE_XML2
4023 static zoom_ret handle_srw_response(ZOOM_connection c,
4024                                     Z_SRW_searchRetrieveResponse *res)
4025 {
4026     ZOOM_resultset resultset = 0;
4027     int i;
4028     NMEM nmem;
4029     ZOOM_Event event;
4030     int *start, *count;
4031     const char *syntax, *elementSetName;
4032
4033     if (!c->tasks)
4034         return zoom_complete;
4035
4036     switch(c->tasks->which)
4037     {
4038     case ZOOM_TASK_SEARCH:
4039         resultset = c->tasks->u.search.resultset;
4040         start = &c->tasks->u.search.start;
4041         count = &c->tasks->u.search.count;
4042         syntax = c->tasks->u.search.syntax;
4043         elementSetName = c->tasks->u.search.elementSetName;
4044
4045         if (!c->tasks->u.search.recv_search_fired)
4046         {
4047             event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH);
4048             ZOOM_connection_put_event(c, event);
4049             c->tasks->u.search.recv_search_fired = 1;
4050         }
4051         break;
4052     case ZOOM_TASK_RETRIEVE:
4053         resultset = c->tasks->u.retrieve.resultset;
4054         start = &c->tasks->u.retrieve.start;
4055         count = &c->tasks->u.retrieve.count;
4056         syntax = c->tasks->u.retrieve.syntax;
4057         elementSetName = c->tasks->u.retrieve.elementSetName;
4058         break;
4059     default:
4060         return zoom_complete;
4061     }
4062
4063     resultset->size = 0;
4064
4065     if (res->resultSetId)
4066         ZOOM_resultset_option_set(resultset, "resultSetId", res->resultSetId);
4067
4068     yaz_log(log_details, "%p handle_srw_response got SRW response OK", c);
4069
4070     if (res->num_diagnostics > 0)
4071     {
4072         set_SRU_error(c, &res->diagnostics[0]);
4073     }
4074     else
4075     {
4076         if (res->numberOfRecords)
4077             resultset->size = *res->numberOfRecords;
4078         for (i = 0; i<res->num_records; i++)
4079         {
4080             int pos;
4081             Z_SRW_record *sru_rec;
4082             Z_SRW_diagnostic *diag = 0;
4083             int num_diag;
4084
4085             Z_NamePlusRecord *npr = (Z_NamePlusRecord *)
4086                 odr_malloc(c->odr_in, sizeof(Z_NamePlusRecord));
4087
4088             if (res->records[i].recordPosition &&
4089                 *res->records[i].recordPosition > 0)
4090                 pos = *res->records[i].recordPosition - 1;
4091             else
4092                 pos = *start + i;
4093
4094             sru_rec = &res->records[i];
4095
4096             npr->databaseName = 0;
4097             npr->which = Z_NamePlusRecord_databaseRecord;
4098             npr->u.databaseRecord = (Z_External *)
4099                 odr_malloc(c->odr_in, sizeof(Z_External));
4100             npr->u.databaseRecord->descriptor = 0;
4101             npr->u.databaseRecord->direct_reference =
4102                 odr_oiddup(c->odr_in, yaz_oid_recsyn_xml);
4103             npr->u.databaseRecord->which = Z_External_octet;
4104
4105             npr->u.databaseRecord->u.octet_aligned = (Odr_oct *)
4106                 odr_malloc(c->odr_in, sizeof(Odr_oct));
4107             npr->u.databaseRecord->u.octet_aligned->buf = (unsigned char*)
4108                 sru_rec->recordData_buf;
4109             npr->u.databaseRecord->u.octet_aligned->len =
4110                 npr->u.databaseRecord->u.octet_aligned->size =
4111                 sru_rec->recordData_len;
4112
4113             if (sru_rec->recordSchema
4114                 && !strcmp(sru_rec->recordSchema,
4115                            "info:srw/schema/1/diagnostics-v1.1"))
4116             {
4117                 sru_decode_surrogate_diagnostics(sru_rec->recordData_buf,
4118                                                  sru_rec->recordData_len,
4119                                                  &diag, &num_diag,
4120                                                  resultset->odr);
4121             }
4122             record_cache_add(resultset, npr, pos, syntax, elementSetName,
4123                              sru_rec->recordSchema, diag);
4124         }
4125         *count -= i;
4126         *start += i;
4127         if (*count + *start > resultset->size)
4128             *count = resultset->size - *start;
4129         if (*count < 0)
4130             *count = 0;
4131
4132         nmem = odr_extract_mem(c->odr_in);
4133         nmem_transfer(odr_getmem(resultset->odr), nmem);
4134         nmem_destroy(nmem);
4135
4136         if (*count > 0)
4137             return ZOOM_connection_srw_send_search(c);
4138     }
4139     return zoom_complete;
4140 }
4141 #endif
4142
4143 #if YAZ_HAVE_XML2
4144 static void handle_srw_scan_response(ZOOM_connection c,
4145                                      Z_SRW_scanResponse *res)
4146 {
4147     NMEM nmem = odr_extract_mem(c->odr_in);
4148     ZOOM_scanset scan;
4149
4150     if (!c->tasks || c->tasks->which != ZOOM_TASK_SCAN)
4151         return;
4152     scan = c->tasks->u.scan.scan;
4153
4154     if (res->num_diagnostics > 0)
4155         set_SRU_error(c, &res->diagnostics[0]);
4156
4157     scan->scan_response = 0;
4158     scan->srw_scan_response = res;
4159     nmem_transfer(odr_getmem(scan->odr), nmem);
4160
4161     ZOOM_options_set_int(scan->options, "number", res->num_terms);
4162     nmem_destroy(nmem);
4163 }
4164 #endif
4165
4166 #if YAZ_HAVE_XML2
4167 static Z_GDU *get_HTTP_Request_url(ODR odr, const char *url)
4168 {
4169     Z_GDU *p = z_get_HTTP_Request(odr);
4170     const char *host = url;
4171     const char *cp0 = strstr(host, "://");
4172     const char *cp1 = 0;
4173     if (cp0)
4174         cp0 = cp0+3;
4175     else
4176         cp0 = host;
4177
4178     cp1 = strchr(cp0, '/');
4179     if (!cp1)
4180         cp1 = cp0 + strlen(cp0);
4181
4182     if (cp0 && cp1)
4183     {
4184         char *h = (char*) odr_malloc(odr, cp1 - cp0 + 1);
4185         memcpy (h, cp0, cp1 - cp0);
4186         h[cp1-cp0] = '\0';
4187         z_HTTP_header_add(odr, &p->u.HTTP_Request->headers, "Host", h);
4188     }
4189     p->u.HTTP_Request->path = odr_strdup(odr, *cp1 ? cp1 : "/");
4190     return p;
4191 }
4192
4193 static zoom_ret send_SRW_redirect(ZOOM_connection c, const char *uri,
4194                                   Z_HTTP_Response *cookie_hres)
4195 {
4196     struct Z_HTTP_Header *h;
4197     Z_GDU *gdu = get_HTTP_Request_url(c->odr_out, uri);
4198
4199     gdu->u.HTTP_Request->method = odr_strdup(c->odr_out, "GET");
4200     z_HTTP_header_add(c->odr_out, &gdu->u.HTTP_Request->headers, "Accept",
4201                       "text/xml");
4202
4203     for (h = cookie_hres->headers; h; h = h->next)
4204     {
4205         if (!strcmp(h->name, "Set-Cookie"))
4206             z_HTTP_header_add(c->odr_out, &gdu->u.HTTP_Request->headers,
4207                               "Cookie", h->value);
4208     }
4209     if (c->user && c->password)
4210     {
4211         z_HTTP_header_add_basic_auth(c->odr_out, &gdu->u.HTTP_Request->headers,
4212                                      c->user, c->password);
4213     }
4214     if (!z_GDU(c->odr_out, &gdu, 0, 0))
4215         return zoom_complete;
4216     if (c->odr_print)
4217         z_GDU(c->odr_print, &gdu, 0, 0);
4218     c->buf_out = odr_getbuf(c->odr_out, &c->len_out, 0);
4219
4220     odr_reset(c->odr_out);
4221     return do_write(c);
4222 }
4223
4224 static void handle_http(ZOOM_connection c, Z_HTTP_Response *hres)
4225 {
4226     zoom_ret cret = zoom_complete;
4227     int ret = -1;
4228     const char *addinfo = 0;
4229     const char *connection_head = z_HTTP_header_lookup(hres->headers,
4230                                                        "Connection");
4231     const char *location;
4232
4233     ZOOM_connection_set_mask(c, 0);
4234     yaz_log(log_details, "%p handle_http", c);
4235
4236     if ((hres->code == 301 || hres->code == 302) && c->sru_mode == zoom_sru_get
4237         && (location = z_HTTP_header_lookup(hres->headers, "Location")))
4238     {
4239         c->no_redirects++;
4240         if (c->no_redirects > 10)
4241         {
4242             set_HTTP_error(c, hres->code, 0, 0);
4243             c->no_redirects = 0;
4244             do_close(c);
4245         }
4246         else
4247         {
4248             /* since redirect may change host we just reconnect. A smarter
4249                implementation might check whether it's the same server */
4250             do_connect_host(c, location, 0);
4251             send_SRW_redirect(c, location, hres);
4252             /* we're OK for now. Operation is not really complete */
4253             ret = 0;
4254             cret = zoom_pending;
4255         }
4256     }
4257     else
4258     {   /* not redirect (normal response) */
4259         if (!yaz_srw_check_content_type(hres))
4260             addinfo = "content-type";
4261         else
4262         {
4263             Z_SOAP *soap_package = 0;
4264             ODR o = c->odr_in;
4265             Z_SOAP_Handler soap_handlers[2] = {
4266                 {YAZ_XMLNS_SRU_v1_1, 0, (Z_SOAP_fun) yaz_srw_codec},
4267                 {0, 0, 0}
4268             };
4269             ret = z_soap_codec(o, &soap_package,
4270                                &hres->content_buf, &hres->content_len,
4271                                soap_handlers);
4272             if (!ret && soap_package->which == Z_SOAP_generic &&
4273                 soap_package->u.generic->no == 0)
4274             {
4275                 Z_SRW_PDU *sr = (Z_SRW_PDU*) soap_package->u.generic->p;
4276
4277                 ZOOM_options_set(c->options, "sru_version", sr->srw_version);
4278                 ZOOM_options_setl(c->options, "sru_extra_response_data",
4279                                   sr->extraResponseData_buf, sr->extraResponseData_len);
4280                 if (sr->which == Z_SRW_searchRetrieve_response)
4281                     cret = handle_srw_response(c, sr->u.response);
4282                 else if (sr->which == Z_SRW_scan_response)
4283                     handle_srw_scan_response(c, sr->u.scan_response);
4284                 else
4285                     ret = -1;
4286             }
4287             else if (!ret && (soap_package->which == Z_SOAP_fault
4288                               || soap_package->which == Z_SOAP_error))
4289             {
4290                 set_HTTP_error(c, hres->code,
4291                                soap_package->u.fault->fault_code,
4292                                soap_package->u.fault->fault_string);
4293             }
4294             else
4295                 ret = -1;
4296         }
4297         if (ret == 0)
4298         {
4299             if (c->no_redirects) /* end of redirect. change hosts again */
4300                 do_close(c);
4301         }
4302         c->no_redirects = 0;
4303     }
4304     if (ret)
4305     {
4306         if (hres->code != 200)
4307             set_HTTP_error(c, hres->code, 0, 0);
4308         else
4309             set_ZOOM_error(c, ZOOM_ERROR_DECODE, addinfo);
4310         do_close(c);
4311     }
4312     if (cret == zoom_complete)
4313         ZOOM_connection_remove_task(c);
4314     if (!strcmp(hres->version, "1.0"))
4315     {
4316         /* HTTP 1.0: only if Keep-Alive we stay alive.. */
4317         if (!connection_head || strcmp(connection_head, "Keep-Alive"))
4318             do_close(c);
4319     }
4320     else
4321     {
4322         /* HTTP 1.1: only if no close we stay alive .. */
4323         if (connection_head && !strcmp(connection_head, "close"))
4324             do_close(c);
4325     }
4326 }
4327 #endif
4328
4329 static int do_read(ZOOM_connection c)
4330 {
4331     int r, more;
4332     ZOOM_Event event;
4333
4334     event = ZOOM_Event_create(ZOOM_EVENT_RECV_DATA);
4335     ZOOM_connection_put_event(c, event);
4336
4337     r = cs_get(c->cs, &c->buf_in, &c->len_in);
4338     more = cs_more(c->cs);
4339     yaz_log(log_details, "%p do_read len=%d more=%d", c, r, more);
4340     if (r == 1)
4341         return 0;
4342     if (r <= 0)
4343     {
4344         if (!ZOOM_test_reconnect(c))
4345         {
4346             set_ZOOM_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
4347             do_close(c);
4348         }
4349     }
4350     else
4351     {
4352         Z_GDU *gdu;
4353         ZOOM_Event event;
4354
4355         odr_reset(c->odr_in);
4356         odr_setbuf(c->odr_in, c->buf_in, r, 0);
4357         event = ZOOM_Event_create(ZOOM_EVENT_RECV_APDU);
4358         ZOOM_connection_put_event(c, event);
4359
4360         if (!z_GDU(c->odr_in, &gdu, 0, 0))
4361         {
4362             int x;
4363             int err = odr_geterrorx(c->odr_in, &x);
4364             char msg[100];
4365             const char *element = odr_getelement(c->odr_in);
4366             yaz_snprintf(msg, sizeof(msg),
4367                     "ODR code %d:%d element=%s offset=%d",
4368                     err, x, element ? element : "<unknown>",
4369                     odr_offset(c->odr_in));
4370             set_ZOOM_error(c, ZOOM_ERROR_DECODE, msg);
4371             if (log_api)
4372             {
4373                 FILE *ber_file = yaz_log_file();
4374                 if (ber_file)
4375                     odr_dumpBER(ber_file, c->buf_in, r);
4376             }
4377             do_close(c);
4378         }
4379         else
4380         {
4381             if (c->odr_print)
4382                 z_GDU(c->odr_print, &gdu, 0, 0);
4383             if (gdu->which == Z_GDU_Z3950)
4384                 recv_apdu(c, gdu->u.z3950);
4385             else if (gdu->which == Z_GDU_HTTP_Response)
4386             {
4387 #if YAZ_HAVE_XML2
4388                 handle_http(c, gdu->u.HTTP_Response);
4389 #else
4390                 set_ZOOM_error(c, ZOOM_ERROR_DECODE, 0);
4391                 do_close(c);
4392 #endif
4393             }
4394         }
4395         c->reconnect_ok = 0;
4396     }
4397     return 1;
4398 }
4399
4400 static zoom_ret do_write_ex(ZOOM_connection c, char *buf_out, int len_out)
4401 {
4402     int r;
4403     ZOOM_Event event;
4404
4405     event = ZOOM_Event_create(ZOOM_EVENT_SEND_DATA);
4406     ZOOM_connection_put_event(c, event);
4407
4408     yaz_log(log_details, "%p do_write_ex len=%d", c, len_out);
4409     if ((r = cs_put(c->cs, buf_out, len_out)) < 0)
4410     {
4411         yaz_log(log_details, "%p do_write_ex write failed", c);
4412         if (ZOOM_test_reconnect(c))
4413         {
4414             return zoom_pending;
4415         }
4416         if (c->state == STATE_CONNECTING)
4417             set_ZOOM_error(c, ZOOM_ERROR_CONNECT, c->host_port);
4418         else
4419             set_ZOOM_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
4420         do_close(c);
4421         return zoom_complete;
4422     }
4423     else if (r == 1)
4424     {
4425         int mask = ZOOM_SELECT_EXCEPT;
4426         if (c->cs->io_pending & CS_WANT_WRITE)
4427             mask += ZOOM_SELECT_WRITE;
4428         if (c->cs->io_pending & CS_WANT_READ)
4429             mask += ZOOM_SELECT_READ;
4430         ZOOM_connection_set_mask(c, mask);
4431         yaz_log(log_details, "%p do_write_ex write incomplete mask=%d",
4432                 c, c->mask);
4433     }
4434     else
4435     {
4436         ZOOM_connection_set_mask(c, ZOOM_SELECT_READ|ZOOM_SELECT_EXCEPT);
4437         yaz_log(log_details, "%p do_write_ex write complete mask=%d",
4438                 c, c->mask);
4439     }
4440     return zoom_pending;
4441 }
4442
4443 static zoom_ret do_write(ZOOM_connection c)
4444 {
4445     return do_write_ex(c, c->buf_out, c->len_out);
4446 }
4447
4448
4449 ZOOM_API(const char *)
4450     ZOOM_connection_option_get(ZOOM_connection c, const char *key)
4451 {
4452     return ZOOM_options_get(c->options, key);
4453 }
4454
4455 ZOOM_API(const char *)
4456     ZOOM_connection_option_getl(ZOOM_connection c, const char *key, int *lenp)
4457 {
4458     return ZOOM_options_getl(c->options, key, lenp);
4459 }
4460
4461 ZOOM_API(void)
4462     ZOOM_connection_option_set(ZOOM_connection c, const char *key,
4463                                const char *val)
4464 {
4465     ZOOM_options_set(c->options, key, val);
4466 }
4467
4468 ZOOM_API(void)
4469     ZOOM_connection_option_setl(ZOOM_connection c, const char *key,
4470                                 const char *val, int len)
4471 {
4472     ZOOM_options_setl(c->options, key, val, len);
4473 }
4474
4475 ZOOM_API(const char *)
4476     ZOOM_resultset_option_get(ZOOM_resultset r, const char *key)
4477 {
4478     return ZOOM_options_get(r->options, key);
4479 }
4480
4481 ZOOM_API(void)
4482     ZOOM_resultset_option_set(ZOOM_resultset r, const char *key,
4483                               const char *val)
4484 {
4485     ZOOM_options_set(r->options, key, val);
4486 }
4487
4488
4489 ZOOM_API(int)
4490     ZOOM_connection_errcode(ZOOM_connection c)
4491 {
4492     return ZOOM_connection_error(c, 0, 0);
4493 }
4494
4495 ZOOM_API(const char *)
4496     ZOOM_connection_errmsg(ZOOM_connection c)
4497 {
4498     const char *msg;
4499     ZOOM_connection_error(c, &msg, 0);
4500     return msg;
4501 }
4502
4503 ZOOM_API(const char *)
4504     ZOOM_connection_addinfo(ZOOM_connection c)
4505 {
4506     const char *addinfo;
4507     ZOOM_connection_error(c, 0, &addinfo);
4508     return addinfo;
4509 }
4510
4511 ZOOM_API(const char *)
4512     ZOOM_connection_diagset(ZOOM_connection c)
4513 {
4514     const char *diagset;
4515     ZOOM_connection_error_x(c, 0, 0, &diagset);
4516     return diagset;
4517 }
4518
4519 ZOOM_API(const char *)
4520     ZOOM_diag_str(int error)
4521 {
4522     switch (error)
4523     {
4524     case ZOOM_ERROR_NONE:
4525         return "No error";
4526     case ZOOM_ERROR_CONNECT:
4527         return "Connect failed";
4528     case ZOOM_ERROR_MEMORY:
4529         return "Out of memory";
4530     case ZOOM_ERROR_ENCODE:
4531         return "Encoding failed";
4532     case ZOOM_ERROR_DECODE:
4533         return "Decoding failed";
4534     case ZOOM_ERROR_CONNECTION_LOST:
4535         return "Connection lost";
4536     case ZOOM_ERROR_INIT:
4537         return "Init rejected";
4538     case ZOOM_ERROR_INTERNAL:
4539         return "Internal failure";
4540     case ZOOM_ERROR_TIMEOUT:
4541         return "Timeout";
4542     case ZOOM_ERROR_UNSUPPORTED_PROTOCOL:
4543         return "Unsupported protocol";
4544     case ZOOM_ERROR_UNSUPPORTED_QUERY:
4545         return "Unsupported query type";
4546     case ZOOM_ERROR_INVALID_QUERY:
4547         return "Invalid query";
4548     case ZOOM_ERROR_CQL_PARSE:
4549         return "CQL parsing error";
4550     case ZOOM_ERROR_CQL_TRANSFORM:
4551         return "CQL transformation error";
4552     case ZOOM_ERROR_CCL_CONFIG:
4553         return "CCL configuration error";
4554     case ZOOM_ERROR_CCL_PARSE:
4555         return "CCL parsing error";
4556     default:
4557         return diagbib1_str(error);
4558     }
4559 }
4560
4561 ZOOM_API(int)
4562     ZOOM_connection_error_x(ZOOM_connection c, const char **cp,
4563                             const char **addinfo, const char **diagset)
4564 {
4565     int error = c->error;
4566     if (cp)
4567     {
4568         if (!c->diagset || !strcmp(c->diagset, "ZOOM"))
4569             *cp = ZOOM_diag_str(error);
4570         else if (!strcmp(c->diagset, "HTTP"))
4571             *cp = z_HTTP_errmsg(c->error);
4572         else if (!strcmp(c->diagset, "Bib-1"))
4573             *cp = ZOOM_diag_str(error);
4574         else if (!strcmp(c->diagset, "info:srw/diagnostic/1"))
4575             *cp = yaz_diag_srw_str(c->error);
4576         else
4577             *cp = "Unknown error and diagnostic set";
4578     }
4579     if (addinfo)
4580         *addinfo = c->addinfo ? c->addinfo : "";
4581     if (diagset)
4582         *diagset = c->diagset ? c->diagset : "";
4583     return c->error;
4584 }
4585
4586 ZOOM_API(int)
4587     ZOOM_connection_error(ZOOM_connection c, const char **cp,
4588                           const char **addinfo)
4589 {
4590     return ZOOM_connection_error_x(c, cp, addinfo, 0);
4591 }
4592
4593 static void ZOOM_connection_do_io(ZOOM_connection c, int mask)
4594 {
4595     ZOOM_Event event = 0;
4596     int r = cs_look(c->cs);
4597     yaz_log(log_details, "%p ZOOM_connection_do_io mask=%d cs_look=%d",
4598             c, mask, r);
4599
4600     if (r == CS_NONE)
4601     {
4602         event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
4603         set_ZOOM_error(c, ZOOM_ERROR_CONNECT, c->host_port);
4604         do_close(c);
4605         ZOOM_connection_put_event(c, event);
4606     }
4607     else if (r == CS_CONNECT)
4608     {
4609         int ret = ret = cs_rcvconnect(c->cs);
4610         yaz_log(log_details, "%p ZOOM_connection_do_io "
4611                 "cs_rcvconnect returned %d", c, ret);
4612         if (ret == 1)
4613         {
4614             int mask = ZOOM_SELECT_EXCEPT;
4615             if (c->cs->io_pending & CS_WANT_WRITE)
4616                 mask += ZOOM_SELECT_WRITE;
4617             if (c->cs->io_pending & CS_WANT_READ)
4618                 mask += ZOOM_SELECT_READ;
4619             ZOOM_connection_set_mask(c, mask);
4620             event = ZOOM_Event_create(ZOOM_EVENT_NONE);
4621             ZOOM_connection_put_event(c, event);
4622         }
4623         else if (ret == 0)
4624         {
4625             event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
4626             ZOOM_connection_put_event(c, event);
4627             get_cert(c);
4628             if (c->proto == PROTO_Z3950)
4629                 ZOOM_connection_send_init(c);
4630             else
4631             {
4632                 /* no init request for SRW .. */
4633                 assert(c->tasks->which == ZOOM_TASK_CONNECT);
4634                 ZOOM_connection_remove_task(c);
4635                 ZOOM_connection_set_mask(c, 0);
4636                 ZOOM_connection_exec_task(c);
4637             }
4638             c->state = STATE_ESTABLISHED;
4639         }
4640         else
4641         {
4642             set_ZOOM_error(c, ZOOM_ERROR_CONNECT, c->host_port);
4643             do_close(c);
4644         }
4645     }
4646     else
4647     {
4648         if (mask & ZOOM_SELECT_EXCEPT)
4649         {
4650             if (!ZOOM_test_reconnect(c))
4651             {
4652                 set_ZOOM_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
4653                 do_close(c);
4654             }
4655             return;
4656         }
4657         if (mask & ZOOM_SELECT_READ)
4658             do_read(c);
4659         if (c->cs && (mask & ZOOM_SELECT_WRITE))
4660             do_write(c);
4661     }
4662 }
4663
4664 ZOOM_API(int)
4665     ZOOM_connection_last_event(ZOOM_connection cs)
4666 {
4667     if (!cs)
4668         return ZOOM_EVENT_NONE;
4669     return cs->last_event;
4670 }
4671
4672
4673 static void cql2pqf_wrbuf_puts(const char *buf, void *client_data)
4674 {
4675     WRBUF wrbuf = (WRBUF) client_data;
4676     wrbuf_puts(wrbuf, buf);
4677 }
4678
4679 /*
4680  * Returns an xmalloc()d string containing RPN that corresponds to the
4681  * CQL passed in.  On error, sets the Connection object's error state
4682  * and returns a null pointer.
4683  * ### We could cache CQL parser and/or transformer in Connection.
4684  */
4685 static char *cql2pqf(ZOOM_connection c, const char *cql)
4686 {
4687     CQL_parser parser;
4688     int error;
4689     const char *cqlfile;
4690     cql_transform_t trans;
4691     char *result = 0;
4692
4693     parser = cql_parser_create();
4694     if ((error = cql_parser_string(parser, cql)) != 0) {
4695         cql_parser_destroy(parser);
4696         set_ZOOM_error(c, ZOOM_ERROR_CQL_PARSE, cql);
4697         return 0;
4698     }
4699
4700     cqlfile = ZOOM_connection_option_get(c, "cqlfile");
4701     if (cqlfile == 0)
4702     {
4703         set_ZOOM_error(c, ZOOM_ERROR_CQL_TRANSFORM, "no CQL transform file");
4704     }
4705     else if ((trans = cql_transform_open_fname(cqlfile)) == 0)
4706     {
4707         char buf[512];
4708         sprintf(buf, "can't open CQL transform file '%.200s': %.200s",
4709                 cqlfile, strerror(errno));
4710         set_ZOOM_error(c, ZOOM_ERROR_CQL_TRANSFORM, buf);
4711     }
4712     else
4713     {
4714         WRBUF wrbuf_result = wrbuf_alloc();
4715         error = cql_transform(trans, cql_parser_result(parser),
4716                               cql2pqf_wrbuf_puts, wrbuf_result);
4717         if (error != 0) {
4718             char buf[512];
4719             const char *addinfo;
4720             error = cql_transform_error(trans, &addinfo);
4721             sprintf(buf, "%.200s (addinfo=%.200s)",
4722                     cql_strerror(error), addinfo);
4723             set_ZOOM_error(c, ZOOM_ERROR_CQL_TRANSFORM, buf);
4724         }
4725         else
4726         {
4727             result = xstrdup(wrbuf_cstr(wrbuf_result));
4728         }
4729         cql_transform_close(trans);
4730         wrbuf_destroy(wrbuf_result);
4731     }
4732     cql_parser_destroy(parser);
4733     return result;
4734 }
4735
4736 ZOOM_API(int) ZOOM_connection_fire_event_timeout(ZOOM_connection c)
4737 {
4738     if (c->mask)
4739     {
4740         ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_TIMEOUT);
4741         /* timeout and this connection was waiting */
4742         set_ZOOM_error(c, ZOOM_ERROR_TIMEOUT, 0);
4743         do_close(c);
4744         ZOOM_connection_put_event(c, event);
4745     }
4746     return 0;
4747 }
4748
4749 ZOOM_API(int)
4750     ZOOM_connection_process(ZOOM_connection c)
4751 {
4752     ZOOM_Event event;
4753     if (!c)
4754         return 0;
4755
4756     event = ZOOM_connection_get_event(c);
4757     if (event)
4758     {
4759         ZOOM_Event_destroy(event);
4760         return 1;
4761     }
4762     ZOOM_connection_exec_task(c);
4763     event = ZOOM_connection_get_event(c);
4764     if (event)
4765     {
4766         ZOOM_Event_destroy(event);
4767         return 1;
4768     }
4769     return 0;
4770 }
4771
4772 ZOOM_API(int)
4773     ZOOM_event_nonblock(int no, ZOOM_connection *cs)
4774 {
4775     int i;
4776
4777     yaz_log(log_details, "ZOOM_process_event(no=%d,cs=%p)", no, cs);
4778
4779     for (i = 0; i<no; i++)
4780     {
4781         ZOOM_connection c = cs[i];
4782
4783         if (c && ZOOM_connection_process(c))
4784             return i+1;
4785     }
4786     return 0;
4787 }
4788
4789 ZOOM_API(int) ZOOM_connection_fire_event_socket(ZOOM_connection c, int mask)
4790 {
4791     if (c->mask && mask)
4792         ZOOM_connection_do_io(c, mask);
4793     return 0;
4794 }
4795
4796 ZOOM_API(int) ZOOM_connection_get_socket(ZOOM_connection c)
4797 {
4798     if (c->cs)
4799         return cs_fileno(c->cs);
4800     return -1;
4801 }
4802
4803 ZOOM_API(int) ZOOM_connection_set_mask(ZOOM_connection c, int mask)
4804 {
4805     c->mask = mask;
4806     if (!c->cs)
4807         return -1;
4808     return 0;
4809 }
4810
4811 ZOOM_API(int) ZOOM_connection_get_mask(ZOOM_connection c)
4812 {
4813     if (c->cs)
4814         return c->mask;
4815     return 0;
4816 }
4817
4818 ZOOM_API(int) ZOOM_connection_get_timeout(ZOOM_connection c)
4819 {
4820     return ZOOM_options_get_int(c->options, "timeout", 30);
4821 }
4822
4823 ZOOM_API(void) ZOOM_connection_close(ZOOM_connection c)
4824 {
4825     do_close(c);
4826 }
4827
4828 /*
4829  * Local variables:
4830  * c-basic-offset: 4
4831  * c-file-style: "Stroustrup"
4832  * indent-tabs-mode: nil
4833  * End:
4834  * vim: shiftwidth=4 tabstop=8 expandtab
4835  */
4836