src/zoom-c.c

   1 /* This file is part of the YAZ toolkit.
   2  * Copyright (C) 1995-2011 Index Data
   3  * See the file LICENSE for details.
   4  */
   5 /**
   6  * \file zoom-c.c
   7  * \brief Implements ZOOM C interface.
   8  */
   9 #if HAVE_CONFIG_H
  10 #include <config.h>
  11 #endif
  12
  13 #include <assert.h>
  14 #include <string.h>
  15 #include <errno.h>
  16 #include "zoom-p.h"
  17
  18 #include <yaz/yaz-util.h>
  19 #include <yaz/xmalloc.h>
  20 #include <yaz/otherinfo.h>
  21 #include <yaz/log.h>
  22 #include <yaz/diagbib1.h>
  23 #include <yaz/charneg.h>
  24 #include <yaz/query-charset.h>
  25 #include <yaz/snprintf.h>
  26 #include <yaz/facet.h>
  27
  28 #include <yaz/shptr.h>
  29
  30 #if SHPTR
  31 YAZ_SHPTR_TYPE(WRBUF)
  32 #endif
  33
  34 static int log_api0 = 0;
  35 static int log_details0 = 0;
  36
  37 static void resultset_destroy(ZOOM_resultset r);
  38 static zoom_ret do_write_ex(ZOOM_connection c, char *buf_out, int len_out);
  39
  40 static void initlog(void)
  41 {
  42     static int log_level_initialized = 0;
  43
  44     if (!log_level_initialized)
  45     {
  46         log_api0 = yaz_log_module_level("zoom");
  47         log_details0 = yaz_log_module_level("zoomdetails");
  48         log_level_initialized = 1;
  49     }
  50 }
  51
  52 void ZOOM_connection_remove_tasks(ZOOM_connection c);
  53
  54 void ZOOM_set_dset_error(ZOOM_connection c, int error,
  55                          const char *dset,
  56                          const char *addinfo, const char *addinfo2)
  57 {
  58     char *cp;
  59
  60     xfree(c->addinfo);
  61     c->addinfo = 0;
  62     c->error = error;
  63     if (!c->diagset || strcmp(dset, c->diagset))
  64     {
  65         xfree(c->diagset);
  66         c->diagset = xstrdup(dset);
  67         /* remove integer part from SRW diagset .. */
  68         if ((cp = strrchr(c->diagset, '/')))
  69             *cp = '\0';
  70     }
  71     if (addinfo && addinfo2)
  72     {
  73         c->addinfo = (char*) xmalloc(strlen(addinfo) + strlen(addinfo2) + 3);
  74         strcpy(c->addinfo, addinfo);
  75         strcat(c->addinfo, ": ");
  76         strcat(c->addinfo, addinfo2);
  77     }
  78     else if (addinfo)
  79         c->addinfo = xstrdup(addinfo);
  80     if (error != ZOOM_ERROR_NONE)
  81     {
  82         yaz_log(c->log_api, "%p set_dset_error %s %s:%d %s %s",
  83                 c, c->host_port ? c->host_port : "<>", dset, error,
  84                 addinfo ? addinfo : "",
  85                 addinfo2 ? addinfo2 : "");
  86         ZOOM_connection_remove_tasks(c);
  87     }
  88 }
  89
  90 int ZOOM_uri_to_code(const char *uri)
  91 {
  92     int code = 0;
  93     const char *cp;
  94     if ((cp = strrchr(uri, '/')))
  95         code = atoi(cp+1);
  96     return code;
  97 }
  98
  99
 100
 101 void ZOOM_set_error(ZOOM_connection c, int error, const char *addinfo)
 102 {
 103     ZOOM_set_dset_error(c, error, "ZOOM", addinfo, 0);
 104 }
 105
 106 static void clear_error(ZOOM_connection c)
 107 {
 108     /*
 109      * If an error is tied to an operation then it's ok to clear: for
 110      * example, a diagnostic returned from a search is cleared by a
 111      * subsequent search.  However, problems such as Connection Lost
 112      * or Init Refused are not cleared, because they are not
 113      * recoverable: doing another search doesn't help.
 114      */
 115
 116     ZOOM_connection_remove_events(c);
 117     switch (c->error)
 118     {
 119     case ZOOM_ERROR_CONNECT:
 120     case ZOOM_ERROR_MEMORY:
 121     case ZOOM_ERROR_DECODE:
 122     case ZOOM_ERROR_CONNECTION_LOST:
 123     case ZOOM_ERROR_INIT:
 124     case ZOOM_ERROR_INTERNAL:
 125     case ZOOM_ERROR_UNSUPPORTED_PROTOCOL:
 126         break;
 127     default:
 128         ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
 129     }
 130 }
 131
 132 void ZOOM_connection_show_task(ZOOM_task task)
 133 {
 134     switch(task->which)
 135     {
 136     case ZOOM_TASK_SEARCH:
 137         yaz_log(YLOG_LOG, "search p=%p", task);
 138         break;
 139     case ZOOM_TASK_RETRIEVE:
 140         yaz_log(YLOG_LOG, "retrieve p=%p", task);
 141         break;
 142     case ZOOM_TASK_CONNECT:
 143         yaz_log(YLOG_LOG, "connect p=%p", task);
 144         break;
 145     case ZOOM_TASK_SCAN:
 146         yaz_log(YLOG_LOG, "scan p=%p", task);
 147         break;
 148     }
 149 }
 150
 151 void ZOOM_connection_show_tasks(ZOOM_connection c)
 152 {
 153     ZOOM_task task;
 154     yaz_log(YLOG_LOG, "connection p=%p tasks", c);
 155     for (task = c->tasks; task; task = task->next)
 156         ZOOM_connection_show_task(task);
 157 }
 158
 159 ZOOM_task ZOOM_connection_add_task(ZOOM_connection c, int which)
 160 {
 161     ZOOM_task *taskp = &c->tasks;
 162     while (*taskp)
 163         taskp = &(*taskp)->next;
 164     *taskp = (ZOOM_task) xmalloc(sizeof(**taskp));
 165     (*taskp)->running = 0;
 166     (*taskp)->which = which;
 167     (*taskp)->next = 0;
 168     clear_error(c);
 169     return *taskp;
 170 }
 171
 172 ZOOM_API(int) ZOOM_connection_is_idle(ZOOM_connection c)
 173 {
 174     return c->tasks ? 0 : 1;
 175 }
 176
 177 ZOOM_task ZOOM_connection_insert_task(ZOOM_connection c, int which)
 178 {
 179     ZOOM_task task = (ZOOM_task) xmalloc(sizeof(*task));
 180
 181     task->next = c->tasks;
 182     c->tasks = task;
 183
 184     task->running = 0;
 185     task->which = which;
 186     return task;
 187 }
 188
 189 void ZOOM_connection_remove_task(ZOOM_connection c)
 190 {
 191     ZOOM_task task = c->tasks;
 192
 193     if (task)
 194     {
 195         c->tasks = task->next;
 196         switch (task->which)
 197         {
 198         case ZOOM_TASK_SEARCH:
 199             resultset_destroy(task->u.search.resultset);
 200             xfree(task->u.search.syntax);
 201             xfree(task->u.search.elementSetName);
 202             break;
 203         case ZOOM_TASK_RETRIEVE:
 204             resultset_destroy(task->u.retrieve.resultset);
 205             xfree(task->u.retrieve.syntax);
 206             xfree(task->u.retrieve.elementSetName);
 207             break;
 208         case ZOOM_TASK_CONNECT:
 209             break;
 210         case ZOOM_TASK_SCAN:
 211             ZOOM_scanset_destroy(task->u.scan.scan);
 212             break;
 213         case ZOOM_TASK_PACKAGE:
 214             ZOOM_package_destroy(task->u.package);
 215             break;
 216         case ZOOM_TASK_SORT:
 217             resultset_destroy(task->u.sort.resultset);
 218             ZOOM_query_destroy(task->u.sort.q);
 219             break;
 220         default:
 221             assert(0);
 222         }
 223         xfree(task);
 224
 225         if (!c->tasks)
 226         {
 227             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_END);
 228             ZOOM_connection_put_event(c, event);
 229         }
 230     }
 231 }
 232
 233 void ZOOM_connection_remove_tasks(ZOOM_connection c)
 234 {
 235     while (c->tasks)
 236         ZOOM_connection_remove_task(c);
 237 }
 238
 239
 240 ZOOM_API(ZOOM_connection)
 241     ZOOM_connection_create(ZOOM_options options)
 242 {
 243     ZOOM_connection c = (ZOOM_connection) xmalloc(sizeof(*c));
 244
 245     initlog();
 246
 247     c->log_api = log_api0;
 248     c->log_details = log_details0;
 249
 250     yaz_log(c->log_api, "%p ZOOM_connection_create", c);
 251
 252     c->proto = PROTO_Z3950;
 253     c->cs = 0;
 254     ZOOM_connection_set_mask(c, 0);
 255     c->reconnect_ok = 0;
 256     c->state = STATE_IDLE;
 257     c->addinfo = 0;
 258     c->diagset = 0;
 259     ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
 260     c->buf_in = 0;
 261     c->len_in = 0;
 262     c->buf_out = 0;
 263     c->len_out = 0;
 264     c->resultsets = 0;
 265
 266     c->options = ZOOM_options_create_with_parent(options);
 267
 268     c->host_port = 0;
 269     c->proxy = 0;
 270
 271     c->charset = c->lang = 0;
 272
 273     c->cookie_out = 0;
 274     c->cookie_in = 0;
 275     c->client_IP = 0;
 276     c->tasks = 0;
 277
 278     c->user = 0;
 279     c->group = 0;
 280     c->password = 0;
 281
 282     c->maximum_record_size = 0;
 283     c->preferred_message_size = 0;
 284
 285     c->odr_in = odr_createmem(ODR_DECODE);
 286     c->odr_out = odr_createmem(ODR_ENCODE);
 287     c->odr_print = 0;
 288
 289     c->async = 0;
 290     c->support_named_resultsets = 0;
 291     c->last_event = ZOOM_EVENT_NONE;
 292
 293     c->m_queue_front = 0;
 294     c->m_queue_back = 0;
 295
 296     c->sru_version = 0;
 297     c->no_redirects = 0;
 298     return c;
 299 }
 300
 301
 302 /* set database names. Take local databases (if set); otherwise
 303    take databases given in ZURL (if set); otherwise use Default */
 304 char **ZOOM_connection_get_databases(ZOOM_connection con, ZOOM_options options,
 305                                      int *num, ODR odr)
 306 {
 307     char **databaseNames;
 308     const char *cp = ZOOM_options_get(options, "databaseName");
 309
 310     if ((!cp || !*cp) && con->host_port)
 311     {
 312         if (strncmp(con->host_port, "unix:", 5) == 0)
 313             cp = strchr(con->host_port+5, ':');
 314         else
 315             cp = strchr(con->host_port, '/');
 316         if (cp)
 317             cp++;
 318     }
 319     if (!cp)
 320         cp = "Default";
 321     nmem_strsplit(odr_getmem(odr), "+", cp,  &databaseNames, num);
 322     return databaseNames;
 323 }
 324
 325 ZOOM_API(ZOOM_connection)
 326     ZOOM_connection_new(const char *host, int portnum)
 327 {
 328     ZOOM_connection c = ZOOM_connection_create(0);
 329
 330     ZOOM_connection_connect(c, host, portnum);
 331     return c;
 332 }
 333
 334 static zoom_sru_mode get_sru_mode_from_string(const char *s)
 335 {
 336     if (!s || !*s)
 337         return zoom_sru_soap;
 338     if (!yaz_matchstr(s, "soap"))
 339         return zoom_sru_soap;
 340     else if (!yaz_matchstr(s, "get"))
 341         return zoom_sru_get;
 342     else if (!yaz_matchstr(s, "post"))
 343         return zoom_sru_post;
 344     else if (!yaz_matchstr(s, "solr"))
 345         return zoom_sru_solr;
 346     return zoom_sru_error;
 347 }
 348
 349 ZOOM_API(void)
 350     ZOOM_connection_connect(ZOOM_connection c,
 351                             const char *host, int portnum)
 352 {
 353     const char *val;
 354
 355     initlog();
 356
 357     yaz_log(c->log_api, "%p ZOOM_connection_connect host=%s portnum=%d",
 358             c, host ? host : "null", portnum);
 359
 360     ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
 361     ZOOM_connection_remove_tasks(c);
 362
 363     if (c->odr_print)
 364     {
 365         odr_setprint(c->odr_print, 0); /* prevent destroy from fclose'ing */
 366         odr_destroy(c->odr_print);
 367     }
 368     if (ZOOM_options_get_bool(c->options, "apdulog", 0))
 369     {
 370         c->odr_print = odr_createmem(ODR_PRINT);
 371         odr_setprint(c->odr_print, yaz_log_file());
 372     }
 373     else
 374         c->odr_print = 0;
 375
 376     if (c->cs)
 377     {
 378         yaz_log(c->log_details, "%p ZOOM_connection_connect reconnect ok", c);
 379         c->reconnect_ok = 1;
 380         return;
 381     }
 382     yaz_log(c->log_details, "%p ZOOM_connection_connect connect", c);
 383     xfree(c->proxy);
 384     c->proxy = 0;
 385     val = ZOOM_options_get(c->options, "proxy");
 386     if (val && *val)
 387     {
 388         yaz_log(c->log_details, "%p ZOOM_connection_connect proxy=%s", c, val);
 389         c->proxy = xstrdup(val);
 390     }
 391
 392     xfree(c->charset);
 393     c->charset = 0;
 394     val = ZOOM_options_get(c->options, "charset");
 395     if (val && *val)
 396     {
 397         yaz_log(c->log_details, "%p ZOOM_connection_connect charset=%s", c, val);
 398         c->charset = xstrdup(val);
 399     }
 400
 401     xfree(c->lang);
 402     val = ZOOM_options_get(c->options, "lang");
 403     if (val && *val)
 404     {
 405         yaz_log(c->log_details, "%p ZOOM_connection_connect lang=%s", c, val);
 406         c->lang = xstrdup(val);
 407     }
 408     else
 409         c->lang = 0;
 410
 411     if (host)
 412     {
 413         xfree(c->host_port);
 414         if (portnum)
 415         {
 416             char hostn[128];
 417             sprintf(hostn, "%.80s:%d", host, portnum);
 418             c->host_port = xstrdup(hostn);
 419         }
 420         else
 421             c->host_port = xstrdup(host);
 422     }
 423
 424     {
 425         /*
 426          * If the "<scheme>:" part of the host string is preceded by one
 427          * or more comma-separated <name>=<value> pairs, these are taken
 428          * to be options to be set on the connection object.  Among other
 429          * applications, this facility can be used to embed authentication
 430          * in a host string:
 431          *          user=admin,password=secret,tcp:localhost:9999
 432          */
 433         char *remainder = c->host_port;
 434         char *pcolon = strchr(remainder, ':');
 435         char *pcomma;
 436         char *pequals;
 437         while ((pcomma = strchr(remainder, ',')) != 0 &&
 438                (pcolon == 0 || pcomma < pcolon)) {
 439             *pcomma = '\0';
 440             if ((pequals = strchr(remainder, '=')) != 0) {
 441                 *pequals = '\0';
 442                 /*printf("# setting '%s'='%s'\n", remainder, pequals+1);*/
 443                 ZOOM_connection_option_set(c, remainder, pequals+1);
 444             }
 445             remainder = pcomma+1;
 446         }
 447
 448         if (remainder != c->host_port) {
 449             xfree(c->host_port);
 450             c->host_port = xstrdup(remainder);
 451             /*printf("# reset hp='%s'\n", remainder);*/
 452         }
 453     }
 454
 455     val = ZOOM_options_get(c->options, "sru");
 456     c->sru_mode = get_sru_mode_from_string(val);
 457
 458     xfree(c->sru_version);
 459     val = ZOOM_options_get(c->options, "sru_version");
 460     c->sru_version = xstrdup(val ? val : "1.2");
 461
 462     ZOOM_options_set(c->options, "host", c->host_port);
 463
 464     xfree(c->cookie_out);
 465     c->cookie_out = 0;
 466     val = ZOOM_options_get(c->options, "cookie");
 467     if (val && *val)
 468     {
 469         yaz_log(c->log_details, "%p ZOOM_connection_connect cookie=%s", c, val);
 470         c->cookie_out = xstrdup(val);
 471     }
 472
 473     xfree(c->client_IP);
 474     c->client_IP = 0;
 475     val = ZOOM_options_get(c->options, "clientIP");
 476     if (val && *val)
 477     {
 478         yaz_log(c->log_details, "%p ZOOM_connection_connect clientIP=%s",
 479                 c, val);
 480         c->client_IP = xstrdup(val);
 481     }
 482
 483     xfree(c->group);
 484     c->group = 0;
 485     val = ZOOM_options_get(c->options, "group");
 486     if (val && *val)
 487         c->group = xstrdup(val);
 488
 489     xfree(c->user);
 490     c->user = 0;
 491     val = ZOOM_options_get(c->options, "user");
 492     if (val && *val)
 493         c->user = xstrdup(val);
 494
 495     xfree(c->password);
 496     c->password = 0;
 497     val = ZOOM_options_get(c->options, "password");
 498     if (!val)
 499         val = ZOOM_options_get(c->options, "pass");
 500
 501     if (val && *val)
 502         c->password = xstrdup(val);
 503
 504     c->maximum_record_size =
 505         ZOOM_options_get_int(c->options, "maximumRecordSize", 1024*1024);
 506     c->preferred_message_size =
 507         ZOOM_options_get_int(c->options, "preferredMessageSize", 1024*1024);
 508
 509     c->async = ZOOM_options_get_bool(c->options, "async", 0);
 510
 511     if (c->sru_mode == zoom_sru_error)
 512     {
 513         ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, val);
 514         ZOOM_connection_remove_tasks(c);
 515         return;
 516     }
 517
 518     yaz_log(c->log_details, "%p ZOOM_connection_connect async=%d", c, c->async);
 519     ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
 520
 521     if (!c->async)
 522     {
 523         while (ZOOM_event(1, &c))
 524             ;
 525     }
 526 }
 527
 528 ZOOM_API(void) ZOOM_resultset_release(ZOOM_resultset r)
 529 {
 530 #if ZOOM_RESULT_LISTS
 531 #else
 532     if (r->connection)
 533     {
 534         /* remove ourselves from the resultsets in connection */
 535         ZOOM_resultset *rp = &r->connection->resultsets;
 536         while (1)
 537         {
 538             assert(*rp);   /* we must be in this list!! */
 539             if (*rp == r)
 540             {   /* OK, we're here - take us out of it */
 541                 *rp = (*rp)->next;
 542                 break;
 543             }
 544             rp = &(*rp)->next;
 545         }
 546         r->connection = 0;
 547     }
 548 #endif
 549 }
 550
 551 ZOOM_API(void)
 552     ZOOM_connection_destroy(ZOOM_connection c)
 553 {
 554 #if ZOOM_RESULT_LISTS
 555     ZOOM_resultsets list;
 556 #else
 557     ZOOM_resultset r;
 558 #endif
 559     if (!c)
 560         return;
 561     yaz_log(c->log_api, "%p ZOOM_connection_destroy", c);
 562     if (c->cs)
 563         cs_close(c->cs);
 564
 565 #if ZOOM_RESULT_LISTS
 566     /* Remove the connection's usage of resultsets */
 567     list = c->resultsets;
 568     while (list) {
 569         ZOOM_resultsets removed = list;
 570         ZOOM_resultset_destroy(list->resultset);
 571         list = list->next;
 572         xfree(removed);
 573     }
 574 #else
 575     for (r = c->resultsets; r; r = r->next)
 576         r->connection = 0;
 577 #endif
 578
 579     xfree(c->buf_in);
 580     xfree(c->addinfo);
 581     xfree(c->diagset);
 582     odr_destroy(c->odr_in);
 583     odr_destroy(c->odr_out);
 584     if (c->odr_print)
 585     {
 586         odr_setprint(c->odr_print, 0); /* prevent destroy from fclose'ing */
 587         odr_destroy(c->odr_print);
 588     }
 589     ZOOM_options_destroy(c->options);
 590     ZOOM_connection_remove_tasks(c);
 591     ZOOM_connection_remove_events(c);
 592     xfree(c->host_port);
 593     xfree(c->proxy);
 594     xfree(c->charset);
 595     xfree(c->lang);
 596     xfree(c->cookie_out);
 597     xfree(c->cookie_in);
 598     xfree(c->client_IP);
 599     xfree(c->user);
 600     xfree(c->group);
 601     xfree(c->password);
 602     xfree(c->sru_version);
 603     xfree(c);
 604 }
 605
 606 void ZOOM_resultset_addref(ZOOM_resultset r)
 607 {
 608     if (r)
 609     {
 610         yaz_mutex_enter(r->mutex);
 611         (r->refcount)++;
 612         yaz_log(log_details0, "%p ZOOM_resultset_addref count=%d",
 613                 r, r->refcount);
 614         yaz_mutex_leave(r->mutex);
 615     }
 616 }
 617
 618 static int g_resultsets = 0;
 619 static YAZ_MUTEX g_resultset_mutex = 0;
 620
 621 /* TODO We need to initialize this before running threaded:
 622  * call resultset_use(0)  */
 623
 624 static int resultset_use(int delta) {
 625     int resultset_count;
 626     if (g_resultset_mutex == 0)
 627         yaz_mutex_create(&g_resultset_mutex);
 628     yaz_mutex_enter(g_resultset_mutex);
 629     g_resultsets += delta;
 630     resultset_count = g_resultsets;
 631     yaz_mutex_leave(g_resultset_mutex);
 632     return resultset_count;
 633 }
 634
 635 int resultsets_count(void) {
 636     return resultset_use(0);
 637 }
 638
 639 ZOOM_resultset ZOOM_resultset_create(void)
 640 {
 641     int i;
 642     ZOOM_resultset r = (ZOOM_resultset) xmalloc(sizeof(*r));
 643
 644     initlog();
 645
 646     yaz_log(log_details0, "%p ZOOM_resultset_create", r);
 647     r->refcount = 1;
 648     r->size = 0;
 649     r->odr = odr_createmem(ODR_ENCODE);
 650     r->piggyback = 1;
 651     r->setname = 0;
 652     r->schema = 0;
 653     r->step = 0;
 654     for (i = 0; i<RECORD_HASH_SIZE; i++)
 655         r->record_hash[i] = 0;
 656     r->r_sort_spec = 0;
 657     r->query = 0;
 658     r->connection = 0;
 659     r->databaseNames = 0;
 660     r->num_databaseNames = 0;
 661     r->facets = 0;
 662     r->num_facets = 0;
 663     r->facets_names = 0;
 664     r->mutex = 0;
 665     yaz_mutex_create(&r->mutex);
 666 #if SHPTR
 667     {
 668         WRBUF w = wrbuf_alloc();
 669         YAZ_SHPTR_INIT(r->record_wrbuf, w);
 670     }
 671 #endif
 672     resultset_use(1);
 673     return r;
 674 }
 675
 676 ZOOM_API(ZOOM_resultset)
 677     ZOOM_connection_search_pqf(ZOOM_connection c, const char *q)
 678 {
 679     ZOOM_resultset r;
 680     ZOOM_query s = ZOOM_query_create();
 681
 682     ZOOM_query_prefix(s, q);
 683
 684     r = ZOOM_connection_search(c, s);
 685     ZOOM_query_destroy(s);
 686     return r;
 687 }
 688
 689 ZOOM_API(ZOOM_resultset)
 690     ZOOM_connection_search(ZOOM_connection c, ZOOM_query q)
 691 {
 692     ZOOM_resultset r = ZOOM_resultset_create();
 693     ZOOM_task task;
 694     const char *cp;
 695     int start, count;
 696     const char *syntax, *elementSetName;
 697 #if ZOOM_RESULT_LISTS
 698     ZOOM_resultsets set;
 699 #endif
 700
 701     yaz_log(c->log_api, "%p ZOOM_connection_search set %p query %p", c, r, q);
 702     r->r_sort_spec = ZOOM_query_get_sortspec(q);
 703     r->query = q;
 704
 705     r->options = ZOOM_options_create_with_parent(c->options);
 706
 707     start = ZOOM_options_get_int(r->options, "start", 0);
 708     count = ZOOM_options_get_int(r->options, "count", 0);
 709     {
 710         /* If "presentChunk" is defined use that; otherwise "step" */
 711         const char *cp = ZOOM_options_get(r->options, "presentChunk");
 712         r->step = ZOOM_options_get_int(r->options,
 713                                        (cp != 0 ? "presentChunk": "step"), 0);
 714     }
 715     r->piggyback = ZOOM_options_get_bool(r->options, "piggyback", 1);
 716     cp = ZOOM_options_get(r->options, "setname");
 717     if (cp)
 718         r->setname = xstrdup(cp);
 719     cp = ZOOM_options_get(r->options, "schema");
 720     if (cp)
 721         r->schema = xstrdup(cp);
 722
 723     r->databaseNames = ZOOM_connection_get_databases(c, c->options, &r->num_databaseNames,
 724                                          r->odr);
 725
 726     r->connection = c;
 727
 728 #if ZOOM_RESULT_LISTS
 729     yaz_log(log_details, "%p ZOOM_connection_search: Adding new resultset (%p) to resultsets (%p) ", c, r, c->resultsets);
 730     set = xmalloc(sizeof(*set));
 731     ZOOM_resultset_addref(r);
 732     set->resultset = r;
 733     set->next = c->resultsets;
 734     c->resultsets = set;
 735 #else
 736     r->next = c->resultsets;
 737     c->resultsets = r;
 738 #endif
 739     if (c->host_port && c->proto == PROTO_HTTP)
 740     {
 741         if (!c->cs)
 742         {
 743             yaz_log(c->log_details, "ZOOM_connection_search: no comstack");
 744             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
 745         }
 746         else
 747         {
 748             yaz_log(c->log_details, "ZOOM_connection_search: reconnect");
 749             c->reconnect_ok = 1;
 750         }
 751     }
 752
 753     task = ZOOM_connection_add_task(c, ZOOM_TASK_SEARCH);
 754     task->u.search.resultset = r;
 755     task->u.search.start = start;
 756     task->u.search.count = count;
 757     task->u.search.recv_search_fired = 0;
 758
 759     syntax = ZOOM_options_get(r->options, "preferredRecordSyntax");
 760     task->u.search.syntax = syntax ? xstrdup(syntax) : 0;
 761     elementSetName = ZOOM_options_get(r->options, "elementSetName");
 762     task->u.search.elementSetName = elementSetName
 763         ? xstrdup(elementSetName) : 0;
 764
 765     ZOOM_resultset_addref(r);
 766
 767     ZOOM_query_addref(q);
 768
 769     if (!c->async)
 770     {
 771         while (ZOOM_event(1, &c))
 772             ;
 773     }
 774     return r;
 775 }
 776
 777 ZOOM_API(void)
 778     ZOOM_resultset_sort(ZOOM_resultset r,
 779                          const char *sort_type, const char *sort_spec)
 780 {
 781     (void) ZOOM_resultset_sort1(r, sort_type, sort_spec);
 782 }
 783
 784 ZOOM_API(int)
 785     ZOOM_resultset_sort1(ZOOM_resultset r,
 786                          const char *sort_type, const char *sort_spec)
 787 {
 788     ZOOM_connection c = r->connection;
 789     ZOOM_task task;
 790     ZOOM_query newq;
 791
 792     newq = ZOOM_query_create();
 793     if (ZOOM_query_sortby(newq, sort_spec) < 0)
 794         return -1;
 795
 796     yaz_log(c->log_api, "%p ZOOM_resultset_sort r=%p sort_type=%s sort_spec=%s",
 797             r, r, sort_type, sort_spec);
 798     if (!c)
 799         return 0;
 800
 801     if (c->host_port && c->proto == PROTO_HTTP)
 802     {
 803         if (!c->cs)
 804         {
 805             yaz_log(c->log_details, "%p ZOOM_resultset_sort: no comstack", r);
 806             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
 807         }
 808         else
 809         {
 810             yaz_log(c->log_details, "%p ZOOM_resultset_sort: prepare reconnect",
 811                     r);
 812             c->reconnect_ok = 1;
 813         }
 814     }
 815
 816     ZOOM_resultset_cache_reset(r);
 817     task = ZOOM_connection_add_task(c, ZOOM_TASK_SORT);
 818     task->u.sort.resultset = r;
 819     task->u.sort.q = newq;
 820
 821     ZOOM_resultset_addref(r);
 822
 823     if (!c->async)
 824     {
 825         while (ZOOM_event(1, &c))
 826             ;
 827     }
 828
 829     return 0;
 830 }
 831
 832 ZOOM_API(void)
 833     ZOOM_resultset_destroy(ZOOM_resultset r)
 834 {
 835     resultset_destroy(r);
 836 }
 837
 838 static void resultset_destroy(ZOOM_resultset r)
 839 {
 840     if (!r)
 841         return;
 842     yaz_mutex_enter(r->mutex);
 843     (r->refcount)--;
 844     yaz_log(log_details0, "%p ZOOM_resultset_destroy r=%p count=%d",
 845             r, r, r->refcount);
 846     if (r->refcount == 0)
 847     {
 848         yaz_mutex_leave(r->mutex);
 849
 850         yaz_log(log_details0, "%p ZOOM_connection resultset_destroy: Deleting resultset (%p) ", r->connection, r);
 851         ZOOM_resultset_cache_reset(r);
 852         ZOOM_resultset_release(r);
 853         ZOOM_query_destroy(r->query);
 854         ZOOM_options_destroy(r->options);
 855         odr_destroy(r->odr);
 856         xfree(r->setname);
 857         xfree(r->schema);
 858         yaz_mutex_destroy(&r->mutex);
 859 #if SHPTR
 860         YAZ_SHPTR_DEC(r->record_wrbuf, wrbuf_destroy);
 861 #endif
 862         resultset_use(-1);
 863         xfree(r);
 864     }
 865     else
 866         yaz_mutex_leave(r->mutex);
 867 }
 868
 869 ZOOM_API(size_t)
 870     ZOOM_resultset_size(ZOOM_resultset r)
 871 {
 872     return r->size;
 873 }
 874
 875 int ZOOM_test_reconnect(ZOOM_connection c)
 876 {
 877     ZOOM_Event event;
 878
 879     if (!c->reconnect_ok)
 880         return 0;
 881     ZOOM_connection_close(c);
 882     c->reconnect_ok = 0;
 883     c->tasks->running = 0;
 884     ZOOM_connection_insert_task(c, ZOOM_TASK_CONNECT);
 885
 886     event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
 887     ZOOM_connection_put_event(c, event);
 888
 889     return 1;
 890 }
 891
 892 static void ZOOM_resultset_retrieve(ZOOM_resultset r,
 893                                     int force_sync, int start, int count)
 894 {
 895     ZOOM_task task;
 896     ZOOM_connection c;
 897     const char *cp;
 898     const char *syntax, *elementSetName;
 899
 900     if (!r)
 901         return;
 902     yaz_log(log_details0, "%p ZOOM_resultset_retrieve force_sync=%d start=%d"
 903             " count=%d", r, force_sync, start, count);
 904     c = r->connection;
 905     if (!c)
 906         return;
 907
 908     if (c->host_port && c->proto == PROTO_HTTP)
 909     {
 910         if (!c->cs)
 911         {
 912             yaz_log(log_details0, "%p ZOOM_resultset_retrieve: no comstack", r);
 913             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
 914         }
 915         else
 916         {
 917             yaz_log(log_details0, "%p ZOOM_resultset_retrieve: prepare "
 918                     "reconnect", r);
 919             c->reconnect_ok = 1;
 920         }
 921     }
 922     task = ZOOM_connection_add_task(c, ZOOM_TASK_RETRIEVE);
 923     task->u.retrieve.resultset = r;
 924     task->u.retrieve.start = start;
 925     task->u.retrieve.count = count;
 926
 927     syntax = ZOOM_options_get(r->options, "preferredRecordSyntax");
 928     task->u.retrieve.syntax = syntax ? xstrdup(syntax) : 0;
 929     elementSetName = ZOOM_options_get(r->options, "elementSetName");
 930     task->u.retrieve.elementSetName = elementSetName
 931         ? xstrdup(elementSetName) : 0;
 932
 933     cp = ZOOM_options_get(r->options, "schema");
 934     if (cp)
 935     {
 936         if (!r->schema || strcmp(r->schema, cp))
 937         {
 938             xfree(r->schema);
 939             r->schema = xstrdup(cp);
 940         }
 941     }
 942
 943     ZOOM_resultset_addref(r);
 944
 945     if (!r->connection->async || force_sync)
 946         while (r->connection && ZOOM_event(1, &r->connection))
 947             ;
 948 }
 949
 950 ZOOM_API(void)
 951     ZOOM_resultset_records(ZOOM_resultset r, ZOOM_record *recs,
 952                            size_t start, size_t count)
 953 {
 954     int force_present = 0;
 955
 956     if (!r)
 957         return ;
 958     yaz_log(log_api0, "%p ZOOM_resultset_records r=%p start=%ld count=%ld",
 959             r, r, (long) start, (long) count);
 960     if (count && recs)
 961         force_present = 1;
 962     ZOOM_resultset_retrieve(r, force_present, start, count);
 963     if (force_present)
 964     {
 965         size_t i;
 966         for (i = 0; i< count; i++)
 967             recs[i] = ZOOM_resultset_record_immediate(r, i+start);
 968     }
 969 }
 970
 971 ZOOM_API(size_t)
 972     ZOOM_resultset_facets_size(ZOOM_resultset r) {
 973     return r->num_facets;
 974 }
 975
 976 ZOOM_API(ZOOM_facet_field)
 977     ZOOM_resultset_get_facet_field(ZOOM_resultset r, const char *name) {
 978     int num = r->num_facets;
 979     ZOOM_facet_field *facets = r->facets;
 980     int index;
 981     for (index = 0; index < num; index++) {
 982         if (!strcmp(facets[index]->facet_name, name)) {
 983             return facets[index];
 984         }
 985     }
 986     return 0;
 987 }
 988
 989
 990 ZOOM_API(ZOOM_facet_field *)
 991     ZOOM_resultset_facets(ZOOM_resultset r)
 992 {
 993     return r->facets;
 994 }
 995
 996 ZOOM_API(const char**)
 997     ZOOM_resultset_facet_names(ZOOM_resultset r)
 998 {
 999     return (const char **) r->facets_names;
1000 }
1001
1002 ZOOM_API(const char*)
1003     ZOOM_facet_field_name(ZOOM_facet_field field)
1004 {
1005     return field->facet_name;
1006 }
1007
1008 ZOOM_API(size_t)
1009     ZOOM_facet_field_term_count(ZOOM_facet_field field)
1010 {
1011     return field->num_terms;
1012 }
1013
1014 ZOOM_API(const char*)
1015     ZOOM_facet_field_get_term(ZOOM_facet_field field, size_t idx, int *freq) {
1016     *freq = field->facet_terms[idx].frequency;
1017     return field->facet_terms[idx].term;
1018 }
1019
1020
1021 static void get_cert(ZOOM_connection c)
1022 {
1023     char *cert_buf;
1024     int cert_len;
1025
1026     if (cs_get_peer_certificate_x509(c->cs, &cert_buf, &cert_len))
1027     {
1028         ZOOM_connection_option_setl(c, "sslPeerCert",
1029                                     cert_buf, cert_len);
1030         xfree(cert_buf);
1031     }
1032 }
1033
1034 static zoom_ret do_connect_host(ZOOM_connection c,
1035                                 const char *logical_url);
1036
1037 static zoom_ret do_connect(ZOOM_connection c)
1038 {
1039     return do_connect_host(c, c->host_port);
1040 }
1041
1042 static zoom_ret do_connect_host(ZOOM_connection c, const char *logical_url)
1043 {
1044     void *add;
1045
1046     if (c->cs)
1047         cs_close(c->cs);
1048     c->cs = cs_create_host_proxy(logical_url, 0, &add, c->proxy);
1049
1050     if (c->cs && c->cs->protocol == PROTO_HTTP)
1051     {
1052 #if YAZ_HAVE_XML2
1053         c->proto = PROTO_HTTP;
1054 #else
1055         ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, "SRW");
1056         ZOOM_connection_close(c);
1057         return zoom_complete;
1058 #endif
1059     }
1060     if (c->cs)
1061     {
1062         int ret = cs_connect(c->cs, add);
1063         if (ret == 0)
1064         {
1065             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1066             ZOOM_connection_put_event(c, event);
1067             get_cert(c);
1068             if (c->proto == PROTO_Z3950)
1069                 ZOOM_connection_Z3950_send_init(c);
1070             else
1071             {
1072                 /* no init request for SRW .. */
1073                 assert(c->tasks->which == ZOOM_TASK_CONNECT);
1074                 ZOOM_connection_remove_task(c);
1075                 ZOOM_connection_set_mask(c, 0);
1076                 ZOOM_connection_exec_task(c);
1077             }
1078             c->state = STATE_ESTABLISHED;
1079             return zoom_pending;
1080         }
1081         else if (ret > 0)
1082         {
1083             int mask = ZOOM_SELECT_EXCEPT;
1084             if (c->cs->io_pending & CS_WANT_WRITE)
1085                 mask += ZOOM_SELECT_WRITE;
1086             if (c->cs->io_pending & CS_WANT_READ)
1087                 mask += ZOOM_SELECT_READ;
1088             ZOOM_connection_set_mask(c, mask);
1089             c->state = STATE_CONNECTING;
1090             return zoom_pending;
1091         }
1092     }
1093     c->state = STATE_IDLE;
1094     ZOOM_set_error(c, ZOOM_ERROR_CONNECT, logical_url);
1095     return zoom_complete;
1096 }
1097
1098 /* returns 1 if PDU was sent OK (still pending )
1099    0 if PDU was not sent OK (nothing to wait for)
1100 */
1101
1102 ZOOM_API(ZOOM_record)
1103     ZOOM_resultset_record_immediate(ZOOM_resultset s,size_t pos)
1104 {
1105     const char *syntax =
1106         ZOOM_options_get(s->options, "preferredRecordSyntax");
1107     const char *elementSetName =
1108         ZOOM_options_get(s->options, "elementSetName");
1109
1110     return ZOOM_record_cache_lookup(s, pos, syntax, elementSetName);
1111 }
1112
1113 ZOOM_API(ZOOM_record)
1114     ZOOM_resultset_record(ZOOM_resultset r, size_t pos)
1115 {
1116     ZOOM_record rec = ZOOM_resultset_record_immediate(r, pos);
1117
1118     if (!rec)
1119     {
1120         /*
1121          * MIKE: I think force_sync should always be zero, but I don't
1122          * want to make this change until I get the go-ahead from
1123          * Adam, in case something depends on the old synchronous
1124          * behaviour.
1125          */
1126         int force_sync = 1;
1127         if (getenv("ZOOM_RECORD_NO_FORCE_SYNC")) force_sync = 0;
1128         ZOOM_resultset_retrieve(r, force_sync, pos, 1);
1129         rec = ZOOM_resultset_record_immediate(r, pos);
1130     }
1131     return rec;
1132 }
1133
1134 ZOOM_API(ZOOM_scanset)
1135     ZOOM_connection_scan(ZOOM_connection c, const char *start)
1136 {
1137     ZOOM_scanset s;
1138     ZOOM_query q = ZOOM_query_create();
1139
1140     ZOOM_query_prefix(q, start);
1141
1142     s = ZOOM_connection_scan1(c, q);
1143     ZOOM_query_destroy(q);
1144     return s;
1145
1146 }
1147
1148 ZOOM_API(ZOOM_scanset)
1149     ZOOM_connection_scan1(ZOOM_connection c, ZOOM_query q)
1150 {
1151     ZOOM_scanset scan = 0;
1152     Z_Query *z_query = ZOOM_query_get_Z_Query(q);
1153
1154     if (!z_query)
1155         return 0;
1156     scan = (ZOOM_scanset) xmalloc(sizeof(*scan));
1157     scan->connection = c;
1158     scan->odr = odr_createmem(ODR_DECODE);
1159     scan->options = ZOOM_options_create_with_parent(c->options);
1160     scan->refcount = 1;
1161     scan->scan_response = 0;
1162     scan->srw_scan_response = 0;
1163
1164     scan->query = q;
1165     ZOOM_query_addref(q);
1166     scan->databaseNames = ZOOM_connection_get_databases(c, c->options,
1167                                             &scan->num_databaseNames,
1168                                             scan->odr);
1169
1170     if (1)
1171     {
1172         ZOOM_task task = ZOOM_connection_add_task(c, ZOOM_TASK_SCAN);
1173         task->u.scan.scan = scan;
1174
1175         (scan->refcount)++;
1176         if (!c->async)
1177         {
1178             while (ZOOM_event(1, &c))
1179                 ;
1180         }
1181     }
1182     return scan;
1183 }
1184
1185 ZOOM_API(void)
1186     ZOOM_scanset_destroy(ZOOM_scanset scan)
1187 {
1188     if (!scan)
1189         return;
1190     (scan->refcount)--;
1191     if (scan->refcount == 0)
1192     {
1193         ZOOM_query_destroy(scan->query);
1194
1195         odr_destroy(scan->odr);
1196
1197         ZOOM_options_destroy(scan->options);
1198         xfree(scan);
1199     }
1200 }
1201
1202 static zoom_ret send_package(ZOOM_connection c)
1203 {
1204     ZOOM_Event event;
1205
1206     yaz_log(c->log_details, "%p send_package", c);
1207     if (!c->tasks)
1208         return zoom_complete;
1209     assert (c->tasks->which == ZOOM_TASK_PACKAGE);
1210
1211     event = ZOOM_Event_create(ZOOM_EVENT_SEND_APDU);
1212     ZOOM_connection_put_event(c, event);
1213
1214     c->buf_out = c->tasks->u.package->buf_out;
1215     c->len_out = c->tasks->u.package->len_out;
1216
1217     return ZOOM_send_buf(c);
1218 }
1219
1220 ZOOM_API(size_t)
1221     ZOOM_scanset_size(ZOOM_scanset scan)
1222 {
1223     if (!scan)
1224         return 0;
1225
1226     if (scan->scan_response && scan->scan_response->entries)
1227         return scan->scan_response->entries->num_entries;
1228     else if (scan->srw_scan_response)
1229         return scan->srw_scan_response->num_terms;
1230     return 0;
1231 }
1232
1233 static void ZOOM_scanset_term_x(ZOOM_scanset scan, size_t pos,
1234                                 size_t *occ,
1235                                 const char **value_term, size_t *value_len,
1236                                 const char **disp_term, size_t *disp_len)
1237 {
1238     size_t noent = ZOOM_scanset_size(scan);
1239
1240     *value_term = 0;
1241     *value_len = 0;
1242
1243     *disp_term = 0;
1244     *disp_len = 0;
1245
1246     *occ = 0;
1247     if (pos >= noent)
1248         return;
1249     if (scan->scan_response)
1250     {
1251         Z_ScanResponse *res = scan->scan_response;
1252         if (res->entries->entries[pos]->which == Z_Entry_termInfo)
1253         {
1254             Z_TermInfo *t = res->entries->entries[pos]->u.termInfo;
1255
1256             *value_term = (const char *) t->term->u.general->buf;
1257             *value_len = t->term->u.general->len;
1258             if (t->displayTerm)
1259             {
1260                 *disp_term = t->displayTerm;
1261                 *disp_len = strlen(*disp_term);
1262             }
1263             else if (t->term->which == Z_Term_general)
1264             {
1265                 *disp_term = (const char *) t->term->u.general->buf;
1266                 *disp_len = t->term->u.general->len;
1267             }
1268             *occ = t->globalOccurrences ? *t->globalOccurrences : 0;
1269         }
1270     }
1271     if (scan->srw_scan_response)
1272     {
1273         Z_SRW_scanResponse *res = scan->srw_scan_response;
1274         Z_SRW_scanTerm *t = res->terms + pos;
1275         if (t)
1276         {
1277             *value_term = t->value;
1278             *value_len = strlen(*value_term);
1279
1280             if (t->displayTerm)
1281                 *disp_term = t->displayTerm;
1282             else
1283                 *disp_term = t->value;
1284             *disp_len = strlen(*disp_term);
1285             *occ = t->numberOfRecords ? *t->numberOfRecords : 0;
1286         }
1287     }
1288 }
1289
1290 ZOOM_API(const char *)
1291     ZOOM_scanset_term(ZOOM_scanset scan, size_t pos,
1292                       size_t *occ, size_t *len)
1293 {
1294     const char *value_term = 0;
1295     size_t value_len = 0;
1296     const char *disp_term = 0;
1297     size_t disp_len = 0;
1298
1299     ZOOM_scanset_term_x(scan, pos, occ, &value_term, &value_len,
1300                         &disp_term, &disp_len);
1301
1302     *len = value_len;
1303     return value_term;
1304 }
1305
1306 ZOOM_API(const char *)
1307     ZOOM_scanset_display_term(ZOOM_scanset scan, size_t pos,
1308                               size_t *occ, size_t *len)
1309 {
1310     const char *value_term = 0;
1311     size_t value_len = 0;
1312     const char *disp_term = 0;
1313     size_t disp_len = 0;
1314
1315     ZOOM_scanset_term_x(scan, pos, occ, &value_term, &value_len,
1316                         &disp_term, &disp_len);
1317
1318     *len = disp_len;
1319     return disp_term;
1320 }
1321
1322 ZOOM_API(const char *)
1323     ZOOM_scanset_option_get(ZOOM_scanset scan, const char *key)
1324 {
1325     return ZOOM_options_get(scan->options, key);
1326 }
1327
1328 ZOOM_API(void)
1329     ZOOM_scanset_option_set(ZOOM_scanset scan, const char *key,
1330                             const char *val)
1331 {
1332     ZOOM_options_set(scan->options, key, val);
1333 }
1334
1335
1336 ZOOM_API(ZOOM_package)
1337     ZOOM_connection_package(ZOOM_connection c, ZOOM_options options)
1338 {
1339     ZOOM_package p = (ZOOM_package) xmalloc(sizeof(*p));
1340
1341     p->connection = c;
1342     p->odr_out = odr_createmem(ODR_ENCODE);
1343     p->options = ZOOM_options_create_with_parent2(options, c->options);
1344     p->refcount = 1;
1345     p->buf_out = 0;
1346     p->len_out = 0;
1347     return p;
1348 }
1349
1350 ZOOM_API(void)
1351     ZOOM_package_destroy(ZOOM_package p)
1352 {
1353     if (!p)
1354         return;
1355     (p->refcount)--;
1356     if (p->refcount == 0)
1357     {
1358         odr_destroy(p->odr_out);
1359         xfree(p->buf_out);
1360
1361         ZOOM_options_destroy(p->options);
1362         xfree(p);
1363     }
1364 }
1365
1366 ZOOM_API(const char *)
1367     ZOOM_package_option_get(ZOOM_package p, const char *key)
1368 {
1369     return ZOOM_options_get(p->options, key);
1370 }
1371
1372 ZOOM_API(const char *)
1373     ZOOM_package_option_getl(ZOOM_package p, const char *key, int *lenp)
1374 {
1375     return ZOOM_options_getl(p->options, key, lenp);
1376 }
1377
1378 ZOOM_API(void)
1379     ZOOM_package_option_set(ZOOM_package p, const char *key,
1380                             const char *val)
1381 {
1382     ZOOM_options_set(p->options, key, val);
1383 }
1384
1385 ZOOM_API(void)
1386     ZOOM_package_option_setl(ZOOM_package p, const char *key,
1387                              const char *val, int len)
1388 {
1389     ZOOM_options_setl(p->options, key, val, len);
1390 }
1391
1392 ZOOM_API(int)
1393     ZOOM_connection_exec_task(ZOOM_connection c)
1394 {
1395     ZOOM_task task = c->tasks;
1396     zoom_ret ret = zoom_complete;
1397
1398     if (!task)
1399         return 0;
1400     yaz_log(c->log_details, "%p ZOOM_connection_exec_task type=%d run=%d",
1401             c, task->which, task->running);
1402     if (c->error != ZOOM_ERROR_NONE)
1403     {
1404         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1405                 "removing tasks because of error = %d", c, c->error);
1406         ZOOM_connection_remove_tasks(c);
1407         return 0;
1408     }
1409     if (task->running)
1410     {
1411         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1412                 "task already running", c);
1413         return 0;
1414     }
1415     task->running = 1;
1416     ret = zoom_complete;
1417     if (c->cs || task->which == ZOOM_TASK_CONNECT)
1418     {
1419         switch (task->which)
1420         {
1421         case ZOOM_TASK_SEARCH:
1422             if (c->proto == PROTO_HTTP)
1423                 ret = ZOOM_connection_srw_send_search(c);
1424             else
1425                 ret = ZOOM_connection_Z3950_send_search(c);
1426             break;
1427         case ZOOM_TASK_RETRIEVE:
1428             if (c->proto == PROTO_HTTP)
1429                 ret = ZOOM_connection_srw_send_search(c);
1430             else
1431                 ret = send_Z3950_present(c);
1432             break;
1433         case ZOOM_TASK_CONNECT:
1434             ret = do_connect(c);
1435             break;
1436         case ZOOM_TASK_SCAN:
1437             if (c->proto == PROTO_HTTP)
1438                 ret = ZOOM_connection_srw_send_scan(c);
1439             else
1440                 ret = ZOOM_connection_Z3950_send_scan(c);
1441             break;
1442         case ZOOM_TASK_PACKAGE:
1443             ret = send_package(c);
1444             break;
1445         case ZOOM_TASK_SORT:
1446             c->tasks->u.sort.resultset->r_sort_spec =
1447                 ZOOM_query_get_sortspec(c->tasks->u.sort.q);
1448             ret = send_Z3950_sort(c, c->tasks->u.sort.resultset);
1449             break;
1450         }
1451     }
1452     else
1453     {
1454         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1455                 "remove tasks because no connection exist", c);
1456         ZOOM_connection_remove_tasks(c);
1457     }
1458     if (ret == zoom_complete)
1459     {
1460         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1461                 "task removed (complete)", c);
1462         ZOOM_connection_remove_task(c);
1463         return 0;
1464     }
1465     yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1466             "task pending", c);
1467     return 1;
1468 }
1469
1470 #if YAZ_HAVE_XML2
1471
1472 static zoom_ret send_HTTP_redirect(ZOOM_connection c, const char *uri,
1473                                   Z_HTTP_Response *cookie_hres)
1474 {
1475     struct Z_HTTP_Header *h;
1476     char *combined_cookies = 0;
1477     int combined_cookies_len = 0;
1478     Z_GDU *gdu = z_get_HTTP_Request_uri(c->odr_out, uri, 0, c->proxy ? 1 : 0);
1479
1480     gdu->u.HTTP_Request->method = odr_strdup(c->odr_out, "GET");
1481     z_HTTP_header_add(c->odr_out, &gdu->u.HTTP_Request->headers, "Accept",
1482                       "text/xml");
1483
1484     for (h = cookie_hres->headers; h; h = h->next)
1485     {
1486         if (!strcmp(h->name, "Set-Cookie"))
1487         {
1488             char *cp;
1489
1490             if (!(cp = strchr(h->value, ';')))
1491                 cp = h->value + strlen(h->value);
1492             if (cp - h->value >= 1) {
1493                 combined_cookies = xrealloc(combined_cookies, combined_cookies_len + cp - h->value + 3);
1494                 memcpy(combined_cookies+combined_cookies_len, h->value, cp - h->value);
1495                 combined_cookies[combined_cookies_len + cp - h->value] = '\0';
1496                 strcat(combined_cookies,"; ");
1497                 combined_cookies_len = strlen(combined_cookies);
1498             }
1499         }
1500     }
1501
1502     if (combined_cookies_len)
1503     {
1504         z_HTTP_header_add(c->odr_out, &gdu->u.HTTP_Request->headers,
1505                           "Cookie", combined_cookies);
1506         xfree(combined_cookies);
1507     }
1508
1509     if (c->user && c->password)
1510     {
1511         z_HTTP_header_add_basic_auth(c->odr_out, &gdu->u.HTTP_Request->headers,
1512                                      c->user, c->password);
1513     }
1514     if (!z_GDU(c->odr_out, &gdu, 0, 0))
1515         return zoom_complete;
1516     if (c->odr_print)
1517         z_GDU(c->odr_print, &gdu, 0, 0);
1518     c->buf_out = odr_getbuf(c->odr_out, &c->len_out, 0);
1519
1520     odr_reset(c->odr_out);
1521     return ZOOM_send_buf(c);
1522 }
1523
1524 #if YAZ_HAVE_XML2
1525 void ZOOM_set_HTTP_error(ZOOM_connection c, int error,
1526                          const char *addinfo, const char *addinfo2)
1527 {
1528     ZOOM_set_dset_error(c, error, "HTTP", addinfo, addinfo2);
1529 }
1530 #endif
1531
1532
1533 static void handle_http(ZOOM_connection c, Z_HTTP_Response *hres)
1534 {
1535     zoom_ret cret = zoom_complete;
1536     int ret = -1;
1537     char *addinfo = 0;
1538     const char *connection_head = z_HTTP_header_lookup(hres->headers,
1539                                                        "Connection");
1540     const char *location;
1541
1542     ZOOM_connection_set_mask(c, 0);
1543     yaz_log(c->log_details, "%p handle_http", c);
1544
1545     if ((hres->code == 301 || hres->code == 302) && c->sru_mode == zoom_sru_get
1546         && (location = z_HTTP_header_lookup(hres->headers, "Location")))
1547     {
1548         c->no_redirects++;
1549         if (c->no_redirects > 10)
1550         {
1551             ZOOM_set_HTTP_error(c, hres->code, 0, 0);
1552             c->no_redirects = 0;
1553             ZOOM_connection_close(c);
1554         }
1555         else
1556         {
1557             /* since redirect may change host we just reconnect. A smarter
1558                implementation might check whether it's the same server */
1559             do_connect_host(c, location);
1560             send_HTTP_redirect(c, location, hres);
1561             /* we're OK for now. Operation is not really complete */
1562             return;
1563         }
1564     }
1565     else
1566     {
1567         ret = ZOOM_handle_sru(c, hres, &cret, &addinfo);
1568         if (ret == 0)
1569         {
1570             if (c->no_redirects) /* end of redirect. change hosts again */
1571                 ZOOM_connection_close(c);
1572         }
1573         c->no_redirects = 0;
1574     }
1575     if (ret)
1576     {
1577         if (hres->code != 200)
1578             ZOOM_set_HTTP_error(c, hres->code, 0, 0);
1579         else
1580         {
1581             yaz_log(YLOG_LOG, "set error... addinfo=%s", addinfo ?
1582                     addinfo : "NULL");
1583             ZOOM_set_error(c, ZOOM_ERROR_DECODE, addinfo);
1584         }
1585         ZOOM_connection_close(c);
1586     }
1587     if (cret == zoom_complete)
1588     {
1589         yaz_log(c->log_details, "removing tasks in handle_http");
1590         ZOOM_connection_remove_task(c);
1591     }
1592     {
1593         int must_close = 0;
1594         if (!strcmp(hres->version, "1.0"))
1595         {
1596             /* HTTP 1.0: only if Keep-Alive we stay alive.. */
1597             if (!connection_head || strcmp(connection_head, "Keep-Alive"))
1598                 must_close = 1;
1599         }
1600         else
1601         {
1602             /* HTTP 1.1: only if no close we stay alive.. */
1603             if (connection_head && !strcmp(connection_head, "close"))
1604                 must_close = 1;
1605         }
1606         if (must_close)
1607         {
1608             ZOOM_connection_close(c);
1609             if (c->tasks)
1610             {
1611                 c->tasks->running = 0;
1612                 ZOOM_connection_insert_task(c, ZOOM_TASK_CONNECT);
1613                 c->reconnect_ok = 0;
1614             }
1615         }
1616         else
1617             c->reconnect_ok = 1; /* if the server closes anyway */
1618     }
1619 }
1620 #endif
1621
1622 static int do_read(ZOOM_connection c)
1623 {
1624     int r, more;
1625     ZOOM_Event event;
1626
1627     event = ZOOM_Event_create(ZOOM_EVENT_RECV_DATA);
1628     ZOOM_connection_put_event(c, event);
1629
1630     r = cs_get(c->cs, &c->buf_in, &c->len_in);
1631     more = cs_more(c->cs);
1632     yaz_log(c->log_details, "%p do_read len=%d more=%d", c, r, more);
1633     if (r == 1)
1634         return 0;
1635     if (r <= 0)
1636     {
1637         if (!ZOOM_test_reconnect(c))
1638         {
1639             ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
1640             ZOOM_connection_close(c);
1641         }
1642     }
1643     else
1644     {
1645         Z_GDU *gdu;
1646         ZOOM_Event event;
1647
1648         odr_reset(c->odr_in);
1649         odr_setbuf(c->odr_in, c->buf_in, r, 0);
1650         event = ZOOM_Event_create(ZOOM_EVENT_RECV_APDU);
1651         ZOOM_connection_put_event(c, event);
1652
1653         if (!z_GDU(c->odr_in, &gdu, 0, 0))
1654         {
1655             int x;
1656             int err = odr_geterrorx(c->odr_in, &x);
1657             char msg[100];
1658             const char *element = odr_getelement(c->odr_in);
1659             yaz_snprintf(msg, sizeof(msg),
1660                     "ODR code %d:%d element=%s offset=%d",
1661                     err, x, element ? element : "<unknown>",
1662                     odr_offset(c->odr_in));
1663             ZOOM_set_error(c, ZOOM_ERROR_DECODE, msg);
1664             if (c->log_api)
1665             {
1666                 FILE *ber_file = yaz_log_file();
1667                 if (ber_file)
1668                     odr_dumpBER(ber_file, c->buf_in, r);
1669             }
1670             ZOOM_connection_close(c);
1671         }
1672         else
1673         {
1674             if (c->odr_print)
1675                 z_GDU(c->odr_print, &gdu, 0, 0);
1676             if (gdu->which == Z_GDU_Z3950)
1677                 ZOOM_handle_Z3950_apdu(c, gdu->u.z3950);
1678             else if (gdu->which == Z_GDU_HTTP_Response)
1679             {
1680 #if YAZ_HAVE_XML2
1681                 handle_http(c, gdu->u.HTTP_Response);
1682 #else
1683                 ZOOM_set_error(c, ZOOM_ERROR_DECODE, 0);
1684                 ZOOM_connection_close(c);
1685 #endif
1686             }
1687         }
1688     }
1689     return 1;
1690 }
1691
1692 static zoom_ret do_write_ex(ZOOM_connection c, char *buf_out, int len_out)
1693 {
1694     int r;
1695     ZOOM_Event event;
1696
1697     event = ZOOM_Event_create(ZOOM_EVENT_SEND_DATA);
1698     ZOOM_connection_put_event(c, event);
1699
1700     yaz_log(c->log_details, "%p do_write_ex len=%d", c, len_out);
1701     if ((r = cs_put(c->cs, buf_out, len_out)) < 0)
1702     {
1703         yaz_log(c->log_details, "%p do_write_ex write failed", c);
1704         if (ZOOM_test_reconnect(c))
1705         {
1706             return zoom_pending;
1707         }
1708         if (c->state == STATE_CONNECTING)
1709             ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
1710         else
1711             ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
1712         ZOOM_connection_close(c);
1713         return zoom_complete;
1714     }
1715     else if (r == 1)
1716     {
1717         int mask = ZOOM_SELECT_EXCEPT;
1718         if (c->cs->io_pending & CS_WANT_WRITE)
1719             mask += ZOOM_SELECT_WRITE;
1720         if (c->cs->io_pending & CS_WANT_READ)
1721             mask += ZOOM_SELECT_READ;
1722         ZOOM_connection_set_mask(c, mask);
1723         yaz_log(c->log_details, "%p do_write_ex write incomplete mask=%d",
1724                 c, c->mask);
1725     }
1726     else
1727     {
1728         ZOOM_connection_set_mask(c, ZOOM_SELECT_READ|ZOOM_SELECT_EXCEPT);
1729         yaz_log(c->log_details, "%p do_write_ex write complete mask=%d",
1730                 c, c->mask);
1731     }
1732     return zoom_pending;
1733 }
1734
1735 zoom_ret ZOOM_send_buf(ZOOM_connection c)
1736 {
1737     return do_write_ex(c, c->buf_out, c->len_out);
1738 }
1739
1740
1741 ZOOM_API(const char *)
1742     ZOOM_connection_option_get(ZOOM_connection c, const char *key)
1743 {
1744     return ZOOM_options_get(c->options, key);
1745 }
1746
1747 ZOOM_API(const char *)
1748     ZOOM_connection_option_getl(ZOOM_connection c, const char *key, int *lenp)
1749 {
1750     return ZOOM_options_getl(c->options, key, lenp);
1751 }
1752
1753 ZOOM_API(void)
1754     ZOOM_connection_option_set(ZOOM_connection c, const char *key,
1755                                const char *val)
1756 {
1757     ZOOM_options_set(c->options, key, val);
1758 }
1759
1760 ZOOM_API(void)
1761     ZOOM_connection_option_setl(ZOOM_connection c, const char *key,
1762                                 const char *val, int len)
1763 {
1764     ZOOM_options_setl(c->options, key, val, len);
1765 }
1766
1767 ZOOM_API(const char *)
1768     ZOOM_resultset_option_get(ZOOM_resultset r, const char *key)
1769 {
1770     return ZOOM_options_get(r->options, key);
1771 }
1772
1773 ZOOM_API(void)
1774     ZOOM_resultset_option_set(ZOOM_resultset r, const char *key,
1775                               const char *val)
1776 {
1777     ZOOM_options_set(r->options, key, val);
1778 }
1779
1780
1781 ZOOM_API(int)
1782     ZOOM_connection_errcode(ZOOM_connection c)
1783 {
1784     return ZOOM_connection_error(c, 0, 0);
1785 }
1786
1787 ZOOM_API(const char *)
1788     ZOOM_connection_errmsg(ZOOM_connection c)
1789 {
1790     const char *msg;
1791     ZOOM_connection_error(c, &msg, 0);
1792     return msg;
1793 }
1794
1795 ZOOM_API(const char *)
1796     ZOOM_connection_addinfo(ZOOM_connection c)
1797 {
1798     const char *addinfo;
1799     ZOOM_connection_error(c, 0, &addinfo);
1800     return addinfo;
1801 }
1802
1803 ZOOM_API(const char *)
1804     ZOOM_connection_diagset(ZOOM_connection c)
1805 {
1806     const char *diagset;
1807     ZOOM_connection_error_x(c, 0, 0, &diagset);
1808     return diagset;
1809 }
1810
1811 ZOOM_API(const char *)
1812     ZOOM_diag_str(int error)
1813 {
1814     switch (error)
1815     {
1816     case ZOOM_ERROR_NONE:
1817         return "No error";
1818     case ZOOM_ERROR_CONNECT:
1819         return "Connect failed";
1820     case ZOOM_ERROR_MEMORY:
1821         return "Out of memory";
1822     case ZOOM_ERROR_ENCODE:
1823         return "Encoding failed";
1824     case ZOOM_ERROR_DECODE:
1825         return "Decoding failed";
1826     case ZOOM_ERROR_CONNECTION_LOST:
1827         return "Connection lost";
1828     case ZOOM_ERROR_INIT:
1829         return "Init rejected";
1830     case ZOOM_ERROR_INTERNAL:
1831         return "Internal failure";
1832     case ZOOM_ERROR_TIMEOUT:
1833         return "Timeout";
1834     case ZOOM_ERROR_UNSUPPORTED_PROTOCOL:
1835         return "Unsupported protocol";
1836     case ZOOM_ERROR_UNSUPPORTED_QUERY:
1837         return "Unsupported query type";
1838     case ZOOM_ERROR_INVALID_QUERY:
1839         return "Invalid query";
1840     case ZOOM_ERROR_CQL_PARSE:
1841         return "CQL parsing error";
1842     case ZOOM_ERROR_CQL_TRANSFORM:
1843         return "CQL transformation error";
1844     case ZOOM_ERROR_CCL_CONFIG:
1845         return "CCL configuration error";
1846     case ZOOM_ERROR_CCL_PARSE:
1847         return "CCL parsing error";
1848     case ZOOM_ERROR_ES_INVALID_ACTION:
1849         return "Extended Service. invalid action";
1850     case ZOOM_ERROR_ES_INVALID_VERSION:
1851         return "Extended Service. invalid version";
1852     case ZOOM_ERROR_ES_INVALID_SYNTAX:
1853         return "Extended Service. invalid syntax";
1854     default:
1855         return diagbib1_str(error);
1856     }
1857 }
1858
1859 ZOOM_API(int)
1860     ZOOM_connection_error_x(ZOOM_connection c, const char **cp,
1861                             const char **addinfo, const char **diagset)
1862 {
1863     int error = c->error;
1864     if (cp)
1865     {
1866         if (!c->diagset || !strcmp(c->diagset, "ZOOM"))
1867             *cp = ZOOM_diag_str(error);
1868         else if (!strcmp(c->diagset, "HTTP"))
1869             *cp = z_HTTP_errmsg(c->error);
1870         else if (!strcmp(c->diagset, "Bib-1"))
1871             *cp = ZOOM_diag_str(error);
1872         else if (!strcmp(c->diagset, "info:srw/diagnostic/1"))
1873             *cp = yaz_diag_srw_str(c->error);
1874         else
1875             *cp = "Unknown error and diagnostic set";
1876     }
1877     if (addinfo)
1878         *addinfo = c->addinfo ? c->addinfo : "";
1879     if (diagset)
1880         *diagset = c->diagset ? c->diagset : "";
1881     return c->error;
1882 }
1883
1884 ZOOM_API(int)
1885     ZOOM_connection_error(ZOOM_connection c, const char **cp,
1886                           const char **addinfo)
1887 {
1888     return ZOOM_connection_error_x(c, cp, addinfo, 0);
1889 }
1890
1891 static void ZOOM_connection_do_io(ZOOM_connection c, int mask)
1892 {
1893     ZOOM_Event event = 0;
1894     int r = cs_look(c->cs);
1895     yaz_log(c->log_details, "%p ZOOM_connection_do_io mask=%d cs_look=%d",
1896             c, mask, r);
1897
1898     if (r == CS_NONE)
1899     {
1900         event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1901         ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
1902         ZOOM_connection_close(c);
1903         ZOOM_connection_put_event(c, event);
1904     }
1905     else if (r == CS_CONNECT)
1906     {
1907         int ret = ret = cs_rcvconnect(c->cs);
1908         yaz_log(c->log_details, "%p ZOOM_connection_do_io "
1909                 "cs_rcvconnect returned %d", c, ret);
1910         if (ret == 1)
1911         {
1912             int mask = ZOOM_SELECT_EXCEPT;
1913             if (c->cs->io_pending & CS_WANT_WRITE)
1914                 mask += ZOOM_SELECT_WRITE;
1915             if (c->cs->io_pending & CS_WANT_READ)
1916                 mask += ZOOM_SELECT_READ;
1917             ZOOM_connection_set_mask(c, mask);
1918             event = ZOOM_Event_create(ZOOM_EVENT_NONE);
1919             ZOOM_connection_put_event(c, event);
1920         }
1921         else if (ret == 0)
1922         {
1923             event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1924             ZOOM_connection_put_event(c, event);
1925             get_cert(c);
1926             if (c->proto == PROTO_Z3950)
1927                 ZOOM_connection_Z3950_send_init(c);
1928             else
1929             {
1930                 /* no init request for SRW .. */
1931                 assert(c->tasks->which == ZOOM_TASK_CONNECT);
1932                 ZOOM_connection_remove_task(c);
1933                 ZOOM_connection_set_mask(c, 0);
1934                 ZOOM_connection_exec_task(c);
1935             }
1936             c->state = STATE_ESTABLISHED;
1937         }
1938         else
1939         {
1940             ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
1941             ZOOM_connection_close(c);
1942         }
1943     }
1944     else
1945     {
1946         if (mask & ZOOM_SELECT_EXCEPT)
1947         {
1948             if (!ZOOM_test_reconnect(c))
1949             {
1950                 ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
1951                 ZOOM_connection_close(c);
1952             }
1953             return;
1954         }
1955         if (mask & ZOOM_SELECT_READ)
1956             do_read(c);
1957         if (c->cs && (mask & ZOOM_SELECT_WRITE))
1958             ZOOM_send_buf(c);
1959     }
1960 }
1961
1962 ZOOM_API(int)
1963     ZOOM_connection_last_event(ZOOM_connection cs)
1964 {
1965     if (!cs)
1966         return ZOOM_EVENT_NONE;
1967     return cs->last_event;
1968 }
1969
1970
1971 ZOOM_API(int) ZOOM_connection_fire_event_timeout(ZOOM_connection c)
1972 {
1973     if (c->mask)
1974     {
1975         ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_TIMEOUT);
1976         /* timeout and this connection was waiting */
1977         ZOOM_set_error(c, ZOOM_ERROR_TIMEOUT, 0);
1978         ZOOM_connection_close(c);
1979         ZOOM_connection_put_event(c, event);
1980     }
1981     return 0;
1982 }
1983
1984 ZOOM_API(int)
1985     ZOOM_connection_process(ZOOM_connection c)
1986 {
1987     ZOOM_Event event;
1988     if (!c)
1989         return 0;
1990
1991     event = ZOOM_connection_get_event(c);
1992     if (event)
1993     {
1994         ZOOM_Event_destroy(event);
1995         return 1;
1996     }
1997     ZOOM_connection_exec_task(c);
1998     event = ZOOM_connection_get_event(c);
1999     if (event)
2000     {
2001         ZOOM_Event_destroy(event);
2002         return 1;
2003     }
2004     return 0;
2005 }
2006
2007 ZOOM_API(int)
2008     ZOOM_event_nonblock(int no, ZOOM_connection *cs)
2009 {
2010     int i;
2011
2012     yaz_log(log_details0, "ZOOM_process_event(no=%d,cs=%p)", no, cs);
2013
2014     for (i = 0; i<no; i++)
2015     {
2016         ZOOM_connection c = cs[i];
2017
2018         if (c && ZOOM_connection_process(c))
2019             return i+1;
2020     }
2021     return 0;
2022 }
2023
2024 ZOOM_API(int) ZOOM_connection_fire_event_socket(ZOOM_connection c, int mask)
2025 {
2026     if (c->mask && mask)
2027         ZOOM_connection_do_io(c, mask);
2028     return 0;
2029 }
2030
2031 ZOOM_API(int) ZOOM_connection_get_socket(ZOOM_connection c)
2032 {
2033     if (c->cs)
2034         return cs_fileno(c->cs);
2035     return -1;
2036 }
2037
2038 ZOOM_API(int) ZOOM_connection_set_mask(ZOOM_connection c, int mask)
2039 {
2040     c->mask = mask;
2041     if (!c->cs)
2042         return -1;
2043     return 0;
2044 }
2045
2046 ZOOM_API(int) ZOOM_connection_get_mask(ZOOM_connection c)
2047 {
2048     if (c->cs)
2049         return c->mask;
2050     return 0;
2051 }
2052
2053 ZOOM_API(int) ZOOM_connection_get_timeout(ZOOM_connection c)
2054 {
2055     return ZOOM_options_get_int(c->options, "timeout", 30);
2056 }
2057
2058 ZOOM_API(void) ZOOM_connection_close(ZOOM_connection c)
2059 {
2060     if (c->cs)
2061         cs_close(c->cs);
2062     c->cs = 0;
2063     ZOOM_connection_set_mask(c, 0);
2064     c->state = STATE_IDLE;
2065 }
2066
2067 /*
2068  * Local variables:
2069  * c-basic-offset: 4
2070  * c-file-style: "Stroustrup"
2071  * indent-tabs-mode: nil
2072  * End:
2073  * vim: shiftwidth=4 tabstop=8 expandtab
2074  */
2075