src/zoom-c.c

   1 /* This file is part of the YAZ toolkit.
   2  * Copyright (C) 1995-2011 Index Data
   3  * See the file LICENSE for details.
   4  */
   5 /**
   6  * \file zoom-c.c
   7  * \brief Implements ZOOM C interface.
   8  */
   9 #if HAVE_CONFIG_H
  10 #include <config.h>
  11 #endif
  12
  13 #include <assert.h>
  14 #include <string.h>
  15 #include <errno.h>
  16 #include "zoom-p.h"
  17
  18 #include <yaz/yaz-util.h>
  19 #include <yaz/xmalloc.h>
  20 #include <yaz/otherinfo.h>
  21 #include <yaz/log.h>
  22 #include <yaz/diagbib1.h>
  23 #include <yaz/charneg.h>
  24 #include <yaz/query-charset.h>
  25 #include <yaz/snprintf.h>
  26 #include <yaz/facet.h>
  27
  28 #include <yaz/shptr.h>
  29
  30 #if SHPTR
  31 YAZ_SHPTR_TYPE(WRBUF)
  32 #endif
  33
  34 static int log_api0 = 0;
  35 static int log_details0 = 0;
  36
  37 static void resultset_destroy(ZOOM_resultset r);
  38 static zoom_ret do_write_ex(ZOOM_connection c, char *buf_out, int len_out);
  39
  40 static void initlog(void)
  41 {
  42     static int log_level_initialized = 0;
  43
  44     if (!log_level_initialized)
  45     {
  46         log_api0 = yaz_log_module_level("zoom");
  47         log_details0 = yaz_log_module_level("zoomdetails");
  48         log_level_initialized = 1;
  49     }
  50 }
  51
  52 void ZOOM_connection_remove_tasks(ZOOM_connection c);
  53
  54 void ZOOM_set_dset_error(ZOOM_connection c, int error,
  55                          const char *dset,
  56                          const char *addinfo, const char *addinfo2)
  57 {
  58     char *cp;
  59
  60     xfree(c->addinfo);
  61     c->addinfo = 0;
  62     c->error = error;
  63     if (!c->diagset || strcmp(dset, c->diagset))
  64     {
  65         xfree(c->diagset);
  66         c->diagset = xstrdup(dset);
  67         /* remove integer part from SRW diagset .. */
  68         if ((cp = strrchr(c->diagset, '/')))
  69             *cp = '\0';
  70     }
  71     if (addinfo && addinfo2)
  72     {
  73         c->addinfo = (char*) xmalloc(strlen(addinfo) + strlen(addinfo2) + 2);
  74         strcpy(c->addinfo, addinfo);
  75         strcat(c->addinfo, addinfo2);
  76     }
  77     else if (addinfo)
  78         c->addinfo = xstrdup(addinfo);
  79     if (error != ZOOM_ERROR_NONE)
  80     {
  81         yaz_log(c->log_api, "%p set_dset_error %s %s:%d %s %s",
  82                 c, c->host_port ? c->host_port : "<>", dset, error,
  83                 addinfo ? addinfo : "",
  84                 addinfo2 ? addinfo2 : "");
  85         ZOOM_connection_remove_tasks(c);
  86     }
  87 }
  88
  89 int ZOOM_uri_to_code(const char *uri)
  90 {
  91     int code = 0;
  92     const char *cp;
  93     if ((cp = strrchr(uri, '/')))
  94         code = atoi(cp+1);
  95     return code;
  96 }
  97
  98
  99
 100 void ZOOM_set_error(ZOOM_connection c, int error, const char *addinfo)
 101 {
 102     ZOOM_set_dset_error(c, error, "ZOOM", addinfo, 0);
 103 }
 104
 105 static void clear_error(ZOOM_connection c)
 106 {
 107     /*
 108      * If an error is tied to an operation then it's ok to clear: for
 109      * example, a diagnostic returned from a search is cleared by a
 110      * subsequent search.  However, problems such as Connection Lost
 111      * or Init Refused are not cleared, because they are not
 112      * recoverable: doing another search doesn't help.
 113      */
 114
 115     ZOOM_connection_remove_events(c);
 116     switch (c->error)
 117     {
 118     case ZOOM_ERROR_CONNECT:
 119     case ZOOM_ERROR_MEMORY:
 120     case ZOOM_ERROR_DECODE:
 121     case ZOOM_ERROR_CONNECTION_LOST:
 122     case ZOOM_ERROR_INIT:
 123     case ZOOM_ERROR_INTERNAL:
 124     case ZOOM_ERROR_UNSUPPORTED_PROTOCOL:
 125         break;
 126     default:
 127         ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
 128     }
 129 }
 130
 131 void ZOOM_connection_show_task(ZOOM_task task)
 132 {
 133     switch(task->which)
 134     {
 135     case ZOOM_TASK_SEARCH:
 136         yaz_log(YLOG_LOG, "search p=%p", task);
 137         break;
 138     case ZOOM_TASK_RETRIEVE:
 139         yaz_log(YLOG_LOG, "retrieve p=%p", task);
 140         break;
 141     case ZOOM_TASK_CONNECT:
 142         yaz_log(YLOG_LOG, "connect p=%p", task);
 143         break;
 144     case ZOOM_TASK_SCAN:
 145         yaz_log(YLOG_LOG, "scan p=%p", task);
 146         break;
 147     }
 148 }
 149
 150 void ZOOM_connection_show_tasks(ZOOM_connection c)
 151 {
 152     ZOOM_task task;
 153     yaz_log(YLOG_LOG, "connection p=%p tasks", c);
 154     for (task = c->tasks; task; task = task->next)
 155         ZOOM_connection_show_task(task);
 156 }
 157
 158 ZOOM_task ZOOM_connection_add_task(ZOOM_connection c, int which)
 159 {
 160     ZOOM_task *taskp = &c->tasks;
 161     while (*taskp)
 162         taskp = &(*taskp)->next;
 163     *taskp = (ZOOM_task) xmalloc(sizeof(**taskp));
 164     (*taskp)->running = 0;
 165     (*taskp)->which = which;
 166     (*taskp)->next = 0;
 167     clear_error(c);
 168     return *taskp;
 169 }
 170
 171 ZOOM_API(int) ZOOM_connection_is_idle(ZOOM_connection c)
 172 {
 173     return c->tasks ? 0 : 1;
 174 }
 175
 176 ZOOM_task ZOOM_connection_insert_task(ZOOM_connection c, int which)
 177 {
 178     ZOOM_task task = (ZOOM_task) xmalloc(sizeof(*task));
 179
 180     task->next = c->tasks;
 181     c->tasks = task;
 182
 183     task->running = 0;
 184     task->which = which;
 185     return task;
 186 }
 187
 188 void ZOOM_connection_remove_task(ZOOM_connection c)
 189 {
 190     ZOOM_task task = c->tasks;
 191
 192     if (task)
 193     {
 194         c->tasks = task->next;
 195         switch (task->which)
 196         {
 197         case ZOOM_TASK_SEARCH:
 198             resultset_destroy(task->u.search.resultset);
 199             xfree(task->u.search.syntax);
 200             xfree(task->u.search.elementSetName);
 201             break;
 202         case ZOOM_TASK_RETRIEVE:
 203             resultset_destroy(task->u.retrieve.resultset);
 204             xfree(task->u.retrieve.syntax);
 205             xfree(task->u.retrieve.elementSetName);
 206             break;
 207         case ZOOM_TASK_CONNECT:
 208             break;
 209         case ZOOM_TASK_SCAN:
 210             ZOOM_scanset_destroy(task->u.scan.scan);
 211             break;
 212         case ZOOM_TASK_PACKAGE:
 213             ZOOM_package_destroy(task->u.package);
 214             break;
 215         case ZOOM_TASK_SORT:
 216             resultset_destroy(task->u.sort.resultset);
 217             ZOOM_query_destroy(task->u.sort.q);
 218             break;
 219         default:
 220             assert(0);
 221         }
 222         xfree(task);
 223
 224         if (!c->tasks)
 225         {
 226             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_END);
 227             ZOOM_connection_put_event(c, event);
 228         }
 229     }
 230 }
 231
 232 void ZOOM_connection_remove_tasks(ZOOM_connection c)
 233 {
 234     while (c->tasks)
 235         ZOOM_connection_remove_task(c);
 236 }
 237
 238
 239 ZOOM_API(ZOOM_connection)
 240     ZOOM_connection_create(ZOOM_options options)
 241 {
 242     ZOOM_connection c = (ZOOM_connection) xmalloc(sizeof(*c));
 243
 244     initlog();
 245
 246     c->log_api = log_api0;
 247     c->log_details = log_details0;
 248
 249     yaz_log(c->log_api, "%p ZOOM_connection_create", c);
 250
 251     c->proto = PROTO_Z3950;
 252     c->cs = 0;
 253     ZOOM_connection_set_mask(c, 0);
 254     c->reconnect_ok = 0;
 255     c->state = STATE_IDLE;
 256     c->addinfo = 0;
 257     c->diagset = 0;
 258     ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
 259     c->buf_in = 0;
 260     c->len_in = 0;
 261     c->buf_out = 0;
 262     c->len_out = 0;
 263     c->resultsets = 0;
 264
 265     c->options = ZOOM_options_create_with_parent(options);
 266
 267     c->host_port = 0;
 268     c->proxy = 0;
 269
 270     c->charset = c->lang = 0;
 271
 272     c->cookie_out = 0;
 273     c->cookie_in = 0;
 274     c->client_IP = 0;
 275     c->tasks = 0;
 276
 277     c->user = 0;
 278     c->group = 0;
 279     c->password = 0;
 280
 281     c->maximum_record_size = 0;
 282     c->preferred_message_size = 0;
 283
 284     c->odr_in = odr_createmem(ODR_DECODE);
 285     c->odr_out = odr_createmem(ODR_ENCODE);
 286     c->odr_print = 0;
 287
 288     c->async = 0;
 289     c->support_named_resultsets = 0;
 290     c->last_event = ZOOM_EVENT_NONE;
 291
 292     c->m_queue_front = 0;
 293     c->m_queue_back = 0;
 294
 295     c->sru_version = 0;
 296     c->no_redirects = 0;
 297     return c;
 298 }
 299
 300
 301 /* set database names. Take local databases (if set); otherwise
 302    take databases given in ZURL (if set); otherwise use Default */
 303 char **ZOOM_connection_get_databases(ZOOM_connection con, ZOOM_options options,
 304                                      int *num, ODR odr)
 305 {
 306     char **databaseNames;
 307     const char *cp = ZOOM_options_get(options, "databaseName");
 308
 309     if ((!cp || !*cp) && con->host_port)
 310     {
 311         if (strncmp(con->host_port, "unix:", 5) == 0)
 312             cp = strchr(con->host_port+5, ':');
 313         else
 314             cp = strchr(con->host_port, '/');
 315         if (cp)
 316             cp++;
 317     }
 318     if (!cp)
 319         cp = "Default";
 320     nmem_strsplit(odr_getmem(odr), "+", cp,  &databaseNames, num);
 321     return databaseNames;
 322 }
 323
 324 ZOOM_API(ZOOM_connection)
 325     ZOOM_connection_new(const char *host, int portnum)
 326 {
 327     ZOOM_connection c = ZOOM_connection_create(0);
 328
 329     ZOOM_connection_connect(c, host, portnum);
 330     return c;
 331 }
 332
 333 static zoom_sru_mode get_sru_mode_from_string(const char *s)
 334 {
 335     if (!s || !*s)
 336         return zoom_sru_soap;
 337     if (!yaz_matchstr(s, "soap"))
 338         return zoom_sru_soap;
 339     else if (!yaz_matchstr(s, "get"))
 340         return zoom_sru_get;
 341     else if (!yaz_matchstr(s, "post"))
 342         return zoom_sru_post;
 343     else if (!yaz_matchstr(s, "solr"))
 344         return zoom_sru_solr;
 345     return zoom_sru_error;
 346 }
 347
 348 ZOOM_API(void)
 349     ZOOM_connection_connect(ZOOM_connection c,
 350                             const char *host, int portnum)
 351 {
 352     const char *val;
 353
 354     initlog();
 355
 356     yaz_log(c->log_api, "%p ZOOM_connection_connect host=%s portnum=%d",
 357             c, host ? host : "null", portnum);
 358
 359     ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
 360     ZOOM_connection_remove_tasks(c);
 361
 362     if (c->odr_print)
 363     {
 364         odr_setprint(c->odr_print, 0); /* prevent destroy from fclose'ing */
 365         odr_destroy(c->odr_print);
 366     }
 367     if (ZOOM_options_get_bool(c->options, "apdulog", 0))
 368     {
 369         c->odr_print = odr_createmem(ODR_PRINT);
 370         odr_setprint(c->odr_print, yaz_log_file());
 371     }
 372     else
 373         c->odr_print = 0;
 374
 375     if (c->cs)
 376     {
 377         yaz_log(c->log_details, "%p ZOOM_connection_connect reconnect ok", c);
 378         c->reconnect_ok = 1;
 379         return;
 380     }
 381     yaz_log(c->log_details, "%p ZOOM_connection_connect connect", c);
 382     xfree(c->proxy);
 383     c->proxy = 0;
 384     val = ZOOM_options_get(c->options, "proxy");
 385     if (val && *val)
 386     {
 387         yaz_log(c->log_details, "%p ZOOM_connection_connect proxy=%s", c, val);
 388         c->proxy = xstrdup(val);
 389     }
 390
 391     xfree(c->charset);
 392     c->charset = 0;
 393     val = ZOOM_options_get(c->options, "charset");
 394     if (val && *val)
 395     {
 396         yaz_log(c->log_details, "%p ZOOM_connection_connect charset=%s", c, val);
 397         c->charset = xstrdup(val);
 398     }
 399
 400     xfree(c->lang);
 401     val = ZOOM_options_get(c->options, "lang");
 402     if (val && *val)
 403     {
 404         yaz_log(c->log_details, "%p ZOOM_connection_connect lang=%s", c, val);
 405         c->lang = xstrdup(val);
 406     }
 407     else
 408         c->lang = 0;
 409
 410     if (host)
 411     {
 412         xfree(c->host_port);
 413         if (portnum)
 414         {
 415             char hostn[128];
 416             sprintf(hostn, "%.80s:%d", host, portnum);
 417             c->host_port = xstrdup(hostn);
 418         }
 419         else
 420             c->host_port = xstrdup(host);
 421     }
 422
 423     {
 424         /*
 425          * If the "<scheme>:" part of the host string is preceded by one
 426          * or more comma-separated <name>=<value> pairs, these are taken
 427          * to be options to be set on the connection object.  Among other
 428          * applications, this facility can be used to embed authentication
 429          * in a host string:
 430          *          user=admin,password=secret,tcp:localhost:9999
 431          */
 432         char *remainder = c->host_port;
 433         char *pcolon = strchr(remainder, ':');
 434         char *pcomma;
 435         char *pequals;
 436         while ((pcomma = strchr(remainder, ',')) != 0 &&
 437                (pcolon == 0 || pcomma < pcolon)) {
 438             *pcomma = '\0';
 439             if ((pequals = strchr(remainder, '=')) != 0) {
 440                 *pequals = '\0';
 441                 /*printf("# setting '%s'='%s'\n", remainder, pequals+1);*/
 442                 ZOOM_connection_option_set(c, remainder, pequals+1);
 443             }
 444             remainder = pcomma+1;
 445         }
 446
 447         if (remainder != c->host_port) {
 448             xfree(c->host_port);
 449             c->host_port = xstrdup(remainder);
 450             /*printf("# reset hp='%s'\n", remainder);*/
 451         }
 452     }
 453
 454     val = ZOOM_options_get(c->options, "sru");
 455     c->sru_mode = get_sru_mode_from_string(val);
 456
 457     xfree(c->sru_version);
 458     val = ZOOM_options_get(c->options, "sru_version");
 459     c->sru_version = xstrdup(val ? val : "1.2");
 460
 461     ZOOM_options_set(c->options, "host", c->host_port);
 462
 463     xfree(c->cookie_out);
 464     c->cookie_out = 0;
 465     val = ZOOM_options_get(c->options, "cookie");
 466     if (val && *val)
 467     {
 468         yaz_log(c->log_details, "%p ZOOM_connection_connect cookie=%s", c, val);
 469         c->cookie_out = xstrdup(val);
 470     }
 471
 472     xfree(c->client_IP);
 473     c->client_IP = 0;
 474     val = ZOOM_options_get(c->options, "clientIP");
 475     if (val && *val)
 476     {
 477         yaz_log(c->log_details, "%p ZOOM_connection_connect clientIP=%s",
 478                 c, val);
 479         c->client_IP = xstrdup(val);
 480     }
 481
 482     xfree(c->group);
 483     c->group = 0;
 484     val = ZOOM_options_get(c->options, "group");
 485     if (val && *val)
 486         c->group = xstrdup(val);
 487
 488     xfree(c->user);
 489     c->user = 0;
 490     val = ZOOM_options_get(c->options, "user");
 491     if (val && *val)
 492         c->user = xstrdup(val);
 493
 494     xfree(c->password);
 495     c->password = 0;
 496     val = ZOOM_options_get(c->options, "password");
 497     if (!val)
 498         val = ZOOM_options_get(c->options, "pass");
 499
 500     if (val && *val)
 501         c->password = xstrdup(val);
 502
 503     c->maximum_record_size =
 504         ZOOM_options_get_int(c->options, "maximumRecordSize", 1024*1024);
 505     c->preferred_message_size =
 506         ZOOM_options_get_int(c->options, "preferredMessageSize", 1024*1024);
 507
 508     c->async = ZOOM_options_get_bool(c->options, "async", 0);
 509
 510     if (c->sru_mode == zoom_sru_error)
 511     {
 512         ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, val);
 513         ZOOM_connection_remove_tasks(c);
 514         return;
 515     }
 516
 517     yaz_log(c->log_details, "%p ZOOM_connection_connect async=%d", c, c->async);
 518     ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
 519
 520     if (!c->async)
 521     {
 522         while (ZOOM_event(1, &c))
 523             ;
 524     }
 525 }
 526
 527 ZOOM_API(void) ZOOM_resultset_release(ZOOM_resultset r)
 528 {
 529 #if ZOOM_RESULT_LISTS
 530 #else
 531     if (r->connection)
 532     {
 533         /* remove ourselves from the resultsets in connection */
 534         ZOOM_resultset *rp = &r->connection->resultsets;
 535         while (1)
 536         {
 537             assert(*rp);   /* we must be in this list!! */
 538             if (*rp == r)
 539             {   /* OK, we're here - take us out of it */
 540                 *rp = (*rp)->next;
 541                 break;
 542             }
 543             rp = &(*rp)->next;
 544         }
 545         r->connection = 0;
 546     }
 547 #endif
 548 }
 549
 550 ZOOM_API(void)
 551     ZOOM_connection_destroy(ZOOM_connection c)
 552 {
 553 #if ZOOM_RESULT_LISTS
 554     ZOOM_resultsets list;
 555 #else
 556     ZOOM_resultset r;
 557 #endif
 558     if (!c)
 559         return;
 560     yaz_log(c->log_api, "%p ZOOM_connection_destroy", c);
 561     if (c->cs)
 562         cs_close(c->cs);
 563
 564 #if ZOOM_RESULT_LISTS
 565     /* Remove the connection's usage of resultsets */
 566     list = c->resultsets;
 567     while (list) {
 568         ZOOM_resultsets removed = list;
 569         ZOOM_resultset_destroy(list->resultset);
 570         list = list->next;
 571         xfree(removed);
 572     }
 573 #else
 574     for (r = c->resultsets; r; r = r->next)
 575         r->connection = 0;
 576 #endif
 577
 578     xfree(c->buf_in);
 579     xfree(c->addinfo);
 580     xfree(c->diagset);
 581     odr_destroy(c->odr_in);
 582     odr_destroy(c->odr_out);
 583     if (c->odr_print)
 584     {
 585         odr_setprint(c->odr_print, 0); /* prevent destroy from fclose'ing */
 586         odr_destroy(c->odr_print);
 587     }
 588     ZOOM_options_destroy(c->options);
 589     ZOOM_connection_remove_tasks(c);
 590     ZOOM_connection_remove_events(c);
 591     xfree(c->host_port);
 592     xfree(c->proxy);
 593     xfree(c->charset);
 594     xfree(c->lang);
 595     xfree(c->cookie_out);
 596     xfree(c->cookie_in);
 597     xfree(c->client_IP);
 598     xfree(c->user);
 599     xfree(c->group);
 600     xfree(c->password);
 601     xfree(c->sru_version);
 602     xfree(c);
 603 }
 604
 605 void ZOOM_resultset_addref(ZOOM_resultset r)
 606 {
 607     if (r)
 608     {
 609         yaz_mutex_enter(r->mutex);
 610         (r->refcount)++;
 611         yaz_log(log_details0, "%p ZOOM_resultset_addref count=%d",
 612                 r, r->refcount);
 613         yaz_mutex_leave(r->mutex);
 614     }
 615 }
 616
 617 static int g_resultsets = 0;
 618 static YAZ_MUTEX g_resultset_mutex = 0;
 619
 620 /* TODO We need to initialize this before running threaded:
 621  * call resultset_use(0)  */
 622
 623 static int resultset_use(int delta) {
 624     int resultset_count;
 625     if (g_resultset_mutex == 0)
 626         yaz_mutex_create(&g_resultset_mutex);
 627     yaz_mutex_enter(g_resultset_mutex);
 628     g_resultsets += delta;
 629     resultset_count = g_resultsets;
 630     yaz_mutex_leave(g_resultset_mutex);
 631     return resultset_count;
 632 }
 633
 634 int resultsets_count(void) {
 635     return resultset_use(0);
 636 }
 637
 638 ZOOM_resultset ZOOM_resultset_create(void)
 639 {
 640     int i;
 641     ZOOM_resultset r = (ZOOM_resultset) xmalloc(sizeof(*r));
 642
 643     initlog();
 644
 645     yaz_log(log_details0, "%p ZOOM_resultset_create", r);
 646     r->refcount = 1;
 647     r->size = 0;
 648     r->odr = odr_createmem(ODR_ENCODE);
 649     r->piggyback = 1;
 650     r->setname = 0;
 651     r->schema = 0;
 652     r->step = 0;
 653     for (i = 0; i<RECORD_HASH_SIZE; i++)
 654         r->record_hash[i] = 0;
 655     r->r_sort_spec = 0;
 656     r->query = 0;
 657     r->connection = 0;
 658     r->databaseNames = 0;
 659     r->num_databaseNames = 0;
 660     r->facets = 0;
 661     r->num_facets = 0;
 662     r->facets_names = 0;
 663     r->mutex = 0;
 664     yaz_mutex_create(&r->mutex);
 665 #if SHPTR
 666     {
 667         WRBUF w = wrbuf_alloc();
 668         YAZ_SHPTR_INIT(r->record_wrbuf, w);
 669     }
 670 #endif
 671     resultset_use(1);
 672     return r;
 673 }
 674
 675 ZOOM_API(ZOOM_resultset)
 676     ZOOM_connection_search_pqf(ZOOM_connection c, const char *q)
 677 {
 678     ZOOM_resultset r;
 679     ZOOM_query s = ZOOM_query_create();
 680
 681     ZOOM_query_prefix(s, q);
 682
 683     r = ZOOM_connection_search(c, s);
 684     ZOOM_query_destroy(s);
 685     return r;
 686 }
 687
 688 ZOOM_API(ZOOM_resultset)
 689     ZOOM_connection_search(ZOOM_connection c, ZOOM_query q)
 690 {
 691     ZOOM_resultset r = ZOOM_resultset_create();
 692     ZOOM_task task;
 693     const char *cp;
 694     int start, count;
 695     const char *syntax, *elementSetName;
 696 #if ZOOM_RESULT_LISTS
 697     ZOOM_resultsets set;
 698 #endif
 699
 700     yaz_log(c->log_api, "%p ZOOM_connection_search set %p query %p", c, r, q);
 701     r->r_sort_spec = ZOOM_query_get_sortspec(q);
 702     r->query = q;
 703
 704     r->options = ZOOM_options_create_with_parent(c->options);
 705
 706     start = ZOOM_options_get_int(r->options, "start", 0);
 707     count = ZOOM_options_get_int(r->options, "count", 0);
 708     {
 709         /* If "presentChunk" is defined use that; otherwise "step" */
 710         const char *cp = ZOOM_options_get(r->options, "presentChunk");
 711         r->step = ZOOM_options_get_int(r->options,
 712                                        (cp != 0 ? "presentChunk": "step"), 0);
 713     }
 714     r->piggyback = ZOOM_options_get_bool(r->options, "piggyback", 1);
 715     cp = ZOOM_options_get(r->options, "setname");
 716     if (cp)
 717         r->setname = xstrdup(cp);
 718     cp = ZOOM_options_get(r->options, "schema");
 719     if (cp)
 720         r->schema = xstrdup(cp);
 721
 722     r->databaseNames = ZOOM_connection_get_databases(c, c->options, &r->num_databaseNames,
 723                                          r->odr);
 724
 725     r->connection = c;
 726
 727 #if ZOOM_RESULT_LISTS
 728     yaz_log(log_details, "%p ZOOM_connection_search: Adding new resultset (%p) to resultsets (%p) ", c, r, c->resultsets);
 729     set = xmalloc(sizeof(*set));
 730     ZOOM_resultset_addref(r);
 731     set->resultset = r;
 732     set->next = c->resultsets;
 733     c->resultsets = set;
 734 #else
 735     r->next = c->resultsets;
 736     c->resultsets = r;
 737 #endif
 738     if (c->host_port && c->proto == PROTO_HTTP)
 739     {
 740         if (!c->cs)
 741         {
 742             yaz_log(c->log_details, "ZOOM_connection_search: no comstack");
 743             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
 744         }
 745         else
 746         {
 747             yaz_log(c->log_details, "ZOOM_connection_search: reconnect");
 748             c->reconnect_ok = 1;
 749         }
 750     }
 751
 752     task = ZOOM_connection_add_task(c, ZOOM_TASK_SEARCH);
 753     task->u.search.resultset = r;
 754     task->u.search.start = start;
 755     task->u.search.count = count;
 756     task->u.search.recv_search_fired = 0;
 757
 758     syntax = ZOOM_options_get(r->options, "preferredRecordSyntax");
 759     task->u.search.syntax = syntax ? xstrdup(syntax) : 0;
 760     elementSetName = ZOOM_options_get(r->options, "elementSetName");
 761     task->u.search.elementSetName = elementSetName
 762         ? xstrdup(elementSetName) : 0;
 763
 764     ZOOM_resultset_addref(r);
 765
 766     ZOOM_query_addref(q);
 767
 768     if (!c->async)
 769     {
 770         while (ZOOM_event(1, &c))
 771             ;
 772     }
 773     return r;
 774 }
 775
 776 ZOOM_API(void)
 777     ZOOM_resultset_sort(ZOOM_resultset r,
 778                          const char *sort_type, const char *sort_spec)
 779 {
 780     (void) ZOOM_resultset_sort1(r, sort_type, sort_spec);
 781 }
 782
 783 ZOOM_API(int)
 784     ZOOM_resultset_sort1(ZOOM_resultset r,
 785                          const char *sort_type, const char *sort_spec)
 786 {
 787     ZOOM_connection c = r->connection;
 788     ZOOM_task task;
 789     ZOOM_query newq;
 790
 791     newq = ZOOM_query_create();
 792     if (ZOOM_query_sortby(newq, sort_spec) < 0)
 793         return -1;
 794
 795     yaz_log(c->log_api, "%p ZOOM_resultset_sort r=%p sort_type=%s sort_spec=%s",
 796             r, r, sort_type, sort_spec);
 797     if (!c)
 798         return 0;
 799
 800     if (c->host_port && c->proto == PROTO_HTTP)
 801     {
 802         if (!c->cs)
 803         {
 804             yaz_log(c->log_details, "%p ZOOM_resultset_sort: no comstack", r);
 805             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
 806         }
 807         else
 808         {
 809             yaz_log(c->log_details, "%p ZOOM_resultset_sort: prepare reconnect",
 810                     r);
 811             c->reconnect_ok = 1;
 812         }
 813     }
 814
 815     ZOOM_resultset_cache_reset(r);
 816     task = ZOOM_connection_add_task(c, ZOOM_TASK_SORT);
 817     task->u.sort.resultset = r;
 818     task->u.sort.q = newq;
 819
 820     ZOOM_resultset_addref(r);
 821
 822     if (!c->async)
 823     {
 824         while (ZOOM_event(1, &c))
 825             ;
 826     }
 827
 828     return 0;
 829 }
 830
 831 ZOOM_API(void)
 832     ZOOM_resultset_destroy(ZOOM_resultset r)
 833 {
 834     resultset_destroy(r);
 835 }
 836
 837 static void resultset_destroy(ZOOM_resultset r)
 838 {
 839     if (!r)
 840         return;
 841     yaz_mutex_enter(r->mutex);
 842     (r->refcount)--;
 843     yaz_log(log_details0, "%p ZOOM_resultset_destroy r=%p count=%d",
 844             r, r, r->refcount);
 845     if (r->refcount == 0)
 846     {
 847         yaz_mutex_leave(r->mutex);
 848
 849         yaz_log(log_details0, "%p ZOOM_connection resultset_destroy: Deleting resultset (%p) ", r->connection, r);
 850         ZOOM_resultset_cache_reset(r);
 851         ZOOM_resultset_release(r);
 852         ZOOM_query_destroy(r->query);
 853         ZOOM_options_destroy(r->options);
 854         odr_destroy(r->odr);
 855         xfree(r->setname);
 856         xfree(r->schema);
 857         yaz_mutex_destroy(&r->mutex);
 858 #if SHPTR
 859         YAZ_SHPTR_DEC(r->record_wrbuf, wrbuf_destroy);
 860 #endif
 861         resultset_use(-1);
 862         xfree(r);
 863     }
 864     else
 865         yaz_mutex_leave(r->mutex);
 866 }
 867
 868 ZOOM_API(size_t)
 869     ZOOM_resultset_size(ZOOM_resultset r)
 870 {
 871     return r->size;
 872 }
 873
 874 int ZOOM_test_reconnect(ZOOM_connection c)
 875 {
 876     ZOOM_Event event;
 877
 878     if (!c->reconnect_ok)
 879         return 0;
 880     ZOOM_connection_close(c);
 881     c->reconnect_ok = 0;
 882     c->tasks->running = 0;
 883     ZOOM_connection_insert_task(c, ZOOM_TASK_CONNECT);
 884
 885     event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
 886     ZOOM_connection_put_event(c, event);
 887
 888     return 1;
 889 }
 890
 891 static void ZOOM_resultset_retrieve(ZOOM_resultset r,
 892                                     int force_sync, int start, int count)
 893 {
 894     ZOOM_task task;
 895     ZOOM_connection c;
 896     const char *cp;
 897     const char *syntax, *elementSetName;
 898
 899     if (!r)
 900         return;
 901     yaz_log(log_details0, "%p ZOOM_resultset_retrieve force_sync=%d start=%d"
 902             " count=%d", r, force_sync, start, count);
 903     c = r->connection;
 904     if (!c)
 905         return;
 906
 907     if (c->host_port && c->proto == PROTO_HTTP)
 908     {
 909         if (!c->cs)
 910         {
 911             yaz_log(log_details0, "%p ZOOM_resultset_retrieve: no comstack", r);
 912             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
 913         }
 914         else
 915         {
 916             yaz_log(log_details0, "%p ZOOM_resultset_retrieve: prepare "
 917                     "reconnect", r);
 918             c->reconnect_ok = 1;
 919         }
 920     }
 921     task = ZOOM_connection_add_task(c, ZOOM_TASK_RETRIEVE);
 922     task->u.retrieve.resultset = r;
 923     task->u.retrieve.start = start;
 924     task->u.retrieve.count = count;
 925
 926     syntax = ZOOM_options_get(r->options, "preferredRecordSyntax");
 927     task->u.retrieve.syntax = syntax ? xstrdup(syntax) : 0;
 928     elementSetName = ZOOM_options_get(r->options, "elementSetName");
 929     task->u.retrieve.elementSetName = elementSetName
 930         ? xstrdup(elementSetName) : 0;
 931
 932     cp = ZOOM_options_get(r->options, "schema");
 933     if (cp)
 934     {
 935         if (!r->schema || strcmp(r->schema, cp))
 936         {
 937             xfree(r->schema);
 938             r->schema = xstrdup(cp);
 939         }
 940     }
 941
 942     ZOOM_resultset_addref(r);
 943
 944     if (!r->connection->async || force_sync)
 945         while (r->connection && ZOOM_event(1, &r->connection))
 946             ;
 947 }
 948
 949 ZOOM_API(void)
 950     ZOOM_resultset_records(ZOOM_resultset r, ZOOM_record *recs,
 951                            size_t start, size_t count)
 952 {
 953     int force_present = 0;
 954
 955     if (!r)
 956         return ;
 957     yaz_log(log_api0, "%p ZOOM_resultset_records r=%p start=%ld count=%ld",
 958             r, r, (long) start, (long) count);
 959     if (count && recs)
 960         force_present = 1;
 961     ZOOM_resultset_retrieve(r, force_present, start, count);
 962     if (force_present)
 963     {
 964         size_t i;
 965         for (i = 0; i< count; i++)
 966             recs[i] = ZOOM_resultset_record_immediate(r, i+start);
 967     }
 968 }
 969
 970 ZOOM_API(size_t)
 971     ZOOM_resultset_facets_size(ZOOM_resultset r) {
 972     return r->num_facets;
 973 }
 974
 975 ZOOM_API(ZOOM_facet_field)
 976     ZOOM_resultset_get_facet_field(ZOOM_resultset r, const char *name) {
 977     int num = r->num_facets;
 978     ZOOM_facet_field *facets = r->facets;
 979     int index;
 980     for (index = 0; index < num; index++) {
 981         if (!strcmp(facets[index]->facet_name, name)) {
 982             return facets[index];
 983         }
 984     }
 985     return 0;
 986 }
 987
 988
 989 ZOOM_API(ZOOM_facet_field *)
 990     ZOOM_resultset_facets(ZOOM_resultset r)
 991 {
 992     return r->facets;
 993 }
 994
 995 ZOOM_API(const char**)
 996     ZOOM_resultset_facet_names(ZOOM_resultset r)
 997 {
 998     return (const char **) r->facets_names;
 999 }
1000
1001 ZOOM_API(const char*)
1002     ZOOM_facet_field_name(ZOOM_facet_field field)
1003 {
1004     return field->facet_name;
1005 }
1006
1007 ZOOM_API(size_t)
1008     ZOOM_facet_field_term_count(ZOOM_facet_field field)
1009 {
1010     return field->num_terms;
1011 }
1012
1013 ZOOM_API(const char*)
1014     ZOOM_facet_field_get_term(ZOOM_facet_field field, size_t idx, int *freq) {
1015     *freq = field->facet_terms[idx].frequency;
1016     return field->facet_terms[idx].term;
1017 }
1018
1019
1020 static void get_cert(ZOOM_connection c)
1021 {
1022     char *cert_buf;
1023     int cert_len;
1024
1025     if (cs_get_peer_certificate_x509(c->cs, &cert_buf, &cert_len))
1026     {
1027         ZOOM_connection_option_setl(c, "sslPeerCert",
1028                                     cert_buf, cert_len);
1029         xfree(cert_buf);
1030     }
1031 }
1032
1033 static zoom_ret do_connect_host(ZOOM_connection c,
1034                                 const char *logical_url);
1035
1036 static zoom_ret do_connect(ZOOM_connection c)
1037 {
1038     return do_connect_host(c, c->host_port);
1039 }
1040
1041 static zoom_ret do_connect_host(ZOOM_connection c, const char *logical_url)
1042 {
1043     void *add;
1044
1045     if (c->cs)
1046         cs_close(c->cs);
1047     c->cs = cs_create_host_proxy(logical_url, 0, &add, c->proxy);
1048
1049     if (c->cs && c->cs->protocol == PROTO_HTTP)
1050     {
1051 #if YAZ_HAVE_XML2
1052         c->proto = PROTO_HTTP;
1053 #else
1054         ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, "SRW");
1055         ZOOM_connection_close(c);
1056         return zoom_complete;
1057 #endif
1058     }
1059     if (c->cs)
1060     {
1061         int ret = cs_connect(c->cs, add);
1062         if (ret == 0)
1063         {
1064             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1065             ZOOM_connection_put_event(c, event);
1066             get_cert(c);
1067             if (c->proto == PROTO_Z3950)
1068                 ZOOM_connection_Z3950_send_init(c);
1069             else
1070             {
1071                 /* no init request for SRW .. */
1072                 assert(c->tasks->which == ZOOM_TASK_CONNECT);
1073                 ZOOM_connection_remove_task(c);
1074                 ZOOM_connection_set_mask(c, 0);
1075                 ZOOM_connection_exec_task(c);
1076             }
1077             c->state = STATE_ESTABLISHED;
1078             return zoom_pending;
1079         }
1080         else if (ret > 0)
1081         {
1082             int mask = ZOOM_SELECT_EXCEPT;
1083             if (c->cs->io_pending & CS_WANT_WRITE)
1084                 mask += ZOOM_SELECT_WRITE;
1085             if (c->cs->io_pending & CS_WANT_READ)
1086                 mask += ZOOM_SELECT_READ;
1087             ZOOM_connection_set_mask(c, mask);
1088             c->state = STATE_CONNECTING;
1089             return zoom_pending;
1090         }
1091     }
1092     c->state = STATE_IDLE;
1093     ZOOM_set_error(c, ZOOM_ERROR_CONNECT, logical_url);
1094     return zoom_complete;
1095 }
1096
1097 /* returns 1 if PDU was sent OK (still pending )
1098    0 if PDU was not sent OK (nothing to wait for)
1099 */
1100
1101 ZOOM_API(ZOOM_record)
1102     ZOOM_resultset_record_immediate(ZOOM_resultset s,size_t pos)
1103 {
1104     const char *syntax =
1105         ZOOM_options_get(s->options, "preferredRecordSyntax");
1106     const char *elementSetName =
1107         ZOOM_options_get(s->options, "elementSetName");
1108
1109     return ZOOM_record_cache_lookup(s, pos, syntax, elementSetName);
1110 }
1111
1112 ZOOM_API(ZOOM_record)
1113     ZOOM_resultset_record(ZOOM_resultset r, size_t pos)
1114 {
1115     ZOOM_record rec = ZOOM_resultset_record_immediate(r, pos);
1116
1117     if (!rec)
1118     {
1119         /*
1120          * MIKE: I think force_sync should always be zero, but I don't
1121          * want to make this change until I get the go-ahead from
1122          * Adam, in case something depends on the old synchronous
1123          * behaviour.
1124          */
1125         int force_sync = 1;
1126         if (getenv("ZOOM_RECORD_NO_FORCE_SYNC")) force_sync = 0;
1127         ZOOM_resultset_retrieve(r, force_sync, pos, 1);
1128         rec = ZOOM_resultset_record_immediate(r, pos);
1129     }
1130     return rec;
1131 }
1132
1133 ZOOM_API(ZOOM_scanset)
1134     ZOOM_connection_scan(ZOOM_connection c, const char *start)
1135 {
1136     ZOOM_scanset s;
1137     ZOOM_query q = ZOOM_query_create();
1138
1139     ZOOM_query_prefix(q, start);
1140
1141     s = ZOOM_connection_scan1(c, q);
1142     ZOOM_query_destroy(q);
1143     return s;
1144
1145 }
1146
1147 ZOOM_API(ZOOM_scanset)
1148     ZOOM_connection_scan1(ZOOM_connection c, ZOOM_query q)
1149 {
1150     ZOOM_scanset scan = 0;
1151     Z_Query *z_query = ZOOM_query_get_Z_Query(q);
1152
1153     if (!z_query)
1154         return 0;
1155     scan = (ZOOM_scanset) xmalloc(sizeof(*scan));
1156     scan->connection = c;
1157     scan->odr = odr_createmem(ODR_DECODE);
1158     scan->options = ZOOM_options_create_with_parent(c->options);
1159     scan->refcount = 1;
1160     scan->scan_response = 0;
1161     scan->srw_scan_response = 0;
1162
1163     scan->query = q;
1164     ZOOM_query_addref(q);
1165     scan->databaseNames = ZOOM_connection_get_databases(c, c->options,
1166                                             &scan->num_databaseNames,
1167                                             scan->odr);
1168
1169     if (1)
1170     {
1171         ZOOM_task task = ZOOM_connection_add_task(c, ZOOM_TASK_SCAN);
1172         task->u.scan.scan = scan;
1173
1174         (scan->refcount)++;
1175         if (!c->async)
1176         {
1177             while (ZOOM_event(1, &c))
1178                 ;
1179         }
1180     }
1181     return scan;
1182 }
1183
1184 ZOOM_API(void)
1185     ZOOM_scanset_destroy(ZOOM_scanset scan)
1186 {
1187     if (!scan)
1188         return;
1189     (scan->refcount)--;
1190     if (scan->refcount == 0)
1191     {
1192         ZOOM_query_destroy(scan->query);
1193
1194         odr_destroy(scan->odr);
1195
1196         ZOOM_options_destroy(scan->options);
1197         xfree(scan);
1198     }
1199 }
1200
1201 static zoom_ret send_package(ZOOM_connection c)
1202 {
1203     ZOOM_Event event;
1204
1205     yaz_log(c->log_details, "%p send_package", c);
1206     if (!c->tasks)
1207         return zoom_complete;
1208     assert (c->tasks->which == ZOOM_TASK_PACKAGE);
1209
1210     event = ZOOM_Event_create(ZOOM_EVENT_SEND_APDU);
1211     ZOOM_connection_put_event(c, event);
1212
1213     c->buf_out = c->tasks->u.package->buf_out;
1214     c->len_out = c->tasks->u.package->len_out;
1215
1216     return ZOOM_send_buf(c);
1217 }
1218
1219 ZOOM_API(size_t)
1220     ZOOM_scanset_size(ZOOM_scanset scan)
1221 {
1222     if (!scan)
1223         return 0;
1224
1225     if (scan->scan_response && scan->scan_response->entries)
1226         return scan->scan_response->entries->num_entries;
1227     else if (scan->srw_scan_response)
1228         return scan->srw_scan_response->num_terms;
1229     return 0;
1230 }
1231
1232 static void ZOOM_scanset_term_x(ZOOM_scanset scan, size_t pos,
1233                                 size_t *occ,
1234                                 const char **value_term, size_t *value_len,
1235                                 const char **disp_term, size_t *disp_len)
1236 {
1237     size_t noent = ZOOM_scanset_size(scan);
1238
1239     *value_term = 0;
1240     *value_len = 0;
1241
1242     *disp_term = 0;
1243     *disp_len = 0;
1244
1245     *occ = 0;
1246     if (pos >= noent)
1247         return;
1248     if (scan->scan_response)
1249     {
1250         Z_ScanResponse *res = scan->scan_response;
1251         if (res->entries->entries[pos]->which == Z_Entry_termInfo)
1252         {
1253             Z_TermInfo *t = res->entries->entries[pos]->u.termInfo;
1254
1255             *value_term = (const char *) t->term->u.general->buf;
1256             *value_len = t->term->u.general->len;
1257             if (t->displayTerm)
1258             {
1259                 *disp_term = t->displayTerm;
1260                 *disp_len = strlen(*disp_term);
1261             }
1262             else if (t->term->which == Z_Term_general)
1263             {
1264                 *disp_term = (const char *) t->term->u.general->buf;
1265                 *disp_len = t->term->u.general->len;
1266             }
1267             *occ = t->globalOccurrences ? *t->globalOccurrences : 0;
1268         }
1269     }
1270     if (scan->srw_scan_response)
1271     {
1272         Z_SRW_scanResponse *res = scan->srw_scan_response;
1273         Z_SRW_scanTerm *t = res->terms + pos;
1274         if (t)
1275         {
1276             *value_term = t->value;
1277             *value_len = strlen(*value_term);
1278
1279             if (t->displayTerm)
1280                 *disp_term = t->displayTerm;
1281             else
1282                 *disp_term = t->value;
1283             *disp_len = strlen(*disp_term);
1284             *occ = t->numberOfRecords ? *t->numberOfRecords : 0;
1285         }
1286     }
1287 }
1288
1289 ZOOM_API(const char *)
1290     ZOOM_scanset_term(ZOOM_scanset scan, size_t pos,
1291                       size_t *occ, size_t *len)
1292 {
1293     const char *value_term = 0;
1294     size_t value_len = 0;
1295     const char *disp_term = 0;
1296     size_t disp_len = 0;
1297
1298     ZOOM_scanset_term_x(scan, pos, occ, &value_term, &value_len,
1299                         &disp_term, &disp_len);
1300
1301     *len = value_len;
1302     return value_term;
1303 }
1304
1305 ZOOM_API(const char *)
1306     ZOOM_scanset_display_term(ZOOM_scanset scan, size_t pos,
1307                               size_t *occ, size_t *len)
1308 {
1309     const char *value_term = 0;
1310     size_t value_len = 0;
1311     const char *disp_term = 0;
1312     size_t disp_len = 0;
1313
1314     ZOOM_scanset_term_x(scan, pos, occ, &value_term, &value_len,
1315                         &disp_term, &disp_len);
1316
1317     *len = disp_len;
1318     return disp_term;
1319 }
1320
1321 ZOOM_API(const char *)
1322     ZOOM_scanset_option_get(ZOOM_scanset scan, const char *key)
1323 {
1324     return ZOOM_options_get(scan->options, key);
1325 }
1326
1327 ZOOM_API(void)
1328     ZOOM_scanset_option_set(ZOOM_scanset scan, const char *key,
1329                             const char *val)
1330 {
1331     ZOOM_options_set(scan->options, key, val);
1332 }
1333
1334
1335 ZOOM_API(ZOOM_package)
1336     ZOOM_connection_package(ZOOM_connection c, ZOOM_options options)
1337 {
1338     ZOOM_package p = (ZOOM_package) xmalloc(sizeof(*p));
1339
1340     p->connection = c;
1341     p->odr_out = odr_createmem(ODR_ENCODE);
1342     p->options = ZOOM_options_create_with_parent2(options, c->options);
1343     p->refcount = 1;
1344     p->buf_out = 0;
1345     p->len_out = 0;
1346     return p;
1347 }
1348
1349 ZOOM_API(void)
1350     ZOOM_package_destroy(ZOOM_package p)
1351 {
1352     if (!p)
1353         return;
1354     (p->refcount)--;
1355     if (p->refcount == 0)
1356     {
1357         odr_destroy(p->odr_out);
1358         xfree(p->buf_out);
1359
1360         ZOOM_options_destroy(p->options);
1361         xfree(p);
1362     }
1363 }
1364
1365 ZOOM_API(const char *)
1366     ZOOM_package_option_get(ZOOM_package p, const char *key)
1367 {
1368     return ZOOM_options_get(p->options, key);
1369 }
1370
1371 ZOOM_API(const char *)
1372     ZOOM_package_option_getl(ZOOM_package p, const char *key, int *lenp)
1373 {
1374     return ZOOM_options_getl(p->options, key, lenp);
1375 }
1376
1377 ZOOM_API(void)
1378     ZOOM_package_option_set(ZOOM_package p, const char *key,
1379                             const char *val)
1380 {
1381     ZOOM_options_set(p->options, key, val);
1382 }
1383
1384 ZOOM_API(void)
1385     ZOOM_package_option_setl(ZOOM_package p, const char *key,
1386                              const char *val, int len)
1387 {
1388     ZOOM_options_setl(p->options, key, val, len);
1389 }
1390
1391 ZOOM_API(int)
1392     ZOOM_connection_exec_task(ZOOM_connection c)
1393 {
1394     ZOOM_task task = c->tasks;
1395     zoom_ret ret = zoom_complete;
1396
1397     if (!task)
1398         return 0;
1399     yaz_log(c->log_details, "%p ZOOM_connection_exec_task type=%d run=%d",
1400             c, task->which, task->running);
1401     if (c->error != ZOOM_ERROR_NONE)
1402     {
1403         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1404                 "removing tasks because of error = %d", c, c->error);
1405         ZOOM_connection_remove_tasks(c);
1406         return 0;
1407     }
1408     if (task->running)
1409     {
1410         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1411                 "task already running", c);
1412         return 0;
1413     }
1414     task->running = 1;
1415     ret = zoom_complete;
1416     if (c->cs || task->which == ZOOM_TASK_CONNECT)
1417     {
1418         switch (task->which)
1419         {
1420         case ZOOM_TASK_SEARCH:
1421             if (c->proto == PROTO_HTTP)
1422                 ret = ZOOM_connection_srw_send_search(c);
1423             else
1424                 ret = ZOOM_connection_Z3950_send_search(c);
1425             break;
1426         case ZOOM_TASK_RETRIEVE:
1427             if (c->proto == PROTO_HTTP)
1428                 ret = ZOOM_connection_srw_send_search(c);
1429             else
1430                 ret = send_Z3950_present(c);
1431             break;
1432         case ZOOM_TASK_CONNECT:
1433             ret = do_connect(c);
1434             break;
1435         case ZOOM_TASK_SCAN:
1436             if (c->proto == PROTO_HTTP)
1437                 ret = ZOOM_connection_srw_send_scan(c);
1438             else
1439                 ret = ZOOM_connection_Z3950_send_scan(c);
1440             break;
1441         case ZOOM_TASK_PACKAGE:
1442             ret = send_package(c);
1443             break;
1444         case ZOOM_TASK_SORT:
1445             c->tasks->u.sort.resultset->r_sort_spec =
1446                 ZOOM_query_get_sortspec(c->tasks->u.sort.q);
1447             ret = send_Z3950_sort(c, c->tasks->u.sort.resultset);
1448             break;
1449         }
1450     }
1451     else
1452     {
1453         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1454                 "remove tasks because no connection exist", c);
1455         ZOOM_connection_remove_tasks(c);
1456     }
1457     if (ret == zoom_complete)
1458     {
1459         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1460                 "task removed (complete)", c);
1461         ZOOM_connection_remove_task(c);
1462         return 0;
1463     }
1464     yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
1465             "task pending", c);
1466     return 1;
1467 }
1468
1469 #if YAZ_HAVE_XML2
1470
1471 static zoom_ret send_HTTP_redirect(ZOOM_connection c, const char *uri,
1472                                   Z_HTTP_Response *cookie_hres)
1473 {
1474     struct Z_HTTP_Header *h;
1475     char *combined_cookies = 0;
1476     int combined_cookies_len = 0;
1477     Z_GDU *gdu = z_get_HTTP_Request_uri(c->odr_out, uri, 0, c->proxy ? 1 : 0);
1478
1479     gdu->u.HTTP_Request->method = odr_strdup(c->odr_out, "GET");
1480     z_HTTP_header_add(c->odr_out, &gdu->u.HTTP_Request->headers, "Accept",
1481                       "text/xml");
1482
1483     for (h = cookie_hres->headers; h; h = h->next)
1484     {
1485         if (!strcmp(h->name, "Set-Cookie"))
1486         {
1487             char *cp;
1488
1489             if (!(cp = strchr(h->value, ';')))
1490                 cp = h->value + strlen(h->value);
1491             if (cp - h->value >= 1) {
1492                 combined_cookies = xrealloc(combined_cookies, combined_cookies_len + cp - h->value + 3);
1493                 memcpy(combined_cookies+combined_cookies_len, h->value, cp - h->value);
1494                 combined_cookies[combined_cookies_len + cp - h->value] = '\0';
1495                 strcat(combined_cookies,"; ");
1496                 combined_cookies_len = strlen(combined_cookies);
1497             }
1498         }
1499     }
1500
1501     if (combined_cookies_len)
1502     {
1503         z_HTTP_header_add(c->odr_out, &gdu->u.HTTP_Request->headers,
1504                           "Cookie", combined_cookies);
1505         xfree(combined_cookies);
1506     }
1507
1508     if (c->user && c->password)
1509     {
1510         z_HTTP_header_add_basic_auth(c->odr_out, &gdu->u.HTTP_Request->headers,
1511                                      c->user, c->password);
1512     }
1513     if (!z_GDU(c->odr_out, &gdu, 0, 0))
1514         return zoom_complete;
1515     if (c->odr_print)
1516         z_GDU(c->odr_print, &gdu, 0, 0);
1517     c->buf_out = odr_getbuf(c->odr_out, &c->len_out, 0);
1518
1519     odr_reset(c->odr_out);
1520     return ZOOM_send_buf(c);
1521 }
1522
1523 #if YAZ_HAVE_XML2
1524 void ZOOM_set_HTTP_error(ZOOM_connection c, int error,
1525                          const char *addinfo, const char *addinfo2)
1526 {
1527     ZOOM_set_dset_error(c, error, "HTTP", addinfo, addinfo2);
1528 }
1529 #endif
1530
1531
1532 static void handle_http(ZOOM_connection c, Z_HTTP_Response *hres)
1533 {
1534     zoom_ret cret = zoom_complete;
1535     int ret = -1;
1536     const char *addinfo = 0;
1537     const char *connection_head = z_HTTP_header_lookup(hres->headers,
1538                                                        "Connection");
1539     const char *location;
1540
1541     ZOOM_connection_set_mask(c, 0);
1542     yaz_log(c->log_details, "%p handle_http", c);
1543
1544     if ((hres->code == 301 || hres->code == 302) && c->sru_mode == zoom_sru_get
1545         && (location = z_HTTP_header_lookup(hres->headers, "Location")))
1546     {
1547         c->no_redirects++;
1548         if (c->no_redirects > 10)
1549         {
1550             ZOOM_set_HTTP_error(c, hres->code, 0, 0);
1551             c->no_redirects = 0;
1552             ZOOM_connection_close(c);
1553         }
1554         else
1555         {
1556             /* since redirect may change host we just reconnect. A smarter
1557                implementation might check whether it's the same server */
1558             do_connect_host(c, location);
1559             send_HTTP_redirect(c, location, hres);
1560             /* we're OK for now. Operation is not really complete */
1561             return;
1562         }
1563     }
1564     else
1565     {
1566         ret = ZOOM_handle_sru(c, hres, &cret);
1567         if (ret == 0)
1568         {
1569             if (c->no_redirects) /* end of redirect. change hosts again */
1570                 ZOOM_connection_close(c);
1571         }
1572         c->no_redirects = 0;
1573     }
1574     if (ret)
1575     {
1576         if (hres->code != 200)
1577             ZOOM_set_HTTP_error(c, hres->code, 0, 0);
1578         else
1579             ZOOM_set_error(c, ZOOM_ERROR_DECODE, addinfo);
1580         ZOOM_connection_close(c);
1581     }
1582     if (cret == zoom_complete)
1583     {
1584         yaz_log(c->log_details, "removing tasks in handle_http");
1585         ZOOM_connection_remove_task(c);
1586     }
1587     {
1588         int must_close = 0;
1589         if (!strcmp(hres->version, "1.0"))
1590         {
1591             /* HTTP 1.0: only if Keep-Alive we stay alive.. */
1592             if (!connection_head || strcmp(connection_head, "Keep-Alive"))
1593                 must_close = 1;
1594         }
1595         else
1596         {
1597             /* HTTP 1.1: only if no close we stay alive.. */
1598             if (connection_head && !strcmp(connection_head, "close"))
1599                 must_close = 1;
1600         }
1601         if (must_close)
1602         {
1603             ZOOM_connection_close(c);
1604             if (c->tasks)
1605             {
1606                 c->tasks->running = 0;
1607                 ZOOM_connection_insert_task(c, ZOOM_TASK_CONNECT);
1608                 c->reconnect_ok = 0;
1609             }
1610         }
1611     }
1612 }
1613 #endif
1614
1615 static int do_read(ZOOM_connection c)
1616 {
1617     int r, more;
1618     ZOOM_Event event;
1619
1620     event = ZOOM_Event_create(ZOOM_EVENT_RECV_DATA);
1621     ZOOM_connection_put_event(c, event);
1622
1623     r = cs_get(c->cs, &c->buf_in, &c->len_in);
1624     more = cs_more(c->cs);
1625     yaz_log(c->log_details, "%p do_read len=%d more=%d", c, r, more);
1626     if (r == 1)
1627         return 0;
1628     if (r <= 0)
1629     {
1630         if (!ZOOM_test_reconnect(c))
1631         {
1632             ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
1633             ZOOM_connection_close(c);
1634         }
1635     }
1636     else
1637     {
1638         Z_GDU *gdu;
1639         ZOOM_Event event;
1640
1641         odr_reset(c->odr_in);
1642         odr_setbuf(c->odr_in, c->buf_in, r, 0);
1643         event = ZOOM_Event_create(ZOOM_EVENT_RECV_APDU);
1644         ZOOM_connection_put_event(c, event);
1645
1646         if (!z_GDU(c->odr_in, &gdu, 0, 0))
1647         {
1648             int x;
1649             int err = odr_geterrorx(c->odr_in, &x);
1650             char msg[100];
1651             const char *element = odr_getelement(c->odr_in);
1652             yaz_snprintf(msg, sizeof(msg),
1653                     "ODR code %d:%d element=%s offset=%d",
1654                     err, x, element ? element : "<unknown>",
1655                     odr_offset(c->odr_in));
1656             ZOOM_set_error(c, ZOOM_ERROR_DECODE, msg);
1657             if (c->log_api)
1658             {
1659                 FILE *ber_file = yaz_log_file();
1660                 if (ber_file)
1661                     odr_dumpBER(ber_file, c->buf_in, r);
1662             }
1663             ZOOM_connection_close(c);
1664         }
1665         else
1666         {
1667             if (c->odr_print)
1668                 z_GDU(c->odr_print, &gdu, 0, 0);
1669             if (gdu->which == Z_GDU_Z3950)
1670                 ZOOM_handle_Z3950_apdu(c, gdu->u.z3950);
1671             else if (gdu->which == Z_GDU_HTTP_Response)
1672             {
1673 #if YAZ_HAVE_XML2
1674                 handle_http(c, gdu->u.HTTP_Response);
1675 #else
1676                 ZOOM_set_error(c, ZOOM_ERROR_DECODE, 0);
1677                 ZOOM_connection_close(c);
1678 #endif
1679             }
1680         }
1681         c->reconnect_ok = 0;
1682     }
1683     return 1;
1684 }
1685
1686 static zoom_ret do_write_ex(ZOOM_connection c, char *buf_out, int len_out)
1687 {
1688     int r;
1689     ZOOM_Event event;
1690
1691     event = ZOOM_Event_create(ZOOM_EVENT_SEND_DATA);
1692     ZOOM_connection_put_event(c, event);
1693
1694     yaz_log(c->log_details, "%p do_write_ex len=%d", c, len_out);
1695     if ((r = cs_put(c->cs, buf_out, len_out)) < 0)
1696     {
1697         yaz_log(c->log_details, "%p do_write_ex write failed", c);
1698         if (ZOOM_test_reconnect(c))
1699         {
1700             return zoom_pending;
1701         }
1702         if (c->state == STATE_CONNECTING)
1703             ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
1704         else
1705             ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
1706         ZOOM_connection_close(c);
1707         return zoom_complete;
1708     }
1709     else if (r == 1)
1710     {
1711         int mask = ZOOM_SELECT_EXCEPT;
1712         if (c->cs->io_pending & CS_WANT_WRITE)
1713             mask += ZOOM_SELECT_WRITE;
1714         if (c->cs->io_pending & CS_WANT_READ)
1715             mask += ZOOM_SELECT_READ;
1716         ZOOM_connection_set_mask(c, mask);
1717         yaz_log(c->log_details, "%p do_write_ex write incomplete mask=%d",
1718                 c, c->mask);
1719     }
1720     else
1721     {
1722         ZOOM_connection_set_mask(c, ZOOM_SELECT_READ|ZOOM_SELECT_EXCEPT);
1723         yaz_log(c->log_details, "%p do_write_ex write complete mask=%d",
1724                 c, c->mask);
1725     }
1726     return zoom_pending;
1727 }
1728
1729 zoom_ret ZOOM_send_buf(ZOOM_connection c)
1730 {
1731     return do_write_ex(c, c->buf_out, c->len_out);
1732 }
1733
1734
1735 ZOOM_API(const char *)
1736     ZOOM_connection_option_get(ZOOM_connection c, const char *key)
1737 {
1738     return ZOOM_options_get(c->options, key);
1739 }
1740
1741 ZOOM_API(const char *)
1742     ZOOM_connection_option_getl(ZOOM_connection c, const char *key, int *lenp)
1743 {
1744     return ZOOM_options_getl(c->options, key, lenp);
1745 }
1746
1747 ZOOM_API(void)
1748     ZOOM_connection_option_set(ZOOM_connection c, const char *key,
1749                                const char *val)
1750 {
1751     ZOOM_options_set(c->options, key, val);
1752 }
1753
1754 ZOOM_API(void)
1755     ZOOM_connection_option_setl(ZOOM_connection c, const char *key,
1756                                 const char *val, int len)
1757 {
1758     ZOOM_options_setl(c->options, key, val, len);
1759 }
1760
1761 ZOOM_API(const char *)
1762     ZOOM_resultset_option_get(ZOOM_resultset r, const char *key)
1763 {
1764     return ZOOM_options_get(r->options, key);
1765 }
1766
1767 ZOOM_API(void)
1768     ZOOM_resultset_option_set(ZOOM_resultset r, const char *key,
1769                               const char *val)
1770 {
1771     ZOOM_options_set(r->options, key, val);
1772 }
1773
1774
1775 ZOOM_API(int)
1776     ZOOM_connection_errcode(ZOOM_connection c)
1777 {
1778     return ZOOM_connection_error(c, 0, 0);
1779 }
1780
1781 ZOOM_API(const char *)
1782     ZOOM_connection_errmsg(ZOOM_connection c)
1783 {
1784     const char *msg;
1785     ZOOM_connection_error(c, &msg, 0);
1786     return msg;
1787 }
1788
1789 ZOOM_API(const char *)
1790     ZOOM_connection_addinfo(ZOOM_connection c)
1791 {
1792     const char *addinfo;
1793     ZOOM_connection_error(c, 0, &addinfo);
1794     return addinfo;
1795 }
1796
1797 ZOOM_API(const char *)
1798     ZOOM_connection_diagset(ZOOM_connection c)
1799 {
1800     const char *diagset;
1801     ZOOM_connection_error_x(c, 0, 0, &diagset);
1802     return diagset;
1803 }
1804
1805 ZOOM_API(const char *)
1806     ZOOM_diag_str(int error)
1807 {
1808     switch (error)
1809     {
1810     case ZOOM_ERROR_NONE:
1811         return "No error";
1812     case ZOOM_ERROR_CONNECT:
1813         return "Connect failed";
1814     case ZOOM_ERROR_MEMORY:
1815         return "Out of memory";
1816     case ZOOM_ERROR_ENCODE:
1817         return "Encoding failed";
1818     case ZOOM_ERROR_DECODE:
1819         return "Decoding failed";
1820     case ZOOM_ERROR_CONNECTION_LOST:
1821         return "Connection lost";
1822     case ZOOM_ERROR_INIT:
1823         return "Init rejected";
1824     case ZOOM_ERROR_INTERNAL:
1825         return "Internal failure";
1826     case ZOOM_ERROR_TIMEOUT:
1827         return "Timeout";
1828     case ZOOM_ERROR_UNSUPPORTED_PROTOCOL:
1829         return "Unsupported protocol";
1830     case ZOOM_ERROR_UNSUPPORTED_QUERY:
1831         return "Unsupported query type";
1832     case ZOOM_ERROR_INVALID_QUERY:
1833         return "Invalid query";
1834     case ZOOM_ERROR_CQL_PARSE:
1835         return "CQL parsing error";
1836     case ZOOM_ERROR_CQL_TRANSFORM:
1837         return "CQL transformation error";
1838     case ZOOM_ERROR_CCL_CONFIG:
1839         return "CCL configuration error";
1840     case ZOOM_ERROR_CCL_PARSE:
1841         return "CCL parsing error";
1842     case ZOOM_ERROR_ES_INVALID_ACTION:
1843         return "Extended Service. invalid action";
1844     case ZOOM_ERROR_ES_INVALID_VERSION:
1845         return "Extended Service. invalid version";
1846     case ZOOM_ERROR_ES_INVALID_SYNTAX:
1847         return "Extended Service. invalid syntax";
1848     default:
1849         return diagbib1_str(error);
1850     }
1851 }
1852
1853 ZOOM_API(int)
1854     ZOOM_connection_error_x(ZOOM_connection c, const char **cp,
1855                             const char **addinfo, const char **diagset)
1856 {
1857     int error = c->error;
1858     if (cp)
1859     {
1860         if (!c->diagset || !strcmp(c->diagset, "ZOOM"))
1861             *cp = ZOOM_diag_str(error);
1862         else if (!strcmp(c->diagset, "HTTP"))
1863             *cp = z_HTTP_errmsg(c->error);
1864         else if (!strcmp(c->diagset, "Bib-1"))
1865             *cp = ZOOM_diag_str(error);
1866         else if (!strcmp(c->diagset, "info:srw/diagnostic/1"))
1867             *cp = yaz_diag_srw_str(c->error);
1868         else
1869             *cp = "Unknown error and diagnostic set";
1870     }
1871     if (addinfo)
1872         *addinfo = c->addinfo ? c->addinfo : "";
1873     if (diagset)
1874         *diagset = c->diagset ? c->diagset : "";
1875     return c->error;
1876 }
1877
1878 ZOOM_API(int)
1879     ZOOM_connection_error(ZOOM_connection c, const char **cp,
1880                           const char **addinfo)
1881 {
1882     return ZOOM_connection_error_x(c, cp, addinfo, 0);
1883 }
1884
1885 static void ZOOM_connection_do_io(ZOOM_connection c, int mask)
1886 {
1887     ZOOM_Event event = 0;
1888     int r = cs_look(c->cs);
1889     yaz_log(c->log_details, "%p ZOOM_connection_do_io mask=%d cs_look=%d",
1890             c, mask, r);
1891
1892     if (r == CS_NONE)
1893     {
1894         event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1895         ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
1896         ZOOM_connection_close(c);
1897         ZOOM_connection_put_event(c, event);
1898     }
1899     else if (r == CS_CONNECT)
1900     {
1901         int ret = ret = cs_rcvconnect(c->cs);
1902         yaz_log(c->log_details, "%p ZOOM_connection_do_io "
1903                 "cs_rcvconnect returned %d", c, ret);
1904         if (ret == 1)
1905         {
1906             int mask = ZOOM_SELECT_EXCEPT;
1907             if (c->cs->io_pending & CS_WANT_WRITE)
1908                 mask += ZOOM_SELECT_WRITE;
1909             if (c->cs->io_pending & CS_WANT_READ)
1910                 mask += ZOOM_SELECT_READ;
1911             ZOOM_connection_set_mask(c, mask);
1912             event = ZOOM_Event_create(ZOOM_EVENT_NONE);
1913             ZOOM_connection_put_event(c, event);
1914         }
1915         else if (ret == 0)
1916         {
1917             event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1918             ZOOM_connection_put_event(c, event);
1919             get_cert(c);
1920             if (c->proto == PROTO_Z3950)
1921                 ZOOM_connection_Z3950_send_init(c);
1922             else
1923             {
1924                 /* no init request for SRW .. */
1925                 assert(c->tasks->which == ZOOM_TASK_CONNECT);
1926                 ZOOM_connection_remove_task(c);
1927                 ZOOM_connection_set_mask(c, 0);
1928                 ZOOM_connection_exec_task(c);
1929             }
1930             c->state = STATE_ESTABLISHED;
1931         }
1932         else
1933         {
1934             ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
1935             ZOOM_connection_close(c);
1936         }
1937     }
1938     else
1939     {
1940         if (mask & ZOOM_SELECT_EXCEPT)
1941         {
1942             if (!ZOOM_test_reconnect(c))
1943             {
1944                 ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
1945                 ZOOM_connection_close(c);
1946             }
1947             return;
1948         }
1949         if (mask & ZOOM_SELECT_READ)
1950             do_read(c);
1951         if (c->cs && (mask & ZOOM_SELECT_WRITE))
1952             ZOOM_send_buf(c);
1953     }
1954 }
1955
1956 ZOOM_API(int)
1957     ZOOM_connection_last_event(ZOOM_connection cs)
1958 {
1959     if (!cs)
1960         return ZOOM_EVENT_NONE;
1961     return cs->last_event;
1962 }
1963
1964
1965 ZOOM_API(int) ZOOM_connection_fire_event_timeout(ZOOM_connection c)
1966 {
1967     if (c->mask)
1968     {
1969         ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_TIMEOUT);
1970         /* timeout and this connection was waiting */
1971         ZOOM_set_error(c, ZOOM_ERROR_TIMEOUT, 0);
1972         ZOOM_connection_close(c);
1973         ZOOM_connection_put_event(c, event);
1974     }
1975     return 0;
1976 }
1977
1978 ZOOM_API(int)
1979     ZOOM_connection_process(ZOOM_connection c)
1980 {
1981     ZOOM_Event event;
1982     if (!c)
1983         return 0;
1984
1985     event = ZOOM_connection_get_event(c);
1986     if (event)
1987     {
1988         ZOOM_Event_destroy(event);
1989         return 1;
1990     }
1991     ZOOM_connection_exec_task(c);
1992     event = ZOOM_connection_get_event(c);
1993     if (event)
1994     {
1995         ZOOM_Event_destroy(event);
1996         return 1;
1997     }
1998     return 0;
1999 }
2000
2001 ZOOM_API(int)
2002     ZOOM_event_nonblock(int no, ZOOM_connection *cs)
2003 {
2004     int i;
2005
2006     yaz_log(log_details0, "ZOOM_process_event(no=%d,cs=%p)", no, cs);
2007
2008     for (i = 0; i<no; i++)
2009     {
2010         ZOOM_connection c = cs[i];
2011
2012         if (c && ZOOM_connection_process(c))
2013             return i+1;
2014     }
2015     return 0;
2016 }
2017
2018 ZOOM_API(int) ZOOM_connection_fire_event_socket(ZOOM_connection c, int mask)
2019 {
2020     if (c->mask && mask)
2021         ZOOM_connection_do_io(c, mask);
2022     return 0;
2023 }
2024
2025 ZOOM_API(int) ZOOM_connection_get_socket(ZOOM_connection c)
2026 {
2027     if (c->cs)
2028         return cs_fileno(c->cs);
2029     return -1;
2030 }
2031
2032 ZOOM_API(int) ZOOM_connection_set_mask(ZOOM_connection c, int mask)
2033 {
2034     c->mask = mask;
2035     if (!c->cs)
2036         return -1;
2037     return 0;
2038 }
2039
2040 ZOOM_API(int) ZOOM_connection_get_mask(ZOOM_connection c)
2041 {
2042     if (c->cs)
2043         return c->mask;
2044     return 0;
2045 }
2046
2047 ZOOM_API(int) ZOOM_connection_get_timeout(ZOOM_connection c)
2048 {
2049     return ZOOM_options_get_int(c->options, "timeout", 30);
2050 }
2051
2052 ZOOM_API(void) ZOOM_connection_close(ZOOM_connection c)
2053 {
2054     if (c->cs)
2055         cs_close(c->cs);
2056     c->cs = 0;
2057     ZOOM_connection_set_mask(c, 0);
2058     c->state = STATE_IDLE;
2059 }
2060
2061 /*
2062  * Local variables:
2063  * c-basic-offset: 4
2064  * c-file-style: "Stroustrup"
2065  * indent-tabs-mode: nil
2066  * End:
2067  * vim: shiftwidth=4 tabstop=8 expandtab
2068  */
2069