src/zoom-c.c

   1 /* This file is part of the YAZ toolkit.
   2  * Copyright (C) 1995-2010 Index Data
   3  * See the file LICENSE for details.
   4  */
   5 /**
   6  * \file zoom-c.c
   7  * \brief Implements ZOOM C interface.
   8  */
   9
  10 #include <assert.h>
  11 #include <string.h>
  12 #include <errno.h>
  13 #include "zoom-p.h"
  14
  15 #include <yaz/yaz-util.h>
  16 #include <yaz/xmalloc.h>
  17 #include <yaz/otherinfo.h>
  18 #include <yaz/log.h>
  19 #include <yaz/pquery.h>
  20 #include <yaz/marcdisp.h>
  21 #include <yaz/diagbib1.h>
  22 #include <yaz/charneg.h>
  23 #include <yaz/ill.h>
  24 #include <yaz/srw.h>
  25 #include <yaz/cql.h>
  26 #include <yaz/ccl.h>
  27 #include <yaz/query-charset.h>
  28 #include <yaz/copy_types.h>
  29 #include <yaz/snprintf.h>
  30 #include <yaz/facet.h>
  31
  32 #include <yaz/shptr.h>
  33
  34 #if SHPTR
  35 YAZ_SHPTR_TYPE(WRBUF)
  36 #endif
  37
  38 static int log_api0 = 0;
  39 static int log_details0 = 0;
  40
  41 static void resultset_destroy(ZOOM_resultset r);
  42 static zoom_ret do_write_ex(ZOOM_connection c, char *buf_out, int len_out);
  43 static char *cql2pqf(ZOOM_connection c, const char *cql);
  44
  45 ZOOM_API(const char *) ZOOM_get_event_str(int event)
  46 {
  47     static const char *ar[] = {
  48         "NONE",
  49         "CONNECT",
  50         "SEND_DATA",
  51         "RECV_DATA",
  52         "TIMEOUT",
  53         "UNKNOWN",
  54         "SEND_APDU",
  55         "RECV_APDU",
  56         "RECV_RECORD",
  57         "RECV_SEARCH",
  58         "END"
  59     };
  60     return ar[event];
  61 }
  62
  63 static void initlog(void)
  64 {
  65     static int log_level_initialized = 0;
  66     if (!log_level_initialized)
  67     {
  68         log_api0 = yaz_log_module_level("zoom");
  69         log_details0 = yaz_log_module_level("zoomdetails");
  70         log_level_initialized = 1;
  71     }
  72 }
  73
  74 ZOOM_Event ZOOM_Event_create(int kind)
  75 {
  76     ZOOM_Event event = (ZOOM_Event) xmalloc(sizeof(*event));
  77     event->kind = kind;
  78     event->next = 0;
  79     event->prev = 0;
  80     return event;
  81 }
  82
  83 static void ZOOM_Event_destroy(ZOOM_Event event)
  84 {
  85     xfree(event);
  86 }
  87
  88 void ZOOM_connection_put_event(ZOOM_connection c, ZOOM_Event event)
  89 {
  90     if (c->m_queue_back)
  91     {
  92         c->m_queue_back->prev = event;
  93         assert(c->m_queue_front);
  94     }
  95     else
  96     {
  97         assert(!c->m_queue_front);
  98         c->m_queue_front = event;
  99     }
 100     event->next = c->m_queue_back;
 101     event->prev = 0;
 102     c->m_queue_back = event;
 103 }
 104
 105 static ZOOM_Event ZOOM_connection_get_event(ZOOM_connection c)
 106 {
 107     ZOOM_Event event = c->m_queue_front;
 108     if (!event)
 109     {
 110         c->last_event = ZOOM_EVENT_NONE;
 111         return 0;
 112     }
 113     assert(c->m_queue_back);
 114     c->m_queue_front = event->prev;
 115     if (c->m_queue_front)
 116     {
 117         assert(c->m_queue_back);
 118         c->m_queue_front->next = 0;
 119     }
 120     else
 121         c->m_queue_back = 0;
 122     c->last_event = event->kind;
 123     return event;
 124 }
 125
 126 static void ZOOM_connection_remove_events(ZOOM_connection c)
 127 {
 128     ZOOM_Event event;
 129     while ((event = ZOOM_connection_get_event(c)))
 130         ZOOM_Event_destroy(event);
 131 }
 132
 133 ZOOM_API(int) ZOOM_connection_peek_event(ZOOM_connection c)
 134 {
 135     ZOOM_Event event = c->m_queue_front;
 136
 137     return event ? event->kind : ZOOM_EVENT_NONE;
 138 }
 139
 140 void ZOOM_connection_remove_tasks(ZOOM_connection c);
 141
 142 void ZOOM_set_dset_error(ZOOM_connection c, int error,
 143                          const char *dset,
 144                          const char *addinfo, const char *addinfo2)
 145 {
 146     char *cp;
 147
 148     xfree(c->addinfo);
 149     c->addinfo = 0;
 150     c->error = error;
 151     if (!c->diagset || strcmp(dset, c->diagset))
 152     {
 153         xfree(c->diagset);
 154         c->diagset = xstrdup(dset);
 155         /* remove integer part from SRW diagset .. */
 156         if ((cp = strrchr(c->diagset, '/')))
 157             *cp = '\0';
 158     }
 159     if (addinfo && addinfo2)
 160     {
 161         c->addinfo = (char*) xmalloc(strlen(addinfo) + strlen(addinfo2) + 2);
 162         strcpy(c->addinfo, addinfo);
 163         strcat(c->addinfo, addinfo2);
 164     }
 165     else if (addinfo)
 166         c->addinfo = xstrdup(addinfo);
 167     if (error != ZOOM_ERROR_NONE)
 168     {
 169         yaz_log(c->log_api, "%p set_dset_error %s %s:%d %s %s",
 170                 c, c->host_port ? c->host_port : "<>", dset, error,
 171                 addinfo ? addinfo : "",
 172                 addinfo2 ? addinfo2 : "");
 173         ZOOM_connection_remove_tasks(c);
 174     }
 175 }
 176
 177 static int uri_to_code(const char *uri)
 178 {
 179     int code = 0;
 180     const char *cp;
 181     if ((cp = strrchr(uri, '/')))
 182         code = atoi(cp+1);
 183     return code;
 184 }
 185
 186 #if YAZ_HAVE_XML2
 187 static void set_HTTP_error(ZOOM_connection c, int error,
 188                            const char *addinfo, const char *addinfo2)
 189 {
 190     ZOOM_set_dset_error(c, error, "HTTP", addinfo, addinfo2);
 191 }
 192
 193 static void set_SRU_error(ZOOM_connection c, Z_SRW_diagnostic *d)
 194 {
 195     const char *uri = d->uri;
 196     if (uri)
 197         ZOOM_set_dset_error(c, uri_to_code(uri), uri, d->details, 0);
 198 }
 199
 200 #endif
 201
 202
 203 void ZOOM_set_error(ZOOM_connection c, int error, const char *addinfo)
 204 {
 205     ZOOM_set_dset_error(c, error, "ZOOM", addinfo, 0);
 206 }
 207
 208 static void clear_error(ZOOM_connection c)
 209 {
 210     /*
 211      * If an error is tied to an operation then it's ok to clear: for
 212      * example, a diagnostic returned from a search is cleared by a
 213      * subsequent search.  However, problems such as Connection Lost
 214      * or Init Refused are not cleared, because they are not
 215      * recoverable: doing another search doesn't help.
 216      */
 217
 218     ZOOM_connection_remove_events(c);
 219     switch (c->error)
 220     {
 221     case ZOOM_ERROR_CONNECT:
 222     case ZOOM_ERROR_MEMORY:
 223     case ZOOM_ERROR_DECODE:
 224     case ZOOM_ERROR_CONNECTION_LOST:
 225     case ZOOM_ERROR_INIT:
 226     case ZOOM_ERROR_INTERNAL:
 227     case ZOOM_ERROR_UNSUPPORTED_PROTOCOL:
 228         break;
 229     default:
 230         ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
 231     }
 232 }
 233
 234 void ZOOM_connection_show_task(ZOOM_task task)
 235 {
 236     switch(task->which)
 237     {
 238     case ZOOM_TASK_SEARCH:
 239         yaz_log(YLOG_LOG, "search p=%p", task);
 240         break;
 241     case ZOOM_TASK_RETRIEVE:
 242         yaz_log(YLOG_LOG, "retrieve p=%p", task);
 243         break;
 244     case ZOOM_TASK_CONNECT:
 245         yaz_log(YLOG_LOG, "connect p=%p", task);
 246         break;
 247     case ZOOM_TASK_SCAN:
 248         yaz_log(YLOG_LOG, "scan p=%p", task);
 249         break;
 250     }
 251 }
 252
 253 void ZOOM_connection_show_tasks(ZOOM_connection c)
 254 {
 255     ZOOM_task task;
 256     yaz_log(YLOG_LOG, "connection p=%p tasks", c);
 257     for (task = c->tasks; task; task = task->next)
 258         ZOOM_connection_show_task(task);
 259 }
 260
 261 ZOOM_task ZOOM_connection_add_task(ZOOM_connection c, int which)
 262 {
 263     ZOOM_task *taskp = &c->tasks;
 264     while (*taskp)
 265         taskp = &(*taskp)->next;
 266     *taskp = (ZOOM_task) xmalloc(sizeof(**taskp));
 267     (*taskp)->running = 0;
 268     (*taskp)->which = which;
 269     (*taskp)->next = 0;
 270     clear_error(c);
 271     return *taskp;
 272 }
 273
 274 ZOOM_API(int) ZOOM_connection_is_idle(ZOOM_connection c)
 275 {
 276     return c->tasks ? 0 : 1;
 277 }
 278
 279 ZOOM_task ZOOM_connection_insert_task(ZOOM_connection c, int which)
 280 {
 281     ZOOM_task task = (ZOOM_task) xmalloc(sizeof(*task));
 282
 283     task->next = c->tasks;
 284     c->tasks = task;
 285
 286     task->running = 0;
 287     task->which = which;
 288     clear_error(c);
 289     return task;
 290 }
 291
 292 void ZOOM_connection_remove_task(ZOOM_connection c)
 293 {
 294     ZOOM_task task = c->tasks;
 295
 296     if (task)
 297     {
 298         c->tasks = task->next;
 299         switch (task->which)
 300         {
 301         case ZOOM_TASK_SEARCH:
 302             resultset_destroy(task->u.search.resultset);
 303             xfree(task->u.search.syntax);
 304             xfree(task->u.search.elementSetName);
 305             break;
 306         case ZOOM_TASK_RETRIEVE:
 307             resultset_destroy(task->u.retrieve.resultset);
 308             xfree(task->u.retrieve.syntax);
 309             xfree(task->u.retrieve.elementSetName);
 310             break;
 311         case ZOOM_TASK_CONNECT:
 312             break;
 313         case ZOOM_TASK_SCAN:
 314             ZOOM_scanset_destroy(task->u.scan.scan);
 315             break;
 316         case ZOOM_TASK_PACKAGE:
 317             ZOOM_package_destroy(task->u.package);
 318             break;
 319         case ZOOM_TASK_SORT:
 320             resultset_destroy(task->u.sort.resultset);
 321             ZOOM_query_destroy(task->u.sort.q);
 322             break;
 323         default:
 324             assert(0);
 325         }
 326         xfree(task);
 327
 328         if (!c->tasks)
 329         {
 330             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_END);
 331             ZOOM_connection_put_event(c, event);
 332         }
 333     }
 334 }
 335
 336 void ZOOM_connection_remove_tasks(ZOOM_connection c)
 337 {
 338     while (c->tasks)
 339         ZOOM_connection_remove_task(c);
 340 }
 341
 342
 343 ZOOM_API(ZOOM_connection)
 344     ZOOM_connection_create(ZOOM_options options)
 345 {
 346     ZOOM_connection c = (ZOOM_connection) xmalloc(sizeof(*c));
 347
 348     initlog();
 349
 350     c->log_api = log_api0;
 351     c->log_details = log_details0;
 352
 353     yaz_log(c->log_api, "%p ZOOM_connection_create", c);
 354
 355     c->proto = PROTO_Z3950;
 356     c->cs = 0;
 357     ZOOM_connection_set_mask(c, 0);
 358     c->reconnect_ok = 0;
 359     c->state = STATE_IDLE;
 360     c->addinfo = 0;
 361     c->diagset = 0;
 362     ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
 363     c->buf_in = 0;
 364     c->len_in = 0;
 365     c->buf_out = 0;
 366     c->len_out = 0;
 367     c->resultsets = 0;
 368
 369     c->options = ZOOM_options_create_with_parent(options);
 370
 371     c->host_port = 0;
 372     c->path = 0;
 373     c->proxy = 0;
 374
 375     c->charset = c->lang = 0;
 376
 377     c->cookie_out = 0;
 378     c->cookie_in = 0;
 379     c->client_IP = 0;
 380     c->tasks = 0;
 381
 382     c->user = 0;
 383     c->group = 0;
 384     c->password = 0;
 385
 386     c->maximum_record_size = 0;
 387     c->preferred_message_size = 0;
 388
 389     c->odr_in = odr_createmem(ODR_DECODE);
 390     c->odr_out = odr_createmem(ODR_ENCODE);
 391     c->odr_print = 0;
 392
 393     c->async = 0;
 394     c->support_named_resultsets = 0;
 395     c->last_event = ZOOM_EVENT_NONE;
 396
 397     c->m_queue_front = 0;
 398     c->m_queue_back = 0;
 399
 400     c->sru_version = 0;
 401     c->no_redirects = 0;
 402     return c;
 403 }
 404
 405
 406 /* set database names. Take local databases (if set); otherwise
 407    take databases given in ZURL (if set); otherwise use Default */
 408 char **ZOOM_connection_get_databases(ZOOM_connection con, ZOOM_options options,
 409                                      int *num, ODR odr)
 410 {
 411     char **databaseNames;
 412     const char *cp = ZOOM_options_get(options, "databaseName");
 413
 414     if ((!cp || !*cp) && con->host_port)
 415     {
 416         if (strncmp(con->host_port, "unix:", 5) == 0)
 417             cp = strchr(con->host_port+5, ':');
 418         else
 419             cp = strchr(con->host_port, '/');
 420         if (cp)
 421             cp++;
 422     }
 423     if (!cp)
 424         cp = "Default";
 425     nmem_strsplit(odr_getmem(odr), "+", cp,  &databaseNames, num);
 426     return databaseNames;
 427 }
 428
 429 ZOOM_API(ZOOM_connection)
 430     ZOOM_connection_new(const char *host, int portnum)
 431 {
 432     ZOOM_connection c = ZOOM_connection_create(0);
 433
 434     ZOOM_connection_connect(c, host, portnum);
 435     return c;
 436 }
 437
 438 static zoom_sru_mode get_sru_mode_from_string(const char *s)
 439 {
 440     if (!s || !*s)
 441         return zoom_sru_soap;
 442     if (!yaz_matchstr(s, "soap"))
 443         return zoom_sru_soap;
 444     else if (!yaz_matchstr(s, "get"))
 445         return zoom_sru_get;
 446     else if (!yaz_matchstr(s, "post"))
 447         return zoom_sru_post;
 448     return zoom_sru_error;
 449 }
 450
 451 ZOOM_API(void)
 452     ZOOM_connection_connect(ZOOM_connection c,
 453                             const char *host, int portnum)
 454 {
 455     const char *val;
 456     ZOOM_task task;
 457
 458     initlog();
 459
 460     yaz_log(c->log_api, "%p ZOOM_connection_connect host=%s portnum=%d",
 461             c, host ? host : "null", portnum);
 462
 463     ZOOM_set_error(c, ZOOM_ERROR_NONE, 0);
 464     ZOOM_connection_remove_tasks(c);
 465
 466     if (c->odr_print)
 467     {
 468         odr_setprint(c->odr_print, 0); /* prevent destroy from fclose'ing */
 469         odr_destroy(c->odr_print);
 470     }
 471     if (ZOOM_options_get_bool(c->options, "apdulog", 0))
 472     {
 473         c->odr_print = odr_createmem(ODR_PRINT);
 474         odr_setprint(c->odr_print, yaz_log_file());
 475     }
 476     else
 477         c->odr_print = 0;
 478
 479     if (c->cs)
 480     {
 481         yaz_log(c->log_details, "%p ZOOM_connection_connect reconnect ok", c);
 482         c->reconnect_ok = 1;
 483         return;
 484     }
 485     yaz_log(c->log_details, "%p ZOOM_connection_connect connect", c);
 486     xfree(c->proxy);
 487     c->proxy = 0;
 488     val = ZOOM_options_get(c->options, "proxy");
 489     if (val && *val)
 490     {
 491         yaz_log(c->log_details, "%p ZOOM_connection_connect proxy=%s", c, val);
 492         c->proxy = xstrdup(val);
 493     }
 494
 495     xfree(c->charset);
 496     c->charset = 0;
 497     val = ZOOM_options_get(c->options, "charset");
 498     if (val && *val)
 499     {
 500         yaz_log(c->log_details, "%p ZOOM_connection_connect charset=%s", c, val);
 501         c->charset = xstrdup(val);
 502     }
 503
 504     xfree(c->lang);
 505     val = ZOOM_options_get(c->options, "lang");
 506     if (val && *val)
 507     {
 508         yaz_log(c->log_details, "%p ZOOM_connection_connect lang=%s", c, val);
 509         c->lang = xstrdup(val);
 510     }
 511     else
 512         c->lang = 0;
 513
 514     if (host)
 515     {
 516         xfree(c->host_port);
 517         if (portnum)
 518         {
 519             char hostn[128];
 520             sprintf(hostn, "%.80s:%d", host, portnum);
 521             c->host_port = xstrdup(hostn);
 522         }
 523         else
 524             c->host_port = xstrdup(host);
 525     }
 526
 527     {
 528         /*
 529          * If the "<scheme>:" part of the host string is preceded by one
 530          * or more comma-separated <name>=<value> pairs, these are taken
 531          * to be options to be set on the connection object.  Among other
 532          * applications, this facility can be used to embed authentication
 533          * in a host string:
 534          *          user=admin,password=secret,tcp:localhost:9999
 535          */
 536         char *remainder = c->host_port;
 537         char *pcolon = strchr(remainder, ':');
 538         char *pcomma;
 539         char *pequals;
 540         while ((pcomma = strchr(remainder, ',')) != 0 &&
 541                (pcolon == 0 || pcomma < pcolon)) {
 542             *pcomma = '\0';
 543             if ((pequals = strchr(remainder, '=')) != 0) {
 544                 *pequals = '\0';
 545                 /*printf("# setting '%s'='%s'\n", remainder, pequals+1);*/
 546                 ZOOM_connection_option_set(c, remainder, pequals+1);
 547             }
 548             remainder = pcomma+1;
 549         }
 550
 551         if (remainder != c->host_port) {
 552             xfree(c->host_port);
 553             c->host_port = xstrdup(remainder);
 554             /*printf("# reset hp='%s'\n", remainder);*/
 555         }
 556     }
 557
 558     val = ZOOM_options_get(c->options, "sru");
 559     c->sru_mode = get_sru_mode_from_string(val);
 560
 561     xfree(c->sru_version);
 562     val = ZOOM_options_get(c->options, "sru_version");
 563     c->sru_version = xstrdup(val ? val : "1.2");
 564
 565     ZOOM_options_set(c->options, "host", c->host_port);
 566
 567     xfree(c->cookie_out);
 568     c->cookie_out = 0;
 569     val = ZOOM_options_get(c->options, "cookie");
 570     if (val && *val)
 571     {
 572         yaz_log(c->log_details, "%p ZOOM_connection_connect cookie=%s", c, val);
 573         c->cookie_out = xstrdup(val);
 574     }
 575
 576     xfree(c->client_IP);
 577     c->client_IP = 0;
 578     val = ZOOM_options_get(c->options, "clientIP");
 579     if (val && *val)
 580     {
 581         yaz_log(c->log_details, "%p ZOOM_connection_connect clientIP=%s",
 582                 c, val);
 583         c->client_IP = xstrdup(val);
 584     }
 585
 586     xfree(c->group);
 587     c->group = 0;
 588     val = ZOOM_options_get(c->options, "group");
 589     if (val && *val)
 590         c->group = xstrdup(val);
 591
 592     xfree(c->user);
 593     c->user = 0;
 594     val = ZOOM_options_get(c->options, "user");
 595     if (val && *val)
 596         c->user = xstrdup(val);
 597
 598     xfree(c->password);
 599     c->password = 0;
 600     val = ZOOM_options_get(c->options, "password");
 601     if (!val)
 602         val = ZOOM_options_get(c->options, "pass");
 603
 604     if (val && *val)
 605         c->password = xstrdup(val);
 606
 607     c->maximum_record_size =
 608         ZOOM_options_get_int(c->options, "maximumRecordSize", 1024*1024);
 609     c->preferred_message_size =
 610         ZOOM_options_get_int(c->options, "preferredMessageSize", 1024*1024);
 611
 612     c->async = ZOOM_options_get_bool(c->options, "async", 0);
 613     yaz_log(c->log_details, "%p ZOOM_connection_connect async=%d", c, c->async);
 614
 615     task = ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
 616
 617     if (!c->async)
 618     {
 619         while (ZOOM_event(1, &c))
 620             ;
 621     }
 622 }
 623
 624 ZOOM_API(ZOOM_query)
 625     ZOOM_query_create(void)
 626 {
 627     ZOOM_query s = (ZOOM_query) xmalloc(sizeof(*s));
 628
 629     s->refcount = 1;
 630     s->z_query = 0;
 631     s->sort_spec = 0;
 632     s->odr = odr_createmem(ODR_ENCODE);
 633     s->query_string = 0;
 634
 635     return s;
 636 }
 637
 638 ZOOM_API(void)
 639     ZOOM_query_destroy(ZOOM_query s)
 640 {
 641     if (!s)
 642         return;
 643
 644     (s->refcount)--;
 645     if (s->refcount == 0)
 646     {
 647         odr_destroy(s->odr);
 648         xfree(s);
 649     }
 650 }
 651
 652 ZOOM_API(int)
 653     ZOOM_query_prefix(ZOOM_query s, const char *str)
 654 {
 655     s->query_string = odr_strdup(s->odr, str);
 656     s->z_query = (Z_Query *) odr_malloc(s->odr, sizeof(*s->z_query));
 657     s->z_query->which = Z_Query_type_1;
 658     s->z_query->u.type_1 =  p_query_rpn(s->odr, str);
 659     if (!s->z_query->u.type_1)
 660     {
 661         s->z_query = 0;
 662         return -1;
 663     }
 664     return 0;
 665 }
 666
 667 ZOOM_API(int)
 668     ZOOM_query_cql(ZOOM_query s, const char *str)
 669 {
 670     Z_External *ext;
 671
 672     s->query_string = odr_strdup(s->odr, str);
 673
 674     ext = (Z_External *) odr_malloc(s->odr, sizeof(*ext));
 675     ext->direct_reference = odr_oiddup(s->odr, yaz_oid_userinfo_cql);
 676     ext->indirect_reference = 0;
 677     ext->descriptor = 0;
 678     ext->which = Z_External_CQL;
 679     ext->u.cql = s->query_string;
 680
 681     s->z_query = (Z_Query *) odr_malloc(s->odr, sizeof(*s->z_query));
 682     s->z_query->which = Z_Query_type_104;
 683     s->z_query->u.type_104 =  ext;
 684
 685     return 0;
 686 }
 687
 688 /*
 689  * Translate the CQL string client-side into RPN which is passed to
 690  * the server.  This is useful for server's that don't themselves
 691  * support CQL, for which ZOOM_query_cql() is useless.  `conn' is used
 692  * only as a place to stash diagnostics if compilation fails; if this
 693  * information is not needed, a null pointer may be used.
 694  */
 695 ZOOM_API(int)
 696     ZOOM_query_cql2rpn(ZOOM_query s, const char *str, ZOOM_connection conn)
 697 {
 698     char *rpn;
 699     int ret;
 700     ZOOM_connection freeme = 0;
 701
 702     if (conn == 0)
 703         conn = freeme = ZOOM_connection_create(0);
 704
 705     rpn = cql2pqf(conn, str);
 706     if (freeme != 0)
 707         ZOOM_connection_destroy(freeme);
 708     if (rpn == 0)
 709         return -1;
 710
 711     ret = ZOOM_query_prefix(s, rpn);
 712     xfree(rpn);
 713     return ret;
 714 }
 715
 716 /*
 717  * Analogous in every way to ZOOM_query_cql2rpn(), except that there
 718  * is no analogous ZOOM_query_ccl() that just sends uninterpreted CCL
 719  * to the server, as the YAZ GFS doesn't know how to handle this.
 720  */
 721 ZOOM_API(int)
 722     ZOOM_query_ccl2rpn(ZOOM_query s, const char *str, const char *config,
 723                        int *ccl_error, const char **error_string,
 724                        int *error_pos)
 725 {
 726     int ret;
 727     struct ccl_rpn_node *rpn;
 728     CCL_bibset bibset = ccl_qual_mk();
 729
 730     if (config)
 731         ccl_qual_buf(bibset, config);
 732
 733     rpn = ccl_find_str(bibset, str, ccl_error, error_pos);
 734     if (!rpn)
 735     {
 736         *error_string = ccl_err_msg(*ccl_error);
 737         ret = -1;
 738     }
 739     else
 740     {
 741         WRBUF wr = wrbuf_alloc();
 742         ccl_pquery(wr, rpn);
 743         ccl_rpn_delete(rpn);
 744         ret = ZOOM_query_prefix(s, wrbuf_cstr(wr));
 745         wrbuf_destroy(wr);
 746     }
 747     ccl_qual_rm(&bibset);
 748     return ret;
 749 }
 750
 751 ZOOM_API(int)
 752     ZOOM_query_sortby(ZOOM_query s, const char *criteria)
 753 {
 754     s->sort_spec = yaz_sort_spec(s->odr, criteria);
 755     if (!s->sort_spec)
 756         return -1;
 757     return 0;
 758 }
 759
 760 ZOOM_API(void) ZOOM_resultset_release(ZOOM_resultset r)
 761 {
 762 #if ZOOM_RESULT_LISTS
 763 #else
 764     if (r->connection)
 765     {
 766         /* remove ourselves from the resultsets in connection */
 767         ZOOM_resultset *rp = &r->connection->resultsets;
 768         while (1)
 769         {
 770             assert(*rp);   /* we must be in this list!! */
 771             if (*rp == r)
 772             {   /* OK, we're here - take us out of it */
 773                 *rp = (*rp)->next;
 774                 break;
 775             }
 776             rp = &(*rp)->next;
 777         }
 778         r->connection = 0;
 779     }
 780 #endif
 781 }
 782
 783 ZOOM_API(void)
 784     ZOOM_connection_destroy(ZOOM_connection c)
 785 {
 786 #if ZOOM_RESULT_LISTS
 787     ZOOM_resultsets list;
 788 #else
 789     ZOOM_resultset r;
 790 #endif
 791     if (!c)
 792         return;
 793     yaz_log(c->log_api, "%p ZOOM_connection_destroy", c);
 794     if (c->cs)
 795         cs_close(c->cs);
 796
 797 #if ZOOM_RESULT_LISTS
 798     // Remove the connection's usage of resultsets
 799     list = c->resultsets;
 800     while (list) {
 801         ZOOM_resultsets removed = list;
 802         ZOOM_resultset_destroy(list->resultset);
 803         list = list->next;
 804         xfree(removed);
 805     }
 806 #else
 807     for (r = c->resultsets; r; r = r->next)
 808         r->connection = 0;
 809 #endif
 810
 811     xfree(c->buf_in);
 812     xfree(c->addinfo);
 813     xfree(c->diagset);
 814     odr_destroy(c->odr_in);
 815     odr_destroy(c->odr_out);
 816     if (c->odr_print)
 817     {
 818         odr_setprint(c->odr_print, 0); /* prevent destroy from fclose'ing */
 819         odr_destroy(c->odr_print);
 820     }
 821     ZOOM_options_destroy(c->options);
 822     ZOOM_connection_remove_tasks(c);
 823     ZOOM_connection_remove_events(c);
 824     xfree(c->host_port);
 825     xfree(c->path);
 826     xfree(c->proxy);
 827     xfree(c->charset);
 828     xfree(c->lang);
 829     xfree(c->cookie_out);
 830     xfree(c->cookie_in);
 831     xfree(c->client_IP);
 832     xfree(c->user);
 833     xfree(c->group);
 834     xfree(c->password);
 835     xfree(c->sru_version);
 836     xfree(c);
 837 }
 838
 839 void ZOOM_resultset_addref(ZOOM_resultset r)
 840 {
 841     if (r)
 842     {
 843         yaz_mutex_enter(r->mutex);
 844         (r->refcount)++;
 845         yaz_log(log_details0, "%p ZOOM_resultset_addref count=%d",
 846                 r, r->refcount);
 847         yaz_mutex_leave(r->mutex);
 848     }
 849 }
 850
 851 ZOOM_resultset ZOOM_resultset_create(void)
 852 {
 853     int i;
 854     ZOOM_resultset r = (ZOOM_resultset) xmalloc(sizeof(*r));
 855
 856     initlog();
 857
 858     yaz_log(log_details0, "%p ZOOM_resultset_create", r);
 859     r->refcount = 1;
 860     r->size = 0;
 861     r->odr = odr_createmem(ODR_ENCODE);
 862     r->piggyback = 1;
 863     r->setname = 0;
 864     r->schema = 0;
 865     r->step = 0;
 866     for (i = 0; i<RECORD_HASH_SIZE; i++)
 867         r->record_hash[i] = 0;
 868     r->r_sort_spec = 0;
 869     r->query = 0;
 870     r->connection = 0;
 871     r->databaseNames = 0;
 872     r->num_databaseNames = 0;
 873     r->facets = 0;
 874     r->num_facets = 0;
 875     r->facets_names = 0;
 876     r->mutex = 0;
 877     yaz_mutex_create(&r->mutex);
 878 #if SHPTR
 879     {
 880         WRBUF w = wrbuf_alloc();
 881         YAZ_SHPTR_INIT(r->record_wrbuf, w);
 882     }
 883 #endif
 884     return r;
 885 }
 886
 887 ZOOM_API(ZOOM_resultset)
 888     ZOOM_connection_search_pqf(ZOOM_connection c, const char *q)
 889 {
 890     ZOOM_resultset r;
 891     ZOOM_query s = ZOOM_query_create();
 892
 893     ZOOM_query_prefix(s, q);
 894
 895     r = ZOOM_connection_search(c, s);
 896     ZOOM_query_destroy(s);
 897     return r;
 898 }
 899
 900 ZOOM_API(ZOOM_resultset)
 901     ZOOM_connection_search(ZOOM_connection c, ZOOM_query q)
 902 {
 903     ZOOM_resultset r = ZOOM_resultset_create();
 904     ZOOM_task task;
 905     const char *cp;
 906     int start, count;
 907     const char *syntax, *elementSetName;
 908 #if ZOOM_RESULT_LISTS
 909     ZOOM_resultsets set;
 910 #endif
 911
 912     yaz_log(c->log_api, "%p ZOOM_connection_search set %p query %p", c, r, q);
 913     r->r_sort_spec = q->sort_spec;
 914     r->query = q;
 915
 916     r->options = ZOOM_options_create_with_parent(c->options);
 917
 918     start = ZOOM_options_get_int(r->options, "start", 0);
 919     count = ZOOM_options_get_int(r->options, "count", 0);
 920     {
 921         /* If "presentChunk" is defined use that; otherwise "step" */
 922         const char *cp = ZOOM_options_get(r->options, "presentChunk");
 923         r->step = ZOOM_options_get_int(r->options,
 924                                        (cp != 0 ? "presentChunk": "step"), 0);
 925     }
 926     r->piggyback = ZOOM_options_get_bool(r->options, "piggyback", 1);
 927     cp = ZOOM_options_get(r->options, "setname");
 928     if (cp)
 929         r->setname = xstrdup(cp);
 930     cp = ZOOM_options_get(r->options, "schema");
 931     if (cp)
 932         r->schema = xstrdup(cp);
 933
 934     r->databaseNames = ZOOM_connection_get_databases(c, c->options, &r->num_databaseNames,
 935                                          r->odr);
 936
 937     r->connection = c;
 938
 939 #if ZOOM_RESULT_LISTS
 940     yaz_log(log_details, "%p ZOOM_connection_search: Adding new resultset (%p) to resultsets (%p) ", c, r, c->resultsets);
 941     set = xmalloc(sizeof(*set));
 942     ZOOM_resultset_addref(r);
 943     set->resultset = r;
 944     set->next = c->resultsets;
 945     c->resultsets = set;
 946 #else
 947     r->next = c->resultsets;
 948     c->resultsets = r;
 949 #endif
 950     if (c->host_port && c->proto == PROTO_HTTP)
 951     {
 952         if (!c->cs)
 953         {
 954             yaz_log(c->log_details, "ZOOM_connection_search: no comstack");
 955             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
 956         }
 957         else
 958         {
 959             yaz_log(c->log_details, "ZOOM_connection_search: reconnect");
 960             c->reconnect_ok = 1;
 961         }
 962     }
 963
 964     task = ZOOM_connection_add_task(c, ZOOM_TASK_SEARCH);
 965     task->u.search.resultset = r;
 966     task->u.search.start = start;
 967     task->u.search.count = count;
 968     task->u.search.recv_search_fired = 0;
 969
 970     syntax = ZOOM_options_get(r->options, "preferredRecordSyntax");
 971     task->u.search.syntax = syntax ? xstrdup(syntax) : 0;
 972     elementSetName = ZOOM_options_get(r->options, "elementSetName");
 973     task->u.search.elementSetName = elementSetName
 974         ? xstrdup(elementSetName) : 0;
 975
 976     ZOOM_resultset_addref(r);
 977
 978     (q->refcount)++;
 979
 980     if (!c->async)
 981     {
 982         while (ZOOM_event(1, &c))
 983             ;
 984     }
 985     return r;
 986 }
 987
 988 ZOOM_API(void)
 989     ZOOM_resultset_sort(ZOOM_resultset r,
 990                          const char *sort_type, const char *sort_spec)
 991 {
 992     (void) ZOOM_resultset_sort1(r, sort_type, sort_spec);
 993 }
 994
 995 ZOOM_API(int)
 996     ZOOM_resultset_sort1(ZOOM_resultset r,
 997                          const char *sort_type, const char *sort_spec)
 998 {
 999     ZOOM_connection c = r->connection;
1000     ZOOM_task task;
1001     ZOOM_query newq;
1002
1003     newq = ZOOM_query_create();
1004     if (ZOOM_query_sortby(newq, sort_spec) < 0)
1005         return -1;
1006
1007     yaz_log(c->log_api, "%p ZOOM_resultset_sort r=%p sort_type=%s sort_spec=%s",
1008             r, r, sort_type, sort_spec);
1009     if (!c)
1010         return 0;
1011
1012     if (c->host_port && c->proto == PROTO_HTTP)
1013     {
1014         if (!c->cs)
1015         {
1016             yaz_log(c->log_details, "%p ZOOM_resultset_sort: no comstack", r);
1017             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
1018         }
1019         else
1020         {
1021             yaz_log(c->log_details, "%p ZOOM_resultset_sort: prepare reconnect",
1022                     r);
1023             c->reconnect_ok = 1;
1024         }
1025     }
1026
1027     ZOOM_resultset_cache_reset(r);
1028     task = ZOOM_connection_add_task(c, ZOOM_TASK_SORT);
1029     task->u.sort.resultset = r;
1030     task->u.sort.q = newq;
1031
1032     ZOOM_resultset_addref(r);
1033
1034     if (!c->async)
1035     {
1036         while (ZOOM_event(1, &c))
1037             ;
1038     }
1039
1040     return 0;
1041 }
1042
1043 static void ZOOM_record_release(ZOOM_record rec)
1044 {
1045     if (!rec)
1046         return;
1047
1048 #if SHPTR
1049     if (rec->record_wrbuf)
1050         YAZ_SHPTR_DEC(rec->record_wrbuf, wrbuf_destroy);
1051 #else
1052     if (rec->wrbuf)
1053         wrbuf_destroy(rec->wrbuf);
1054 #endif
1055
1056     if (rec->odr)
1057         odr_destroy(rec->odr);
1058 }
1059
1060 ZOOM_API(void)
1061     ZOOM_resultset_cache_reset(ZOOM_resultset r)
1062 {
1063     int i;
1064     for (i = 0; i<RECORD_HASH_SIZE; i++)
1065     {
1066         ZOOM_record_cache rc;
1067         for (rc = r->record_hash[i]; rc; rc = rc->next)
1068         {
1069             ZOOM_record_release(&rc->rec);
1070         }
1071         r->record_hash[i] = 0;
1072     }
1073 }
1074
1075 ZOOM_API(void)
1076     ZOOM_resultset_destroy(ZOOM_resultset r)
1077 {
1078     resultset_destroy(r);
1079 }
1080
1081 static void resultset_destroy(ZOOM_resultset r)
1082 {
1083     if (!r)
1084         return;
1085     yaz_mutex_enter(r->mutex);
1086     (r->refcount)--;
1087     yaz_log(log_details0, "%p ZOOM_resultset_destroy r=%p count=%d",
1088             r, r, r->refcount);
1089     if (r->refcount == 0)
1090     {
1091         yaz_mutex_leave(r->mutex);
1092
1093         yaz_log(log_details0, "%p ZOOM_connection resultset_destroy: Deleting resultset (%p) ", r->connection, r);
1094         ZOOM_resultset_cache_reset(r);
1095         ZOOM_resultset_release(r);
1096         ZOOM_query_destroy(r->query);
1097         ZOOM_options_destroy(r->options);
1098         odr_destroy(r->odr);
1099         xfree(r->setname);
1100         xfree(r->schema);
1101         yaz_mutex_destroy(&r->mutex);
1102 #if SHPTR
1103         YAZ_SHPTR_DEC(r->record_wrbuf, wrbuf_destroy);
1104 #endif
1105         xfree(r);
1106     }
1107     else
1108         yaz_mutex_leave(r->mutex);
1109 }
1110
1111 ZOOM_API(size_t)
1112     ZOOM_resultset_size(ZOOM_resultset r)
1113 {
1114     return r->size;
1115 }
1116
1117 int ZOOM_test_reconnect(ZOOM_connection c)
1118 {
1119     ZOOM_Event event;
1120
1121     if (!c->reconnect_ok)
1122         return 0;
1123     ZOOM_connection_close(c);
1124     c->reconnect_ok = 0;
1125     c->tasks->running = 0;
1126     ZOOM_connection_insert_task(c, ZOOM_TASK_CONNECT);
1127
1128     event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1129     ZOOM_connection_put_event(c, event);
1130
1131     return 1;
1132 }
1133
1134 static void ZOOM_resultset_retrieve(ZOOM_resultset r,
1135                                     int force_sync, int start, int count)
1136 {
1137     ZOOM_task task;
1138     ZOOM_connection c;
1139     const char *cp;
1140     const char *syntax, *elementSetName;
1141
1142     if (!r)
1143         return;
1144     yaz_log(log_details0, "%p ZOOM_resultset_retrieve force_sync=%d start=%d"
1145             " count=%d", r, force_sync, start, count);
1146     c = r->connection;
1147     if (!c)
1148         return;
1149
1150     if (c->host_port && c->proto == PROTO_HTTP)
1151     {
1152         if (!c->cs)
1153         {
1154             yaz_log(log_details0, "%p ZOOM_resultset_retrieve: no comstack", r);
1155             ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
1156         }
1157         else
1158         {
1159             yaz_log(log_details0, "%p ZOOM_resultset_retrieve: prepare "
1160                     "reconnect", r);
1161             c->reconnect_ok = 1;
1162         }
1163     }
1164     task = ZOOM_connection_add_task(c, ZOOM_TASK_RETRIEVE);
1165     task->u.retrieve.resultset = r;
1166     task->u.retrieve.start = start;
1167     task->u.retrieve.count = count;
1168
1169     syntax = ZOOM_options_get(r->options, "preferredRecordSyntax");
1170     task->u.retrieve.syntax = syntax ? xstrdup(syntax) : 0;
1171     elementSetName = ZOOM_options_get(r->options, "elementSetName");
1172     task->u.retrieve.elementSetName = elementSetName
1173         ? xstrdup(elementSetName) : 0;
1174
1175     cp = ZOOM_options_get(r->options, "schema");
1176     if (cp)
1177     {
1178         if (!r->schema || strcmp(r->schema, cp))
1179         {
1180             xfree(r->schema);
1181             r->schema = xstrdup(cp);
1182         }
1183     }
1184
1185     ZOOM_resultset_addref(r);
1186
1187     if (!r->connection->async || force_sync)
1188         while (r->connection && ZOOM_event(1, &r->connection))
1189             ;
1190 }
1191
1192 ZOOM_API(void)
1193     ZOOM_resultset_records(ZOOM_resultset r, ZOOM_record *recs,
1194                            size_t start, size_t count)
1195 {
1196     int force_present = 0;
1197
1198     if (!r)
1199         return ;
1200     yaz_log(log_api0, "%p ZOOM_resultset_records r=%p start=%ld count=%ld",
1201             r, r, (long) start, (long) count);
1202     if (count && recs)
1203         force_present = 1;
1204     ZOOM_resultset_retrieve(r, force_present, start, count);
1205     if (force_present)
1206     {
1207         size_t i;
1208         for (i = 0; i< count; i++)
1209             recs[i] = ZOOM_resultset_record_immediate(r, i+start);
1210     }
1211 }
1212
1213 ZOOM_API(size_t)
1214     ZOOM_resultset_facets_size(ZOOM_resultset r) {
1215     return r->num_facets;
1216 }
1217
1218 ZOOM_API(ZOOM_facet_field)
1219     ZOOM_resultset_get_facet_field(ZOOM_resultset r, const char *name) {
1220     int num = r->num_facets;
1221     ZOOM_facet_field *facets = r->facets;
1222     int index;
1223     for (index = 0; index < num; index++) {
1224         if (!strcmp(facets[index]->facet_name, name)) {
1225             return facets[index];
1226         }
1227     }
1228     return 0;
1229 }
1230
1231
1232 ZOOM_API(ZOOM_facet_field *)
1233     ZOOM_resultset_facets(ZOOM_resultset r)
1234 {
1235     return r->facets;
1236 }
1237
1238 ZOOM_API(const char**)
1239     ZOOM_resultset_facet_names(ZOOM_resultset r)
1240 {
1241     return (const char **) r->facets_names;
1242 }
1243
1244 ZOOM_API(const char*)
1245     ZOOM_facet_field_name(ZOOM_facet_field field)
1246 {
1247     return field->facet_name;
1248 }
1249
1250 ZOOM_API(size_t)
1251     ZOOM_facet_field_term_count(ZOOM_facet_field field)
1252 {
1253     return field->num_terms;
1254 }
1255
1256 ZOOM_API(const char*)
1257     ZOOM_facet_field_get_term(ZOOM_facet_field field, size_t idx, int *freq) {
1258     *freq = field->facet_terms[idx].frequency;
1259     return field->facet_terms[idx].term;
1260 }
1261
1262
1263 static void get_cert(ZOOM_connection c)
1264 {
1265     char *cert_buf;
1266     int cert_len;
1267
1268     if (cs_get_peer_certificate_x509(c->cs, &cert_buf, &cert_len))
1269     {
1270         ZOOM_connection_option_setl(c, "sslPeerCert",
1271                                     cert_buf, cert_len);
1272         xfree(cert_buf);
1273     }
1274 }
1275
1276 static zoom_ret do_connect_host(ZOOM_connection c,
1277                                 const char *effective_host,
1278                                 const char *logical_url);
1279
1280 static zoom_ret do_connect(ZOOM_connection c)
1281 {
1282     const char *effective_host;
1283
1284     if (c->proxy)
1285         effective_host = c->proxy;
1286     else
1287         effective_host = c->host_port;
1288     return do_connect_host(c, effective_host, c->host_port);
1289 }
1290
1291 static zoom_ret do_connect_host(ZOOM_connection c, const char *effective_host,
1292     const char *logical_url)
1293 {
1294     void *add;
1295
1296     yaz_log(c->log_details, "%p do_connect effective_host=%s", c, effective_host);
1297
1298     if (c->cs)
1299         cs_close(c->cs);
1300     c->cs = cs_create_host(effective_host, 0, &add);
1301
1302     if (c->cs && c->cs->protocol == PROTO_HTTP)
1303     {
1304 #if YAZ_HAVE_XML2
1305         if (logical_url)
1306         {
1307             const char *db = 0;
1308
1309             c->proto = PROTO_HTTP;
1310             cs_get_host_args(logical_url, &db);
1311             xfree(c->path);
1312
1313             c->path = xmalloc(strlen(db) * 3 + 2);
1314             yaz_encode_sru_dbpath_buf(c->path, db);
1315         }
1316 #else
1317         ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, "SRW");
1318         ZOOM_connection_close(c);
1319         return zoom_complete;
1320 #endif
1321     }
1322     if (c->cs)
1323     {
1324         int ret = cs_connect(c->cs, add);
1325         if (ret == 0)
1326         {
1327             ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
1328             ZOOM_connection_put_event(c, event);
1329             get_cert(c);
1330             if (c->proto == PROTO_Z3950)
1331                 ZOOM_connection_Z3950_send_init(c);
1332             else
1333             {
1334                 /* no init request for SRW .. */
1335                 assert(c->tasks->which == ZOOM_TASK_CONNECT);
1336                 ZOOM_connection_remove_task(c);
1337                 ZOOM_connection_set_mask(c, 0);
1338                 ZOOM_connection_exec_task(c);
1339             }
1340             c->state = STATE_ESTABLISHED;
1341             return zoom_pending;
1342         }
1343         else if (ret > 0)
1344         {
1345             int mask = ZOOM_SELECT_EXCEPT;
1346             if (c->cs->io_pending & CS_WANT_WRITE)
1347                 mask += ZOOM_SELECT_WRITE;
1348             if (c->cs->io_pending & CS_WANT_READ)
1349                 mask += ZOOM_SELECT_READ;
1350             ZOOM_connection_set_mask(c, mask);
1351             c->state = STATE_CONNECTING;
1352             return zoom_pending;
1353         }
1354     }
1355     c->state = STATE_IDLE;
1356     ZOOM_set_error(c, ZOOM_ERROR_CONNECT, logical_url);
1357     return zoom_complete;
1358 }
1359
1360 /* returns 1 if PDU was sent OK (still pending )
1361    0 if PDU was not sent OK (nothing to wait for)
1362 */
1363
1364 #if YAZ_HAVE_XML2
1365 static zoom_ret send_srw(ZOOM_connection c, Z_SRW_PDU *sr)
1366 {
1367     Z_GDU *gdu;
1368     ZOOM_Event event;
1369     const char *database =  ZOOM_options_get(c->options, "databaseName");
1370     char *fdatabase = 0;
1371
1372     if (database)
1373         fdatabase = yaz_encode_sru_dbpath_odr(c->odr_out, database);
1374     gdu = z_get_HTTP_Request_host_path(c->odr_out, c->host_port,
1375                                        fdatabase ? fdatabase : c->path);
1376
1377     if (c->sru_mode == zoom_sru_get)
1378     {
1379         yaz_sru_get_encode(gdu->u.HTTP_Request, sr, c->odr_out, c->charset);
1380     }
1381     else if (c->sru_mode == zoom_sru_post)
1382     {
1383         yaz_sru_post_encode(gdu->u.HTTP_Request, sr, c->odr_out, c->charset);
1384     }
1385     else if (c->sru_mode == zoom_sru_soap)
1386     {
1387         yaz_sru_soap_encode(gdu->u.HTTP_Request, sr, c->odr_out, c->charset);
1388     }
1389     if (!z_GDU(c->odr_out, &gdu, 0, 0))
1390         return zoom_complete;
1391     if (c->odr_print)
1392         z_GDU(c->odr_print, &gdu, 0, 0);
1393     c->buf_out = odr_getbuf(c->odr_out, &c->len_out, 0);
1394
1395     event = ZOOM_Event_create(ZOOM_EVENT_SEND_APDU);
1396     ZOOM_connection_put_event(c, event);
1397     odr_reset(c->odr_out);
1398     return ZOOM_send_buf(c);
1399 }
1400 #endif
1401
1402 #if YAZ_HAVE_XML2
1403 static Z_SRW_PDU *ZOOM_srw_get_pdu(ZOOM_connection c, int type)
1404 {
1405     Z_SRW_PDU *sr = yaz_srw_get_pdu(c->odr_out, type, c->sru_version);
1406     sr->username = c->user;
1407     sr->password = c->password;
1408     return sr;
1409 }
1410 #endif
1411
1412 #if YAZ_HAVE_XML2
1413 static zoom_ret ZOOM_connection_srw_send_search(ZOOM_connection c)
1414 {
1415     int i;
1416     int *start, *count;
1417     ZOOM_resultset resultset = 0;
1418     Z_SRW_PDU *sr = 0;
1419     const char *option_val = 0;
1420
1421     if (c->error)                  /* don't continue on error */
1422         return zoom_complete;
1423     assert(c->tasks);
1424     switch(c->tasks->which)
1425     {
1426     case ZOOM_TASK_SEARCH:
1427         resultset = c->tasks->u.search.resultset;
1428         if (!resultset->setname)
1429             resultset->setname = xstrdup("default");
1430         ZOOM_options_set(resultset->options, "setname", resultset->setname);
1431         start = &c->tasks->u.search.start;
1432         count = &c->tasks->u.search.count;
1433         break;
1434     case ZOOM_TASK_RETRIEVE:
1435         resultset = c->tasks->u.retrieve.resultset;
1436
1437         start = &c->tasks->u.retrieve.start;
1438         count = &c->tasks->u.retrieve.count;
1439
1440         if (*start >= resultset->size)
1441             return zoom_complete;
1442         if (*start + *count > resultset->size)
1443             *count = resultset->size - *start;
1444
1445         for (i = 0; i < *count; i++)
1446         {
1447             ZOOM_record rec =
1448                 ZOOM_record_cache_lookup(resultset, i + *start,
1449                                          c->tasks->u.retrieve.syntax,
1450                                          c->tasks->u.retrieve.elementSetName);
1451             if (!rec)
1452                 break;
1453             else
1454             {
1455                 ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_RECV_RECORD);
1456                 ZOOM_connection_put_event(c, event);
1457             }
1458         }
1459         *start += i;
1460         *count -= i;
1461
1462         if (*count == 0)
1463             return zoom_complete;
1464         break;
1465     default:
1466         return zoom_complete;
1467     }
1468     assert(resultset->query);
1469
1470     sr = ZOOM_srw_get_pdu(c, Z_SRW_searchRetrieve_request);
1471     if (resultset->query->z_query->which == Z_Query_type_104
1472         && resultset->query->z_query->u.type_104->which == Z_External_CQL)
1473     {
1474         sr->u.request->query_type = Z_SRW_query_type_cql;
1475         sr->u.request->query.cql =resultset->query->z_query->u.type_104->u.cql;
1476     }
1477     else if (resultset->query->z_query->which == Z_Query_type_1 &&
1478              resultset->query->z_query->u.type_1)
1479     {
1480         sr->u.request->query_type = Z_SRW_query_type_pqf;
1481         sr->u.request->query.pqf = resultset->query->query_string;
1482     }
1483     else
1484     {
1485         ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_QUERY, 0);
1486         return zoom_complete;
1487     }
1488     sr->u.request->startRecord = odr_intdup(c->odr_out, *start + 1);
1489     sr->u.request->maximumRecords = odr_intdup(
1490         c->odr_out, (resultset->step > 0 && resultset->step < *count) ?
1491         resultset->step : *count);
1492     sr->u.request->recordSchema = resultset->schema;
1493
1494     option_val = ZOOM_resultset_option_get(resultset, "recordPacking");
1495     if (option_val)
1496         sr->u.request->recordPacking = odr_strdup(c->odr_out, option_val);
1497
1498     option_val = ZOOM_resultset_option_get(resultset, "extraArgs");
1499     yaz_encode_sru_extra(sr, c->odr_out, option_val);
1500     return send_srw(c, sr);
1501 }
1502 #else
1503 static zoom_ret ZOOM_connection_srw_send_search(ZOOM_connection c)
1504 {
1505     return zoom_complete;
1506 }
1507 #endif
1508
1509 ZOOM_API(ZOOM_record)
1510     ZOOM_record_clone(ZOOM_record srec)
1511 {
1512     char *buf;
1513     int size;
1514     ODR odr_enc;
1515     ZOOM_record nrec;
1516
1517     odr_enc = odr_createmem(ODR_ENCODE);
1518     if (!z_NamePlusRecord(odr_enc, &srec->npr, 0, 0))
1519         return 0;
1520     buf = odr_getbuf(odr_enc, &size, 0);
1521
1522     nrec = (ZOOM_record) xmalloc(sizeof(*nrec));
1523     yaz_log(log_details0, "ZOOM_record create");
1524     nrec->odr = odr_createmem(ODR_DECODE);
1525 #if SHPTR
1526     nrec->record_wrbuf = 0;
1527 #else
1528     nrec->wrbuf = 0;
1529 #endif
1530     odr_setbuf(nrec->odr, buf, size, 0);
1531     z_NamePlusRecord(nrec->odr, &nrec->npr, 0, 0);
1532
1533     nrec->schema = odr_strdup_null(nrec->odr, srec->schema);
1534     nrec->diag_uri = odr_strdup_null(nrec->odr, srec->diag_uri);
1535     nrec->diag_message = odr_strdup_null(nrec->odr, srec->diag_message);
1536     nrec->diag_details = odr_strdup_null(nrec->odr, srec->diag_details);
1537     nrec->diag_set = odr_strdup_null(nrec->odr, srec->diag_set);
1538     odr_destroy(odr_enc);
1539     return nrec;
1540 }
1541
1542 ZOOM_API(ZOOM_record)
1543     ZOOM_resultset_record_immediate(ZOOM_resultset s,size_t pos)
1544 {
1545     const char *syntax =
1546         ZOOM_options_get(s->options, "preferredRecordSyntax");
1547     const char *elementSetName =
1548         ZOOM_options_get(s->options, "elementSetName");
1549
1550     return ZOOM_record_cache_lookup(s, pos, syntax, elementSetName);
1551 }
1552
1553 ZOOM_API(ZOOM_record)
1554     ZOOM_resultset_record(ZOOM_resultset r, size_t pos)
1555 {
1556     ZOOM_record rec = ZOOM_resultset_record_immediate(r, pos);
1557
1558     if (!rec)
1559     {
1560         /*
1561          * MIKE: I think force_sync should always be zero, but I don't
1562          * want to make this change until I get the go-ahead from
1563          * Adam, in case something depends on the old synchronous
1564          * behaviour.
1565          */
1566         int force_sync = 1;
1567         if (getenv("ZOOM_RECORD_NO_FORCE_SYNC")) force_sync = 0;
1568         ZOOM_resultset_retrieve(r, force_sync, pos, 1);
1569         rec = ZOOM_resultset_record_immediate(r, pos);
1570     }
1571     return rec;
1572 }
1573
1574 ZOOM_API(void)
1575     ZOOM_record_destroy(ZOOM_record rec)
1576 {
1577     ZOOM_record_release(rec);
1578     yaz_log(log_details0, "ZOOM_record destroy");
1579     xfree(rec);
1580 }
1581
1582
1583 static yaz_iconv_t iconv_create_charset(const char *record_charset)
1584 {
1585     char to[40];
1586     char from[40];
1587     yaz_iconv_t cd = 0;
1588
1589     *from = '\0';
1590     strcpy(to, "UTF-8");
1591     if (record_charset && *record_charset)
1592     {
1593         /* Use "from,to" or just "from" */
1594         const char *cp = strchr(record_charset, ',');
1595         size_t clen = strlen(record_charset);
1596         if (cp && cp[1])
1597         {
1598             strncpy( to, cp+1, sizeof(to)-1);
1599             to[sizeof(to)-1] = '\0';
1600             clen = cp - record_charset;
1601         }
1602         if (clen > sizeof(from)-1)
1603             clen = sizeof(from)-1;
1604
1605         if (clen)
1606             strncpy(from, record_charset, clen);
1607         from[clen] = '\0';
1608     }
1609     if (*from && *to)
1610         cd = yaz_iconv_open(to, from);
1611     return cd;
1612 }
1613
1614 static const char *return_marc_record(WRBUF wrbuf,
1615                                       int marc_type,
1616                                       int *len,
1617                                       const char *buf, int sz,
1618                                       const char *record_charset)
1619 {
1620     yaz_iconv_t cd = iconv_create_charset(record_charset);
1621     yaz_marc_t mt = yaz_marc_create();
1622     const char *ret_string = 0;
1623
1624     if (cd)
1625         yaz_marc_iconv(mt, cd);
1626     yaz_marc_xml(mt, marc_type);
1627     if (yaz_marc_decode_wrbuf(mt, buf, sz, wrbuf) > 0)
1628     {
1629         if (len)
1630             *len = wrbuf_len(wrbuf);
1631         ret_string = wrbuf_cstr(wrbuf);
1632     }
1633     yaz_marc_destroy(mt);
1634     if (cd)
1635         yaz_iconv_close(cd);
1636     return ret_string;
1637 }
1638
1639 static const char *return_opac_record(WRBUF wrbuf,
1640                                       int marc_type,
1641                                       int *len,
1642                                       Z_OPACRecord *opac_rec,
1643                                       const char *record_charset)
1644 {
1645     yaz_iconv_t cd = iconv_create_charset(record_charset);
1646     yaz_marc_t mt = yaz_marc_create();
1647
1648     if (cd)
1649         yaz_marc_iconv(mt, cd);
1650     yaz_marc_xml(mt, marc_type);
1651
1652     yaz_opac_decode_wrbuf(mt, opac_rec, wrbuf);
1653     yaz_marc_destroy(mt);
1654
1655     if (cd)
1656         yaz_iconv_close(cd);
1657     if (len)
1658         *len = wrbuf_len(wrbuf);
1659     return wrbuf_cstr(wrbuf);
1660 }
1661
1662 static const char *return_string_record(WRBUF wrbuf,
1663                                         int *len,
1664                                         const char *buf, int sz,
1665                                         const char *record_charset)
1666 {
1667     yaz_iconv_t cd = iconv_create_charset(record_charset);
1668
1669     if (cd)
1670     {
1671         wrbuf_iconv_write(wrbuf, cd, buf, sz);
1672         wrbuf_iconv_reset(wrbuf, cd);
1673
1674         buf = wrbuf_cstr(wrbuf);
1675         sz = wrbuf_len(wrbuf);
1676         yaz_iconv_close(cd);
1677     }
1678     if (len)
1679         *len = sz;
1680     return buf;
1681 }
1682
1683 static const char *return_record_wrbuf(WRBUF wrbuf, int *len,
1684                                        Z_NamePlusRecord *npr,
1685                                        int marctype, const char *charset)
1686 {
1687     Z_External *r = (Z_External *) npr->u.databaseRecord;
1688     const Odr_oid *oid = r->direct_reference;
1689
1690     wrbuf_rewind(wrbuf);
1691     /* render bibliographic record .. */
1692     if (r->which == Z_External_OPAC)
1693     {
1694         return return_opac_record(wrbuf, marctype, len,
1695                                   r->u.opac, charset);
1696     }
1697     if (r->which == Z_External_sutrs)
1698         return return_string_record(wrbuf, len,
1699                                     (char*) r->u.sutrs->buf,
1700                                     r->u.sutrs->len,
1701                                     charset);
1702     else if (r->which == Z_External_octet)
1703     {
1704         if (yaz_oid_is_iso2709(oid))
1705         {
1706             const char *ret_buf = return_marc_record(
1707                 wrbuf, marctype, len,
1708                 (const char *) r->u.octet_aligned->buf,
1709                 r->u.octet_aligned->len,
1710                 charset);
1711             if (ret_buf)
1712                 return ret_buf;
1713             /* bad ISO2709. Return fail unless raw (ISO2709) is wanted */
1714             if (marctype != YAZ_MARC_ISO2709)
1715                 return 0;
1716         }
1717         return return_string_record(wrbuf, len,
1718                                     (const char *) r->u.octet_aligned->buf,
1719                                     r->u.octet_aligned->len,
1720                                     charset);
1721     }
1722     else if (r->which == Z_External_grs1)
1723     {
1724         yaz_display_grs1(wrbuf, r->u.grs1, 0);
1725         return return_string_record(wrbuf, len,
1726                                     wrbuf_buf(wrbuf),
1727                                     wrbuf_len(wrbuf),
1728                                     charset);
1729     }
1730     return 0;
1731 }
1732
1733 ZOOM_API(int)
1734     ZOOM_record_error(ZOOM_record rec, const char **cp,
1735                       const char **addinfo, const char **diagset)
1736 {
1737     Z_NamePlusRecord *npr;
1738
1739     if (!rec)
1740         return 0;
1741
1742     npr = rec->npr;
1743     if (rec->diag_uri)
1744     {
1745         if (cp)
1746             *cp = rec->diag_message;
1747         if (addinfo)
1748             *addinfo = rec->diag_details;
1749         if (diagset)
1750             *diagset = rec->diag_set;
1751         return uri_to_code(rec->diag_uri);
1752     }
1753     if (npr && npr->which == Z_NamePlusRecord_surrogateDiagnostic)
1754     {
1755         Z_DiagRec *diag_rec = npr->u.surrogateDiagnostic;
1756         int error = YAZ_BIB1_UNSPECIFIED_ERROR;
1757         const char *add = 0;
1758
1759         if (diag_rec->which == Z_DiagRec_defaultFormat)
1760         {
1761             Z_DefaultDiagFormat *ddf = diag_rec->u.defaultFormat;
1762             oid_class oclass;
1763
1764             error = *ddf->condition;
1765             switch (ddf->which)
1766             {
1767             case Z_DefaultDiagFormat_v2Addinfo:
1768                 add = ddf->u.v2Addinfo;
1769                 break;
1770             case Z_DefaultDiagFormat_v3Addinfo:
1771                 add = ddf->u.v3Addinfo;
1772                 break;
1773             }
1774             if (diagset)
1775                 *diagset =
1776                     yaz_oid_to_string(yaz_oid_std(),
1777                                       ddf->diagnosticSetId, &oclass);
1778         }
1779         else
1780         {
1781             if (diagset)
1782                 *diagset = "Bib-1";
1783         }
1784         if (addinfo)
1785             *addinfo = add ? add : "";
1786         if (cp)
1787             *cp = diagbib1_str(error);
1788         return error;
1789     }
1790     return 0;
1791 }
1792
1793 static const char *get_record_format(WRBUF wrbuf, int *len,
1794                                      Z_NamePlusRecord *npr,
1795                                      int marctype, const char *charset,
1796                                      const char *format)
1797 {
1798     const char *res = return_record_wrbuf(wrbuf, len, npr, marctype, charset);
1799 #if YAZ_HAVE_XML2
1800     if (*format == '1' && len)
1801     {
1802         /* try to XML format res */
1803         xmlDocPtr doc;
1804         xmlKeepBlanksDefault(0); /* get get xmlDocFormatMemory to work! */
1805         doc = xmlParseMemory(res, *len);
1806         if (doc)
1807         {
1808             xmlChar *xml_mem;
1809             int xml_size;
1810             xmlDocDumpFormatMemory(doc, &xml_mem, &xml_size, 1);
1811             wrbuf_rewind(wrbuf);
1812             wrbuf_write(wrbuf, (const char *) xml_mem, xml_size);
1813             xmlFree(xml_mem);
1814             xmlFreeDoc(doc);
1815             res = wrbuf_cstr(wrbuf);
1816             *len = wrbuf_len(wrbuf);
1817         }
1818     }
1819 #endif
1820     return res;
1821 }
1822
1823
1824 static const char *npr_format(Z_NamePlusRecord *npr, const char *schema,
1825                               WRBUF wrbuf,
1826                               const char *type_spec, int *len)
1827 {
1828     size_t i;
1829     char type[40];
1830     char charset[40];
1831     char format[3];
1832     const char *cp = type_spec;
1833
1834     for (i = 0; cp[i] && cp[i] != ';' && cp[i] != ' ' && i < sizeof(type)-1;
1835          i++)
1836         type[i] = cp[i];
1837     type[i] = '\0';
1838     charset[0] = '\0';
1839     format[0] = '\0';
1840     while (1)
1841     {
1842         while (cp[i] == ' ')
1843             i++;
1844         if (cp[i] != ';')
1845             break;
1846         i++;
1847         while (cp[i] == ' ')
1848             i++;
1849         if (!strncmp(cp + i, "charset=", 8))
1850         {
1851             size_t j = 0;
1852             i = i + 8; /* skip charset= */
1853             for (j = 0; cp[i] && cp[i] != ';' && cp[i] != ' '; i++)
1854             {
1855                 if (j < sizeof(charset)-1)
1856                     charset[j++] = cp[i];
1857             }
1858             charset[j] = '\0';
1859         }
1860         else if (!strncmp(cp + i, "format=", 7))
1861         {
1862             size_t j = 0;
1863             i = i + 7;
1864             for (j = 0; cp[i] && cp[i] != ';' && cp[i] != ' '; i++)
1865             {
1866                 if (j < sizeof(format)-1)
1867                     format[j++] = cp[i];
1868             }
1869             format[j] = '\0';
1870         }
1871     }
1872     if (!strcmp(type, "database"))
1873     {
1874         if (len)
1875             *len = (npr->databaseName ? strlen(npr->databaseName) : 0);
1876         return npr->databaseName;
1877     }
1878     else if (!strcmp(type, "schema"))
1879     {
1880         if (len)
1881             *len = schema ? strlen(schema) : 0;
1882         return schema;
1883     }
1884     else if (!strcmp(type, "syntax"))
1885     {
1886         const char *desc = 0;
1887         if (npr->which == Z_NamePlusRecord_databaseRecord)
1888         {
1889             Z_External *r = (Z_External *) npr->u.databaseRecord;
1890             desc = yaz_oid_to_string(yaz_oid_std(), r->direct_reference, 0);
1891         }
1892         if (!desc)
1893             desc = "none";
1894         if (len)
1895             *len = strlen(desc);
1896         return desc;
1897     }
1898     if (npr->which != Z_NamePlusRecord_databaseRecord)
1899         return 0;
1900
1901     /* from now on - we have a database record .. */
1902     if (!strcmp(type, "render"))
1903     {
1904         return get_record_format(wrbuf, len, npr, YAZ_MARC_LINE, charset, format);
1905     }
1906     else if (!strcmp(type, "xml"))
1907     {
1908         return get_record_format(wrbuf, len, npr, YAZ_MARC_MARCXML, charset,
1909                                  format);
1910     }
1911     else if (!strcmp(type, "txml"))
1912     {
1913         return get_record_format(wrbuf, len, npr, YAZ_MARC_TURBOMARC, charset,
1914                                  format);
1915     }
1916     else if (!strcmp(type, "raw"))
1917     {
1918         return get_record_format(wrbuf, len, npr, YAZ_MARC_ISO2709, charset,
1919             format);
1920     }
1921     else if (!strcmp(type, "ext"))
1922     {
1923         if (len) *len = -1;
1924         return (const char *) npr->u.databaseRecord;
1925     }
1926     else if (!strcmp(type, "opac"))
1927     {
1928         if (npr->u.databaseRecord->which == Z_External_OPAC)
1929             return get_record_format(wrbuf, len, npr, YAZ_MARC_MARCXML, charset,
1930                                      format);
1931     }
1932     return 0;
1933 }
1934
1935 ZOOM_API(const char *)
1936     ZOOM_record_get(ZOOM_record rec, const char *type_spec, int *len)
1937 {
1938     WRBUF wrbuf;
1939
1940     if (len)
1941         *len = 0; /* default return */
1942
1943     if (!rec || !rec->npr)
1944         return 0;
1945
1946 #if SHPTR
1947     if (!rec->record_wrbuf)
1948     {
1949         WRBUF w = wrbuf_alloc();
1950         YAZ_SHPTR_INIT(rec->record_wrbuf, w);
1951     }
1952     wrbuf = rec->record_wrbuf->ptr;
1953 #else
1954     if (!rec->wrbuf)
1955         rec->wrbuf = wrbuf_alloc();
1956     wrbuf = rec->wrbuf;
1957 #endif
1958     return npr_format(rec->npr, rec->schema, wrbuf, type_spec, len);
1959 }
1960
1961 static int strcmp_null(const char *v1, const char *v2)
1962 {
1963     if (!v1 && !v2)
1964         return 0;
1965     if (!v1 || !v2)
1966         return -1;
1967     return strcmp(v1, v2);
1968 }
1969
1970 static size_t record_hash(int pos)
1971 {
1972     if (pos < 0)
1973         pos = 0;
1974     return pos % RECORD_HASH_SIZE;
1975 }
1976
1977 void ZOOM_record_cache_add(ZOOM_resultset r, Z_NamePlusRecord *npr,
1978                            int pos,
1979                            const char *syntax, const char *elementSetName,
1980                            const char *schema,
1981                            Z_SRW_diagnostic *diag)
1982 {
1983     ZOOM_record_cache rc = 0;
1984
1985     ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_RECV_RECORD);
1986     ZOOM_connection_put_event(r->connection, event);
1987
1988     for (rc = r->record_hash[record_hash(pos)]; rc; rc = rc->next)
1989     {
1990         if (pos == rc->pos
1991             && strcmp_null(r->schema, rc->schema) == 0
1992             && strcmp_null(elementSetName,rc->elementSetName) == 0
1993             && strcmp_null(syntax, rc->syntax) == 0)
1994             break;
1995     }
1996     if (!rc)
1997     {
1998         rc = (ZOOM_record_cache) odr_malloc(r->odr, sizeof(*rc));
1999         rc->rec.odr = 0;
2000 #if SHPTR
2001         YAZ_SHPTR_INC(r->record_wrbuf);
2002         rc->rec.record_wrbuf = r->record_wrbuf;
2003 #else
2004         rc->rec.wrbuf = 0;
2005 #endif
2006         rc->elementSetName = odr_strdup_null(r->odr, elementSetName);
2007
2008         rc->syntax = odr_strdup_null(r->odr, syntax);
2009
2010         rc->schema = odr_strdup_null(r->odr, r->schema);
2011
2012         rc->pos = pos;
2013         rc->next = r->record_hash[record_hash(pos)];
2014         r->record_hash[record_hash(pos)] = rc;
2015     }
2016     rc->rec.npr = npr;
2017     rc->rec.schema = odr_strdup_null(r->odr, schema);
2018     rc->rec.diag_set = 0;
2019     rc->rec.diag_uri = 0;
2020     rc->rec.diag_message = 0;
2021     rc->rec.diag_details = 0;
2022     if (diag)
2023     {
2024         if (diag->uri)
2025         {
2026             char *cp;
2027             rc->rec.diag_set = odr_strdup(r->odr, diag->uri);
2028             if ((cp = strrchr(rc->rec.diag_set, '/')))
2029                 *cp = '\0';
2030             rc->rec.diag_uri = odr_strdup(r->odr, diag->uri);
2031         }
2032         rc->rec.diag_message = odr_strdup_null(r->odr, diag->message);
2033         rc->rec.diag_details = odr_strdup_null(r->odr, diag->details);
2034     }
2035 }
2036
2037 ZOOM_record ZOOM_record_cache_lookup(ZOOM_resultset r, int pos,
2038                                      const char *syntax,
2039                                      const char *elementSetName)
2040 {
2041     ZOOM_record_cache rc;
2042
2043     for (rc = r->record_hash[record_hash(pos)]; rc; rc = rc->next)
2044     {
2045         if (pos == rc->pos)
2046         {
2047             if (strcmp_null(r->schema, rc->schema))
2048                 continue;
2049             if (strcmp_null(elementSetName,rc->elementSetName))
2050                 continue;
2051             if (strcmp_null(syntax, rc->syntax))
2052                 continue;
2053             return &rc->rec;
2054         }
2055     }
2056     return 0;
2057 }
2058
2059
2060 ZOOM_API(ZOOM_scanset)
2061     ZOOM_connection_scan(ZOOM_connection c, const char *start)
2062 {
2063     ZOOM_scanset s;
2064     ZOOM_query q = ZOOM_query_create();
2065
2066     ZOOM_query_prefix(q, start);
2067
2068     s = ZOOM_connection_scan1(c, q);
2069     ZOOM_query_destroy(q);
2070     return s;
2071
2072 }
2073
2074 ZOOM_API(ZOOM_scanset)
2075     ZOOM_connection_scan1(ZOOM_connection c, ZOOM_query q)
2076 {
2077     ZOOM_scanset scan = 0;
2078
2079     if (!q->z_query)
2080         return 0;
2081     scan = (ZOOM_scanset) xmalloc(sizeof(*scan));
2082     scan->connection = c;
2083     scan->odr = odr_createmem(ODR_DECODE);
2084     scan->options = ZOOM_options_create_with_parent(c->options);
2085     scan->refcount = 1;
2086     scan->scan_response = 0;
2087     scan->srw_scan_response = 0;
2088
2089     scan->query = q;
2090     (q->refcount)++;
2091     scan->databaseNames = ZOOM_connection_get_databases(c, c->options,
2092                                             &scan->num_databaseNames,
2093                                             scan->odr);
2094
2095     if (1)
2096     {
2097         ZOOM_task task = ZOOM_connection_add_task(c, ZOOM_TASK_SCAN);
2098         task->u.scan.scan = scan;
2099
2100         (scan->refcount)++;
2101         if (!c->async)
2102         {
2103             while (ZOOM_event(1, &c))
2104                 ;
2105         }
2106     }
2107     return scan;
2108 }
2109
2110 ZOOM_API(void)
2111     ZOOM_scanset_destroy(ZOOM_scanset scan)
2112 {
2113     if (!scan)
2114         return;
2115     (scan->refcount)--;
2116     if (scan->refcount == 0)
2117     {
2118         ZOOM_query_destroy(scan->query);
2119
2120         odr_destroy(scan->odr);
2121
2122         ZOOM_options_destroy(scan->options);
2123         xfree(scan);
2124     }
2125 }
2126
2127 static zoom_ret send_package(ZOOM_connection c)
2128 {
2129     ZOOM_Event event;
2130
2131     yaz_log(c->log_details, "%p send_package", c);
2132     if (!c->tasks)
2133         return zoom_complete;
2134     assert (c->tasks->which == ZOOM_TASK_PACKAGE);
2135
2136     event = ZOOM_Event_create(ZOOM_EVENT_SEND_APDU);
2137     ZOOM_connection_put_event(c, event);
2138
2139     c->buf_out = c->tasks->u.package->buf_out;
2140     c->len_out = c->tasks->u.package->len_out;
2141
2142     return ZOOM_send_buf(c);
2143 }
2144
2145 #if YAZ_HAVE_XML2
2146 static zoom_ret ZOOM_connection_srw_send_scan(ZOOM_connection c)
2147 {
2148     ZOOM_scanset scan;
2149     Z_SRW_PDU *sr = 0;
2150     const char *option_val = 0;
2151
2152     if (!c->tasks)
2153         return zoom_complete;
2154     assert (c->tasks->which == ZOOM_TASK_SCAN);
2155     scan = c->tasks->u.scan.scan;
2156
2157     sr = ZOOM_srw_get_pdu(c, Z_SRW_scan_request);
2158
2159     /* SRU scan can only carry CQL and PQF */
2160     if (scan->query->z_query->which == Z_Query_type_104)
2161     {
2162         sr->u.scan_request->query_type = Z_SRW_query_type_cql;
2163         sr->u.scan_request->scanClause.cql = scan->query->query_string;
2164     }
2165     else if (scan->query->z_query->which == Z_Query_type_1
2166              || scan->query->z_query->which == Z_Query_type_101)
2167     {
2168         sr->u.scan_request->query_type = Z_SRW_query_type_pqf;
2169         sr->u.scan_request->scanClause.pqf = scan->query->query_string;
2170     }
2171     else
2172     {
2173         ZOOM_set_error(c, ZOOM_ERROR_UNSUPPORTED_QUERY, 0);
2174         return zoom_complete;
2175     }
2176
2177     sr->u.scan_request->maximumTerms = odr_intdup(
2178         c->odr_out, ZOOM_options_get_int(scan->options, "number", 10));
2179
2180     sr->u.scan_request->responsePosition = odr_intdup(
2181         c->odr_out, ZOOM_options_get_int(scan->options, "position", 1));
2182
2183     option_val = ZOOM_options_get(scan->options, "extraArgs");
2184     yaz_encode_sru_extra(sr, c->odr_out, option_val);
2185     return send_srw(c, sr);
2186 }
2187 #else
2188 static zoom_ret ZOOM_connection_srw_send_scan(ZOOM_connection c)
2189 {
2190     return zoom_complete;
2191 }
2192 #endif
2193
2194
2195 ZOOM_API(size_t)
2196     ZOOM_scanset_size(ZOOM_scanset scan)
2197 {
2198     if (!scan)
2199         return 0;
2200
2201     if (scan->scan_response && scan->scan_response->entries)
2202         return scan->scan_response->entries->num_entries;
2203     else if (scan->srw_scan_response)
2204         return scan->srw_scan_response->num_terms;
2205     return 0;
2206 }
2207
2208 static void ZOOM_scanset_term_x(ZOOM_scanset scan, size_t pos,
2209                                 size_t *occ,
2210                                 const char **value_term, size_t *value_len,
2211                                 const char **disp_term, size_t *disp_len)
2212 {
2213     size_t noent = ZOOM_scanset_size(scan);
2214
2215     *value_term = 0;
2216     *value_len = 0;
2217
2218     *disp_term = 0;
2219     *disp_len = 0;
2220
2221     *occ = 0;
2222     if (pos >= noent)
2223         return;
2224     if (scan->scan_response)
2225     {
2226         Z_ScanResponse *res = scan->scan_response;
2227         if (res->entries->entries[pos]->which == Z_Entry_termInfo)
2228         {
2229             Z_TermInfo *t = res->entries->entries[pos]->u.termInfo;
2230
2231             *value_term = (const char *) t->term->u.general->buf;
2232             *value_len = t->term->u.general->len;
2233             if (t->displayTerm)
2234             {
2235                 *disp_term = t->displayTerm;
2236                 *disp_len = strlen(*disp_term);
2237             }
2238             else if (t->term->which == Z_Term_general)
2239             {
2240                 *disp_term = (const char *) t->term->u.general->buf;
2241                 *disp_len = t->term->u.general->len;
2242             }
2243             *occ = t->globalOccurrences ? *t->globalOccurrences : 0;
2244         }
2245     }
2246     if (scan->srw_scan_response)
2247     {
2248         Z_SRW_scanResponse *res = scan->srw_scan_response;
2249         Z_SRW_scanTerm *t = res->terms + pos;
2250         if (t)
2251         {
2252             *value_term = t->value;
2253             *value_len = strlen(*value_term);
2254
2255             if (t->displayTerm)
2256                 *disp_term = t->displayTerm;
2257             else
2258                 *disp_term = t->value;
2259             *disp_len = strlen(*disp_term);
2260             *occ = t->numberOfRecords ? *t->numberOfRecords : 0;
2261         }
2262     }
2263 }
2264
2265 ZOOM_API(const char *)
2266     ZOOM_scanset_term(ZOOM_scanset scan, size_t pos,
2267                       size_t *occ, size_t *len)
2268 {
2269     const char *value_term = 0;
2270     size_t value_len = 0;
2271     const char *disp_term = 0;
2272     size_t disp_len = 0;
2273
2274     ZOOM_scanset_term_x(scan, pos, occ, &value_term, &value_len,
2275                         &disp_term, &disp_len);
2276
2277     *len = value_len;
2278     return value_term;
2279 }
2280
2281 ZOOM_API(const char *)
2282     ZOOM_scanset_display_term(ZOOM_scanset scan, size_t pos,
2283                               size_t *occ, size_t *len)
2284 {
2285     const char *value_term = 0;
2286     size_t value_len = 0;
2287     const char *disp_term = 0;
2288     size_t disp_len = 0;
2289
2290     ZOOM_scanset_term_x(scan, pos, occ, &value_term, &value_len,
2291                         &disp_term, &disp_len);
2292
2293     *len = disp_len;
2294     return disp_term;
2295 }
2296
2297 ZOOM_API(const char *)
2298     ZOOM_scanset_option_get(ZOOM_scanset scan, const char *key)
2299 {
2300     return ZOOM_options_get(scan->options, key);
2301 }
2302
2303 ZOOM_API(void)
2304     ZOOM_scanset_option_set(ZOOM_scanset scan, const char *key,
2305                             const char *val)
2306 {
2307     ZOOM_options_set(scan->options, key, val);
2308 }
2309
2310
2311 ZOOM_API(ZOOM_package)
2312     ZOOM_connection_package(ZOOM_connection c, ZOOM_options options)
2313 {
2314     ZOOM_package p = (ZOOM_package) xmalloc(sizeof(*p));
2315
2316     p->connection = c;
2317     p->odr_out = odr_createmem(ODR_ENCODE);
2318     p->options = ZOOM_options_create_with_parent2(options, c->options);
2319     p->refcount = 1;
2320     p->buf_out = 0;
2321     p->len_out = 0;
2322     return p;
2323 }
2324
2325 ZOOM_API(void)
2326     ZOOM_package_destroy(ZOOM_package p)
2327 {
2328     if (!p)
2329         return;
2330     (p->refcount)--;
2331     if (p->refcount == 0)
2332     {
2333         odr_destroy(p->odr_out);
2334         xfree(p->buf_out);
2335
2336         ZOOM_options_destroy(p->options);
2337         xfree(p);
2338     }
2339 }
2340
2341 ZOOM_API(const char *)
2342     ZOOM_package_option_get(ZOOM_package p, const char *key)
2343 {
2344     return ZOOM_options_get(p->options, key);
2345 }
2346
2347 ZOOM_API(const char *)
2348     ZOOM_package_option_getl(ZOOM_package p, const char *key, int *lenp)
2349 {
2350     return ZOOM_options_getl(p->options, key, lenp);
2351 }
2352
2353 ZOOM_API(void)
2354     ZOOM_package_option_set(ZOOM_package p, const char *key,
2355                             const char *val)
2356 {
2357     ZOOM_options_set(p->options, key, val);
2358 }
2359
2360 ZOOM_API(void)
2361     ZOOM_package_option_setl(ZOOM_package p, const char *key,
2362                              const char *val, int len)
2363 {
2364     ZOOM_options_setl(p->options, key, val, len);
2365 }
2366
2367 ZOOM_API(int)
2368     ZOOM_connection_exec_task(ZOOM_connection c)
2369 {
2370     ZOOM_task task = c->tasks;
2371     zoom_ret ret = zoom_complete;
2372
2373     if (!task)
2374         return 0;
2375     yaz_log(c->log_details, "%p ZOOM_connection_exec_task type=%d run=%d",
2376             c, task->which, task->running);
2377     if (c->error != ZOOM_ERROR_NONE)
2378     {
2379         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
2380                 "removing tasks because of error = %d", c, c->error);
2381         ZOOM_connection_remove_tasks(c);
2382         return 0;
2383     }
2384     if (task->running)
2385     {
2386         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
2387                 "task already running", c);
2388         return 0;
2389     }
2390     task->running = 1;
2391     ret = zoom_complete;
2392     if (c->cs || task->which == ZOOM_TASK_CONNECT)
2393     {
2394         switch (task->which)
2395         {
2396         case ZOOM_TASK_SEARCH:
2397             if (c->proto == PROTO_HTTP)
2398                 ret = ZOOM_connection_srw_send_search(c);
2399             else
2400                 ret = ZOOM_connection_Z3950_send_search(c);
2401             break;
2402         case ZOOM_TASK_RETRIEVE:
2403             if (c->proto == PROTO_HTTP)
2404                 ret = ZOOM_connection_srw_send_search(c);
2405             else
2406                 ret = send_Z3950_present(c);
2407             break;
2408         case ZOOM_TASK_CONNECT:
2409             ret = do_connect(c);
2410             break;
2411         case ZOOM_TASK_SCAN:
2412             if (c->proto == PROTO_HTTP)
2413                 ret = ZOOM_connection_srw_send_scan(c);
2414             else
2415                 ret = ZOOM_connection_Z3950_send_scan(c);
2416             break;
2417         case ZOOM_TASK_PACKAGE:
2418             ret = send_package(c);
2419             break;
2420         case ZOOM_TASK_SORT:
2421             c->tasks->u.sort.resultset->r_sort_spec =
2422                 c->tasks->u.sort.q->sort_spec;
2423             ret = send_Z3950_sort(c, c->tasks->u.sort.resultset);
2424             break;
2425         }
2426     }
2427     else
2428     {
2429         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
2430                 "remove tasks because no connection exist", c);
2431         ZOOM_connection_remove_tasks(c);
2432     }
2433     if (ret == zoom_complete)
2434     {
2435         yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
2436                 "task removed (complete)", c);
2437         ZOOM_connection_remove_task(c);
2438         return 0;
2439     }
2440     yaz_log(c->log_details, "%p ZOOM_connection_exec_task "
2441             "task pending", c);
2442     return 1;
2443 }
2444
2445 #if YAZ_HAVE_XML2
2446 static zoom_ret handle_srw_response(ZOOM_connection c,
2447                                     Z_SRW_searchRetrieveResponse *res)
2448 {
2449     ZOOM_resultset resultset = 0;
2450     int i;
2451     NMEM nmem;
2452     ZOOM_Event event;
2453     int *start, *count;
2454     const char *syntax, *elementSetName;
2455
2456     if (!c->tasks)
2457         return zoom_complete;
2458
2459     switch(c->tasks->which)
2460     {
2461     case ZOOM_TASK_SEARCH:
2462         resultset = c->tasks->u.search.resultset;
2463         start = &c->tasks->u.search.start;
2464         count = &c->tasks->u.search.count;
2465         syntax = c->tasks->u.search.syntax;
2466         elementSetName = c->tasks->u.search.elementSetName;
2467
2468         if (!c->tasks->u.search.recv_search_fired)
2469         {
2470             event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH);
2471             ZOOM_connection_put_event(c, event);
2472             c->tasks->u.search.recv_search_fired = 1;
2473         }
2474         break;
2475     case ZOOM_TASK_RETRIEVE:
2476         resultset = c->tasks->u.retrieve.resultset;
2477         start = &c->tasks->u.retrieve.start;
2478         count = &c->tasks->u.retrieve.count;
2479         syntax = c->tasks->u.retrieve.syntax;
2480         elementSetName = c->tasks->u.retrieve.elementSetName;
2481         break;
2482     default:
2483         return zoom_complete;
2484     }
2485
2486     resultset->size = 0;
2487
2488     if (res->resultSetId)
2489         ZOOM_resultset_option_set(resultset, "resultSetId", res->resultSetId);
2490
2491     yaz_log(c->log_details, "%p handle_srw_response got SRW response OK", c);
2492
2493     if (res->num_diagnostics > 0)
2494     {
2495         set_SRU_error(c, &res->diagnostics[0]);
2496     }
2497     else
2498     {
2499         if (res->numberOfRecords)
2500             resultset->size = *res->numberOfRecords;
2501         for (i = 0; i<res->num_records; i++)
2502         {
2503             int pos;
2504             Z_SRW_record *sru_rec;
2505             Z_SRW_diagnostic *diag = 0;
2506             int num_diag;
2507
2508             Z_NamePlusRecord *npr = (Z_NamePlusRecord *)
2509                 odr_malloc(c->odr_in, sizeof(Z_NamePlusRecord));
2510
2511             if (res->records[i].recordPosition &&
2512                 *res->records[i].recordPosition > 0)
2513                 pos = *res->records[i].recordPosition - 1;
2514             else
2515                 pos = *start + i;
2516
2517             sru_rec = &res->records[i];
2518
2519             npr->databaseName = 0;
2520             npr->which = Z_NamePlusRecord_databaseRecord;
2521             npr->u.databaseRecord = (Z_External *)
2522                 odr_malloc(c->odr_in, sizeof(Z_External));
2523             npr->u.databaseRecord->descriptor = 0;
2524             npr->u.databaseRecord->direct_reference =
2525                 odr_oiddup(c->odr_in, yaz_oid_recsyn_xml);
2526             npr->u.databaseRecord->which = Z_External_octet;
2527
2528             npr->u.databaseRecord->u.octet_aligned = (Odr_oct *)
2529                 odr_malloc(c->odr_in, sizeof(Odr_oct));
2530             npr->u.databaseRecord->u.octet_aligned->buf = (unsigned char*)
2531                 sru_rec->recordData_buf;
2532             npr->u.databaseRecord->u.octet_aligned->len =
2533                 npr->u.databaseRecord->u.octet_aligned->size =
2534                 sru_rec->recordData_len;
2535
2536             if (sru_rec->recordSchema
2537                 && !strcmp(sru_rec->recordSchema,
2538                            "info:srw/schema/1/diagnostics-v1.1"))
2539             {
2540                 sru_decode_surrogate_diagnostics(sru_rec->recordData_buf,
2541                                                  sru_rec->recordData_len,
2542                                                  &diag, &num_diag,
2543                                                  resultset->odr);
2544             }
2545             ZOOM_record_cache_add(resultset, npr, pos, syntax, elementSetName,
2546                                   sru_rec->recordSchema, diag);
2547         }
2548         *count -= i;
2549         *start += i;
2550         if (*count + *start > resultset->size)
2551             *count = resultset->size - *start;
2552         if (*count < 0)
2553             *count = 0;
2554
2555         nmem = odr_extract_mem(c->odr_in);
2556         nmem_transfer(odr_getmem(resultset->odr), nmem);
2557         nmem_destroy(nmem);
2558
2559         if (*count > 0)
2560             return ZOOM_connection_srw_send_search(c);
2561     }
2562     return zoom_complete;
2563 }
2564 #endif
2565
2566 #if YAZ_HAVE_XML2
2567 static void handle_srw_scan_response(ZOOM_connection c,
2568                                      Z_SRW_scanResponse *res)
2569 {
2570     NMEM nmem = odr_extract_mem(c->odr_in);
2571     ZOOM_scanset scan;
2572
2573     if (!c->tasks || c->tasks->which != ZOOM_TASK_SCAN)
2574         return;
2575     scan = c->tasks->u.scan.scan;
2576
2577     if (res->num_diagnostics > 0)
2578         set_SRU_error(c, &res->diagnostics[0]);
2579
2580     scan->scan_response = 0;
2581     scan->srw_scan_response = res;
2582     nmem_transfer(odr_getmem(scan->odr), nmem);
2583
2584     ZOOM_options_set_int(scan->options, "number", res->num_terms);
2585     nmem_destroy(nmem);
2586 }
2587 #endif
2588
2589 #if YAZ_HAVE_XML2
2590 static Z_GDU *get_HTTP_Request_url(ODR odr, const char *url)
2591 {
2592     Z_GDU *p = z_get_HTTP_Request(odr);
2593     const char *host = url;
2594     const char *cp0 = strstr(host, "://");
2595     const char *cp1 = 0;
2596     if (cp0)
2597         cp0 = cp0+3;
2598     else
2599         cp0 = host;
2600
2601     cp1 = strchr(cp0, '/');
2602     if (!cp1)
2603         cp1 = cp0 + strlen(cp0);
2604
2605     if (cp0 && cp1)
2606     {
2607         char *h = (char*) odr_malloc(odr, cp1 - cp0 + 1);
2608         memcpy (h, cp0, cp1 - cp0);
2609         h[cp1-cp0] = '\0';
2610         z_HTTP_header_add(odr, &p->u.HTTP_Request->headers, "Host", h);
2611     }
2612     p->u.HTTP_Request->path = odr_strdup(odr, *cp1 ? cp1 : "/");
2613     return p;
2614 }
2615
2616 static zoom_ret send_SRW_redirect(ZOOM_connection c, const char *uri,
2617                                   Z_HTTP_Response *cookie_hres)
2618 {
2619     struct Z_HTTP_Header *h;
2620     Z_GDU *gdu = get_HTTP_Request_url(c->odr_out, uri);
2621     char *combined_cookies;
2622     int combined_cookies_len = 0;
2623
2624     gdu->u.HTTP_Request->method = odr_strdup(c->odr_out, "GET");
2625     z_HTTP_header_add(c->odr_out, &gdu->u.HTTP_Request->headers, "Accept",
2626                       "text/xml");
2627
2628     for (h = cookie_hres->headers; h; h = h->next)
2629     {
2630         if (!strcmp(h->name, "Set-Cookie"))
2631         {
2632             char *cp;
2633
2634             if (!(cp = strchr(h->value, ';')))
2635                 cp = h->value + strlen(h->value);
2636             if (cp - h->value >= 1) {
2637                 combined_cookies = xrealloc(combined_cookies, combined_cookies_len + cp - h->value + 3);
2638                 memcpy(combined_cookies+combined_cookies_len, h->value, cp - h->value);
2639                 combined_cookies[combined_cookies_len + cp - h->value] = '\0';
2640                 strcat(combined_cookies,"; ");
2641                 combined_cookies_len = strlen(combined_cookies);
2642             }
2643         }
2644     }
2645
2646     if (combined_cookies_len)
2647     {
2648         z_HTTP_header_add(c->odr_out, &gdu->u.HTTP_Request->headers,
2649                           "Cookie", combined_cookies);
2650         xfree(combined_cookies);
2651     }
2652
2653     if (c->user && c->password)
2654     {
2655         z_HTTP_header_add_basic_auth(c->odr_out, &gdu->u.HTTP_Request->headers,
2656                                      c->user, c->password);
2657     }
2658     if (!z_GDU(c->odr_out, &gdu, 0, 0))
2659         return zoom_complete;
2660     if (c->odr_print)
2661         z_GDU(c->odr_print, &gdu, 0, 0);
2662     c->buf_out = odr_getbuf(c->odr_out, &c->len_out, 0);
2663
2664     odr_reset(c->odr_out);
2665     return ZOOM_send_buf(c);
2666 }
2667
2668 static void handle_http(ZOOM_connection c, Z_HTTP_Response *hres)
2669 {
2670     zoom_ret cret = zoom_complete;
2671     int ret = -1;
2672     const char *addinfo = 0;
2673     const char *connection_head = z_HTTP_header_lookup(hres->headers,
2674                                                        "Connection");
2675     const char *location;
2676
2677     ZOOM_connection_set_mask(c, 0);
2678     yaz_log(c->log_details, "%p handle_http", c);
2679
2680     if ((hres->code == 301 || hres->code == 302) && c->sru_mode == zoom_sru_get
2681         && (location = z_HTTP_header_lookup(hres->headers, "Location")))
2682     {
2683         c->no_redirects++;
2684         if (c->no_redirects > 10)
2685         {
2686             set_HTTP_error(c, hres->code, 0, 0);
2687             c->no_redirects = 0;
2688             ZOOM_connection_close(c);
2689         }
2690         else
2691         {
2692             /* since redirect may change host we just reconnect. A smarter
2693                implementation might check whether it's the same server */
2694             do_connect_host(c, location, 0);
2695             send_SRW_redirect(c, location, hres);
2696             /* we're OK for now. Operation is not really complete */
2697             ret = 0;
2698             cret = zoom_pending;
2699         }
2700     }
2701     else
2702     {   /* not redirect (normal response) */
2703         if (!yaz_srw_check_content_type(hres))
2704             addinfo = "content-type";
2705         else
2706         {
2707             Z_SOAP *soap_package = 0;
2708             ODR o = c->odr_in;
2709             Z_SOAP_Handler soap_handlers[2] = {
2710                 {YAZ_XMLNS_SRU_v1_1, 0, (Z_SOAP_fun) yaz_srw_codec},
2711                 {0, 0, 0}
2712             };
2713             ret = z_soap_codec(o, &soap_package,
2714                                &hres->content_buf, &hres->content_len,
2715                                soap_handlers);
2716             if (!ret && soap_package->which == Z_SOAP_generic &&
2717                 soap_package->u.generic->no == 0)
2718             {
2719                 Z_SRW_PDU *sr = (Z_SRW_PDU*) soap_package->u.generic->p;
2720
2721                 ZOOM_options_set(c->options, "sru_version", sr->srw_version);
2722                 ZOOM_options_setl(c->options, "sru_extra_response_data",
2723                                   sr->extraResponseData_buf, sr->extraResponseData_len);
2724                 if (sr->which == Z_SRW_searchRetrieve_response)
2725                     cret = handle_srw_response(c, sr->u.response);
2726                 else if (sr->which == Z_SRW_scan_response)
2727                     handle_srw_scan_response(c, sr->u.scan_response);
2728                 else
2729                     ret = -1;
2730             }
2731             else if (!ret && (soap_package->which == Z_SOAP_fault
2732                               || soap_package->which == Z_SOAP_error))
2733             {
2734                 set_HTTP_error(c, hres->code,
2735                                soap_package->u.fault->fault_code,
2736                                soap_package->u.fault->fault_string);
2737             }
2738             else
2739                 ret = -1;
2740         }
2741         if (ret == 0)
2742         {
2743             if (c->no_redirects) /* end of redirect. change hosts again */
2744                 ZOOM_connection_close(c);
2745         }
2746         c->no_redirects = 0;
2747     }
2748     if (ret)
2749     {
2750         if (hres->code != 200)
2751             set_HTTP_error(c, hres->code, 0, 0);
2752         else
2753             ZOOM_set_error(c, ZOOM_ERROR_DECODE, addinfo);
2754         ZOOM_connection_close(c);
2755     }
2756     if (cret == zoom_complete)
2757     {
2758         yaz_log(YLOG_LOG, "removing tasks in handle_http");
2759         ZOOM_connection_remove_task(c);
2760     }
2761     {
2762         int must_close = 0;
2763         if (!strcmp(hres->version, "1.0"))
2764         {
2765             /* HTTP 1.0: only if Keep-Alive we stay alive.. */
2766             if (!connection_head || strcmp(connection_head, "Keep-Alive"))
2767                 must_close = 1;
2768         }
2769         else
2770         {
2771             /* HTTP 1.1: only if no close we stay alive.. */
2772             if (connection_head && !strcmp(connection_head, "close"))
2773                 must_close = 1;
2774         }
2775         if (must_close)
2776         {
2777             ZOOM_connection_close(c);
2778             if (c->tasks)
2779             {
2780                 c->tasks->running = 0;
2781                 ZOOM_connection_insert_task(c, ZOOM_TASK_CONNECT);
2782                 c->reconnect_ok = 0;
2783             }
2784         }
2785     }
2786 }
2787 #endif
2788
2789 static int do_read(ZOOM_connection c)
2790 {
2791     int r, more;
2792     ZOOM_Event event;
2793
2794     event = ZOOM_Event_create(ZOOM_EVENT_RECV_DATA);
2795     ZOOM_connection_put_event(c, event);
2796
2797     r = cs_get(c->cs, &c->buf_in, &c->len_in);
2798     more = cs_more(c->cs);
2799     yaz_log(c->log_details, "%p do_read len=%d more=%d", c, r, more);
2800     if (r == 1)
2801         return 0;
2802     if (r <= 0)
2803     {
2804         if (!ZOOM_test_reconnect(c))
2805         {
2806             ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
2807             ZOOM_connection_close(c);
2808         }
2809     }
2810     else
2811     {
2812         Z_GDU *gdu;
2813         ZOOM_Event event;
2814
2815         odr_reset(c->odr_in);
2816         odr_setbuf(c->odr_in, c->buf_in, r, 0);
2817         event = ZOOM_Event_create(ZOOM_EVENT_RECV_APDU);
2818         ZOOM_connection_put_event(c, event);
2819
2820         if (!z_GDU(c->odr_in, &gdu, 0, 0))
2821         {
2822             int x;
2823             int err = odr_geterrorx(c->odr_in, &x);
2824             char msg[100];
2825             const char *element = odr_getelement(c->odr_in);
2826             yaz_snprintf(msg, sizeof(msg),
2827                     "ODR code %d:%d element=%s offset=%d",
2828                     err, x, element ? element : "<unknown>",
2829                     odr_offset(c->odr_in));
2830             ZOOM_set_error(c, ZOOM_ERROR_DECODE, msg);
2831             if (c->log_api)
2832             {
2833                 FILE *ber_file = yaz_log_file();
2834                 if (ber_file)
2835                     odr_dumpBER(ber_file, c->buf_in, r);
2836             }
2837             ZOOM_connection_close(c);
2838         }
2839         else
2840         {
2841             if (c->odr_print)
2842                 z_GDU(c->odr_print, &gdu, 0, 0);
2843             if (gdu->which == Z_GDU_Z3950)
2844                 ZOOM_handle_Z3950_apdu(c, gdu->u.z3950);
2845             else if (gdu->which == Z_GDU_HTTP_Response)
2846             {
2847 #if YAZ_HAVE_XML2
2848                 handle_http(c, gdu->u.HTTP_Response);
2849 #else
2850                 ZOOM_set_error(c, ZOOM_ERROR_DECODE, 0);
2851                 ZOOM_connection_close(c);
2852 #endif
2853             }
2854         }
2855         c->reconnect_ok = 0;
2856     }
2857     return 1;
2858 }
2859
2860 static zoom_ret do_write_ex(ZOOM_connection c, char *buf_out, int len_out)
2861 {
2862     int r;
2863     ZOOM_Event event;
2864
2865     event = ZOOM_Event_create(ZOOM_EVENT_SEND_DATA);
2866     ZOOM_connection_put_event(c, event);
2867
2868     yaz_log(c->log_details, "%p do_write_ex len=%d", c, len_out);
2869     if ((r = cs_put(c->cs, buf_out, len_out)) < 0)
2870     {
2871         yaz_log(c->log_details, "%p do_write_ex write failed", c);
2872         if (ZOOM_test_reconnect(c))
2873         {
2874             return zoom_pending;
2875         }
2876         if (c->state == STATE_CONNECTING)
2877             ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
2878         else
2879             ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
2880         ZOOM_connection_close(c);
2881         return zoom_complete;
2882     }
2883     else if (r == 1)
2884     {
2885         int mask = ZOOM_SELECT_EXCEPT;
2886         if (c->cs->io_pending & CS_WANT_WRITE)
2887             mask += ZOOM_SELECT_WRITE;
2888         if (c->cs->io_pending & CS_WANT_READ)
2889             mask += ZOOM_SELECT_READ;
2890         ZOOM_connection_set_mask(c, mask);
2891         yaz_log(c->log_details, "%p do_write_ex write incomplete mask=%d",
2892                 c, c->mask);
2893     }
2894     else
2895     {
2896         ZOOM_connection_set_mask(c, ZOOM_SELECT_READ|ZOOM_SELECT_EXCEPT);
2897         yaz_log(c->log_details, "%p do_write_ex write complete mask=%d",
2898                 c, c->mask);
2899     }
2900     return zoom_pending;
2901 }
2902
2903 zoom_ret ZOOM_send_buf(ZOOM_connection c)
2904 {
2905     return do_write_ex(c, c->buf_out, c->len_out);
2906 }
2907
2908
2909 ZOOM_API(const char *)
2910     ZOOM_connection_option_get(ZOOM_connection c, const char *key)
2911 {
2912     return ZOOM_options_get(c->options, key);
2913 }
2914
2915 ZOOM_API(const char *)
2916     ZOOM_connection_option_getl(ZOOM_connection c, const char *key, int *lenp)
2917 {
2918     return ZOOM_options_getl(c->options, key, lenp);
2919 }
2920
2921 ZOOM_API(void)
2922     ZOOM_connection_option_set(ZOOM_connection c, const char *key,
2923                                const char *val)
2924 {
2925     ZOOM_options_set(c->options, key, val);
2926 }
2927
2928 ZOOM_API(void)
2929     ZOOM_connection_option_setl(ZOOM_connection c, const char *key,
2930                                 const char *val, int len)
2931 {
2932     ZOOM_options_setl(c->options, key, val, len);
2933 }
2934
2935 ZOOM_API(const char *)
2936     ZOOM_resultset_option_get(ZOOM_resultset r, const char *key)
2937 {
2938     return ZOOM_options_get(r->options, key);
2939 }
2940
2941 ZOOM_API(void)
2942     ZOOM_resultset_option_set(ZOOM_resultset r, const char *key,
2943                               const char *val)
2944 {
2945     ZOOM_options_set(r->options, key, val);
2946 }
2947
2948
2949 ZOOM_API(int)
2950     ZOOM_connection_errcode(ZOOM_connection c)
2951 {
2952     return ZOOM_connection_error(c, 0, 0);
2953 }
2954
2955 ZOOM_API(const char *)
2956     ZOOM_connection_errmsg(ZOOM_connection c)
2957 {
2958     const char *msg;
2959     ZOOM_connection_error(c, &msg, 0);
2960     return msg;
2961 }
2962
2963 ZOOM_API(const char *)
2964     ZOOM_connection_addinfo(ZOOM_connection c)
2965 {
2966     const char *addinfo;
2967     ZOOM_connection_error(c, 0, &addinfo);
2968     return addinfo;
2969 }
2970
2971 ZOOM_API(const char *)
2972     ZOOM_connection_diagset(ZOOM_connection c)
2973 {
2974     const char *diagset;
2975     ZOOM_connection_error_x(c, 0, 0, &diagset);
2976     return diagset;
2977 }
2978
2979 ZOOM_API(const char *)
2980     ZOOM_diag_str(int error)
2981 {
2982     switch (error)
2983     {
2984     case ZOOM_ERROR_NONE:
2985         return "No error";
2986     case ZOOM_ERROR_CONNECT:
2987         return "Connect failed";
2988     case ZOOM_ERROR_MEMORY:
2989         return "Out of memory";
2990     case ZOOM_ERROR_ENCODE:
2991         return "Encoding failed";
2992     case ZOOM_ERROR_DECODE:
2993         return "Decoding failed";
2994     case ZOOM_ERROR_CONNECTION_LOST:
2995         return "Connection lost";
2996     case ZOOM_ERROR_INIT:
2997         return "Init rejected";
2998     case ZOOM_ERROR_INTERNAL:
2999         return "Internal failure";
3000     case ZOOM_ERROR_TIMEOUT:
3001         return "Timeout";
3002     case ZOOM_ERROR_UNSUPPORTED_PROTOCOL:
3003         return "Unsupported protocol";
3004     case ZOOM_ERROR_UNSUPPORTED_QUERY:
3005         return "Unsupported query type";
3006     case ZOOM_ERROR_INVALID_QUERY:
3007         return "Invalid query";
3008     case ZOOM_ERROR_CQL_PARSE:
3009         return "CQL parsing error";
3010     case ZOOM_ERROR_CQL_TRANSFORM:
3011         return "CQL transformation error";
3012     case ZOOM_ERROR_CCL_CONFIG:
3013         return "CCL configuration error";
3014     case ZOOM_ERROR_CCL_PARSE:
3015         return "CCL parsing error";
3016     default:
3017         return diagbib1_str(error);
3018     }
3019 }
3020
3021 ZOOM_API(int)
3022     ZOOM_connection_error_x(ZOOM_connection c, const char **cp,
3023                             const char **addinfo, const char **diagset)
3024 {
3025     int error = c->error;
3026     if (cp)
3027     {
3028         if (!c->diagset || !strcmp(c->diagset, "ZOOM"))
3029             *cp = ZOOM_diag_str(error);
3030         else if (!strcmp(c->diagset, "HTTP"))
3031             *cp = z_HTTP_errmsg(c->error);
3032         else if (!strcmp(c->diagset, "Bib-1"))
3033             *cp = ZOOM_diag_str(error);
3034         else if (!strcmp(c->diagset, "info:srw/diagnostic/1"))
3035             *cp = yaz_diag_srw_str(c->error);
3036         else
3037             *cp = "Unknown error and diagnostic set";
3038     }
3039     if (addinfo)
3040         *addinfo = c->addinfo ? c->addinfo : "";
3041     if (diagset)
3042         *diagset = c->diagset ? c->diagset : "";
3043     return c->error;
3044 }
3045
3046 ZOOM_API(int)
3047     ZOOM_connection_error(ZOOM_connection c, const char **cp,
3048                           const char **addinfo)
3049 {
3050     return ZOOM_connection_error_x(c, cp, addinfo, 0);
3051 }
3052
3053 static void ZOOM_connection_do_io(ZOOM_connection c, int mask)
3054 {
3055     ZOOM_Event event = 0;
3056     int r = cs_look(c->cs);
3057     yaz_log(c->log_details, "%p ZOOM_connection_do_io mask=%d cs_look=%d",
3058             c, mask, r);
3059
3060     if (r == CS_NONE)
3061     {
3062         event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
3063         ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
3064         ZOOM_connection_close(c);
3065         ZOOM_connection_put_event(c, event);
3066     }
3067     else if (r == CS_CONNECT)
3068     {
3069         int ret = ret = cs_rcvconnect(c->cs);
3070         yaz_log(c->log_details, "%p ZOOM_connection_do_io "
3071                 "cs_rcvconnect returned %d", c, ret);
3072         if (ret == 1)
3073         {
3074             int mask = ZOOM_SELECT_EXCEPT;
3075             if (c->cs->io_pending & CS_WANT_WRITE)
3076                 mask += ZOOM_SELECT_WRITE;
3077             if (c->cs->io_pending & CS_WANT_READ)
3078                 mask += ZOOM_SELECT_READ;
3079             ZOOM_connection_set_mask(c, mask);
3080             event = ZOOM_Event_create(ZOOM_EVENT_NONE);
3081             ZOOM_connection_put_event(c, event);
3082         }
3083         else if (ret == 0)
3084         {
3085             event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
3086             ZOOM_connection_put_event(c, event);
3087             get_cert(c);
3088             if (c->proto == PROTO_Z3950)
3089                 ZOOM_connection_Z3950_send_init(c);
3090             else
3091             {
3092                 /* no init request for SRW .. */
3093                 assert(c->tasks->which == ZOOM_TASK_CONNECT);
3094                 ZOOM_connection_remove_task(c);
3095                 ZOOM_connection_set_mask(c, 0);
3096                 ZOOM_connection_exec_task(c);
3097             }
3098             c->state = STATE_ESTABLISHED;
3099         }
3100         else
3101         {
3102             ZOOM_set_error(c, ZOOM_ERROR_CONNECT, c->host_port);
3103             ZOOM_connection_close(c);
3104         }
3105     }
3106     else
3107     {
3108         if (mask & ZOOM_SELECT_EXCEPT)
3109         {
3110             if (!ZOOM_test_reconnect(c))
3111             {
3112                 ZOOM_set_error(c, ZOOM_ERROR_CONNECTION_LOST, c->host_port);
3113                 ZOOM_connection_close(c);
3114             }
3115             return;
3116         }
3117         if (mask & ZOOM_SELECT_READ)
3118             do_read(c);
3119         if (c->cs && (mask & ZOOM_SELECT_WRITE))
3120             ZOOM_send_buf(c);
3121     }
3122 }
3123
3124 ZOOM_API(int)
3125     ZOOM_connection_last_event(ZOOM_connection cs)
3126 {
3127     if (!cs)
3128         return ZOOM_EVENT_NONE;
3129     return cs->last_event;
3130 }
3131
3132
3133 static void cql2pqf_wrbuf_puts(const char *buf, void *client_data)
3134 {
3135     WRBUF wrbuf = (WRBUF) client_data;
3136     wrbuf_puts(wrbuf, buf);
3137 }
3138
3139 /*
3140  * Returns an xmalloc()d string containing RPN that corresponds to the
3141  * CQL passed in.  On error, sets the Connection object's error state
3142  * and returns a null pointer.
3143  * ### We could cache CQL parser and/or transformer in Connection.
3144  */
3145 static char *cql2pqf(ZOOM_connection c, const char *cql)
3146 {
3147     CQL_parser parser;
3148     int error;
3149     const char *cqlfile;
3150     cql_transform_t trans;
3151     char *result = 0;
3152
3153     parser = cql_parser_create();
3154     if ((error = cql_parser_string(parser, cql)) != 0) {
3155         cql_parser_destroy(parser);
3156         ZOOM_set_error(c, ZOOM_ERROR_CQL_PARSE, cql);
3157         return 0;
3158     }
3159
3160     cqlfile = ZOOM_connection_option_get(c, "cqlfile");
3161     if (cqlfile == 0)
3162     {
3163         ZOOM_set_error(c, ZOOM_ERROR_CQL_TRANSFORM, "no CQL transform file");
3164     }
3165     else if ((trans = cql_transform_open_fname(cqlfile)) == 0)
3166     {
3167         char buf[512];
3168         sprintf(buf, "can't open CQL transform file '%.200s': %.200s",
3169                 cqlfile, strerror(errno));
3170         ZOOM_set_error(c, ZOOM_ERROR_CQL_TRANSFORM, buf);
3171     }
3172     else
3173     {
3174         WRBUF wrbuf_result = wrbuf_alloc();
3175         error = cql_transform(trans, cql_parser_result(parser),
3176                               cql2pqf_wrbuf_puts, wrbuf_result);
3177         if (error != 0) {
3178             char buf[512];
3179             const char *addinfo;
3180             error = cql_transform_error(trans, &addinfo);
3181             sprintf(buf, "%.200s (addinfo=%.200s)",
3182                     cql_strerror(error), addinfo);
3183             ZOOM_set_error(c, ZOOM_ERROR_CQL_TRANSFORM, buf);
3184         }
3185         else
3186         {
3187             result = xstrdup(wrbuf_cstr(wrbuf_result));
3188         }
3189         cql_transform_close(trans);
3190         wrbuf_destroy(wrbuf_result);
3191     }
3192     cql_parser_destroy(parser);
3193     return result;
3194 }
3195
3196 ZOOM_API(int) ZOOM_connection_fire_event_timeout(ZOOM_connection c)
3197 {
3198     if (c->mask)
3199     {
3200         ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_TIMEOUT);
3201         /* timeout and this connection was waiting */
3202         ZOOM_set_error(c, ZOOM_ERROR_TIMEOUT, 0);
3203         ZOOM_connection_close(c);
3204         ZOOM_connection_put_event(c, event);
3205     }
3206     return 0;
3207 }
3208
3209 ZOOM_API(int)
3210     ZOOM_connection_process(ZOOM_connection c)
3211 {
3212     ZOOM_Event event;
3213     if (!c)
3214         return 0;
3215
3216     event = ZOOM_connection_get_event(c);
3217     if (event)
3218     {
3219         ZOOM_Event_destroy(event);
3220         return 1;
3221     }
3222     ZOOM_connection_exec_task(c);
3223     event = ZOOM_connection_get_event(c);
3224     if (event)
3225     {
3226         ZOOM_Event_destroy(event);
3227         return 1;
3228     }
3229     return 0;
3230 }
3231
3232 ZOOM_API(int)
3233     ZOOM_event_nonblock(int no, ZOOM_connection *cs)
3234 {
3235     int i;
3236
3237     yaz_log(log_details0, "ZOOM_process_event(no=%d,cs=%p)", no, cs);
3238
3239     for (i = 0; i<no; i++)
3240     {
3241         ZOOM_connection c = cs[i];
3242
3243         if (c && ZOOM_connection_process(c))
3244             return i+1;
3245     }
3246     return 0;
3247 }
3248
3249 ZOOM_API(int) ZOOM_connection_fire_event_socket(ZOOM_connection c, int mask)
3250 {
3251     if (c->mask && mask)
3252         ZOOM_connection_do_io(c, mask);
3253     return 0;
3254 }
3255
3256 ZOOM_API(int) ZOOM_connection_get_socket(ZOOM_connection c)
3257 {
3258     if (c->cs)
3259         return cs_fileno(c->cs);
3260     return -1;
3261 }
3262
3263 ZOOM_API(int) ZOOM_connection_set_mask(ZOOM_connection c, int mask)
3264 {
3265     c->mask = mask;
3266     if (!c->cs)
3267         return -1;
3268     return 0;
3269 }
3270
3271 ZOOM_API(int) ZOOM_connection_get_mask(ZOOM_connection c)
3272 {
3273     if (c->cs)
3274         return c->mask;
3275     return 0;
3276 }
3277
3278 ZOOM_API(int) ZOOM_connection_get_timeout(ZOOM_connection c)
3279 {
3280     return ZOOM_options_get_int(c->options, "timeout", 30);
3281 }
3282
3283 ZOOM_API(void) ZOOM_connection_close(ZOOM_connection c)
3284 {
3285     if (c->cs)
3286         cs_close(c->cs);
3287     c->cs = 0;
3288     ZOOM_connection_set_mask(c, 0);
3289     c->state = STATE_IDLE;
3290 }
3291
3292 /*
3293  * Local variables:
3294  * c-basic-offset: 4
3295  * c-file-style: "Stroustrup"
3296  * indent-tabs-mode: nil
3297  * End:
3298  * vim: shiftwidth=4 tabstop=8 expandtab
3299  */
3300