src/client.c

   1 /* $Id: client.c,v 1.5 2007-06-02 03:37:55 quinn Exp $
   2    Copyright (c) 2006-2007, Index Data.
   3
   4 This file is part of Pazpar2.
   5
   6 Pazpar2 is free software; you can redistribute it and/or modify it under
   7 the terms of the GNU General Public License as published by the Free
   8 Software Foundation; either version 2, or (at your option) any later
   9 version.
  10
  11 Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY
  12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14 for more details.
  15
  16 You should have received a copy of the GNU General Public License
  17 along with Pazpar2; see the file LICENSE.  If not, write to the
  18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  19 02111-1307, USA.
  20  */
  21
  22 /** \file client.c
  23     \brief Z39.50 client
  24 */
  25
  26 #include <stdlib.h>
  27 #include <stdio.h>
  28 #include <string.h>
  29 #include <sys/time.h>
  30 #include <unistd.h>
  31 #include <sys/socket.h>
  32 #include <netdb.h>
  33 #include <signal.h>
  34 #include <ctype.h>
  35 #include <assert.h>
  36
  37 #include <yaz/marcdisp.h>
  38 #include <yaz/comstack.h>
  39 #include <yaz/tcpip.h>
  40 #include <yaz/proto.h>
  41 #include <yaz/readconf.h>
  42 #include <yaz/pquery.h>
  43 #include <yaz/otherinfo.h>
  44 #include <yaz/yaz-util.h>
  45 #include <yaz/nmem.h>
  46 #include <yaz/query-charset.h>
  47 #include <yaz/querytowrbuf.h>
  48 #if YAZ_VERSIONL >= 0x020163
  49 #include <yaz/oid_db.h>
  50 #endif
  51
  52 #if HAVE_CONFIG_H
  53 #include "cconfig.h"
  54 #endif
  55
  56 #define USE_TIMING 0
  57 #if USE_TIMING
  58 #include <yaz/timing.h>
  59 #endif
  60
  61 #include <netinet/in.h>
  62
  63 #include "pazpar2.h"
  64
  65 #include "client.h"
  66 #include "connection.h"
  67 #include "settings.h"
  68
  69 /** \brief Represents client state for a connection to one search target */
  70 struct client {
  71     struct session_database *database;
  72     struct connection *connection;
  73     struct session *session;
  74     char *pquery; // Current search
  75     int hits;
  76     int records;
  77     int setno;
  78     int requestid;            // ID of current outstanding request
  79     int diagnostic;
  80     enum client_state state;
  81     struct client *next;     // next client in session or next in free list
  82 };
  83
  84 static const char *client_states[] = {
  85     "Client_Connecting",
  86     "Client_Connected",
  87     "Client_Idle",
  88     "Client_Initializing",
  89     "Client_Searching",
  90     "Client_Presenting",
  91     "Client_Error",
  92     "Client_Failed",
  93     "Client_Disconnected",
  94     "Client_Stopped"
  95 };
  96
  97 static struct client *client_freelist = 0;
  98
  99 static int send_apdu(struct client *c, Z_APDU *a)
 100 {
 101     return connection_send_apdu(client_get_connection(c), a);
 102 }
 103
 104
 105 const char *client_get_state_str(struct client *cl)
 106 {
 107     return client_states[cl->state];
 108 }
 109
 110 enum client_state client_get_state(struct client *cl)
 111 {
 112     return cl->state;
 113 }
 114
 115 void client_set_state(struct client *cl, enum client_state st)
 116 {
 117     cl->state = st;
 118 }
 119
 120 // Close connection and set state to error
 121 void client_fatal(struct client *cl)
 122 {
 123     yaz_log(YLOG_WARN, "Fatal error from %s", client_get_url(cl));
 124     connection_destroy(cl->connection);
 125     cl->state = Client_Error;
 126 }
 127
 128 struct connection *client_get_connection(struct client *cl)
 129 {
 130     return cl->connection;
 131 }
 132
 133 struct session_database *client_get_database(struct client *cl)
 134 {
 135     return cl->database;
 136 }
 137
 138 struct session *client_get_session(struct client *cl)
 139 {
 140     return cl->session;
 141 }
 142
 143 const char *client_get_pquery(struct client *cl)
 144 {
 145     return cl->pquery;
 146 }
 147
 148 void client_set_requestid(struct client *cl, int id)
 149 {
 150     cl->requestid = id;
 151 }
 152
 153 void client_send_present(struct client *cl)
 154 {
 155     struct session_database *sdb = client_get_database(cl);
 156     Z_APDU *a = zget_APDU(global_parameters.odr_out, Z_APDU_presentRequest);
 157     int toget;
 158     int start = cl->records + 1;
 159     char *recsyn;
 160
 161     toget = global_parameters.chunk;
 162     if (toget > global_parameters.toget - cl->records)
 163         toget = global_parameters.toget - cl->records;
 164     if (toget > cl->hits - cl->records)
 165         toget = cl->hits - cl->records;
 166
 167     yaz_log(YLOG_DEBUG, "Trying to present %d records\n", toget);
 168
 169     a->u.presentRequest->resultSetStartPoint = &start;
 170     a->u.presentRequest->numberOfRecordsRequested = &toget;
 171
 172     a->u.presentRequest->resultSetId = "Default";
 173
 174     if ((recsyn = session_setting_oneval(sdb, PZ_REQUESTSYNTAX)))
 175     {
 176 #if YAZ_VERSIONL >= 0x020163
 177         a->u.presentRequest->preferredRecordSyntax =
 178             yaz_string_to_oid_odr(yaz_oid_std(),
 179                                   CLASS_RECSYN, recsyn,
 180                                   global_parameters.odr_out);
 181 #else
 182         a->u.presentRequest->preferredRecordSyntax =
 183             yaz_str_to_z3950oid(global_parameters.odr_out,
 184                                 CLASS_RECSYN, recsyn);
 185 #endif
 186     }
 187
 188     if (send_apdu(cl, a) >= 0)
 189         cl->state = Client_Presenting;
 190     else
 191         cl->state = Client_Error;
 192     odr_reset(global_parameters.odr_out);
 193 }
 194
 195
 196 void client_send_search(struct client *cl)
 197 {
 198     struct session *se = client_get_session(cl);
 199     struct session_database *sdb = client_get_database(cl);
 200     Z_APDU *a = zget_APDU(global_parameters.odr_out, Z_APDU_searchRequest);
 201     int ndb;
 202     char **databaselist;
 203     Z_Query *zquery;
 204     int ssub = 0, lslb = 100000, mspn = 10;
 205     char *recsyn = 0;
 206     char *piggyback = 0;
 207     char *queryenc = 0;
 208     yaz_iconv_t iconv = 0;
 209
 210     yaz_log(YLOG_DEBUG, "Sending search to %s", sdb->database->url);
 211
 212     // constructing RPN query
 213     a->u.searchRequest->query = zquery = odr_malloc(global_parameters.odr_out,
 214                                                     sizeof(Z_Query));
 215     zquery->which = Z_Query_type_1;
 216     zquery->u.type_1 = p_query_rpn(global_parameters.odr_out,
 217                                    client_get_pquery(cl));
 218
 219     // converting to target encoding
 220     if ((queryenc = session_setting_oneval(sdb, PZ_QUERYENCODING))){
 221         iconv = yaz_iconv_open(queryenc, "UTF-8");
 222         if (iconv){
 223             yaz_query_charset_convert_rpnquery(zquery->u.type_1,
 224                                                global_parameters.odr_out,
 225                                                iconv);
 226             yaz_iconv_close(iconv);
 227         } else
 228             yaz_log(YLOG_WARN, "Query encoding failed %s %s",
 229                     client_get_database(cl)->database->url, queryenc);
 230     }
 231
 232     for (ndb = 0; sdb->database->databases[ndb]; ndb++)
 233         ;
 234     databaselist = odr_malloc(global_parameters.odr_out, sizeof(char*) * ndb);
 235     for (ndb = 0; sdb->database->databases[ndb]; ndb++)
 236         databaselist[ndb] = sdb->database->databases[ndb];
 237
 238     if (!(piggyback = session_setting_oneval(sdb, PZ_PIGGYBACK)) || *piggyback == '1')
 239     {
 240         if ((recsyn = session_setting_oneval(sdb, PZ_REQUESTSYNTAX)))
 241         {
 242 #if YAZ_VERSIONL >= 0x020163
 243             a->u.searchRequest->preferredRecordSyntax =
 244                 yaz_string_to_oid_odr(yaz_oid_std(),
 245                                       CLASS_RECSYN, recsyn,
 246                                       global_parameters.odr_out);
 247 #else
 248             a->u.searchRequest->preferredRecordSyntax =
 249                 yaz_str_to_z3950oid(global_parameters.odr_out,
 250                                     CLASS_RECSYN, recsyn);
 251 #endif
 252         }
 253         a->u.searchRequest->smallSetUpperBound = &ssub;
 254         a->u.searchRequest->largeSetLowerBound = &lslb;
 255         a->u.searchRequest->mediumSetPresentNumber = &mspn;
 256     }
 257     a->u.searchRequest->resultSetName = "Default";
 258     a->u.searchRequest->databaseNames = databaselist;
 259     a->u.searchRequest->num_databaseNames = ndb;
 260
 261
 262     {  //scope for sending and logging queries
 263         WRBUF wbquery = wrbuf_alloc();
 264         yaz_query_to_wrbuf(wbquery, a->u.searchRequest->query);
 265
 266
 267         if (send_apdu(cl, a) >= 0)
 268         {
 269             client_set_state(cl, Client_Searching);
 270             client_set_requestid(cl, se->requestid);
 271             yaz_log(YLOG_LOG, "SearchRequest %s %s %s",
 272                     client_get_database(cl)->database->url,
 273                     queryenc ? queryenc : "UTF-8",
 274                     wrbuf_cstr(wbquery));
 275         }
 276         else {
 277             client_set_state(cl, Client_Error);
 278             yaz_log(YLOG_WARN, "Failed SearchRequest %s  %s %s",
 279                     client_get_database(cl)->database->url,
 280                     queryenc ? queryenc : "UTF-8",
 281                     wrbuf_cstr(wbquery));
 282         }
 283
 284         wrbuf_destroy(wbquery);
 285     }
 286
 287     odr_reset(global_parameters.odr_out);
 288 }
 289
 290 void client_init_response(struct client *cl, Z_APDU *a)
 291 {
 292     Z_InitResponse *r = a->u.initResponse;
 293
 294     yaz_log(YLOG_DEBUG, "Init response %s", cl->database->database->url);
 295
 296     if (*r->result)
 297     {
 298         cl->state = Client_Idle;
 299     }
 300     else
 301         cl->state = Client_Failed; // FIXME need to do something to the connection
 302 }
 303
 304
 305 static void ingest_records(struct client *cl, Z_Records *r)
 306 {
 307 #if USE_TIMING
 308     yaz_timing_t t = yaz_timing_create();
 309 #endif
 310     struct record *rec;
 311     struct session *s = client_get_session(cl);
 312     Z_NamePlusRecordList *rlist;
 313     int i;
 314
 315     if (r->which != Z_Records_DBOSD)
 316         return;
 317     rlist = r->u.databaseOrSurDiagnostics;
 318     for (i = 0; i < rlist->num_records; i++)
 319     {
 320         Z_NamePlusRecord *npr = rlist->records[i];
 321
 322         cl->records++;
 323         if (npr->which != Z_NamePlusRecord_databaseRecord)
 324         {
 325             yaz_log(YLOG_WARN,
 326                     "Unexpected record type, probably diagnostic %s",
 327                     cl->database->database->url);
 328             continue;
 329         }
 330
 331         rec = ingest_record(cl, npr->u.databaseRecord, cl->records);
 332         if (!rec)
 333             continue;
 334     }
 335     if (rlist->num_records)
 336         session_alert_watch(s, SESSION_WATCH_RECORDS);
 337
 338 #if USE_TIMING
 339     yaz_timing_stop(t);
 340     yaz_log(YLOG_LOG, "ingest_records %6.5f %3.2f %3.2f",
 341             yaz_timing_get_real(t), yaz_timing_get_user(t),
 342             yaz_timing_get_sys(t));
 343     yaz_timing_destroy(&t);
 344 #endif
 345 }
 346
 347
 348 void client_search_response(struct client *cl, Z_APDU *a)
 349 {
 350     struct session *se = cl->session;
 351     Z_SearchResponse *r = a->u.searchResponse;
 352
 353     yaz_log(YLOG_DEBUG, "Search response %s (status=%d)",
 354             cl->database->database->url, *r->searchStatus);
 355
 356     if (*r->searchStatus)
 357     {
 358         cl->hits = *r->resultCount;
 359         se->total_hits += cl->hits;
 360         if (r->presentStatus && !*r->presentStatus && r->records)
 361         {
 362             yaz_log(YLOG_DEBUG, "Records in search response %s",
 363                     cl->database->database->url);
 364             ingest_records(cl, r->records);
 365         }
 366         cl->state = Client_Idle;
 367     }
 368     else
 369     {          /*"FAILED"*/
 370         cl->hits = 0;
 371         cl->state = Client_Error;
 372         if (r->records) {
 373             Z_Records *recs = r->records;
 374             if (recs->which == Z_Records_NSD)
 375             {
 376                 yaz_log(YLOG_WARN,
 377                     "Search response: Non-surrogate diagnostic %s (%d)",
 378                     cl->database->database->url,
 379                     *recs->u.nonSurrogateDiagnostic->condition);
 380                 cl->diagnostic = *recs->u.nonSurrogateDiagnostic->condition;
 381                 cl->state = Client_Error;
 382             }
 383         }
 384     }
 385 }
 386
 387 void client_present_response(struct client *cl, Z_APDU *a)
 388 {
 389     Z_PresentResponse *r = a->u.presentResponse;
 390
 391     if (r->records) {
 392         Z_Records *recs = r->records;
 393         if (recs->which == Z_Records_NSD)
 394         {
 395             yaz_log(YLOG_WARN, "Non-surrogate diagnostic %s",
 396                     cl->database->database->url);
 397             cl->diagnostic = *recs->u.nonSurrogateDiagnostic->condition;
 398             cl->state = Client_Error;
 399         }
 400     }
 401
 402     if (!*r->presentStatus && cl->state != Client_Error)
 403     {
 404         yaz_log(YLOG_DEBUG, "Good Present response %s",
 405                 cl->database->database->url);
 406         ingest_records(cl, r->records);
 407         cl->state = Client_Idle;
 408     }
 409     else if (*r->presentStatus)
 410     {
 411         yaz_log(YLOG_WARN, "Bad Present response %s",
 412                 cl->database->database->url);
 413         cl->state = Client_Error;
 414     }
 415 }
 416
 417 void client_close_response(struct client *cl, Z_APDU *a)
 418 {
 419     struct connection *co = cl->connection;
 420     /* Z_Close *r = a->u.close; */
 421
 422     yaz_log(YLOG_WARN, "Close response %s", cl->database->database->url);
 423
 424     cl->state = Client_Failed;
 425     connection_destroy(co);
 426 }
 427
 428 int client_is_our_response(struct client *cl)
 429 {
 430     struct session *se = client_get_session(cl);
 431
 432     if (cl && (cl->requestid == se->requestid ||
 433                cl->state == Client_Initializing))
 434         return 1;
 435     return 0;
 436 }
 437
 438 // Set authentication token in init if one is set for the client
 439 // TODO: Extend this to handle other schemes than open (should be simple)
 440 static void init_authentication(struct client *cl, Z_InitRequest *req)
 441 {
 442     struct session_database *sdb = client_get_database(cl);
 443     char *auth = session_setting_oneval(sdb, PZ_AUTHENTICATION);
 444
 445     if (*auth)
 446     {
 447         Z_IdAuthentication *idAuth = odr_malloc(global_parameters.odr_out,
 448                 sizeof(*idAuth));
 449         idAuth->which = Z_IdAuthentication_open;
 450         idAuth->u.open = auth;
 451         req->idAuthentication = idAuth;
 452     }
 453 }
 454
 455 static void client_init_request(struct client *cl)
 456 {
 457     Z_APDU *a = zget_APDU(global_parameters.odr_out, Z_APDU_initRequest);
 458
 459     a->u.initRequest->implementationId = global_parameters.implementationId;
 460     a->u.initRequest->implementationName = global_parameters.implementationName;
 461     a->u.initRequest->implementationVersion =
 462         global_parameters.implementationVersion;
 463     ODR_MASK_SET(a->u.initRequest->options, Z_Options_search);
 464     ODR_MASK_SET(a->u.initRequest->options, Z_Options_present);
 465     ODR_MASK_SET(a->u.initRequest->options, Z_Options_namedResultSets);
 466
 467     ODR_MASK_SET(a->u.initRequest->protocolVersion, Z_ProtocolVersion_1);
 468     ODR_MASK_SET(a->u.initRequest->protocolVersion, Z_ProtocolVersion_2);
 469     ODR_MASK_SET(a->u.initRequest->protocolVersion, Z_ProtocolVersion_3);
 470
 471     init_authentication(cl, a->u.initRequest);
 472
 473     /* add virtual host if tunneling through Z39.50 proxy */
 474
 475     if (0 < strlen(global_parameters.zproxy_override)
 476         && 0 < strlen(client_get_database(cl)->database->url))
 477     {
 478 #if YAZ_VERSIONL >= 0x020163
 479         yaz_oi_set_string_oid(&a->u.initRequest->otherInfo,
 480                               global_parameters.odr_out,
 481                               yaz_oid_userinfo_proxy,
 482                               1, client_get_database(cl)->database->url);
 483 #else
 484         yaz_oi_set_string_oidval(&a->u.initRequest->otherInfo,
 485                                  global_parameters.odr_out, VAL_PROXY,
 486                                  1, client_get_database(cl->database)->url);
 487 #endif
 488     }
 489
 490     if (send_apdu(cl, a) >= 0)
 491         client_set_state(cl, Client_Initializing);
 492     else
 493         client_set_state(cl, Client_Error);
 494     odr_reset(global_parameters.odr_out);
 495 }
 496
 497 void client_continue(struct client *cl)
 498 {
 499     if (cl->state == Client_Connected) {
 500         client_init_request(cl);
 501     }
 502
 503     if (cl->state == Client_Idle)
 504     {
 505         struct session *se = client_get_session(cl);
 506         if (cl->requestid != se->requestid && cl->pquery) {
 507             client_send_search(cl);
 508         }
 509         else if (cl->hits > 0 && cl->records < global_parameters.toget &&
 510             cl->records < cl->hits) {
 511             client_send_present(cl);
 512         }
 513     }
 514 }
 515
 516 struct client *client_create(void)
 517 {
 518     struct client *r;
 519     if (client_freelist)
 520     {
 521         r = client_freelist;
 522         client_freelist = client_freelist->next;
 523     }
 524     else
 525         r = xmalloc(sizeof(struct client));
 526     r->pquery = 0;
 527     r->database = 0;
 528     r->connection = 0;
 529     r->session = 0;
 530     r->hits = 0;
 531     r->records = 0;
 532     r->setno = 0;
 533     r->requestid = -1;
 534     r->diagnostic = 0;
 535     r->state = Client_Disconnected;
 536     r->next = 0;
 537     return r;
 538 }
 539
 540 void client_destroy(struct client *c)
 541 {
 542     struct session *se = c->session;
 543     if (c == se->clients)
 544         se->clients = c->next;
 545     else
 546     {
 547         struct client *cc;
 548         for (cc = se->clients; cc && cc->next != c; cc = cc->next)
 549             ;
 550         if (cc)
 551             cc->next = c->next;
 552     }
 553     if (c->connection)
 554         connection_release(c->connection);
 555     c->next = client_freelist;
 556     client_freelist = c;
 557 }
 558
 559 void client_set_connection(struct client *cl, struct connection *con)
 560 {
 561     cl->connection = con;
 562 }
 563
 564 void client_disconnect(struct client *cl)
 565 {
 566     if (cl->state != Client_Idle)
 567         cl->state = Client_Disconnected;
 568     client_set_connection(cl, 0);
 569 }
 570
 571 // Extract terms from query into null-terminated termlist
 572 static void extract_terms(NMEM nmem, struct ccl_rpn_node *query, char **termlist)
 573 {
 574     int num = 0;
 575
 576     pull_terms(nmem, query, termlist, &num);
 577     termlist[num] = 0;
 578 }
 579
 580 // Initialize CCL map for a target
 581 static CCL_bibset prepare_cclmap(struct client *cl)
 582 {
 583     struct session_database *sdb = client_get_database(cl);
 584     struct setting *s;
 585     CCL_bibset res;
 586
 587     if (!sdb->settings)
 588         return 0;
 589     res = ccl_qual_mk();
 590     for (s = sdb->settings[PZ_CCLMAP]; s; s = s->next)
 591     {
 592         char *p = strchr(s->name + 3, ':');
 593         if (!p)
 594         {
 595             yaz_log(YLOG_WARN, "Malformed cclmap name: %s", s->name);
 596             ccl_qual_rm(&res);
 597             return 0;
 598         }
 599         p++;
 600         ccl_qual_fitem(res, s->value, p);
 601     }
 602     return res;
 603 }
 604
 605 // Parse the query given the settings specific to this client
 606 int client_parse_query(struct client *cl, const char *query)
 607 {
 608     struct session *se = client_get_session(cl);
 609     struct ccl_rpn_node *cn;
 610     int cerror, cpos;
 611     CCL_bibset ccl_map = prepare_cclmap(cl);
 612
 613     if (!ccl_map)
 614         return -1;
 615     cn = ccl_find_str(ccl_map, query, &cerror, &cpos);
 616     ccl_qual_rm(&ccl_map);
 617     if (!cn)
 618     {
 619         cl->state = Client_Error;
 620         yaz_log(YLOG_WARN, "Failed to parse query for %s",
 621                          client_get_database(cl)->database->url);
 622         return -1;
 623     }
 624     wrbuf_rewind(se->wrbuf);
 625     ccl_pquery(se->wrbuf, cn);
 626     xfree(cl->pquery);
 627     cl->pquery = xstrdup(wrbuf_cstr(se->wrbuf));
 628
 629     if (!se->relevance)
 630     {
 631         // Initialize relevance structure with query terms
 632         char *p[512];
 633         extract_terms(se->nmem, cn, p);
 634         se->relevance = relevance_create(client_get_database(cl)->pct,
 635                                          se->nmem, (const char **) p,
 636                                          se->expected_maxrecs);
 637     }
 638
 639     ccl_rpn_delete(cn);
 640     return 0;
 641 }
 642
 643 void client_set_session(struct client *cl, struct session *se)
 644 {
 645     cl->session = se;
 646     cl->next = se->clients;
 647     se->clients = cl;
 648 }
 649
 650 int client_is_active(struct client *cl)
 651 {
 652     if (cl->connection && (cl->state == Client_Connecting ||
 653                            cl->state == Client_Initializing ||
 654                            cl->state == Client_Searching ||
 655                            cl->state == Client_Presenting))
 656         return 1;
 657     return 0;
 658 }
 659
 660 struct client *client_next_in_session(struct client *cl)
 661 {
 662     if (cl)
 663         return cl->next;
 664     return 0;
 665
 666 }
 667
 668 int client_get_hits(struct client *cl)
 669 {
 670     return cl->hits;
 671 }
 672
 673 int client_get_num_records(struct client *cl)
 674 {
 675     return cl->records;
 676 }
 677
 678 int client_get_diagnostic(struct client *cl)
 679 {
 680     return cl->diagnostic;
 681 }
 682
 683 void client_set_database(struct client *cl, struct session_database *db)
 684 {
 685     cl->database = db;
 686 }
 687
 688 struct host *client_get_host(struct client *cl)
 689 {
 690     return client_get_database(cl)->database->host;
 691 }
 692
 693 const char *client_get_url(struct client *cl)
 694 {
 695     return client_get_database(cl)->database->url;
 696 }
 697
 698 /*
 699  * Local variables:
 700  * c-basic-offset: 4
 701  * indent-tabs-mode: nil
 702  * End:
 703  * vim: shiftwidth=4 tabstop=8 expandtab
 704  */