src/client.c

   1 /* $Id: client.c,v 1.4 2007-05-15 08:52:35 adam Exp $
   2    Copyright (c) 2006-2007, Index Data.
   3
   4 This file is part of Pazpar2.
   5
   6 Pazpar2 is free software; you can redistribute it and/or modify it under
   7 the terms of the GNU General Public License as published by the Free
   8 Software Foundation; either version 2, or (at your option) any later
   9 version.
  10
  11 Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY
  12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14 for more details.
  15
  16 You should have received a copy of the GNU General Public License
  17 along with Pazpar2; see the file LICENSE.  If not, write to the
  18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  19 02111-1307, USA.
  20  */
  21
  22 /** \file client.c
  23     \brief Z39.50 client
  24 */
  25
  26 #include <stdlib.h>
  27 #include <stdio.h>
  28 #include <string.h>
  29 #include <sys/time.h>
  30 #include <unistd.h>
  31 #include <sys/socket.h>
  32 #include <netdb.h>
  33 #include <signal.h>
  34 #include <ctype.h>
  35 #include <assert.h>
  36
  37 #include <yaz/marcdisp.h>
  38 #include <yaz/comstack.h>
  39 #include <yaz/tcpip.h>
  40 #include <yaz/proto.h>
  41 #include <yaz/readconf.h>
  42 #include <yaz/pquery.h>
  43 #include <yaz/otherinfo.h>
  44 #include <yaz/yaz-util.h>
  45 #include <yaz/nmem.h>
  46 #include <yaz/query-charset.h>
  47 #include <yaz/querytowrbuf.h>
  48 #if YAZ_VERSIONL >= 0x020163
  49 #include <yaz/oid_db.h>
  50 #endif
  51
  52 #if HAVE_CONFIG_H
  53 #include "cconfig.h"
  54 #endif
  55
  56 #define USE_TIMING 0
  57 #if USE_TIMING
  58 #include <yaz/timing.h>
  59 #endif
  60
  61 #include <netinet/in.h>
  62
  63 #include "pazpar2.h"
  64
  65 #include "client.h"
  66 #include "connection.h"
  67 #include "settings.h"
  68
  69 /** \brief Represents client state for a connection to one search target */
  70 struct client {
  71     struct session_database *database;
  72     struct connection *connection;
  73     struct session *session;
  74     char *pquery; // Current search
  75     int hits;
  76     int records;
  77     int setno;
  78     int requestid;            // ID of current outstanding request
  79     int diagnostic;
  80     enum client_state state;
  81     struct client *next;     // next client in session or next in free list
  82 };
  83
  84 static const char *client_states[] = {
  85     "Client_Connecting",
  86     "Client_Connected",
  87     "Client_Idle",
  88     "Client_Initializing",
  89     "Client_Searching",
  90     "Client_Presenting",
  91     "Client_Error",
  92     "Client_Failed",
  93     "Client_Disconnected",
  94     "Client_Stopped"
  95 };
  96
  97 static struct client *client_freelist = 0;
  98
  99 static int send_apdu(struct client *c, Z_APDU *a)
 100 {
 101     return connection_send_apdu(client_get_connection(c), a);
 102 }
 103
 104
 105 const char *client_get_state_str(struct client *cl)
 106 {
 107     return client_states[cl->state];
 108 }
 109
 110 enum client_state client_get_state(struct client *cl)
 111 {
 112     return cl->state;
 113 }
 114
 115 void client_set_state(struct client *cl, enum client_state st)
 116 {
 117     cl->state = st;
 118 }
 119
 120 // Close connection and set state to error
 121 void client_fatal(struct client *cl)
 122 {
 123     yaz_log(YLOG_WARN, "Fatal error from %s", client_get_url(cl));
 124     connection_destroy(cl->connection);
 125     cl->state = Client_Error;
 126 }
 127
 128 struct connection *client_get_connection(struct client *cl)
 129 {
 130     return cl->connection;
 131 }
 132
 133 struct session_database *client_get_database(struct client *cl)
 134 {
 135     return cl->database;
 136 }
 137
 138 struct session *client_get_session(struct client *cl)
 139 {
 140     return cl->session;
 141 }
 142
 143 const char *client_get_pquery(struct client *cl)
 144 {
 145     return cl->pquery;
 146 }
 147
 148 void client_set_requestid(struct client *cl, int id)
 149 {
 150     cl->requestid = id;
 151 }
 152
 153 void client_send_present(struct client *cl)
 154 {
 155     struct session_database *sdb = client_get_database(cl);
 156     Z_APDU *a = zget_APDU(global_parameters.odr_out, Z_APDU_presentRequest);
 157     int toget;
 158     int start = cl->records + 1;
 159     char *recsyn;
 160
 161     toget = global_parameters.chunk;
 162     if (toget > global_parameters.toget - cl->records)
 163         toget = global_parameters.toget - cl->records;
 164     if (toget > cl->hits - cl->records)
 165         toget = cl->hits - cl->records;
 166
 167     yaz_log(YLOG_DEBUG, "Trying to present %d records\n", toget);
 168
 169     a->u.presentRequest->resultSetStartPoint = &start;
 170     a->u.presentRequest->numberOfRecordsRequested = &toget;
 171
 172     a->u.presentRequest->resultSetId = "Default";
 173
 174     if ((recsyn = session_setting_oneval(sdb, PZ_REQUESTSYNTAX)))
 175     {
 176 #if YAZ_VERSIONL >= 0x020163
 177         a->u.presentRequest->preferredRecordSyntax =
 178             yaz_string_to_oid_odr(yaz_oid_std(),
 179                                   CLASS_RECSYN, recsyn,
 180                                   global_parameters.odr_out);
 181 #else
 182         a->u.presentRequest->preferredRecordSyntax =
 183             yaz_str_to_z3950oid(global_parameters.odr_out,
 184                                 CLASS_RECSYN, recsyn);
 185 #endif
 186     }
 187
 188     if (send_apdu(cl, a) >= 0)
 189         cl->state = Client_Presenting;
 190     else
 191         cl->state = Client_Error;
 192     odr_reset(global_parameters.odr_out);
 193 }
 194
 195
 196 void client_send_search(struct client *cl)
 197 {
 198     struct session *se = client_get_session(cl);
 199     struct session_database *sdb = client_get_database(cl);
 200     Z_APDU *a = zget_APDU(global_parameters.odr_out, Z_APDU_searchRequest);
 201     int ndb;
 202     char **databaselist;
 203     Z_Query *zquery;
 204     int ssub = 0, lslb = 100000, mspn = 10;
 205     char *recsyn = 0;
 206     char *piggyback = 0;
 207     char *queryenc = 0;
 208     yaz_iconv_t iconv = 0;
 209
 210     yaz_log(YLOG_DEBUG, "Sending search to %s", sdb->database->url);
 211
 212     // constructing RPN query
 213     a->u.searchRequest->query = zquery = odr_malloc(global_parameters.odr_out,
 214                                                     sizeof(Z_Query));
 215     zquery->which = Z_Query_type_1;
 216     zquery->u.type_1 = p_query_rpn(global_parameters.odr_out,
 217                                    client_get_pquery(cl));
 218
 219     // converting to target encoding
 220     if ((queryenc = session_setting_oneval(sdb, PZ_QUERYENCODING))){
 221         iconv = yaz_iconv_open(queryenc, "UTF-8");
 222         if (iconv){
 223             yaz_query_charset_convert_rpnquery(zquery->u.type_1,
 224                                                global_parameters.odr_out,
 225                                                iconv);
 226             yaz_iconv_close(iconv);
 227         } else
 228             yaz_log(YLOG_WARN, "Query encoding failed %s %s",
 229                     client_get_database(cl)->database->url, queryenc);
 230     }
 231
 232     for (ndb = 0; sdb->database->databases[ndb]; ndb++)
 233         ;
 234     databaselist = odr_malloc(global_parameters.odr_out, sizeof(char*) * ndb);
 235     for (ndb = 0; sdb->database->databases[ndb]; ndb++)
 236         databaselist[ndb] = sdb->database->databases[ndb];
 237
 238     if (!(piggyback = session_setting_oneval(sdb, PZ_PIGGYBACK)) || *piggyback == '1')
 239     {
 240         if ((recsyn = session_setting_oneval(sdb, PZ_REQUESTSYNTAX)))
 241         {
 242 #if YAZ_VERSIONL >= 0x020163
 243             a->u.searchRequest->preferredRecordSyntax =
 244                 yaz_string_to_oid_odr(yaz_oid_std(),
 245                                       CLASS_RECSYN, recsyn,
 246                                       global_parameters.odr_out);
 247 #else
 248             a->u.searchRequest->preferredRecordSyntax =
 249                 yaz_str_to_z3950oid(global_parameters.odr_out,
 250                                     CLASS_RECSYN, recsyn);
 251 #endif
 252         }
 253         a->u.searchRequest->smallSetUpperBound = &ssub;
 254         a->u.searchRequest->largeSetLowerBound = &lslb;
 255         a->u.searchRequest->mediumSetPresentNumber = &mspn;
 256     }
 257     a->u.searchRequest->resultSetName = "Default";
 258     a->u.searchRequest->databaseNames = databaselist;
 259     a->u.searchRequest->num_databaseNames = ndb;
 260
 261
 262     {  //scope for sending and logging queries
 263         WRBUF wbquery = wrbuf_alloc();
 264         yaz_query_to_wrbuf(wbquery, a->u.searchRequest->query);
 265
 266
 267         if (send_apdu(cl, a) >= 0)
 268         {
 269             client_set_state(cl, Client_Searching);
 270             client_set_requestid(cl, se->requestid);
 271             yaz_log(YLOG_LOG, "SearchRequest %s %s %s",
 272                     client_get_database(cl)->database->url,
 273                     queryenc ? queryenc : "UTF-8",
 274                     wrbuf_cstr(wbquery));
 275         }
 276         else {
 277             client_set_state(cl, Client_Error);
 278             yaz_log(YLOG_WARN, "Failed SearchRequest %s  %s %s",
 279                     client_get_database(cl)->database->url,
 280                     queryenc ? queryenc : "UTF-8",
 281                     wrbuf_cstr(wbquery));
 282         }
 283
 284         wrbuf_destroy(wbquery);
 285     }
 286
 287     odr_reset(global_parameters.odr_out);
 288 }
 289
 290 void client_init_response(struct client *cl, Z_APDU *a)
 291 {
 292     Z_InitResponse *r = a->u.initResponse;
 293
 294     yaz_log(YLOG_DEBUG, "Init response %s", cl->database->database->url);
 295
 296     if (*r->result)
 297     {
 298         cl->state = Client_Idle;
 299     }
 300     else
 301         cl->state = Client_Failed; // FIXME need to do something to the connection
 302 }
 303
 304
 305 static void ingest_records(struct client *cl, Z_Records *r)
 306 {
 307 #if USE_TIMING
 308     yaz_timing_t t = yaz_timing_create();
 309 #endif
 310     struct record *rec;
 311     struct session *s = client_get_session(cl);
 312     Z_NamePlusRecordList *rlist;
 313     int i;
 314
 315     if (r->which != Z_Records_DBOSD)
 316         return;
 317     rlist = r->u.databaseOrSurDiagnostics;
 318     for (i = 0; i < rlist->num_records; i++)
 319     {
 320         Z_NamePlusRecord *npr = rlist->records[i];
 321
 322         cl->records++;
 323         if (npr->which != Z_NamePlusRecord_databaseRecord)
 324         {
 325             yaz_log(YLOG_WARN,
 326                     "Unexpected record type, probably diagnostic %s",
 327                     cl->database->database->url);
 328             continue;
 329         }
 330
 331         rec = ingest_record(cl, npr->u.databaseRecord, cl->records);
 332         if (!rec)
 333             continue;
 334     }
 335     if (rlist->num_records)
 336         session_alert_watch(s, SESSION_WATCH_RECORDS);
 337
 338 #if USE_TIMING
 339     yaz_timing_stop(t);
 340     yaz_log(YLOG_LOG, "ingest_records %6.5f %3.2f %3.2f",
 341             yaz_timing_get_real(t), yaz_timing_get_user(t),
 342             yaz_timing_get_sys(t));
 343     yaz_timing_destroy(&t);
 344 #endif
 345 }
 346
 347
 348 void client_search_response(struct client *cl, Z_APDU *a)
 349 {
 350     struct session *se = cl->session;
 351     Z_SearchResponse *r = a->u.searchResponse;
 352
 353     yaz_log(YLOG_DEBUG, "Search response %s (status=%d)",
 354             cl->database->database->url, *r->searchStatus);
 355
 356     if (*r->searchStatus)
 357     {
 358         cl->hits = *r->resultCount;
 359         se->total_hits += cl->hits;
 360         if (r->presentStatus && !*r->presentStatus && r->records)
 361         {
 362             yaz_log(YLOG_DEBUG, "Records in search response %s",
 363                     cl->database->database->url);
 364             ingest_records(cl, r->records);
 365         }
 366         cl->state = Client_Idle;
 367     }
 368     else
 369     {          /*"FAILED"*/
 370         cl->hits = 0;
 371         cl->state = Client_Error;
 372         if (r->records) {
 373             Z_Records *recs = r->records;
 374             if (recs->which == Z_Records_NSD)
 375             {
 376                 yaz_log(YLOG_WARN,
 377                         "Search response: Non-surrogate diagnostic %s",
 378                         cl->database->database->url);
 379                 cl->diagnostic = *recs->u.nonSurrogateDiagnostic->condition;
 380                 cl->state = Client_Error;
 381             }
 382         }
 383     }
 384 }
 385
 386 void client_present_response(struct client *cl, Z_APDU *a)
 387 {
 388     Z_PresentResponse *r = a->u.presentResponse;
 389
 390     if (r->records) {
 391         Z_Records *recs = r->records;
 392         if (recs->which == Z_Records_NSD)
 393         {
 394             yaz_log(YLOG_WARN, "Non-surrogate diagnostic %s",
 395                     cl->database->database->url);
 396             cl->diagnostic = *recs->u.nonSurrogateDiagnostic->condition;
 397             cl->state = Client_Error;
 398         }
 399     }
 400
 401     if (!*r->presentStatus && cl->state != Client_Error)
 402     {
 403         yaz_log(YLOG_DEBUG, "Good Present response %s",
 404                 cl->database->database->url);
 405         ingest_records(cl, r->records);
 406         cl->state = Client_Idle;
 407     }
 408     else if (*r->presentStatus)
 409     {
 410         yaz_log(YLOG_WARN, "Bad Present response %s",
 411                 cl->database->database->url);
 412         cl->state = Client_Error;
 413     }
 414 }
 415
 416 void client_close_response(struct client *cl, Z_APDU *a)
 417 {
 418     struct connection *co = cl->connection;
 419     /* Z_Close *r = a->u.close; */
 420
 421     yaz_log(YLOG_WARN, "Close response %s", cl->database->database->url);
 422
 423     cl->state = Client_Failed;
 424     connection_destroy(co);
 425 }
 426
 427 int client_is_our_response(struct client *cl)
 428 {
 429     struct session *se = client_get_session(cl);
 430
 431     if (cl && (cl->requestid == se->requestid ||
 432                cl->state == Client_Initializing))
 433         return 1;
 434     return 0;
 435 }
 436
 437 // Set authentication token in init if one is set for the client
 438 // TODO: Extend this to handle other schemes than open (should be simple)
 439 static void init_authentication(struct client *cl, Z_InitRequest *req)
 440 {
 441     struct session_database *sdb = client_get_database(cl);
 442     char *auth = session_setting_oneval(sdb, PZ_AUTHENTICATION);
 443
 444     if (auth)
 445     {
 446         Z_IdAuthentication *idAuth = odr_malloc(global_parameters.odr_out,
 447                 sizeof(*idAuth));
 448         idAuth->which = Z_IdAuthentication_open;
 449         idAuth->u.open = auth;
 450         req->idAuthentication = idAuth;
 451     }
 452 }
 453
 454 static void client_init_request(struct client *cl)
 455 {
 456     Z_APDU *a = zget_APDU(global_parameters.odr_out, Z_APDU_initRequest);
 457
 458     a->u.initRequest->implementationId = global_parameters.implementationId;
 459     a->u.initRequest->implementationName = global_parameters.implementationName;
 460     a->u.initRequest->implementationVersion =
 461         global_parameters.implementationVersion;
 462     ODR_MASK_SET(a->u.initRequest->options, Z_Options_search);
 463     ODR_MASK_SET(a->u.initRequest->options, Z_Options_present);
 464     ODR_MASK_SET(a->u.initRequest->options, Z_Options_namedResultSets);
 465
 466     ODR_MASK_SET(a->u.initRequest->protocolVersion, Z_ProtocolVersion_1);
 467     ODR_MASK_SET(a->u.initRequest->protocolVersion, Z_ProtocolVersion_2);
 468     ODR_MASK_SET(a->u.initRequest->protocolVersion, Z_ProtocolVersion_3);
 469
 470     init_authentication(cl, a->u.initRequest);
 471
 472     /* add virtual host if tunneling through Z39.50 proxy */
 473
 474     if (0 < strlen(global_parameters.zproxy_override)
 475         && 0 < strlen(client_get_database(cl)->database->url))
 476     {
 477 #if YAZ_VERSIONL >= 0x020163
 478         yaz_oi_set_string_oid(&a->u.initRequest->otherInfo,
 479                               global_parameters.odr_out,
 480                               yaz_oid_userinfo_proxy,
 481                               1, client_get_database(cl)->database->url);
 482 #else
 483         yaz_oi_set_string_oidval(&a->u.initRequest->otherInfo,
 484                                  global_parameters.odr_out, VAL_PROXY,
 485                                  1, client_get_database(cl->database)->url);
 486 #endif
 487     }
 488
 489     if (send_apdu(cl, a) >= 0)
 490         client_set_state(cl, Client_Initializing);
 491     else
 492         client_set_state(cl, Client_Error);
 493     odr_reset(global_parameters.odr_out);
 494 }
 495
 496 void client_continue(struct client *cl)
 497 {
 498     if (cl->state == Client_Connected) {
 499         client_init_request(cl);
 500     }
 501
 502     if (cl->state == Client_Idle)
 503     {
 504         struct session *se = client_get_session(cl);
 505         if (cl->requestid != se->requestid && cl->pquery) {
 506             client_send_search(cl);
 507         }
 508         else if (cl->hits > 0 && cl->records < global_parameters.toget &&
 509             cl->records < cl->hits) {
 510             client_send_present(cl);
 511         }
 512     }
 513 }
 514
 515 struct client *client_create(void)
 516 {
 517     struct client *r;
 518     if (client_freelist)
 519     {
 520         r = client_freelist;
 521         client_freelist = client_freelist->next;
 522     }
 523     else
 524         r = xmalloc(sizeof(struct client));
 525     r->pquery = 0;
 526     r->database = 0;
 527     r->connection = 0;
 528     r->session = 0;
 529     r->hits = 0;
 530     r->records = 0;
 531     r->setno = 0;
 532     r->requestid = -1;
 533     r->diagnostic = 0;
 534     r->state = Client_Disconnected;
 535     r->next = 0;
 536     return r;
 537 }
 538
 539 void client_destroy(struct client *c)
 540 {
 541     struct session *se = c->session;
 542     if (c == se->clients)
 543         se->clients = c->next;
 544     else
 545     {
 546         struct client *cc;
 547         for (cc = se->clients; cc && cc->next != c; cc = cc->next)
 548             ;
 549         if (cc)
 550             cc->next = c->next;
 551     }
 552     if (c->connection)
 553         connection_release(c->connection);
 554     c->next = client_freelist;
 555     client_freelist = c;
 556 }
 557
 558 void client_set_connection(struct client *cl, struct connection *con)
 559 {
 560     cl->connection = con;
 561 }
 562
 563 void client_disconnect(struct client *cl)
 564 {
 565     if (cl->state != Client_Idle)
 566         cl->state = Client_Disconnected;
 567     client_set_connection(cl, 0);
 568 }
 569
 570 // Extract terms from query into null-terminated termlist
 571 static void extract_terms(NMEM nmem, struct ccl_rpn_node *query, char **termlist)
 572 {
 573     int num = 0;
 574
 575     pull_terms(nmem, query, termlist, &num);
 576     termlist[num] = 0;
 577 }
 578
 579 // Initialize CCL map for a target
 580 static CCL_bibset prepare_cclmap(struct client *cl)
 581 {
 582     struct session_database *sdb = client_get_database(cl);
 583     struct setting *s;
 584     CCL_bibset res;
 585
 586     if (!sdb->settings)
 587         return 0;
 588     res = ccl_qual_mk();
 589     for (s = sdb->settings[PZ_CCLMAP]; s; s = s->next)
 590     {
 591         char *p = strchr(s->name + 3, ':');
 592         if (!p)
 593         {
 594             yaz_log(YLOG_WARN, "Malformed cclmap name: %s", s->name);
 595             ccl_qual_rm(&res);
 596             return 0;
 597         }
 598         p++;
 599         ccl_qual_fitem(res, s->value, p);
 600     }
 601     return res;
 602 }
 603
 604 // Parse the query given the settings specific to this client
 605 int client_parse_query(struct client *cl, const char *query)
 606 {
 607     struct session *se = client_get_session(cl);
 608     struct ccl_rpn_node *cn;
 609     int cerror, cpos;
 610     CCL_bibset ccl_map = prepare_cclmap(cl);
 611
 612     if (!ccl_map)
 613         return -1;
 614     cn = ccl_find_str(ccl_map, query, &cerror, &cpos);
 615     ccl_qual_rm(&ccl_map);
 616     if (!cn)
 617     {
 618         cl->state = Client_Error;
 619         yaz_log(YLOG_WARN, "Failed to parse query for %s",
 620                          client_get_database(cl)->database->url);
 621         return -1;
 622     }
 623     wrbuf_rewind(se->wrbuf);
 624     ccl_pquery(se->wrbuf, cn);
 625     xfree(cl->pquery);
 626     cl->pquery = xstrdup(wrbuf_cstr(se->wrbuf));
 627
 628     if (!se->relevance)
 629     {
 630         // Initialize relevance structure with query terms
 631         char *p[512];
 632         extract_terms(se->nmem, cn, p);
 633         se->relevance = relevance_create(client_get_database(cl)->pct,
 634                                          se->nmem, (const char **) p,
 635                                          se->expected_maxrecs);
 636     }
 637
 638     ccl_rpn_delete(cn);
 639     return 0;
 640 }
 641
 642 void client_set_session(struct client *cl, struct session *se)
 643 {
 644     cl->session = se;
 645     cl->next = se->clients;
 646     se->clients = cl;
 647 }
 648
 649 int client_is_active(struct client *cl)
 650 {
 651     if (cl->connection && (cl->state == Client_Connecting ||
 652                            cl->state == Client_Initializing ||
 653                            cl->state == Client_Searching ||
 654                            cl->state == Client_Presenting))
 655         return 1;
 656     return 0;
 657 }
 658
 659 struct client *client_next_in_session(struct client *cl)
 660 {
 661     if (cl)
 662         return cl->next;
 663     return 0;
 664
 665 }
 666
 667 int client_get_hits(struct client *cl)
 668 {
 669     return cl->hits;
 670 }
 671
 672 int client_get_num_records(struct client *cl)
 673 {
 674     return cl->records;
 675 }
 676
 677 int client_get_diagnostic(struct client *cl)
 678 {
 679     return cl->diagnostic;
 680 }
 681
 682 void client_set_database(struct client *cl, struct session_database *db)
 683 {
 684     cl->database = db;
 685 }
 686
 687 struct host *client_get_host(struct client *cl)
 688 {
 689     return client_get_database(cl)->database->host;
 690 }
 691
 692 const char *client_get_url(struct client *cl)
 693 {
 694     return client_get_database(cl)->database->url;
 695 }
 696
 697 /*
 698  * Local variables:
 699  * c-basic-offset: 4
 700  * indent-tabs-mode: nil
 701  * End:
 702  * vim: shiftwidth=4 tabstop=8 expandtab
 703  */