src/client.c

   1 /* $Id: client.c,v 1.8 2007-06-06 11:56:35 marc Exp $
   2    Copyright (c) 2006-2007, Index Data.
   3
   4 This file is part of Pazpar2.
   5
   6 Pazpar2 is free software; you can redistribute it and/or modify it under
   7 the terms of the GNU General Public License as published by the Free
   8 Software Foundation; either version 2, or (at your option) any later
   9 version.
  10
  11 Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY
  12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14 for more details.
  15
  16 You should have received a copy of the GNU General Public License
  17 along with Pazpar2; see the file LICENSE.  If not, write to the
  18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  19 02111-1307, USA.
  20  */
  21
  22 /** \file client.c
  23     \brief Z39.50 client
  24 */
  25
  26 #include <stdlib.h>
  27 #include <stdio.h>
  28 #include <string.h>
  29 #include <sys/time.h>
  30 #include <unistd.h>
  31 #include <sys/socket.h>
  32 #include <netdb.h>
  33 #include <signal.h>
  34 #include <ctype.h>
  35 #include <assert.h>
  36
  37 #include <yaz/marcdisp.h>
  38 #include <yaz/comstack.h>
  39 #include <yaz/tcpip.h>
  40 #include <yaz/proto.h>
  41 #include <yaz/readconf.h>
  42 #include <yaz/pquery.h>
  43 #include <yaz/otherinfo.h>
  44 #include <yaz/yaz-util.h>
  45 #include <yaz/nmem.h>
  46 #include <yaz/query-charset.h>
  47 #include <yaz/querytowrbuf.h>
  48 #if YAZ_VERSIONL >= 0x020163
  49 #include <yaz/oid_db.h>
  50 #endif
  51
  52 #if HAVE_CONFIG_H
  53 #include "cconfig.h"
  54 #endif
  55
  56 #define USE_TIMING 0
  57 #if USE_TIMING
  58 #include <yaz/timing.h>
  59 #endif
  60
  61 #include <netinet/in.h>
  62
  63 #include "pazpar2.h"
  64
  65 #include "client.h"
  66 #include "connection.h"
  67 #include "settings.h"
  68
  69 /** \brief Represents client state for a connection to one search target */
  70 struct client {
  71     struct session_database *database;
  72     struct connection *connection;
  73     struct session *session;
  74     char *pquery; // Current search
  75     int hits;
  76     int records;
  77     int setno;
  78     int requestid;            // ID of current outstanding request
  79     int diagnostic;
  80     enum client_state state;
  81     struct client *next;     // next client in session or next in free list
  82 };
  83
  84 static const char *client_states[] = {
  85     "Client_Connecting",
  86     "Client_Connected",
  87     "Client_Idle",
  88     "Client_Initializing",
  89     "Client_Searching",
  90     "Client_Presenting",
  91     "Client_Error",
  92     "Client_Failed",
  93     "Client_Disconnected",
  94     "Client_Stopped"
  95 };
  96
  97 static struct client *client_freelist = 0;
  98
  99 static int send_apdu(struct client *c, Z_APDU *a)
 100 {
 101     return connection_send_apdu(client_get_connection(c), a);
 102 }
 103
 104
 105 const char *client_get_state_str(struct client *cl)
 106 {
 107     return client_states[cl->state];
 108 }
 109
 110 enum client_state client_get_state(struct client *cl)
 111 {
 112     return cl->state;
 113 }
 114
 115 void client_set_state(struct client *cl, enum client_state st)
 116 {
 117     cl->state = st;
 118 }
 119
 120 // Close connection and set state to error
 121 void client_fatal(struct client *cl)
 122 {
 123     yaz_log(YLOG_WARN, "Fatal error from %s", client_get_url(cl));
 124     connection_destroy(cl->connection);
 125     cl->state = Client_Error;
 126 }
 127
 128 struct connection *client_get_connection(struct client *cl)
 129 {
 130     return cl->connection;
 131 }
 132
 133 struct session_database *client_get_database(struct client *cl)
 134 {
 135     return cl->database;
 136 }
 137
 138 struct session *client_get_session(struct client *cl)
 139 {
 140     return cl->session;
 141 }
 142
 143 const char *client_get_pquery(struct client *cl)
 144 {
 145     return cl->pquery;
 146 }
 147
 148 void client_set_requestid(struct client *cl, int id)
 149 {
 150     cl->requestid = id;
 151 }
 152
 153 void client_send_present(struct client *cl)
 154 {
 155     struct session_database *sdb = client_get_database(cl);
 156     Z_APDU *a = zget_APDU(global_parameters.odr_out, Z_APDU_presentRequest);
 157     int toget;
 158     int start = cl->records + 1;
 159     char *recsyn;
 160
 161     toget = global_parameters.chunk;
 162     if (toget > global_parameters.toget - cl->records)
 163         toget = global_parameters.toget - cl->records;
 164     if (toget > cl->hits - cl->records)
 165         toget = cl->hits - cl->records;
 166
 167     yaz_log(YLOG_DEBUG, "Trying to present %d records\n", toget);
 168
 169     a->u.presentRequest->resultSetStartPoint = &start;
 170     a->u.presentRequest->numberOfRecordsRequested = &toget;
 171
 172     a->u.presentRequest->resultSetId = "Default";
 173
 174     if ((recsyn = session_setting_oneval(sdb, PZ_REQUESTSYNTAX)))
 175     {
 176 #if YAZ_VERSIONL >= 0x020163
 177         a->u.presentRequest->preferredRecordSyntax =
 178             yaz_string_to_oid_odr(yaz_oid_std(),
 179                                   CLASS_RECSYN, recsyn,
 180                                   global_parameters.odr_out);
 181 #else
 182         a->u.presentRequest->preferredRecordSyntax =
 183             yaz_str_to_z3950oid(global_parameters.odr_out,
 184                                 CLASS_RECSYN, recsyn);
 185 #endif
 186     }
 187
 188     if (send_apdu(cl, a) >= 0)
 189         cl->state = Client_Presenting;
 190     else
 191         cl->state = Client_Error;
 192     odr_reset(global_parameters.odr_out);
 193 }
 194
 195
 196 void client_send_search(struct client *cl)
 197 {
 198     struct session *se = client_get_session(cl);
 199     struct session_database *sdb = client_get_database(cl);
 200     Z_APDU *a = zget_APDU(global_parameters.odr_out, Z_APDU_searchRequest);
 201     int ndb;
 202     char **databaselist;
 203     Z_Query *zquery;
 204     int ssub = 0, lslb = 100000, mspn = 10;
 205     char *recsyn = 0;
 206     char *piggyback = 0;
 207     char *queryenc = 0;
 208     yaz_iconv_t iconv = 0;
 209
 210     yaz_log(YLOG_DEBUG, "Sending search to %s", sdb->database->url);
 211
 212     // constructing RPN query
 213     a->u.searchRequest->query = zquery = odr_malloc(global_parameters.odr_out,
 214                                                     sizeof(Z_Query));
 215     zquery->which = Z_Query_type_1;
 216     zquery->u.type_1 = p_query_rpn(global_parameters.odr_out,
 217                                    client_get_pquery(cl));
 218
 219     // converting to target encoding
 220     if ((queryenc = session_setting_oneval(sdb, PZ_QUERYENCODING))){
 221         iconv = yaz_iconv_open(queryenc, "UTF-8");
 222         if (iconv){
 223             yaz_query_charset_convert_rpnquery(zquery->u.type_1,
 224                                                global_parameters.odr_out,
 225                                                iconv);
 226             yaz_iconv_close(iconv);
 227         } else
 228             yaz_log(YLOG_WARN, "Query encoding failed %s %s",
 229                     client_get_database(cl)->database->url, queryenc);
 230     }
 231
 232     for (ndb = 0; sdb->database->databases[ndb]; ndb++)
 233         ;
 234     databaselist = odr_malloc(global_parameters.odr_out, sizeof(char*) * ndb);
 235     for (ndb = 0; sdb->database->databases[ndb]; ndb++)
 236         databaselist[ndb] = sdb->database->databases[ndb];
 237
 238     if (!(piggyback = session_setting_oneval(sdb, PZ_PIGGYBACK))
 239         || *piggyback == '1')
 240     {
 241         if ((recsyn = session_setting_oneval(sdb, PZ_REQUESTSYNTAX)))
 242         {
 243 #if YAZ_VERSIONL >= 0x020163
 244             a->u.searchRequest->preferredRecordSyntax =
 245                 yaz_string_to_oid_odr(yaz_oid_std(),
 246                                       CLASS_RECSYN, recsyn,
 247                                       global_parameters.odr_out);
 248 #else
 249             a->u.searchRequest->preferredRecordSyntax =
 250                 yaz_str_to_z3950oid(global_parameters.odr_out,
 251                                     CLASS_RECSYN, recsyn);
 252 #endif
 253         }
 254         a->u.searchRequest->smallSetUpperBound = &ssub;
 255         a->u.searchRequest->largeSetLowerBound = &lslb;
 256         a->u.searchRequest->mediumSetPresentNumber = &mspn;
 257     }
 258     a->u.searchRequest->resultSetName = "Default";
 259     a->u.searchRequest->databaseNames = databaselist;
 260     a->u.searchRequest->num_databaseNames = ndb;
 261
 262
 263     {  //scope for sending and logging queries
 264         WRBUF wbquery = wrbuf_alloc();
 265         yaz_query_to_wrbuf(wbquery, a->u.searchRequest->query);
 266
 267
 268         if (send_apdu(cl, a) >= 0)
 269         {
 270             client_set_state(cl, Client_Searching);
 271             client_set_requestid(cl, se->requestid);
 272             yaz_log(YLOG_LOG, "SearchRequest %s %s %s",
 273                     client_get_database(cl)->database->url,
 274                     queryenc ? queryenc : "UTF-8",
 275                     wrbuf_cstr(wbquery));
 276         }
 277         else {
 278             client_set_state(cl, Client_Error);
 279             yaz_log(YLOG_WARN, "Failed SearchRequest %s  %s %s",
 280                     client_get_database(cl)->database->url,
 281                     queryenc ? queryenc : "UTF-8",
 282                     wrbuf_cstr(wbquery));
 283         }
 284
 285         wrbuf_destroy(wbquery);
 286     }
 287
 288     odr_reset(global_parameters.odr_out);
 289 }
 290
 291 void client_init_response(struct client *cl, Z_APDU *a)
 292 {
 293     Z_InitResponse *r = a->u.initResponse;
 294
 295     yaz_log(YLOG_DEBUG, "Init response %s", cl->database->database->url);
 296
 297     if (*r->result)
 298     {
 299         cl->state = Client_Idle;
 300     }
 301     else
 302         cl->state = Client_Failed; // FIXME need to do something to the connection
 303 }
 304
 305
 306 static void ingest_records(struct client *cl, Z_Records *r)
 307 {
 308 #if USE_TIMING
 309     yaz_timing_t t = yaz_timing_create();
 310 #endif
 311     struct record *rec;
 312     struct session *s = client_get_session(cl);
 313     Z_NamePlusRecordList *rlist;
 314     int i;
 315
 316     if (r->which != Z_Records_DBOSD)
 317         return;
 318     rlist = r->u.databaseOrSurDiagnostics;
 319     for (i = 0; i < rlist->num_records; i++)
 320     {
 321         Z_NamePlusRecord *npr = rlist->records[i];
 322
 323         cl->records++;
 324         if (npr->which != Z_NamePlusRecord_databaseRecord)
 325         {
 326             yaz_log(YLOG_WARN,
 327                     "Unexpected record type, probably diagnostic %s",
 328                     cl->database->database->url);
 329             continue;
 330         }
 331
 332         rec = ingest_record(cl, npr->u.databaseRecord, cl->records);
 333         if (!rec)
 334             continue;
 335     }
 336     if (rlist->num_records)
 337         session_alert_watch(s, SESSION_WATCH_RECORDS);
 338
 339 #if USE_TIMING
 340     yaz_timing_stop(t);
 341     yaz_log(YLOG_LOG, "ingest_records %6.5f %3.2f %3.2f",
 342             yaz_timing_get_real(t), yaz_timing_get_user(t),
 343             yaz_timing_get_sys(t));
 344     yaz_timing_destroy(&t);
 345 #endif
 346 }
 347
 348
 349 void client_search_response(struct client *cl, Z_APDU *a)
 350 {
 351     struct session *se = cl->session;
 352     Z_SearchResponse *r = a->u.searchResponse;
 353
 354     yaz_log(YLOG_DEBUG, "Search response %s (status=%d)",
 355             cl->database->database->url, *r->searchStatus);
 356
 357     if (*r->searchStatus)
 358     {
 359         cl->hits = *r->resultCount;
 360         se->total_hits += cl->hits;
 361         if (r->presentStatus && !*r->presentStatus && r->records)
 362         {
 363             yaz_log(YLOG_DEBUG, "Records in search response %s",
 364                     cl->database->database->url);
 365             ingest_records(cl, r->records);
 366         }
 367         cl->state = Client_Idle;
 368     }
 369     else
 370     {          /*"FAILED"*/
 371         cl->hits = 0;
 372         cl->state = Client_Error;
 373         if (r->records) {
 374             Z_Records *recs = r->records;
 375             if (recs->which == Z_Records_NSD)
 376             {
 377                 yaz_log(YLOG_WARN,
 378                     "Search response: Non-surrogate diagnostic %s (%d)",
 379                     cl->database->database->url,
 380                     *recs->u.nonSurrogateDiagnostic->condition);
 381                 cl->diagnostic = *recs->u.nonSurrogateDiagnostic->condition;
 382                 cl->state = Client_Error;
 383             }
 384         }
 385     }
 386 }
 387
 388 void client_present_response(struct client *cl, Z_APDU *a)
 389 {
 390     Z_PresentResponse *r = a->u.presentResponse;
 391
 392     if (r->records) {
 393         Z_Records *recs = r->records;
 394         if (recs->which == Z_Records_NSD)
 395         {
 396             yaz_log(YLOG_WARN, "Non-surrogate diagnostic %s",
 397                     cl->database->database->url);
 398             cl->diagnostic = *recs->u.nonSurrogateDiagnostic->condition;
 399             cl->state = Client_Error;
 400         }
 401     }
 402
 403     if (!*r->presentStatus && cl->state != Client_Error)
 404     {
 405         yaz_log(YLOG_DEBUG, "Good Present response %s",
 406                 cl->database->database->url);
 407         ingest_records(cl, r->records);
 408         cl->state = Client_Idle;
 409     }
 410     else if (*r->presentStatus)
 411     {
 412         yaz_log(YLOG_WARN, "Bad Present response %s",
 413                 cl->database->database->url);
 414         cl->state = Client_Error;
 415     }
 416 }
 417
 418 void client_close_response(struct client *cl, Z_APDU *a)
 419 {
 420     struct connection *co = cl->connection;
 421     /* Z_Close *r = a->u.close; */
 422
 423     yaz_log(YLOG_WARN, "Close response %s", cl->database->database->url);
 424
 425     cl->state = Client_Failed;
 426     connection_destroy(co);
 427 }
 428
 429 int client_is_our_response(struct client *cl)
 430 {
 431     struct session *se = client_get_session(cl);
 432
 433     if (cl && (cl->requestid == se->requestid ||
 434                cl->state == Client_Initializing))
 435         return 1;
 436     return 0;
 437 }
 438
 439 // Set authentication token in init if one is set for the client
 440 // TODO: Extend this to handle other schemes than open (should be simple)
 441 static void init_authentication(struct client *cl, Z_InitRequest *req)
 442 {
 443     struct session_database *sdb = client_get_database(cl);
 444     char *auth = session_setting_oneval(sdb, PZ_AUTHENTICATION);
 445
 446     if (*auth)
 447     {
 448         struct connection *co = client_get_connection(cl);
 449         struct session *se = client_get_session(cl);
 450         Z_IdAuthentication *idAuth = odr_malloc(global_parameters.odr_out,
 451                 sizeof(*idAuth));
 452         idAuth->which = Z_IdAuthentication_open;
 453         idAuth->u.open = auth;
 454         req->idAuthentication = idAuth;
 455         connection_set_authentication(co, nmem_strdup(se->session_nmem, auth));
 456     }
 457 }
 458
 459 static void init_zproxy(struct client *cl, Z_InitRequest *req)
 460 {
 461     struct session_database *sdb = client_get_database(cl);
 462     char *ztarget = sdb->database->url;
 463     //char *ztarget = sdb->url;
 464     char *zproxy = session_setting_oneval(sdb, PZ_ZPROXY);
 465
 466     if (*zproxy)
 467 #if YAZ_VERSIONL >= 0x020163
 468         yaz_oi_set_string_oid(&req->otherInfo,
 469                               global_parameters.odr_out,
 470                               yaz_oid_userinfo_proxy,
 471                               1, ztarget);
 472 #else
 473         yaz_oi_set_string_oidval(&req->otherInfo,
 474                                  global_parameters.odr_out, VAL_PROXY,
 475                                  1, ztarget);
 476 #endif
 477 }
 478
 479
 480 static void client_init_request(struct client *cl)
 481 {
 482     Z_APDU *a = zget_APDU(global_parameters.odr_out, Z_APDU_initRequest);
 483
 484     a->u.initRequest->implementationId = global_parameters.implementationId;
 485     a->u.initRequest->implementationName = global_parameters.implementationName;
 486     a->u.initRequest->implementationVersion =
 487         global_parameters.implementationVersion;
 488     ODR_MASK_SET(a->u.initRequest->options, Z_Options_search);
 489     ODR_MASK_SET(a->u.initRequest->options, Z_Options_present);
 490     ODR_MASK_SET(a->u.initRequest->options, Z_Options_namedResultSets);
 491
 492     ODR_MASK_SET(a->u.initRequest->protocolVersion, Z_ProtocolVersion_1);
 493     ODR_MASK_SET(a->u.initRequest->protocolVersion, Z_ProtocolVersion_2);
 494     ODR_MASK_SET(a->u.initRequest->protocolVersion, Z_ProtocolVersion_3);
 495
 496     init_authentication(cl, a->u.initRequest);
 497     init_zproxy(cl, a->u.initRequest);
 498
 499
 500
 501     if (send_apdu(cl, a) >= 0)
 502         client_set_state(cl, Client_Initializing);
 503     else
 504         client_set_state(cl, Client_Error);
 505     odr_reset(global_parameters.odr_out);
 506 }
 507
 508 void client_continue(struct client *cl)
 509 {
 510     if (cl->state == Client_Connected) {
 511         client_init_request(cl);
 512     }
 513
 514     if (cl->state == Client_Idle)
 515     {
 516         struct session *se = client_get_session(cl);
 517         if (cl->requestid != se->requestid && cl->pquery) {
 518             client_send_search(cl);
 519         }
 520         else if (cl->hits > 0 && cl->records < global_parameters.toget &&
 521             cl->records < cl->hits) {
 522             client_send_present(cl);
 523         }
 524     }
 525 }
 526
 527 struct client *client_create(void)
 528 {
 529     struct client *r;
 530     if (client_freelist)
 531     {
 532         r = client_freelist;
 533         client_freelist = client_freelist->next;
 534     }
 535     else
 536         r = xmalloc(sizeof(struct client));
 537     r->pquery = 0;
 538     r->database = 0;
 539     r->connection = 0;
 540     r->session = 0;
 541     r->hits = 0;
 542     r->records = 0;
 543     r->setno = 0;
 544     r->requestid = -1;
 545     r->diagnostic = 0;
 546     r->state = Client_Disconnected;
 547     r->next = 0;
 548     return r;
 549 }
 550
 551 void client_destroy(struct client *c)
 552 {
 553     struct session *se = c->session;
 554     if (c == se->clients)
 555         se->clients = c->next;
 556     else
 557     {
 558         struct client *cc;
 559         for (cc = se->clients; cc && cc->next != c; cc = cc->next)
 560             ;
 561         if (cc)
 562             cc->next = c->next;
 563     }
 564     if (c->connection)
 565         connection_release(c->connection);
 566     c->next = client_freelist;
 567     client_freelist = c;
 568 }
 569
 570 void client_set_connection(struct client *cl, struct connection *con)
 571 {
 572     cl->connection = con;
 573 }
 574
 575 void client_disconnect(struct client *cl)
 576 {
 577     if (cl->state != Client_Idle)
 578         cl->state = Client_Disconnected;
 579     client_set_connection(cl, 0);
 580 }
 581
 582 // Extract terms from query into null-terminated termlist
 583 static void extract_terms(NMEM nmem, struct ccl_rpn_node *query, char **termlist)
 584 {
 585     int num = 0;
 586
 587     pull_terms(nmem, query, termlist, &num);
 588     termlist[num] = 0;
 589 }
 590
 591 // Initialize CCL map for a target
 592 static CCL_bibset prepare_cclmap(struct client *cl)
 593 {
 594     struct session_database *sdb = client_get_database(cl);
 595     struct setting *s;
 596     CCL_bibset res;
 597
 598     if (!sdb->settings)
 599         return 0;
 600     res = ccl_qual_mk();
 601     for (s = sdb->settings[PZ_CCLMAP]; s; s = s->next)
 602     {
 603         char *p = strchr(s->name + 3, ':');
 604         if (!p)
 605         {
 606             yaz_log(YLOG_WARN, "Malformed cclmap name: %s", s->name);
 607             ccl_qual_rm(&res);
 608             return 0;
 609         }
 610         p++;
 611         ccl_qual_fitem(res, s->value, p);
 612     }
 613     return res;
 614 }
 615
 616 // Parse the query given the settings specific to this client
 617 int client_parse_query(struct client *cl, const char *query)
 618 {
 619     struct session *se = client_get_session(cl);
 620     struct ccl_rpn_node *cn;
 621     int cerror, cpos;
 622     CCL_bibset ccl_map = prepare_cclmap(cl);
 623
 624     if (!ccl_map)
 625         return -1;
 626     cn = ccl_find_str(ccl_map, query, &cerror, &cpos);
 627     ccl_qual_rm(&ccl_map);
 628     if (!cn)
 629     {
 630         cl->state = Client_Error;
 631         yaz_log(YLOG_WARN, "Failed to parse query for %s",
 632                          client_get_database(cl)->database->url);
 633         return -1;
 634     }
 635     wrbuf_rewind(se->wrbuf);
 636     ccl_pquery(se->wrbuf, cn);
 637     xfree(cl->pquery);
 638     cl->pquery = xstrdup(wrbuf_cstr(se->wrbuf));
 639
 640     if (!se->relevance)
 641     {
 642         // Initialize relevance structure with query terms
 643         char *p[512];
 644         extract_terms(se->nmem, cn, p);
 645         se->relevance = relevance_create(client_get_database(cl)->pct,
 646                                          se->nmem, (const char **) p,
 647                                          se->expected_maxrecs);
 648     }
 649
 650     ccl_rpn_delete(cn);
 651     return 0;
 652 }
 653
 654 void client_set_session(struct client *cl, struct session *se)
 655 {
 656     cl->session = se;
 657     cl->next = se->clients;
 658     se->clients = cl;
 659 }
 660
 661 int client_is_active(struct client *cl)
 662 {
 663     if (cl->connection && (cl->state == Client_Connecting ||
 664                            cl->state == Client_Initializing ||
 665                            cl->state == Client_Searching ||
 666                            cl->state == Client_Presenting))
 667         return 1;
 668     return 0;
 669 }
 670
 671 struct client *client_next_in_session(struct client *cl)
 672 {
 673     if (cl)
 674         return cl->next;
 675     return 0;
 676
 677 }
 678
 679 int client_get_hits(struct client *cl)
 680 {
 681     return cl->hits;
 682 }
 683
 684 int client_get_num_records(struct client *cl)
 685 {
 686     return cl->records;
 687 }
 688
 689 int client_get_diagnostic(struct client *cl)
 690 {
 691     return cl->diagnostic;
 692 }
 693
 694 void client_set_database(struct client *cl, struct session_database *db)
 695 {
 696     cl->database = db;
 697 }
 698
 699 struct host *client_get_host(struct client *cl)
 700 {
 701     return client_get_database(cl)->database->host;
 702 }
 703
 704 const char *client_get_url(struct client *cl)
 705 {
 706     return client_get_database(cl)->database->url;
 707 }
 708
 709 /*
 710  * Local variables:
 711  * c-basic-offset: 4
 712  * indent-tabs-mode: nil
 713  * End:
 714  * vim: shiftwidth=4 tabstop=8 expandtab
 715  */