src/client.c

   1 /* $Id: client.c,v 1.6 2007-06-02 04:32:28 quinn Exp $
   2    Copyright (c) 2006-2007, Index Data.
   3
   4 This file is part of Pazpar2.
   5
   6 Pazpar2 is free software; you can redistribute it and/or modify it under
   7 the terms of the GNU General Public License as published by the Free
   8 Software Foundation; either version 2, or (at your option) any later
   9 version.
  10
  11 Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY
  12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14 for more details.
  15
  16 You should have received a copy of the GNU General Public License
  17 along with Pazpar2; see the file LICENSE.  If not, write to the
  18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  19 02111-1307, USA.
  20  */
  21
  22 /** \file client.c
  23     \brief Z39.50 client
  24 */
  25
  26 #include <stdlib.h>
  27 #include <stdio.h>
  28 #include <string.h>
  29 #include <sys/time.h>
  30 #include <unistd.h>
  31 #include <sys/socket.h>
  32 #include <netdb.h>
  33 #include <signal.h>
  34 #include <ctype.h>
  35 #include <assert.h>
  36
  37 #include <yaz/marcdisp.h>
  38 #include <yaz/comstack.h>
  39 #include <yaz/tcpip.h>
  40 #include <yaz/proto.h>
  41 #include <yaz/readconf.h>
  42 #include <yaz/pquery.h>
  43 #include <yaz/otherinfo.h>
  44 #include <yaz/yaz-util.h>
  45 #include <yaz/nmem.h>
  46 #include <yaz/query-charset.h>
  47 #include <yaz/querytowrbuf.h>
  48 #if YAZ_VERSIONL >= 0x020163
  49 #include <yaz/oid_db.h>
  50 #endif
  51
  52 #if HAVE_CONFIG_H
  53 #include "cconfig.h"
  54 #endif
  55
  56 #define USE_TIMING 0
  57 #if USE_TIMING
  58 #include <yaz/timing.h>
  59 #endif
  60
  61 #include <netinet/in.h>
  62
  63 #include "pazpar2.h"
  64
  65 #include "client.h"
  66 #include "connection.h"
  67 #include "settings.h"
  68
  69 /** \brief Represents client state for a connection to one search target */
  70 struct client {
  71     struct session_database *database;
  72     struct connection *connection;
  73     struct session *session;
  74     char *pquery; // Current search
  75     int hits;
  76     int records;
  77     int setno;
  78     int requestid;            // ID of current outstanding request
  79     int diagnostic;
  80     enum client_state state;
  81     struct client *next;     // next client in session or next in free list
  82 };
  83
  84 static const char *client_states[] = {
  85     "Client_Connecting",
  86     "Client_Connected",
  87     "Client_Idle",
  88     "Client_Initializing",
  89     "Client_Searching",
  90     "Client_Presenting",
  91     "Client_Error",
  92     "Client_Failed",
  93     "Client_Disconnected",
  94     "Client_Stopped"
  95 };
  96
  97 static struct client *client_freelist = 0;
  98
  99 static int send_apdu(struct client *c, Z_APDU *a)
 100 {
 101     return connection_send_apdu(client_get_connection(c), a);
 102 }
 103
 104
 105 const char *client_get_state_str(struct client *cl)
 106 {
 107     return client_states[cl->state];
 108 }
 109
 110 enum client_state client_get_state(struct client *cl)
 111 {
 112     return cl->state;
 113 }
 114
 115 void client_set_state(struct client *cl, enum client_state st)
 116 {
 117     cl->state = st;
 118 }
 119
 120 // Close connection and set state to error
 121 void client_fatal(struct client *cl)
 122 {
 123     yaz_log(YLOG_WARN, "Fatal error from %s", client_get_url(cl));
 124     connection_destroy(cl->connection);
 125     cl->state = Client_Error;
 126 }
 127
 128 struct connection *client_get_connection(struct client *cl)
 129 {
 130     return cl->connection;
 131 }
 132
 133 struct session_database *client_get_database(struct client *cl)
 134 {
 135     return cl->database;
 136 }
 137
 138 struct session *client_get_session(struct client *cl)
 139 {
 140     return cl->session;
 141 }
 142
 143 const char *client_get_pquery(struct client *cl)
 144 {
 145     return cl->pquery;
 146 }
 147
 148 void client_set_requestid(struct client *cl, int id)
 149 {
 150     cl->requestid = id;
 151 }
 152
 153 void client_send_present(struct client *cl)
 154 {
 155     struct session_database *sdb = client_get_database(cl);
 156     Z_APDU *a = zget_APDU(global_parameters.odr_out, Z_APDU_presentRequest);
 157     int toget;
 158     int start = cl->records + 1;
 159     char *recsyn;
 160
 161     toget = global_parameters.chunk;
 162     if (toget > global_parameters.toget - cl->records)
 163         toget = global_parameters.toget - cl->records;
 164     if (toget > cl->hits - cl->records)
 165         toget = cl->hits - cl->records;
 166
 167     yaz_log(YLOG_DEBUG, "Trying to present %d records\n", toget);
 168
 169     a->u.presentRequest->resultSetStartPoint = &start;
 170     a->u.presentRequest->numberOfRecordsRequested = &toget;
 171
 172     a->u.presentRequest->resultSetId = "Default";
 173
 174     if ((recsyn = session_setting_oneval(sdb, PZ_REQUESTSYNTAX)))
 175     {
 176 #if YAZ_VERSIONL >= 0x020163
 177         a->u.presentRequest->preferredRecordSyntax =
 178             yaz_string_to_oid_odr(yaz_oid_std(),
 179                                   CLASS_RECSYN, recsyn,
 180                                   global_parameters.odr_out);
 181 #else
 182         a->u.presentRequest->preferredRecordSyntax =
 183             yaz_str_to_z3950oid(global_parameters.odr_out,
 184                                 CLASS_RECSYN, recsyn);
 185 #endif
 186     }
 187
 188     if (send_apdu(cl, a) >= 0)
 189         cl->state = Client_Presenting;
 190     else
 191         cl->state = Client_Error;
 192     odr_reset(global_parameters.odr_out);
 193 }
 194
 195
 196 void client_send_search(struct client *cl)
 197 {
 198     struct session *se = client_get_session(cl);
 199     struct session_database *sdb = client_get_database(cl);
 200     Z_APDU *a = zget_APDU(global_parameters.odr_out, Z_APDU_searchRequest);
 201     int ndb;
 202     char **databaselist;
 203     Z_Query *zquery;
 204     int ssub = 0, lslb = 100000, mspn = 10;
 205     char *recsyn = 0;
 206     char *piggyback = 0;
 207     char *queryenc = 0;
 208     yaz_iconv_t iconv = 0;
 209
 210     yaz_log(YLOG_DEBUG, "Sending search to %s", sdb->database->url);
 211
 212     // constructing RPN query
 213     a->u.searchRequest->query = zquery = odr_malloc(global_parameters.odr_out,
 214                                                     sizeof(Z_Query));
 215     zquery->which = Z_Query_type_1;
 216     zquery->u.type_1 = p_query_rpn(global_parameters.odr_out,
 217                                    client_get_pquery(cl));
 218
 219     // converting to target encoding
 220     if ((queryenc = session_setting_oneval(sdb, PZ_QUERYENCODING))){
 221         iconv = yaz_iconv_open(queryenc, "UTF-8");
 222         if (iconv){
 223             yaz_query_charset_convert_rpnquery(zquery->u.type_1,
 224                                                global_parameters.odr_out,
 225                                                iconv);
 226             yaz_iconv_close(iconv);
 227         } else
 228             yaz_log(YLOG_WARN, "Query encoding failed %s %s",
 229                     client_get_database(cl)->database->url, queryenc);
 230     }
 231
 232     for (ndb = 0; sdb->database->databases[ndb]; ndb++)
 233         ;
 234     databaselist = odr_malloc(global_parameters.odr_out, sizeof(char*) * ndb);
 235     for (ndb = 0; sdb->database->databases[ndb]; ndb++)
 236         databaselist[ndb] = sdb->database->databases[ndb];
 237
 238     if (!(piggyback = session_setting_oneval(sdb, PZ_PIGGYBACK)) || *piggyback == '1')
 239     {
 240         if ((recsyn = session_setting_oneval(sdb, PZ_REQUESTSYNTAX)))
 241         {
 242 #if YAZ_VERSIONL >= 0x020163
 243             a->u.searchRequest->preferredRecordSyntax =
 244                 yaz_string_to_oid_odr(yaz_oid_std(),
 245                                       CLASS_RECSYN, recsyn,
 246                                       global_parameters.odr_out);
 247 #else
 248             a->u.searchRequest->preferredRecordSyntax =
 249                 yaz_str_to_z3950oid(global_parameters.odr_out,
 250                                     CLASS_RECSYN, recsyn);
 251 #endif
 252         }
 253         a->u.searchRequest->smallSetUpperBound = &ssub;
 254         a->u.searchRequest->largeSetLowerBound = &lslb;
 255         a->u.searchRequest->mediumSetPresentNumber = &mspn;
 256     }
 257     a->u.searchRequest->resultSetName = "Default";
 258     a->u.searchRequest->databaseNames = databaselist;
 259     a->u.searchRequest->num_databaseNames = ndb;
 260
 261
 262     {  //scope for sending and logging queries
 263         WRBUF wbquery = wrbuf_alloc();
 264         yaz_query_to_wrbuf(wbquery, a->u.searchRequest->query);
 265
 266
 267         if (send_apdu(cl, a) >= 0)
 268         {
 269             client_set_state(cl, Client_Searching);
 270             client_set_requestid(cl, se->requestid);
 271             yaz_log(YLOG_LOG, "SearchRequest %s %s %s",
 272                     client_get_database(cl)->database->url,
 273                     queryenc ? queryenc : "UTF-8",
 274                     wrbuf_cstr(wbquery));
 275         }
 276         else {
 277             client_set_state(cl, Client_Error);
 278             yaz_log(YLOG_WARN, "Failed SearchRequest %s  %s %s",
 279                     client_get_database(cl)->database->url,
 280                     queryenc ? queryenc : "UTF-8",
 281                     wrbuf_cstr(wbquery));
 282         }
 283
 284         wrbuf_destroy(wbquery);
 285     }
 286
 287     odr_reset(global_parameters.odr_out);
 288 }
 289
 290 void client_init_response(struct client *cl, Z_APDU *a)
 291 {
 292     Z_InitResponse *r = a->u.initResponse;
 293
 294     yaz_log(YLOG_DEBUG, "Init response %s", cl->database->database->url);
 295
 296     if (*r->result)
 297     {
 298         cl->state = Client_Idle;
 299     }
 300     else
 301         cl->state = Client_Failed; // FIXME need to do something to the connection
 302 }
 303
 304
 305 static void ingest_records(struct client *cl, Z_Records *r)
 306 {
 307 #if USE_TIMING
 308     yaz_timing_t t = yaz_timing_create();
 309 #endif
 310     struct record *rec;
 311     struct session *s = client_get_session(cl);
 312     Z_NamePlusRecordList *rlist;
 313     int i;
 314
 315     if (r->which != Z_Records_DBOSD)
 316         return;
 317     rlist = r->u.databaseOrSurDiagnostics;
 318     for (i = 0; i < rlist->num_records; i++)
 319     {
 320         Z_NamePlusRecord *npr = rlist->records[i];
 321
 322         cl->records++;
 323         if (npr->which != Z_NamePlusRecord_databaseRecord)
 324         {
 325             yaz_log(YLOG_WARN,
 326                     "Unexpected record type, probably diagnostic %s",
 327                     cl->database->database->url);
 328             continue;
 329         }
 330
 331         rec = ingest_record(cl, npr->u.databaseRecord, cl->records);
 332         if (!rec)
 333             continue;
 334     }
 335     if (rlist->num_records)
 336         session_alert_watch(s, SESSION_WATCH_RECORDS);
 337
 338 #if USE_TIMING
 339     yaz_timing_stop(t);
 340     yaz_log(YLOG_LOG, "ingest_records %6.5f %3.2f %3.2f",
 341             yaz_timing_get_real(t), yaz_timing_get_user(t),
 342             yaz_timing_get_sys(t));
 343     yaz_timing_destroy(&t);
 344 #endif
 345 }
 346
 347
 348 void client_search_response(struct client *cl, Z_APDU *a)
 349 {
 350     struct session *se = cl->session;
 351     Z_SearchResponse *r = a->u.searchResponse;
 352
 353     yaz_log(YLOG_DEBUG, "Search response %s (status=%d)",
 354             cl->database->database->url, *r->searchStatus);
 355
 356     if (*r->searchStatus)
 357     {
 358         cl->hits = *r->resultCount;
 359         se->total_hits += cl->hits;
 360         if (r->presentStatus && !*r->presentStatus && r->records)
 361         {
 362             yaz_log(YLOG_DEBUG, "Records in search response %s",
 363                     cl->database->database->url);
 364             ingest_records(cl, r->records);
 365         }
 366         cl->state = Client_Idle;
 367     }
 368     else
 369     {          /*"FAILED"*/
 370         cl->hits = 0;
 371         cl->state = Client_Error;
 372         if (r->records) {
 373             Z_Records *recs = r->records;
 374             if (recs->which == Z_Records_NSD)
 375             {
 376                 yaz_log(YLOG_WARN,
 377                     "Search response: Non-surrogate diagnostic %s (%d)",
 378                     cl->database->database->url,
 379                     *recs->u.nonSurrogateDiagnostic->condition);
 380                 cl->diagnostic = *recs->u.nonSurrogateDiagnostic->condition;
 381                 cl->state = Client_Error;
 382             }
 383         }
 384     }
 385 }
 386
 387 void client_present_response(struct client *cl, Z_APDU *a)
 388 {
 389     Z_PresentResponse *r = a->u.presentResponse;
 390
 391     if (r->records) {
 392         Z_Records *recs = r->records;
 393         if (recs->which == Z_Records_NSD)
 394         {
 395             yaz_log(YLOG_WARN, "Non-surrogate diagnostic %s",
 396                     cl->database->database->url);
 397             cl->diagnostic = *recs->u.nonSurrogateDiagnostic->condition;
 398             cl->state = Client_Error;
 399         }
 400     }
 401
 402     if (!*r->presentStatus && cl->state != Client_Error)
 403     {
 404         yaz_log(YLOG_DEBUG, "Good Present response %s",
 405                 cl->database->database->url);
 406         ingest_records(cl, r->records);
 407         cl->state = Client_Idle;
 408     }
 409     else if (*r->presentStatus)
 410     {
 411         yaz_log(YLOG_WARN, "Bad Present response %s",
 412                 cl->database->database->url);
 413         cl->state = Client_Error;
 414     }
 415 }
 416
 417 void client_close_response(struct client *cl, Z_APDU *a)
 418 {
 419     struct connection *co = cl->connection;
 420     /* Z_Close *r = a->u.close; */
 421
 422     yaz_log(YLOG_WARN, "Close response %s", cl->database->database->url);
 423
 424     cl->state = Client_Failed;
 425     connection_destroy(co);
 426 }
 427
 428 int client_is_our_response(struct client *cl)
 429 {
 430     struct session *se = client_get_session(cl);
 431
 432     if (cl && (cl->requestid == se->requestid ||
 433                cl->state == Client_Initializing))
 434         return 1;
 435     return 0;
 436 }
 437
 438 // Set authentication token in init if one is set for the client
 439 // TODO: Extend this to handle other schemes than open (should be simple)
 440 static void init_authentication(struct client *cl, Z_InitRequest *req)
 441 {
 442     struct session_database *sdb = client_get_database(cl);
 443     char *auth = session_setting_oneval(sdb, PZ_AUTHENTICATION);
 444
 445     if (*auth)
 446     {
 447         struct connection *co = client_get_connection(cl);
 448         struct session *se = client_get_session(cl);
 449         Z_IdAuthentication *idAuth = odr_malloc(global_parameters.odr_out,
 450                 sizeof(*idAuth));
 451         idAuth->which = Z_IdAuthentication_open;
 452         idAuth->u.open = auth;
 453         req->idAuthentication = idAuth;
 454         connection_set_authentication(co, nmem_strdup(se->session_nmem, auth));
 455     }
 456 }
 457
 458 static void client_init_request(struct client *cl)
 459 {
 460     Z_APDU *a = zget_APDU(global_parameters.odr_out, Z_APDU_initRequest);
 461
 462     a->u.initRequest->implementationId = global_parameters.implementationId;
 463     a->u.initRequest->implementationName = global_parameters.implementationName;
 464     a->u.initRequest->implementationVersion =
 465         global_parameters.implementationVersion;
 466     ODR_MASK_SET(a->u.initRequest->options, Z_Options_search);
 467     ODR_MASK_SET(a->u.initRequest->options, Z_Options_present);
 468     ODR_MASK_SET(a->u.initRequest->options, Z_Options_namedResultSets);
 469
 470     ODR_MASK_SET(a->u.initRequest->protocolVersion, Z_ProtocolVersion_1);
 471     ODR_MASK_SET(a->u.initRequest->protocolVersion, Z_ProtocolVersion_2);
 472     ODR_MASK_SET(a->u.initRequest->protocolVersion, Z_ProtocolVersion_3);
 473
 474     init_authentication(cl, a->u.initRequest);
 475
 476     /* add virtual host if tunneling through Z39.50 proxy */
 477
 478     if (0 < strlen(global_parameters.zproxy_override)
 479         && 0 < strlen(client_get_database(cl)->database->url))
 480     {
 481 #if YAZ_VERSIONL >= 0x020163
 482         yaz_oi_set_string_oid(&a->u.initRequest->otherInfo,
 483                               global_parameters.odr_out,
 484                               yaz_oid_userinfo_proxy,
 485                               1, client_get_database(cl)->database->url);
 486 #else
 487         yaz_oi_set_string_oidval(&a->u.initRequest->otherInfo,
 488                                  global_parameters.odr_out, VAL_PROXY,
 489                                  1, client_get_database(cl->database)->url);
 490 #endif
 491     }
 492
 493     if (send_apdu(cl, a) >= 0)
 494         client_set_state(cl, Client_Initializing);
 495     else
 496         client_set_state(cl, Client_Error);
 497     odr_reset(global_parameters.odr_out);
 498 }
 499
 500 void client_continue(struct client *cl)
 501 {
 502     if (cl->state == Client_Connected) {
 503         client_init_request(cl);
 504     }
 505
 506     if (cl->state == Client_Idle)
 507     {
 508         struct session *se = client_get_session(cl);
 509         if (cl->requestid != se->requestid && cl->pquery) {
 510             client_send_search(cl);
 511         }
 512         else if (cl->hits > 0 && cl->records < global_parameters.toget &&
 513             cl->records < cl->hits) {
 514             client_send_present(cl);
 515         }
 516     }
 517 }
 518
 519 struct client *client_create(void)
 520 {
 521     struct client *r;
 522     if (client_freelist)
 523     {
 524         r = client_freelist;
 525         client_freelist = client_freelist->next;
 526     }
 527     else
 528         r = xmalloc(sizeof(struct client));
 529     r->pquery = 0;
 530     r->database = 0;
 531     r->connection = 0;
 532     r->session = 0;
 533     r->hits = 0;
 534     r->records = 0;
 535     r->setno = 0;
 536     r->requestid = -1;
 537     r->diagnostic = 0;
 538     r->state = Client_Disconnected;
 539     r->next = 0;
 540     return r;
 541 }
 542
 543 void client_destroy(struct client *c)
 544 {
 545     struct session *se = c->session;
 546     if (c == se->clients)
 547         se->clients = c->next;
 548     else
 549     {
 550         struct client *cc;
 551         for (cc = se->clients; cc && cc->next != c; cc = cc->next)
 552             ;
 553         if (cc)
 554             cc->next = c->next;
 555     }
 556     if (c->connection)
 557         connection_release(c->connection);
 558     c->next = client_freelist;
 559     client_freelist = c;
 560 }
 561
 562 void client_set_connection(struct client *cl, struct connection *con)
 563 {
 564     cl->connection = con;
 565 }
 566
 567 void client_disconnect(struct client *cl)
 568 {
 569     if (cl->state != Client_Idle)
 570         cl->state = Client_Disconnected;
 571     client_set_connection(cl, 0);
 572 }
 573
 574 // Extract terms from query into null-terminated termlist
 575 static void extract_terms(NMEM nmem, struct ccl_rpn_node *query, char **termlist)
 576 {
 577     int num = 0;
 578
 579     pull_terms(nmem, query, termlist, &num);
 580     termlist[num] = 0;
 581 }
 582
 583 // Initialize CCL map for a target
 584 static CCL_bibset prepare_cclmap(struct client *cl)
 585 {
 586     struct session_database *sdb = client_get_database(cl);
 587     struct setting *s;
 588     CCL_bibset res;
 589
 590     if (!sdb->settings)
 591         return 0;
 592     res = ccl_qual_mk();
 593     for (s = sdb->settings[PZ_CCLMAP]; s; s = s->next)
 594     {
 595         char *p = strchr(s->name + 3, ':');
 596         if (!p)
 597         {
 598             yaz_log(YLOG_WARN, "Malformed cclmap name: %s", s->name);
 599             ccl_qual_rm(&res);
 600             return 0;
 601         }
 602         p++;
 603         ccl_qual_fitem(res, s->value, p);
 604     }
 605     return res;
 606 }
 607
 608 // Parse the query given the settings specific to this client
 609 int client_parse_query(struct client *cl, const char *query)
 610 {
 611     struct session *se = client_get_session(cl);
 612     struct ccl_rpn_node *cn;
 613     int cerror, cpos;
 614     CCL_bibset ccl_map = prepare_cclmap(cl);
 615
 616     if (!ccl_map)
 617         return -1;
 618     cn = ccl_find_str(ccl_map, query, &cerror, &cpos);
 619     ccl_qual_rm(&ccl_map);
 620     if (!cn)
 621     {
 622         cl->state = Client_Error;
 623         yaz_log(YLOG_WARN, "Failed to parse query for %s",
 624                          client_get_database(cl)->database->url);
 625         return -1;
 626     }
 627     wrbuf_rewind(se->wrbuf);
 628     ccl_pquery(se->wrbuf, cn);
 629     xfree(cl->pquery);
 630     cl->pquery = xstrdup(wrbuf_cstr(se->wrbuf));
 631
 632     if (!se->relevance)
 633     {
 634         // Initialize relevance structure with query terms
 635         char *p[512];
 636         extract_terms(se->nmem, cn, p);
 637         se->relevance = relevance_create(client_get_database(cl)->pct,
 638                                          se->nmem, (const char **) p,
 639                                          se->expected_maxrecs);
 640     }
 641
 642     ccl_rpn_delete(cn);
 643     return 0;
 644 }
 645
 646 void client_set_session(struct client *cl, struct session *se)
 647 {
 648     cl->session = se;
 649     cl->next = se->clients;
 650     se->clients = cl;
 651 }
 652
 653 int client_is_active(struct client *cl)
 654 {
 655     if (cl->connection && (cl->state == Client_Connecting ||
 656                            cl->state == Client_Initializing ||
 657                            cl->state == Client_Searching ||
 658                            cl->state == Client_Presenting))
 659         return 1;
 660     return 0;
 661 }
 662
 663 struct client *client_next_in_session(struct client *cl)
 664 {
 665     if (cl)
 666         return cl->next;
 667     return 0;
 668
 669 }
 670
 671 int client_get_hits(struct client *cl)
 672 {
 673     return cl->hits;
 674 }
 675
 676 int client_get_num_records(struct client *cl)
 677 {
 678     return cl->records;
 679 }
 680
 681 int client_get_diagnostic(struct client *cl)
 682 {
 683     return cl->diagnostic;
 684 }
 685
 686 void client_set_database(struct client *cl, struct session_database *db)
 687 {
 688     cl->database = db;
 689 }
 690
 691 struct host *client_get_host(struct client *cl)
 692 {
 693     return client_get_database(cl)->database->host;
 694 }
 695
 696 const char *client_get_url(struct client *cl)
 697 {
 698     return client_get_database(cl)->database->url;
 699 }
 700
 701 /*
 702  * Local variables:
 703  * c-basic-offset: 4
 704  * indent-tabs-mode: nil
 705  * End:
 706  * vim: shiftwidth=4 tabstop=8 expandtab
 707  */