index/retrieve.c

   1 /* $Id: retrieve.c,v 1.85 2008-03-05 09:21:48 adam Exp $
   2    Copyright (C) 1995-2007
   3    Index Data ApS
   4
   5 This file is part of the Zebra server.
   6
   7 Zebra is free software; you can redistribute it and/or modify it under
   8 the terms of the GNU General Public License as published by the Free
   9 Software Foundation; either version 2, or (at your option) any later
  10 version.
  11
  12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 for more details.
  16
  17 You should have received a copy of the GNU General Public License
  18 along with this program; if not, write to the Free Software
  19 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  20
  21 */
  22
  23 #include <stdio.h>
  24 #include <assert.h>
  25
  26 #include <fcntl.h>
  27 #ifdef WIN32
  28 #include <io.h>
  29 #include <process.h>
  30 #endif
  31 #if HAVE_UNISTD_H
  32 #include <unistd.h>
  33 #endif
  34
  35 #include "index.h"
  36 #include <yaz/diagbib1.h>
  37 #include <yaz/snprintf.h>
  38 #include <direntz.h>
  39 #include <yaz/oid_db.h>
  40 #include <zebra_strmap.h>
  41
  42 #define MAX_SYSNOS_PER_RECORD 40
  43
  44 #define ZEBRA_XML_HEADER_STR "<record xmlns=\"http://www.indexdata.com/zebra/\""
  45
  46 static int zebra_create_record_stream(ZebraHandle zh,
  47                                       Record *rec,
  48                                       struct ZebraRecStream *stream)
  49 {
  50     RecordAttr *recordAttr = rec_init_attr(zh->reg->zei, *rec);
  51
  52     if ((*rec)->size[recInfo_storeData] > 0
  53         || (*rec)->info[recInfo_filename] == 0)
  54         zebra_create_stream_mem(stream, (*rec)->info[recInfo_storeData],
  55                                 (*rec)->size[recInfo_storeData]);
  56     else
  57     {
  58         char full_rep[1024];
  59         int fd;
  60
  61         if (zh->path_reg && !yaz_is_abspath((*rec)->info[recInfo_filename])){
  62             strcpy(full_rep, zh->path_reg);
  63             strcat(full_rep, "/");
  64             strcat(full_rep, (*rec)->info[recInfo_filename]);
  65         }
  66         else
  67             strcpy(full_rep, (*rec)->info[recInfo_filename]);
  68
  69         if ((fd = open(full_rep, O_BINARY|O_RDONLY)) == -1){
  70             yaz_log(YLOG_WARN|YLOG_ERRNO, "Retrieve fail; missing file: %s",
  71                      full_rep);
  72             rec_free(rec);
  73             return YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
  74         }
  75         zebra_create_stream_fd(stream, fd, recordAttr->recordOffset);
  76     }
  77     return 0;
  78 }
  79
  80
  81 struct index_spec {
  82     const char *index_name;
  83     const char *index_type;
  84     const char *extra;
  85     struct index_spec *next;
  86 };
  87
  88
  89 struct index_spec *parse_index_spec(const char *elem, NMEM nmem,
  90                                     int *error)
  91 {
  92     struct index_spec *first = 0;
  93     struct index_spec **last = &first;
  94     const char *cp = elem;
  95
  96     *error = 0;
  97     if (cp[0] == ':' && cp[1] == ':')
  98     {
  99
 100         cp++; /* skip first ':' */
 101
 102         for (;;)
 103         {
 104             const char *cp0;
 105             struct index_spec *spec = nmem_malloc(nmem, sizeof(*spec));
 106             spec->index_type = 0;
 107             spec->next = 0;
 108             spec->extra = 0;
 109
 110             if (!first)
 111                 first = spec;
 112             *last = spec;
 113             last = &spec->next;
 114
 115             cp++; /* skip ',' or second ':' */
 116             cp0 = cp;
 117             while (*cp != ':' && *cp != '\0' && *cp != ',')
 118                 cp++;
 119             spec->index_name = nmem_strdupn(nmem, cp0, cp - cp0);
 120             if (*cp == ':') /* type as well */
 121             {
 122                 cp++;
 123                 cp0 = cp;
 124
 125                 while (*cp != '\0' && *cp != ',' && *cp != ':')
 126                     cp++;
 127                 spec->index_type = nmem_strdupn(nmem, cp0, cp - cp0);
 128             }
 129             if (*cp == ':') /* extra arguments */
 130             {
 131                 cp++;
 132                 cp0 = cp;
 133
 134                 while (*cp != '\0' && *cp != ',' && *cp != ':')
 135                     cp++;
 136                 spec->extra = nmem_strdupn(nmem, cp0, cp - cp0);
 137             }
 138             if (*cp != ',')
 139                 break;
 140         }
 141     }
 142     if (*cp != '\0')
 143         *error = 1;
 144     return first;
 145 }
 146
 147 static int parse_zebra_elem(const char *elem,
 148                             const char **index, size_t *index_len,
 149                             const char **type, size_t *type_len)
 150 {
 151     *index = 0;
 152     *index_len = 0;
 153
 154     *type = 0;
 155     *type_len = 0;
 156
 157     if (elem && *elem)
 158     {
 159         char *cp;
 160         /* verify that '::' is in the beginning of *elem
 161            and something more follows */
 162         if (':' != *elem
 163             || !(elem +1) || ':' != *(elem +1)
 164             || !(elem +2) || '\0' == *(elem +2))
 165             return 0;
 166
 167         /* pick out info from string after '::' */
 168         elem = elem + 2;
 169         cp = strchr(elem, ':');
 170
 171         if (!cp) /* index, no colon, no type */
 172         {
 173             *index = elem;
 174             *index_len = strlen(elem);
 175         }
 176         else if (cp[1] == '\0') /* colon, but no following type */
 177         {
 178             return 0;
 179         }
 180         else  /* index, colon and type */
 181         {
 182             *index = elem;
 183             *index_len = cp - elem;
 184             *type = cp+1;
 185             *type_len = strlen(cp+1);
 186         }
 187     }
 188     return 1;
 189 }
 190
 191
 192 int zebra_special_sort_fetch(ZebraHandle zh, zint sysno, ODR odr,
 193                              const char *elemsetname,
 194                              const Odr_oid *input_format,
 195                              const Odr_oid **output_format,
 196                              char **rec_bufp, int *rec_lenp)
 197 {
 198     const char *retrieval_index;
 199     size_t retrieval_index_len;
 200     const char *retrieval_type;
 201     size_t retrieval_type_len;
 202     char retrieval_index_cstr[256];
 203     char retrieval_type_cstr[256];
 204     int ord;
 205
 206     /* only accept XML and SUTRS requests */
 207     if (oid_oidcmp(input_format, yaz_oid_recsyn_xml)
 208         && oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
 209     {
 210         yaz_log(YLOG_WARN, "unsupported format for element set zebra::%s",
 211                 elemsetname);
 212         *output_format = 0;
 213         return YAZ_BIB1_NO_SYNTAXES_AVAILABLE_FOR_THIS_REQUEST;
 214     }
 215
 216     if (!parse_zebra_elem(elemsetname,
 217                           &retrieval_index, &retrieval_index_len,
 218                           &retrieval_type,  &retrieval_type_len))
 219     {
 220         return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
 221     }
 222
 223     if (retrieval_type_len == 0)
 224         return -1;   /* must have a register type specified */
 225     if (!retrieval_index_len ||
 226         retrieval_index_len >= sizeof(retrieval_index_cstr)-1)
 227     {
 228         return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
 229     }
 230
 231     memcpy(retrieval_index_cstr, retrieval_index, retrieval_index_len);
 232     retrieval_index_cstr[retrieval_index_len] = '\0';
 233
 234     memcpy(retrieval_type_cstr, retrieval_type, retrieval_type_len);
 235     retrieval_type_cstr[retrieval_type_len] = '\0';
 236
 237     ord = zebraExplain_lookup_attr_str(zh->reg->zei,
 238                                        zinfo_index_category_sort,
 239                                        retrieval_type_cstr,
 240                                        retrieval_index_cstr);
 241     if (ord == -1)
 242         return -1;  /* is not a sort index */
 243     else
 244     {
 245         char dst_buf[IT_MAX_WORD];
 246         char str[IT_MAX_WORD];
 247         const char *index_type;
 248         const char *db = 0;
 249         const char *string_index = 0;
 250         WRBUF wrbuf = wrbuf_alloc();
 251
 252         zebra_sort_sysno(zh->reg->sort_index, sysno);
 253         zebra_sort_type(zh->reg->sort_index, ord);
 254         zebra_sort_read(zh->reg->sort_index, str);
 255
 256         zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type, &db, &string_index);
 257
 258         zebra_term_untrans(zh, index_type, dst_buf, str);
 259
 260         if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
 261         {
 262             *output_format = yaz_oid_recsyn_xml;
 263             wrbuf_printf(wrbuf, ZEBRA_XML_HEADER_STR
 264                          " sysno=\"" ZINT_FORMAT "\""
 265                          " set=\"zebra::index%s/\">\n",
 266                          sysno, elemsetname);
 267
 268             wrbuf_printf(wrbuf, "  <index name=\"%s\"",
 269                          string_index);
 270             wrbuf_printf(wrbuf, " type=\"%s\">", index_type);
 271             wrbuf_xmlputs(wrbuf, dst_buf);
 272             wrbuf_printf(wrbuf, "</index>\n");
 273             wrbuf_printf(wrbuf, "</record>\n");
 274         }
 275         else if (!oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
 276         {
 277             *output_format = yaz_oid_recsyn_sutrs;
 278
 279             wrbuf_printf(wrbuf, "%s %s %s\n", string_index, index_type,
 280                          dst_buf);
 281         }
 282         *rec_lenp = wrbuf_len(wrbuf);
 283         *rec_bufp = odr_malloc(odr, *rec_lenp);
 284         memcpy(*rec_bufp, wrbuf_buf(wrbuf), *rec_lenp);
 285         wrbuf_destroy(wrbuf);
 286         return 0;
 287     }
 288 }
 289
 290 int zebra_special_index_fetch(ZebraHandle zh, zint sysno, ODR odr,
 291                               Record rec,
 292                               const char *elemsetname,
 293                               const Odr_oid *input_format,
 294                               const Odr_oid **output_format,
 295                               char **rec_bufp, int *rec_lenp)
 296 {
 297     const char *retrieval_index;
 298     size_t retrieval_index_len;
 299     const char *retrieval_type;
 300     size_t retrieval_type_len;
 301     zebra_rec_keys_t keys;
 302     int ret_code = 0;
 303     char retrieval_type_cstr[256];
 304
 305     /* set output variables before processing possible error states */
 306     /* *rec_lenp = 0; */
 307
 308     /* only accept XML and SUTRS requests */
 309     if (oid_oidcmp(input_format, yaz_oid_recsyn_xml)
 310         && oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
 311     {
 312         yaz_log(YLOG_WARN, "unsupported format for element set zebra::%s",
 313                 elemsetname);
 314         *output_format = 0;
 315         return YAZ_BIB1_NO_SYNTAXES_AVAILABLE_FOR_THIS_REQUEST;
 316     }
 317
 318     if (!parse_zebra_elem(elemsetname,
 319                      &retrieval_index, &retrieval_index_len,
 320                      &retrieval_type,  &retrieval_type_len))
 321         return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
 322
 323     if (retrieval_type_len)
 324     {
 325         memcpy(retrieval_type_cstr, retrieval_type, retrieval_type_len);
 326         retrieval_type_cstr[retrieval_type_len] = '\0';
 327     }
 328
 329     if (retrieval_index_len)
 330     {
 331         char retrieval_index_cstr[256];
 332
 333         if (retrieval_index_len < sizeof(retrieval_index_cstr) -1)
 334         {
 335             memcpy(retrieval_index_cstr, retrieval_index, retrieval_index_len);
 336             retrieval_index_cstr[retrieval_index_len] = '\0';
 337
 338             if (zebraExplain_lookup_attr_str(zh->reg->zei,
 339                                              zinfo_index_category_index,
 340                                              (retrieval_type_len == 0 ? 0 :
 341                                               retrieval_type_cstr),
 342                                              retrieval_index_cstr) == -1)
 343                 return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
 344         }
 345     }
 346
 347     keys = zebra_rec_keys_open();
 348     zebra_rec_keys_set_buf(keys, rec->info[recInfo_delKeys],
 349                            rec->size[recInfo_delKeys], 0);
 350
 351     if (!zebra_rec_keys_rewind(keys))
 352     {
 353         ret_code = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
 354     }
 355     else
 356     {
 357         size_t slen;
 358         const char *str;
 359         struct it_key key_in;
 360         WRBUF wrbuf = wrbuf_alloc();
 361
 362         if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
 363         {
 364             *output_format = input_format;
 365             wrbuf_printf(wrbuf, ZEBRA_XML_HEADER_STR
 366                          " sysno=\"" ZINT_FORMAT "\""
 367                          " set=\"zebra::index%s/\">\n",
 368                          sysno, elemsetname);
 369         }
 370         else if (!oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
 371             *output_format = input_format;
 372
 373         while (zebra_rec_keys_read(keys, &str, &slen, &key_in))
 374         {
 375             int i;
 376             int ord = CAST_ZINT_TO_INT(key_in.mem[0]);
 377             const char *index_type;
 378             const char *db = 0;
 379             const char *string_index = 0;
 380             size_t string_index_len;
 381             char dst_buf[IT_MAX_WORD];
 382
 383             zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type, &db,
 384                                     &string_index);
 385             string_index_len = strlen(string_index);
 386
 387             /* process only if index is not defined,
 388                or if defined and matching */
 389             if (retrieval_index == 0
 390                 || (string_index_len == retrieval_index_len
 391                     && !memcmp(string_index, retrieval_index,
 392                                string_index_len)))
 393             {
 394                 /* process only if type is not defined, or is matching */
 395                 if (retrieval_type == 0
 396                     || !strcmp(retrieval_type_cstr, index_type))
 397                 {
 398                     if (zebra_term_untrans(zh, index_type, dst_buf, str))
 399                         *dst_buf = '\0'; /* untrans failed */
 400
 401                     if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
 402                     {
 403                         wrbuf_printf(wrbuf, "  <index name=\"%s\"",
 404                                      string_index);
 405
 406                         wrbuf_printf(wrbuf, " type=\"%s\"", index_type);
 407
 408                         wrbuf_printf(wrbuf, " seq=\"" ZINT_FORMAT "\">",
 409                                      key_in.mem[key_in.len -1]);
 410                         wrbuf_xmlputs(wrbuf, dst_buf);
 411                         wrbuf_printf(wrbuf, "</index>\n");
 412                     }
 413                     else
 414                     {
 415                         wrbuf_printf(wrbuf, "%s ", string_index);
 416
 417                         wrbuf_printf(wrbuf, "%s", index_type);
 418
 419                         for (i = 1; i < key_in.len; i++)
 420                             wrbuf_printf(wrbuf, " " ZINT_FORMAT,
 421                                              key_in.mem[i]);
 422
 423                         wrbuf_printf(wrbuf, " %s", dst_buf);
 424
 425                         wrbuf_printf(wrbuf, "\n");
 426
 427                     }
 428
 429                 }
 430             }
 431         }
 432         if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
 433             wrbuf_printf(wrbuf, "</record>\n");
 434         *rec_lenp = wrbuf_len(wrbuf);
 435         *rec_bufp = odr_malloc(odr, *rec_lenp);
 436         memcpy(*rec_bufp, wrbuf_buf(wrbuf), *rec_lenp);
 437         wrbuf_destroy(wrbuf);
 438     }
 439     zebra_rec_keys_close(keys);
 440     return ret_code;
 441 }
 442
 443
 444 static void retrieve_puts_attr(WRBUF wrbuf, const char *name,
 445                                const char *value)
 446 {
 447     if (value)
 448     {
 449         wrbuf_printf(wrbuf, " %s=\"", name);
 450         wrbuf_xmlputs(wrbuf, value);
 451         wrbuf_printf(wrbuf, "\"");
 452     }
 453 }
 454
 455 static void retrieve_puts_attr_int(WRBUF wrbuf, const char *name,
 456                                const int value)
 457 {
 458     wrbuf_printf(wrbuf, " %s=\"%i\"", name, value);
 459 }
 460
 461 static void retrieve_puts_str(WRBUF wrbuf, const char *name,
 462                                const char *value)
 463 {
 464     if (value)
 465         wrbuf_printf(wrbuf, "%s %s\n", name, value);
 466 }
 467
 468 static void retrieve_puts_int(WRBUF wrbuf, const char *name,
 469                                const int value)
 470 {
 471     wrbuf_printf(wrbuf, "%s %i\n", name, value);
 472 }
 473
 474
 475 static void snippet_xml_record(ZebraHandle zh, WRBUF wrbuf, zebra_snippets *doc)
 476 {
 477     const zebra_snippet_word *doc_w;
 478     int mark_state = 0;
 479
 480     wrbuf_printf(wrbuf, "%s>\n", ZEBRA_XML_HEADER_STR);
 481     for (doc_w = zebra_snippets_constlist(doc); doc_w; doc_w = doc_w->next)
 482     {
 483         if (doc_w->mark)
 484         {
 485             const char *index_type;
 486             const char *db = 0;
 487             const char *string_index = 0;
 488
 489             zebraExplain_lookup_ord(zh->reg->zei, doc_w->ord,
 490                                     &index_type, &db, &string_index);
 491
 492             if (mark_state == 0)
 493             {
 494                 wrbuf_printf(wrbuf, "  <snippet name=\"%s\"",  string_index);
 495                 wrbuf_printf(wrbuf, " type=\"%s\">", index_type);
 496             }
 497             if (doc_w->match)
 498                 wrbuf_puts(wrbuf, "<s>");
 499             /* not printing leading ws */
 500             if (mark_state || !doc_w->ws || doc_w->match)
 501                 wrbuf_xmlputs(wrbuf, doc_w->term);
 502             if (doc_w->match)
 503                 wrbuf_puts(wrbuf, "</s>");
 504         }
 505         else if (mark_state == 1)
 506         {
 507             wrbuf_puts(wrbuf, "</snippet>\n");
 508         }
 509         mark_state = doc_w->mark;
 510     }
 511     if (mark_state == 1)
 512     {
 513         wrbuf_puts(wrbuf, "</snippet>\n");
 514     }
 515     wrbuf_printf(wrbuf, "</record>");
 516 }
 517
 518 int zebra_get_rec_snippets(ZebraHandle zh, zint sysno,
 519                            zebra_snippets *snippets)
 520 {
 521     int return_code = 0;
 522     Record rec = rec_get(zh->reg->records, sysno);
 523     if (!rec)
 524     {
 525         yaz_log(YLOG_WARN, "rec_get fail on sysno=" ZINT_FORMAT, sysno);
 526         return_code = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
 527     }
 528     else
 529     {
 530         const char *file_type = rec->info[recInfo_fileType];
 531         void *recTypeClientData;
 532         RecType rt = recType_byName(zh->reg->recTypes, zh->res,
 533                                     file_type, &recTypeClientData);
 534
 535         if (!rt)
 536             return_code = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
 537         else
 538         {
 539             struct ZebraRecStream stream;
 540             return_code = zebra_create_record_stream(zh, &rec, &stream);
 541             if (return_code == 0)
 542             {
 543                 extract_snippet(zh, snippets, &stream,
 544                                 rt, recTypeClientData);
 545
 546                 stream.destroy(&stream);
 547             }
 548         }
 549         rec_free(&rec);
 550     }
 551     return return_code;
 552 }
 553
 554 static int snippet_fetch(ZebraHandle zh, const char *setname,
 555                          zint sysno, ODR odr,
 556                          const char *elemsetname,
 557                          const Odr_oid *input_format,
 558                          const Odr_oid **output_format,
 559                          char **rec_bufp, int *rec_lenp)
 560 {
 561     zebra_snippets *rec_snippets = zebra_snippets_create();
 562     int return_code = zebra_get_rec_snippets(zh, sysno, rec_snippets);
 563
 564     if (!return_code)
 565     {
 566         WRBUF wrbuf = wrbuf_alloc();
 567         zebra_snippets *hit_snippet = zebra_snippets_create();
 568
 569         zebra_snippets_hit_vector(zh, setname, sysno, hit_snippet);
 570
 571 #if 0
 572         /* for debugging purposes */
 573         yaz_log(YLOG_LOG, "---------------------------");
 574         yaz_log(YLOG_LOG, "REC SNIPPET:");
 575         zebra_snippets_log(rec_snippet, YLOG_LOG, 1);
 576         yaz_log(YLOG_LOG, "---------------------------");
 577         yaz_log(YLOG_LOG, "HIT SNIPPET:");
 578         zebra_snippets_log(hit_snippet, YLOG_LOG, 1);
 579 #endif
 580
 581         zebra_snippets_ring(rec_snippets, hit_snippet, 5, 5);
 582
 583 #if 0
 584         yaz_log(YLOG_LOG, "---------------------------");
 585         yaz_log(YLOG_LOG, "RING SNIPPET:");
 586         zebra_snippets_log(rec_snippets, YLOG_LOG, 1);
 587 #endif
 588         snippet_xml_record(zh, wrbuf, rec_snippets);
 589
 590         *output_format = yaz_oid_recsyn_xml;
 591
 592         if (return_code == 0)
 593         {
 594             *rec_lenp = wrbuf_len(wrbuf);
 595             *rec_bufp = odr_strdup(odr, wrbuf_cstr(wrbuf));
 596         }
 597         wrbuf_destroy(wrbuf);
 598         zebra_snippets_destroy(hit_snippet);
 599     }
 600     zebra_snippets_destroy(rec_snippets);
 601     return return_code;
 602 }
 603
 604 struct term_collect {
 605     const char *term;
 606     int oc;
 607     zint set_occur;
 608 };
 609
 610 zint freq_term(ZebraHandle zh, int ord, const char *term, RSET rset_set)
 611 {
 612     struct rset_key_control *kc = zebra_key_control_create(zh);
 613     char ord_buf[IT_MAX_WORD];
 614     int ord_len = key_SU_encode(ord, ord_buf);
 615     char *info;
 616     zint hits = 0;
 617     NMEM nmem = nmem_create();
 618
 619     strcpy(ord_buf + ord_len, term);
 620
 621     info = dict_lookup(zh->reg->dict, ord_buf);
 622     if (info)
 623     {
 624         ISAM_P isam_p;
 625         RSET rsets[2], rset;
 626         memcpy(&isam_p, info+1, sizeof(ISAM_P));
 627
 628         rsets[0] = zebra_create_rset_isam(zh, nmem, kc, kc->scope, isam_p, 0);
 629         rsets[1] = rset_dup(rset_set);
 630
 631         rset = rset_create_and(nmem, kc, kc->scope, 2, rsets);
 632
 633         zebra_count_set(zh, rset, &hits, zh->approx_limit);
 634
 635         rset_delete(rsets[0]);
 636         rset_delete(rset);
 637     }
 638     (*kc->dec)(kc);
 639     nmem_destroy(nmem);
 640     return hits;
 641 }
 642
 643 int term_qsort_handle(const void *a, const void *b)
 644 {
 645     const struct term_collect *l = a;
 646     const struct term_collect *r = b;
 647     if (l->set_occur < r->set_occur)
 648         return 1;
 649     else if (l->set_occur > r->set_occur)
 650         return -1;
 651     else
 652     {
 653         const char *lterm = l->term ? l->term : "";
 654         const char *rterm = r->term ? r->term : "";
 655         return strcmp(lterm, rterm);
 656     }
 657 }
 658
 659 void term_collect_freq(ZebraHandle zh,
 660                        struct term_collect *col, int no_terms_collect,
 661                        int ord, RSET rset)
 662 {
 663     int i;
 664     for (i = 0; i < no_terms_collect; i++)
 665     {
 666         if (col[i].term)
 667             col[i].set_occur = freq_term(zh, ord, col[i].term, rset);
 668     }
 669     qsort(col, no_terms_collect, sizeof(*col), term_qsort_handle);
 670 }
 671
 672 struct term_collect *term_collect_create(zebra_strmap_t sm,
 673                                          int no_terms_collect,
 674                                          NMEM nmem)
 675 {
 676     const char *term;
 677     void *data_buf;
 678     size_t data_len;
 679     zebra_strmap_it it;
 680     struct term_collect *col = nmem_malloc(nmem,
 681                                            sizeof *col *no_terms_collect);
 682     int i;
 683     for (i = 0; i < no_terms_collect; i++)
 684     {
 685         col[i].term = 0;
 686         col[i].oc = 0;
 687         col[i].set_occur = 0;
 688     }
 689     /* iterate over terms and collect the most frequent ones */
 690     it = zebra_strmap_it_create(sm);
 691     while ((term = zebra_strmap_it_next(it, &data_buf, &data_len)))
 692     {
 693         /* invariant:
 694            col[0] has lowest oc .  col[no_terms_collect-1] has highest oc */
 695         int oc = *(int*) data_buf;
 696         int j = 0;
 697         /* insertion may be slow but terms terms will be "infrequent" and
 698            thus number of iterations should be small below
 699         */
 700         while (j < no_terms_collect && oc > col[j].oc)
 701             j++;
 702         if (j)
 703         {   /* oc <= col[j] and oc > col[j-1] */
 704             --j;
 705             memmove(col, col+1, sizeof(*col) * j);
 706             col[j].term = term;
 707             col[j].oc = oc;
 708         }
 709     }
 710     zebra_strmap_it_destroy(it);
 711     return col;
 712 }
 713
 714 static ZEBRA_RES facet_fetch(ZebraHandle zh, const char *setname,
 715                              ODR odr,
 716                              const char *elemsetname,
 717                              const Odr_oid *input_format,
 718                              const Odr_oid **output_format,
 719                              char **rec_bufp, int *rec_lenp)
 720 {
 721     zint *pos_array;
 722     int i;
 723     int num_recs = 10; /* number of records to analyze */
 724     int max_chunks = 2;
 725     ZebraMetaRecord *poset;
 726     ZEBRA_RES ret = ZEBRA_OK;
 727     int *ord_array;
 728     WRBUF wr = wrbuf_alloc();
 729     int use_xml = 0;
 730     int no_ord = 0;
 731     struct index_spec *spec, *spec_list;
 732     int error;
 733
 734     res_get_int(zh->res, "facetNumRecs", &num_recs);
 735     res_get_int(zh->res, "facetMaxChunks", &max_chunks);
 736
 737     /* see if XML is required for response */
 738     if (oid_oidcmp(input_format, yaz_oid_recsyn_xml) == 0)
 739         use_xml = 1;
 740
 741     spec_list = parse_index_spec(elemsetname, odr_getmem(odr), &error);
 742
 743     if (!spec_list || error)
 744     {
 745         zebra_setError(
 746             zh,
 747             YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_,
 748             0);
 749         return ZEBRA_FAIL;
 750     }
 751
 752     for (spec = spec_list; spec; spec = spec->next)
 753     {
 754         if (!spec->index_type)
 755         {
 756             zebra_setError(
 757                 zh,
 758                 YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_,
 759                 0);
 760             return ZEBRA_FAIL;
 761         }
 762         no_ord++;
 763     }
 764
 765     ord_array = odr_malloc(odr, sizeof(*ord_array) * no_ord);
 766
 767     for (spec = spec_list, i = 0; spec; spec = spec->next, i++)
 768     {
 769         int ord = zebraExplain_lookup_attr_str(zh->reg->zei,
 770                                                zinfo_index_category_index,
 771                                                spec->index_type,
 772                                                spec->index_name);
 773         if (ord == -1)
 774         {
 775             zebra_setError(
 776                 zh,
 777                 YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_,
 778                 0);
 779             return ZEBRA_FAIL;
 780         }
 781         ord_array[i] = ord;
 782     }
 783     pos_array = (zint *) odr_malloc(odr, num_recs * sizeof(*pos_array));
 784     for (i = 0; i < num_recs; i++)
 785         pos_array[i] = i+1;
 786     poset = zebra_meta_records_create(zh, setname, num_recs, pos_array);
 787     if (!poset)
 788     {
 789         zebra_setError(zh, YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
 790                        setname);
 791         ret = ZEBRA_FAIL;
 792     }
 793     else
 794     {
 795         yaz_timing_t timing = yaz_timing_create();
 796         zebra_strmap_t *map_array
 797             = odr_malloc(odr, sizeof *map_array * no_ord);
 798         for (i = 0; i < no_ord; i++)
 799             map_array[i] = zebra_strmap_create();
 800
 801         for (i = 0; i < num_recs; i++)
 802         {
 803             int j;
 804             zint sysnos[MAX_SYSNOS_PER_RECORD];
 805             int no_sysnos = MAX_SYSNOS_PER_RECORD;
 806             if (!poset[i].sysno)
 807                 continue;
 808             ret = zebra_result_recid_to_sysno(zh,  setname,
 809                                               poset[i].sysno,
 810                                               sysnos, &no_sysnos);
 811             assert(no_sysnos > 0);
 812             yaz_log(YLOG_DEBUG, "Analyzing rec=%d ISAM sysno=" ZINT_FORMAT " chunks=%d",
 813                     i, poset[i].sysno, no_sysnos);
 814             for (j = 0; j < no_sysnos && j < max_chunks; j++)
 815             {
 816                 size_t slen;
 817                 const char *str;
 818                 struct it_key key_in;
 819                 Record rec = rec_get(zh->reg->records, sysnos[j]);
 820                 zebra_rec_keys_t keys = zebra_rec_keys_open();
 821                 zebra_rec_keys_set_buf(keys, rec->info[recInfo_delKeys],
 822                                        rec->size[recInfo_delKeys], 0);
 823
 824                 yaz_log(YLOG_DEBUG, "rec %d " ZINT_FORMAT " %s",
 825                         j, sysnos[j], zebra_rec_keys_empty(keys) ? "empty" : "non-empty");
 826                 if (zebra_rec_keys_rewind(keys))
 827                 {
 828                     while (zebra_rec_keys_read(keys, &str, &slen, &key_in))
 829                     {
 830                         int i;
 831                         struct index_spec *spec;
 832                         for (spec = spec_list, i = 0; i < no_ord;
 833                              i++, spec = spec->next)
 834                         {
 835                             int ord = CAST_ZINT_TO_INT(key_in.mem[0]);
 836                             if (ord == ord_array[i] &&
 837                                 str[0] != FIRST_IN_FIELD_CHAR)
 838                             {
 839                                 int *freq;
 840                                 zebra_strmap_t sm = map_array[i];
 841
 842                                 freq = zebra_strmap_lookup(sm, str, 0, 0);
 843                                 if (freq)
 844                                     (*freq)++;
 845                                 else
 846                                 {
 847                                     int v = 1;
 848                                     zebra_strmap_add(sm, str, &v, sizeof v);
 849                                 }
 850                             }
 851                         }
 852                     }
 853                 }
 854                 zebra_rec_keys_close(keys);
 855                 rec_free(&rec);
 856             }
 857         }
 858         yaz_timing_stop(timing);
 859         yaz_log(YLOG_LOG, "facet first phase real=%4.2f",
 860                 yaz_timing_get_real(timing));
 861         yaz_timing_start(timing);
 862         if (use_xml)
 863             wrbuf_puts(wr, "<facets>\n");
 864         for (spec = spec_list, i = 0; i < no_ord; i++, spec = spec->next)
 865         {
 866             int j;
 867             NMEM nmem = nmem_create();
 868             struct term_collect *col;
 869             int no_collect_terms = 20;
 870
 871             if (spec->extra)
 872                 no_collect_terms = atoi(spec->extra);
 873             if (no_collect_terms < 1)
 874                 no_collect_terms = 1;
 875             col = term_collect_create(map_array[i], no_collect_terms, nmem);
 876             term_collect_freq(zh, col, no_collect_terms, ord_array[i],
 877                               resultSetRef(zh, setname));
 878
 879             if (use_xml)
 880                 wrbuf_printf(wr, "  <facet type=\"%s\" index=\"%s\">\n",
 881                              spec->index_type, spec->index_name);
 882             else
 883                 wrbuf_printf(wr, "facet %s %s\n",
 884                              spec->index_type, spec->index_name);
 885             for (j = 0; j < no_collect_terms; j++)
 886             {
 887                 if (col[j].term)
 888                 {
 889                     char dst_buf[IT_MAX_WORD];
 890                     zebra_term_untrans(zh, spec->index_type, dst_buf, col[j].term);
 891                     if (use_xml)
 892                     {
 893                         wrbuf_printf(wr, "    <term coccur=\"%d\"", col[j].oc);
 894                         if (col[j].set_occur)
 895                             wrbuf_printf(wr, " occur=\"" ZINT_FORMAT "\"",
 896                                          col[j].set_occur);
 897                         wrbuf_printf(wr, ">");
 898                         wrbuf_xmlputs(wr, dst_buf);
 899                         wrbuf_printf(wr, "</term>\n");
 900                     }
 901                     else
 902                     {
 903                         wrbuf_printf(wr, "term %d", col[j].oc);
 904                         if (col[j].set_occur)
 905                             wrbuf_printf(wr, " " ZINT_FORMAT,
 906                                          col[j].set_occur);
 907                         wrbuf_printf(wr, ": %s\n", dst_buf);
 908                     }
 909                 }
 910             }
 911             if (use_xml)
 912                 wrbuf_puts(wr, "  </facet>\n");
 913             nmem_destroy(nmem);
 914         }
 915         if (use_xml)
 916             wrbuf_puts(wr, "</facets>\n");
 917         for (i = 0; i < no_ord; i++)
 918             zebra_strmap_destroy(map_array[i]);
 919         yaz_timing_stop(timing);
 920         yaz_log(YLOG_LOG, "facet second phase real=%4.2f",
 921                 yaz_timing_get_real(timing));
 922         yaz_timing_destroy(&timing);
 923     }
 924     *rec_bufp = odr_strdup(odr, wrbuf_cstr(wr));
 925     wrbuf_destroy(wr);
 926     *rec_lenp = strlen(*rec_bufp);
 927     *output_format = yaz_oid_recsyn_xml;
 928
 929     zebra_meta_records_destroy(zh, poset, num_recs);
 930     return ret;
 931 }
 932
 933 int zebra_special_fetch(ZebraHandle zh, const char *setname,
 934                         zint sysno, int score, ODR odr,
 935                         const char *elemsetname,
 936                         const Odr_oid *input_format,
 937                         const Odr_oid **output_format,
 938                         char **rec_bufp, int *rec_lenp)
 939 {
 940     Record rec;
 941
 942     /* set output variables before processing possible error states */
 943     /* *rec_lenp = 0; */
 944
 945     if (elemsetname && 0 == strncmp(elemsetname, "facet", 5))
 946     {
 947         return facet_fetch(zh, setname, odr,
 948                            elemsetname + 5,
 949                            input_format, output_format,
 950                            rec_bufp, rec_lenp);
 951     }
 952
 953     if (elemsetname && 0 == strcmp(elemsetname, "snippet"))
 954     {
 955         return snippet_fetch(zh, setname, sysno, odr,
 956                              elemsetname + 7,
 957                              input_format, output_format,
 958                              rec_bufp, rec_lenp);
 959     }
 960
 961     /* processing zebra::meta::sysno elemset without fetching binary data */
 962     if (elemsetname && 0 == strcmp(elemsetname, "meta::sysno"))
 963     {
 964         int ret = 0;
 965         WRBUF wrbuf = wrbuf_alloc();
 966         if (!oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
 967         {
 968             wrbuf_printf(wrbuf, ZINT_FORMAT, sysno);
 969             *output_format = input_format;
 970         }
 971         else if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
 972         {
 973             wrbuf_printf(wrbuf, ZEBRA_XML_HEADER_STR
 974                          " sysno=\"" ZINT_FORMAT "\"/>\n",
 975                          sysno);
 976             *output_format = input_format;
 977         }
 978         *rec_lenp = wrbuf_len(wrbuf);
 979         if (*rec_lenp)
 980             *rec_bufp = odr_strdup(odr, wrbuf_cstr(wrbuf));
 981         else
 982             ret = YAZ_BIB1_NO_SYNTAXES_AVAILABLE_FOR_THIS_REQUEST;
 983         wrbuf_destroy(wrbuf);
 984         return ret;
 985     }
 986
 987     /* processing special elementsetname zebra::index:: for sort elements */
 988     if (elemsetname && 0 == strncmp(elemsetname, "index", 5))
 989     {
 990         int ret = zebra_special_sort_fetch(zh, sysno, odr,
 991                                            elemsetname + 5,
 992                                            input_format, output_format,
 993                                            rec_bufp, rec_lenp);
 994         if (ret != -1)
 995             return ret;
 996         /* not a sort index so we continue to get the full record */
 997     }
 998
 999
1000     /* fetching binary record up for all other display elementsets */
1001     rec = rec_get(zh->reg->records, sysno);
1002     if (!rec)
1003     {
1004         yaz_log(YLOG_WARN, "rec_get fail on sysno=" ZINT_FORMAT, sysno);
1005         return YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
1006     }
1007
1008     /* processing special elementsetnames zebra::data */
1009     if (elemsetname && 0 == strcmp(elemsetname, "data"))
1010     {
1011         struct ZebraRecStream stream;
1012         RecordAttr *recordAttr = rec_init_attr(zh->reg->zei, rec);
1013         zebra_create_record_stream(zh, &rec, &stream);
1014         *output_format = input_format;
1015         *rec_lenp = recordAttr->recordSize;
1016         *rec_bufp = (char *) odr_malloc(odr, *rec_lenp);
1017         stream.readf(&stream, *rec_bufp, *rec_lenp);
1018         stream.destroy(&stream);
1019         rec_free(&rec);
1020         return 0;
1021     }
1022
1023     /* only accept XML and SUTRS requests from now */
1024     if (oid_oidcmp(input_format, yaz_oid_recsyn_xml)
1025         && oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
1026     {
1027         yaz_log(YLOG_WARN, "unsupported format for element set zebra::%s",
1028                 elemsetname);
1029         return YAZ_BIB1_NO_SYNTAXES_AVAILABLE_FOR_THIS_REQUEST;
1030     }
1031
1032
1033     /* processing special elementsetnames zebra::meta:: */
1034     if (elemsetname && 0 == strcmp(elemsetname, "meta"))
1035     {
1036         int ret = 0;
1037         WRBUF wrbuf = wrbuf_alloc();
1038         RecordAttr *recordAttr = rec_init_attr(zh->reg->zei, rec);
1039
1040         if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
1041         {
1042             *output_format = input_format;
1043
1044             wrbuf_printf(wrbuf, ZEBRA_XML_HEADER_STR
1045                          " sysno=\"" ZINT_FORMAT "\"", sysno);
1046             retrieve_puts_attr(wrbuf, "base", rec->info[recInfo_databaseName]);
1047             retrieve_puts_attr(wrbuf, "file", rec->info[recInfo_filename]);
1048             retrieve_puts_attr(wrbuf, "type", rec->info[recInfo_fileType]);
1049             if (score >= 0)
1050                 retrieve_puts_attr_int(wrbuf, "score", score);
1051
1052             wrbuf_printf(wrbuf,
1053                          " rank=\"" ZINT_FORMAT "\""
1054                          " size=\"%i\""
1055                          " set=\"zebra::%s\"/>\n",
1056                          recordAttr->staticrank,
1057                          recordAttr->recordSize,
1058                          elemsetname);
1059         }
1060         else if (!oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
1061         {
1062             *output_format = input_format;
1063             wrbuf_printf(wrbuf, "sysno " ZINT_FORMAT "\n", sysno);
1064             retrieve_puts_str(wrbuf, "base", rec->info[recInfo_databaseName]);
1065             retrieve_puts_str(wrbuf, "file", rec->info[recInfo_filename]);
1066             retrieve_puts_str(wrbuf, "type", rec->info[recInfo_fileType]);
1067             if (score >= 0)
1068                 retrieve_puts_int(wrbuf, "score", score);
1069
1070             wrbuf_printf(wrbuf,
1071                          "rank " ZINT_FORMAT "\n"
1072                          "size %i\n"
1073                          "set zebra::%s\n",
1074                          recordAttr->staticrank,
1075                          recordAttr->recordSize,
1076                          elemsetname);
1077         }
1078         *rec_lenp = wrbuf_len(wrbuf);
1079         if (*rec_lenp)
1080             *rec_bufp = odr_strdup(odr, wrbuf_cstr(wrbuf));
1081         else
1082             ret = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
1083
1084         wrbuf_destroy(wrbuf);
1085         rec_free(&rec);
1086         return ret;
1087     }
1088
1089     /* processing special elementsetnames zebra::index:: */
1090     if (elemsetname && 0 == strncmp(elemsetname, "index", 5))
1091     {
1092         int ret = zebra_special_index_fetch(zh, sysno, odr, rec,
1093                                             elemsetname + 5,
1094                                             input_format, output_format,
1095                                             rec_bufp, rec_lenp);
1096
1097         rec_free(&rec);
1098         return ret;
1099     }
1100
1101     if (rec)
1102         rec_free(&rec);
1103     return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
1104 }
1105
1106
1107 int zebra_record_fetch(ZebraHandle zh, const char *setname,
1108                        zint sysno, int score,
1109                        ODR odr,
1110                        const Odr_oid *input_format, Z_RecordComposition *comp,
1111                        const Odr_oid **output_format,
1112                        char **rec_bufp, int *rec_lenp, char **basenamep,
1113                        char **addinfo)
1114 {
1115     Record rec;
1116     char *fname, *file_type, *basename;
1117     const char *elemsetname;
1118     struct ZebraRecStream stream;
1119     RecordAttr *recordAttr;
1120     void *clientData;
1121     int return_code = 0;
1122     zint sysnos[MAX_SYSNOS_PER_RECORD];
1123     int no_sysnos = MAX_SYSNOS_PER_RECORD;
1124     ZEBRA_RES res;
1125
1126     res = zebra_result_recid_to_sysno(zh, setname, sysno, sysnos, &no_sysnos);
1127     if (res != ZEBRA_OK)
1128         return ZEBRA_FAIL;
1129
1130     sysno = sysnos[0];
1131     *basenamep = 0;
1132     *addinfo = 0;
1133     elemsetname = yaz_get_esn(comp);
1134
1135     /* processing zebra special elementset names of form 'zebra:: */
1136     if (elemsetname && 0 == strncmp(elemsetname, "zebra::", 7))
1137         return  zebra_special_fetch(zh, setname, sysno, score, odr,
1138                                     elemsetname + 7,
1139                                     input_format, output_format,
1140                                     rec_bufp, rec_lenp);
1141
1142
1143     /* processing all other element set names */
1144     rec = rec_get(zh->reg->records, sysno);
1145     if (!rec)
1146     {
1147         yaz_log(YLOG_WARN, "rec_get fail on sysno=" ZINT_FORMAT, sysno);
1148         *basenamep = 0;
1149         return YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
1150     }
1151
1152
1153     recordAttr = rec_init_attr(zh->reg->zei, rec);
1154
1155     file_type = rec->info[recInfo_fileType];
1156     fname = rec->info[recInfo_filename];
1157     basename = rec->info[recInfo_databaseName];
1158     *basenamep = (char *) odr_malloc(odr, strlen(basename)+1);
1159     strcpy(*basenamep, basename);
1160
1161     yaz_log(YLOG_DEBUG, "retrieve localno=" ZINT_FORMAT " score=%d",
1162             sysno, score);
1163
1164     return_code = zebra_create_record_stream(zh, &rec, &stream);
1165
1166     if (rec)
1167     {
1168         RecType rt;
1169         struct recRetrieveCtrl retrieveCtrl;
1170
1171         retrieveCtrl.stream = &stream;
1172         retrieveCtrl.fname = fname;
1173         retrieveCtrl.localno = sysno;
1174         retrieveCtrl.staticrank = recordAttr->staticrank;
1175         retrieveCtrl.score = score;
1176         retrieveCtrl.recordSize = recordAttr->recordSize;
1177         retrieveCtrl.odr = odr;
1178         retrieveCtrl.input_format = retrieveCtrl.output_format = input_format;
1179         retrieveCtrl.comp = comp;
1180         retrieveCtrl.encoding = zh->record_encoding;
1181         retrieveCtrl.diagnostic = 0;
1182         retrieveCtrl.addinfo = 0;
1183         retrieveCtrl.dh = zh->reg->dh;
1184         retrieveCtrl.res = zh->res;
1185         retrieveCtrl.rec_buf = 0;
1186         retrieveCtrl.rec_len = -1;
1187
1188         if (!(rt = recType_byName(zh->reg->recTypes, zh->res,
1189                                   file_type, &clientData)))
1190         {
1191             char addinfo_str[100];
1192
1193             sprintf(addinfo_str, "Could not handle record type %.40s",
1194                     file_type);
1195
1196             *addinfo = odr_strdup(odr, addinfo_str);
1197             return_code = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
1198         }
1199         else
1200         {
1201             (*rt->retrieve)(clientData, &retrieveCtrl);
1202             return_code = retrieveCtrl.diagnostic;
1203
1204             *output_format = retrieveCtrl.output_format;
1205             *rec_bufp = (char *) retrieveCtrl.rec_buf;
1206             *rec_lenp = retrieveCtrl.rec_len;
1207             *addinfo = retrieveCtrl.addinfo;
1208         }
1209
1210         stream.destroy(&stream);
1211         rec_free(&rec);
1212     }
1213
1214     return return_code;
1215 }
1216
1217 /*
1218  * Local variables:
1219  * c-basic-offset: 4
1220  * indent-tabs-mode: nil
1221  * End:
1222  * vim: shiftwidth=4 tabstop=8 expandtab
1223  */
1224