index/retrieve.c

   1 /* This file is part of the Zebra server.
   2    Copyright (C) 1995-2008 Index Data
   3
   4 Zebra is free software; you can redistribute it and/or modify it under
   5 the terms of the GNU General Public License as published by the Free
   6 Software Foundation; either version 2, or (at your option) any later
   7 version.
   8
   9 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 for more details.
  13
  14 You should have received a copy of the GNU General Public License
  15 along with this program; if not, write to the Free Software
  16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  17
  18 */
  19
  20 #include <stdio.h>
  21 #include <assert.h>
  22
  23 #include <fcntl.h>
  24 #ifdef WIN32
  25 #include <io.h>
  26 #include <process.h>
  27 #endif
  28 #if HAVE_UNISTD_H
  29 #include <unistd.h>
  30 #endif
  31
  32 #include "index.h"
  33 #include <yaz/diagbib1.h>
  34 #include <yaz/snprintf.h>
  35 #include <direntz.h>
  36 #include <yaz/oid_db.h>
  37 #include <zebra_strmap.h>
  38
  39 #define MAX_SYSNOS_PER_RECORD 40
  40
  41 #define ZEBRA_XML_HEADER_STR "<record xmlns=\"http://www.indexdata.com/zebra/\""
  42
  43 static int zebra_create_record_stream(ZebraHandle zh,
  44                                       Record *rec,
  45                                       struct ZebraRecStream *stream)
  46 {
  47     RecordAttr *recordAttr = rec_init_attr(zh->reg->zei, *rec);
  48
  49     if ((*rec)->size[recInfo_storeData] > 0
  50         || (*rec)->info[recInfo_filename] == 0)
  51         zebra_create_stream_mem(stream, (*rec)->info[recInfo_storeData],
  52                                 (*rec)->size[recInfo_storeData]);
  53     else
  54     {
  55         char full_rep[1024];
  56         int fd;
  57
  58         if (zh->path_reg && !yaz_is_abspath((*rec)->info[recInfo_filename])){
  59             strcpy(full_rep, zh->path_reg);
  60             strcat(full_rep, "/");
  61             strcat(full_rep, (*rec)->info[recInfo_filename]);
  62         }
  63         else
  64             strcpy(full_rep, (*rec)->info[recInfo_filename]);
  65
  66         if ((fd = open(full_rep, O_BINARY|O_RDONLY)) == -1){
  67             yaz_log(YLOG_WARN|YLOG_ERRNO, "Retrieve fail; missing file: %s",
  68                      full_rep);
  69             rec_free(rec);
  70             return YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
  71         }
  72         zebra_create_stream_fd(stream, fd, recordAttr->recordOffset);
  73     }
  74     return 0;
  75 }
  76
  77
  78 struct index_spec {
  79     const char *index_name;
  80     const char *index_type;
  81     const char *extra;
  82     struct index_spec *next;
  83 };
  84
  85
  86 struct index_spec *parse_index_spec(const char *elem, NMEM nmem,
  87                                     int *error)
  88 {
  89     struct index_spec *first = 0;
  90     struct index_spec **last = &first;
  91     const char *cp = elem;
  92
  93     *error = 0;
  94     if (cp[0] == ':' && cp[1] == ':')
  95     {
  96
  97         cp++; /* skip first ':' */
  98
  99         for (;;)
 100         {
 101             const char *cp0;
 102             struct index_spec *spec = nmem_malloc(nmem, sizeof(*spec));
 103             spec->index_type = 0;
 104             spec->next = 0;
 105             spec->extra = 0;
 106
 107             if (!first)
 108                 first = spec;
 109             *last = spec;
 110             last = &spec->next;
 111
 112             cp++; /* skip ',' or second ':' */
 113             cp0 = cp;
 114             while (*cp != ':' && *cp != '\0' && *cp != ',')
 115                 cp++;
 116             spec->index_name = nmem_strdupn(nmem, cp0, cp - cp0);
 117             if (*cp == ':') /* type as well */
 118             {
 119                 cp++;
 120                 cp0 = cp;
 121
 122                 while (*cp != '\0' && *cp != ',' && *cp != ':')
 123                     cp++;
 124                 spec->index_type = nmem_strdupn(nmem, cp0, cp - cp0);
 125             }
 126             if (*cp == ':') /* extra arguments */
 127             {
 128                 cp++;
 129                 cp0 = cp;
 130
 131                 while (*cp != '\0' && *cp != ',' && *cp != ':')
 132                     cp++;
 133                 spec->extra = nmem_strdupn(nmem, cp0, cp - cp0);
 134             }
 135             if (*cp != ',')
 136                 break;
 137         }
 138     }
 139     if (*cp != '\0')
 140         *error = 1;
 141     return first;
 142 }
 143
 144 static int parse_zebra_elem(const char *elem,
 145                             const char **index, size_t *index_len,
 146                             const char **type, size_t *type_len)
 147 {
 148     *index = 0;
 149     *index_len = 0;
 150
 151     *type = 0;
 152     *type_len = 0;
 153
 154     if (elem && *elem)
 155     {
 156         char *cp;
 157         /* verify that '::' is in the beginning of *elem
 158            and something more follows */
 159         if (':' != *elem
 160             || !(elem +1) || ':' != *(elem +1)
 161             || !(elem +2) || '\0' == *(elem +2))
 162             return 0;
 163
 164         /* pick out info from string after '::' */
 165         elem = elem + 2;
 166         cp = strchr(elem, ':');
 167
 168         if (!cp) /* index, no colon, no type */
 169         {
 170             *index = elem;
 171             *index_len = strlen(elem);
 172         }
 173         else if (cp[1] == '\0') /* colon, but no following type */
 174         {
 175             return 0;
 176         }
 177         else  /* index, colon and type */
 178         {
 179             *index = elem;
 180             *index_len = cp - elem;
 181             *type = cp+1;
 182             *type_len = strlen(cp+1);
 183         }
 184     }
 185     return 1;
 186 }
 187
 188
 189 int zebra_special_sort_fetch(ZebraHandle zh, zint sysno, ODR odr,
 190                              const char *elemsetname,
 191                              const Odr_oid *input_format,
 192                              const Odr_oid **output_format,
 193                              char **rec_bufp, int *rec_lenp)
 194 {
 195     const char *retrieval_index;
 196     size_t retrieval_index_len;
 197     const char *retrieval_type;
 198     size_t retrieval_type_len;
 199     char retrieval_index_cstr[256];
 200     char retrieval_type_cstr[256];
 201     int ord;
 202
 203     /* only accept XML and SUTRS requests */
 204     if (oid_oidcmp(input_format, yaz_oid_recsyn_xml)
 205         && oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
 206     {
 207         yaz_log(YLOG_WARN, "unsupported format for element set zebra::%s",
 208                 elemsetname);
 209         *output_format = 0;
 210         return YAZ_BIB1_NO_SYNTAXES_AVAILABLE_FOR_THIS_REQUEST;
 211     }
 212
 213     if (!parse_zebra_elem(elemsetname,
 214                           &retrieval_index, &retrieval_index_len,
 215                           &retrieval_type,  &retrieval_type_len))
 216     {
 217         return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
 218     }
 219
 220     if (retrieval_type_len == 0)
 221         return -1;   /* must have a register type specified */
 222     if (!retrieval_index_len ||
 223         retrieval_index_len >= sizeof(retrieval_index_cstr)-1)
 224     {
 225         return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
 226     }
 227
 228     memcpy(retrieval_index_cstr, retrieval_index, retrieval_index_len);
 229     retrieval_index_cstr[retrieval_index_len] = '\0';
 230
 231     memcpy(retrieval_type_cstr, retrieval_type, retrieval_type_len);
 232     retrieval_type_cstr[retrieval_type_len] = '\0';
 233
 234     ord = zebraExplain_lookup_attr_str(zh->reg->zei,
 235                                        zinfo_index_category_sort,
 236                                        retrieval_type_cstr,
 237                                        retrieval_index_cstr);
 238     if (ord == -1)
 239         return -1;  /* is not a sort index */
 240     else
 241     {
 242         char dst_buf[IT_MAX_WORD];
 243         char str[IT_MAX_WORD];
 244         const char *index_type;
 245         const char *db = 0;
 246         const char *string_index = 0;
 247         WRBUF wrbuf = wrbuf_alloc();
 248
 249         zebra_sort_sysno(zh->reg->sort_index, sysno);
 250         zebra_sort_type(zh->reg->sort_index, ord);
 251         zebra_sort_read(zh->reg->sort_index, str);
 252
 253         zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type, &db, &string_index);
 254
 255         zebra_term_untrans(zh, index_type, dst_buf, str);
 256
 257         if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
 258         {
 259             *output_format = yaz_oid_recsyn_xml;
 260             wrbuf_printf(wrbuf, ZEBRA_XML_HEADER_STR
 261                          " sysno=\"" ZINT_FORMAT "\""
 262                          " set=\"zebra::index%s/\">\n",
 263                          sysno, elemsetname);
 264
 265             wrbuf_printf(wrbuf, "  <index name=\"%s\"",
 266                          string_index);
 267             wrbuf_printf(wrbuf, " type=\"%s\">", index_type);
 268             wrbuf_xmlputs(wrbuf, dst_buf);
 269             wrbuf_printf(wrbuf, "</index>\n");
 270             wrbuf_printf(wrbuf, "</record>\n");
 271         }
 272         else if (!oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
 273         {
 274             *output_format = yaz_oid_recsyn_sutrs;
 275
 276             wrbuf_printf(wrbuf, "%s %s %s\n", string_index, index_type,
 277                          dst_buf);
 278         }
 279         *rec_lenp = wrbuf_len(wrbuf);
 280         *rec_bufp = odr_malloc(odr, *rec_lenp);
 281         memcpy(*rec_bufp, wrbuf_buf(wrbuf), *rec_lenp);
 282         wrbuf_destroy(wrbuf);
 283         return 0;
 284     }
 285 }
 286
 287 int zebra_special_index_fetch(ZebraHandle zh, zint sysno, ODR odr,
 288                               Record rec,
 289                               const char *elemsetname,
 290                               const Odr_oid *input_format,
 291                               const Odr_oid **output_format,
 292                               char **rec_bufp, int *rec_lenp)
 293 {
 294     const char *retrieval_index;
 295     size_t retrieval_index_len;
 296     const char *retrieval_type;
 297     size_t retrieval_type_len;
 298     zebra_rec_keys_t keys;
 299     int ret_code = 0;
 300     char retrieval_type_cstr[256];
 301
 302     /* set output variables before processing possible error states */
 303     /* *rec_lenp = 0; */
 304
 305     /* only accept XML and SUTRS requests */
 306     if (oid_oidcmp(input_format, yaz_oid_recsyn_xml)
 307         && oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
 308     {
 309         yaz_log(YLOG_WARN, "unsupported format for element set zebra::%s",
 310                 elemsetname);
 311         *output_format = 0;
 312         return YAZ_BIB1_NO_SYNTAXES_AVAILABLE_FOR_THIS_REQUEST;
 313     }
 314
 315     if (!parse_zebra_elem(elemsetname,
 316                      &retrieval_index, &retrieval_index_len,
 317                      &retrieval_type,  &retrieval_type_len))
 318         return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
 319
 320     if (retrieval_type_len)
 321     {
 322         memcpy(retrieval_type_cstr, retrieval_type, retrieval_type_len);
 323         retrieval_type_cstr[retrieval_type_len] = '\0';
 324     }
 325
 326     if (retrieval_index_len)
 327     {
 328         char retrieval_index_cstr[256];
 329
 330         if (retrieval_index_len < sizeof(retrieval_index_cstr) -1)
 331         {
 332             memcpy(retrieval_index_cstr, retrieval_index, retrieval_index_len);
 333             retrieval_index_cstr[retrieval_index_len] = '\0';
 334
 335             if (zebraExplain_lookup_attr_str(zh->reg->zei,
 336                                              zinfo_index_category_index,
 337                                              (retrieval_type_len == 0 ? 0 :
 338                                               retrieval_type_cstr),
 339                                              retrieval_index_cstr) == -1)
 340                 return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
 341         }
 342     }
 343
 344     keys = zebra_rec_keys_open();
 345     zebra_rec_keys_set_buf(keys, rec->info[recInfo_delKeys],
 346                            rec->size[recInfo_delKeys], 0);
 347
 348     if (!zebra_rec_keys_rewind(keys))
 349     {
 350         ret_code = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
 351     }
 352     else
 353     {
 354         size_t slen;
 355         const char *str;
 356         struct it_key key_in;
 357         WRBUF wrbuf = wrbuf_alloc();
 358
 359         if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
 360         {
 361             *output_format = input_format;
 362             wrbuf_printf(wrbuf, ZEBRA_XML_HEADER_STR
 363                          " sysno=\"" ZINT_FORMAT "\""
 364                          " set=\"zebra::index%s/\">\n",
 365                          sysno, elemsetname);
 366         }
 367         else if (!oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
 368             *output_format = input_format;
 369
 370         while (zebra_rec_keys_read(keys, &str, &slen, &key_in))
 371         {
 372             int i;
 373             int ord = CAST_ZINT_TO_INT(key_in.mem[0]);
 374             const char *index_type;
 375             const char *db = 0;
 376             const char *string_index = 0;
 377             size_t string_index_len;
 378             char dst_buf[IT_MAX_WORD];
 379
 380             zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type, &db,
 381                                     &string_index);
 382             string_index_len = strlen(string_index);
 383
 384             /* process only if index is not defined,
 385                or if defined and matching */
 386             if (retrieval_index == 0
 387                 || (string_index_len == retrieval_index_len
 388                     && !memcmp(string_index, retrieval_index,
 389                                string_index_len)))
 390             {
 391                 /* process only if type is not defined, or is matching */
 392                 if (retrieval_type == 0
 393                     || !strcmp(retrieval_type_cstr, index_type))
 394                 {
 395                     if (zebra_term_untrans(zh, index_type, dst_buf, str))
 396                         *dst_buf = '\0'; /* untrans failed */
 397
 398                     if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
 399                     {
 400                         wrbuf_printf(wrbuf, "  <index name=\"%s\"",
 401                                      string_index);
 402
 403                         wrbuf_printf(wrbuf, " type=\"%s\"", index_type);
 404
 405                         wrbuf_printf(wrbuf, " seq=\"" ZINT_FORMAT "\">",
 406                                      key_in.mem[key_in.len -1]);
 407                         wrbuf_xmlputs(wrbuf, dst_buf);
 408                         wrbuf_printf(wrbuf, "</index>\n");
 409                     }
 410                     else
 411                     {
 412                         wrbuf_printf(wrbuf, "%s ", string_index);
 413
 414                         wrbuf_printf(wrbuf, "%s", index_type);
 415
 416                         for (i = 1; i < key_in.len; i++)
 417                             wrbuf_printf(wrbuf, " " ZINT_FORMAT,
 418                                              key_in.mem[i]);
 419
 420                         wrbuf_printf(wrbuf, " %s", dst_buf);
 421
 422                         wrbuf_printf(wrbuf, "\n");
 423
 424                     }
 425
 426                 }
 427             }
 428         }
 429         if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
 430             wrbuf_printf(wrbuf, "</record>\n");
 431         *rec_lenp = wrbuf_len(wrbuf);
 432         *rec_bufp = odr_malloc(odr, *rec_lenp);
 433         memcpy(*rec_bufp, wrbuf_buf(wrbuf), *rec_lenp);
 434         wrbuf_destroy(wrbuf);
 435     }
 436     zebra_rec_keys_close(keys);
 437     return ret_code;
 438 }
 439
 440
 441 static void retrieve_puts_attr(WRBUF wrbuf, const char *name,
 442                                const char *value)
 443 {
 444     if (value)
 445     {
 446         wrbuf_printf(wrbuf, " %s=\"", name);
 447         wrbuf_xmlputs(wrbuf, value);
 448         wrbuf_printf(wrbuf, "\"");
 449     }
 450 }
 451
 452 static void retrieve_puts_attr_int(WRBUF wrbuf, const char *name,
 453                                const int value)
 454 {
 455     wrbuf_printf(wrbuf, " %s=\"%i\"", name, value);
 456 }
 457
 458 static void retrieve_puts_str(WRBUF wrbuf, const char *name,
 459                                const char *value)
 460 {
 461     if (value)
 462         wrbuf_printf(wrbuf, "%s %s\n", name, value);
 463 }
 464
 465 static void retrieve_puts_int(WRBUF wrbuf, const char *name,
 466                                const int value)
 467 {
 468     wrbuf_printf(wrbuf, "%s %i\n", name, value);
 469 }
 470
 471
 472 static void snippet_xml_record(ZebraHandle zh, WRBUF wrbuf, zebra_snippets *doc)
 473 {
 474     const zebra_snippet_word *doc_w;
 475     int mark_state = 0;
 476
 477     wrbuf_printf(wrbuf, "%s>\n", ZEBRA_XML_HEADER_STR);
 478     for (doc_w = zebra_snippets_constlist(doc); doc_w; doc_w = doc_w->next)
 479     {
 480         if (doc_w->mark)
 481         {
 482             const char *index_type;
 483             const char *db = 0;
 484             const char *string_index = 0;
 485
 486             zebraExplain_lookup_ord(zh->reg->zei, doc_w->ord,
 487                                     &index_type, &db, &string_index);
 488
 489             if (mark_state == 0)
 490             {
 491                 wrbuf_printf(wrbuf, "  <snippet name=\"%s\"",  string_index);
 492                 wrbuf_printf(wrbuf, " type=\"%s\">", index_type);
 493             }
 494             if (doc_w->match)
 495                 wrbuf_puts(wrbuf, "<s>");
 496             /* not printing leading ws */
 497             if (mark_state || !doc_w->ws || doc_w->match)
 498                 wrbuf_xmlputs(wrbuf, doc_w->term);
 499             if (doc_w->match)
 500                 wrbuf_puts(wrbuf, "</s>");
 501         }
 502         else if (mark_state == 1)
 503         {
 504             wrbuf_puts(wrbuf, "</snippet>\n");
 505         }
 506         mark_state = doc_w->mark;
 507     }
 508     if (mark_state == 1)
 509     {
 510         wrbuf_puts(wrbuf, "</snippet>\n");
 511     }
 512     wrbuf_printf(wrbuf, "</record>");
 513 }
 514
 515 int zebra_get_rec_snippets(ZebraHandle zh, zint sysno,
 516                            zebra_snippets *snippets)
 517 {
 518     int return_code = 0;
 519     Record rec = rec_get(zh->reg->records, sysno);
 520     if (!rec)
 521     {
 522         yaz_log(YLOG_WARN, "rec_get fail on sysno=" ZINT_FORMAT, sysno);
 523         return_code = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
 524     }
 525     else
 526     {
 527         const char *file_type = rec->info[recInfo_fileType];
 528         void *recTypeClientData;
 529         RecType rt = recType_byName(zh->reg->recTypes, zh->res,
 530                                     file_type, &recTypeClientData);
 531
 532         if (!rt)
 533             return_code = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
 534         else
 535         {
 536             struct ZebraRecStream stream;
 537             return_code = zebra_create_record_stream(zh, &rec, &stream);
 538             if (return_code == 0)
 539             {
 540                 extract_snippet(zh, snippets, &stream,
 541                                 rt, recTypeClientData);
 542
 543                 stream.destroy(&stream);
 544             }
 545         }
 546         rec_free(&rec);
 547     }
 548     return return_code;
 549 }
 550
 551 static int snippet_fetch(ZebraHandle zh, const char *setname,
 552                          zint sysno, ODR odr,
 553                          const char *elemsetname,
 554                          const Odr_oid *input_format,
 555                          const Odr_oid **output_format,
 556                          char **rec_bufp, int *rec_lenp)
 557 {
 558     zebra_snippets *rec_snippets = zebra_snippets_create();
 559     int return_code = zebra_get_rec_snippets(zh, sysno, rec_snippets);
 560
 561     if (!return_code)
 562     {
 563         WRBUF wrbuf = wrbuf_alloc();
 564         zebra_snippets *hit_snippet = zebra_snippets_create();
 565
 566         zebra_snippets_hit_vector(zh, setname, sysno, hit_snippet);
 567
 568 #if 0
 569         /* for debugging purposes */
 570         yaz_log(YLOG_LOG, "---------------------------");
 571         yaz_log(YLOG_LOG, "REC SNIPPET:");
 572         zebra_snippets_log(rec_snippet, YLOG_LOG, 1);
 573         yaz_log(YLOG_LOG, "---------------------------");
 574         yaz_log(YLOG_LOG, "HIT SNIPPET:");
 575         zebra_snippets_log(hit_snippet, YLOG_LOG, 1);
 576 #endif
 577
 578         zebra_snippets_ring(rec_snippets, hit_snippet, 5, 5);
 579
 580 #if 0
 581         yaz_log(YLOG_LOG, "---------------------------");
 582         yaz_log(YLOG_LOG, "RING SNIPPET:");
 583         zebra_snippets_log(rec_snippets, YLOG_LOG, 1);
 584 #endif
 585         snippet_xml_record(zh, wrbuf, rec_snippets);
 586
 587         *output_format = yaz_oid_recsyn_xml;
 588
 589         if (return_code == 0)
 590         {
 591             *rec_lenp = wrbuf_len(wrbuf);
 592             *rec_bufp = odr_strdup(odr, wrbuf_cstr(wrbuf));
 593         }
 594         wrbuf_destroy(wrbuf);
 595         zebra_snippets_destroy(hit_snippet);
 596     }
 597     zebra_snippets_destroy(rec_snippets);
 598     return return_code;
 599 }
 600
 601 struct term_collect {
 602     const char *term;
 603     int oc;
 604     zint set_occur;
 605 };
 606
 607 zint freq_term(ZebraHandle zh, int ord, const char *term, RSET rset_set)
 608 {
 609     struct rset_key_control *kc = zebra_key_control_create(zh);
 610     char ord_buf[IT_MAX_WORD];
 611     int ord_len = key_SU_encode(ord, ord_buf);
 612     char *info;
 613     zint hits = 0;
 614     NMEM nmem = nmem_create();
 615
 616     strcpy(ord_buf + ord_len, term);
 617
 618     info = dict_lookup(zh->reg->dict, ord_buf);
 619     if (info)
 620     {
 621         ISAM_P isam_p;
 622         RSET rsets[2], rset;
 623         memcpy(&isam_p, info+1, sizeof(ISAM_P));
 624
 625         rsets[0] = zebra_create_rset_isam(zh, nmem, kc, kc->scope, isam_p, 0);
 626         rsets[1] = rset_dup(rset_set);
 627
 628         rset = rset_create_and(nmem, kc, kc->scope, 2, rsets);
 629
 630         zebra_count_set(zh, rset, &hits, zh->approx_limit);
 631
 632         rset_delete(rsets[0]);
 633         rset_delete(rset);
 634     }
 635     (*kc->dec)(kc);
 636     nmem_destroy(nmem);
 637     return hits;
 638 }
 639
 640 int term_qsort_handle(const void *a, const void *b)
 641 {
 642     const struct term_collect *l = a;
 643     const struct term_collect *r = b;
 644     if (l->set_occur < r->set_occur)
 645         return 1;
 646     else if (l->set_occur > r->set_occur)
 647         return -1;
 648     else
 649     {
 650         const char *lterm = l->term ? l->term : "";
 651         const char *rterm = r->term ? r->term : "";
 652         return strcmp(lterm, rterm);
 653     }
 654 }
 655
 656 void term_collect_freq(ZebraHandle zh,
 657                        struct term_collect *col, int no_terms_collect,
 658                        int ord, RSET rset)
 659 {
 660     int i;
 661     for (i = 0; i < no_terms_collect; i++)
 662     {
 663         if (col[i].term)
 664             col[i].set_occur = freq_term(zh, ord, col[i].term, rset);
 665     }
 666     qsort(col, no_terms_collect, sizeof(*col), term_qsort_handle);
 667 }
 668
 669 struct term_collect *term_collect_create(zebra_strmap_t sm,
 670                                          int no_terms_collect,
 671                                          NMEM nmem)
 672 {
 673     const char *term;
 674     void *data_buf;
 675     size_t data_len;
 676     zebra_strmap_it it;
 677     struct term_collect *col = nmem_malloc(nmem,
 678                                            sizeof *col *no_terms_collect);
 679     int i;
 680     for (i = 0; i < no_terms_collect; i++)
 681     {
 682         col[i].term = 0;
 683         col[i].oc = 0;
 684         col[i].set_occur = 0;
 685     }
 686     /* iterate over terms and collect the most frequent ones */
 687     it = zebra_strmap_it_create(sm);
 688     while ((term = zebra_strmap_it_next(it, &data_buf, &data_len)))
 689     {
 690         /* invariant:
 691            col[0] has lowest oc .  col[no_terms_collect-1] has highest oc */
 692         int oc = *(int*) data_buf;
 693         int j = 0;
 694         /* insertion may be slow but terms terms will be "infrequent" and
 695            thus number of iterations should be small below
 696         */
 697         while (j < no_terms_collect && oc > col[j].oc)
 698             j++;
 699         if (j)
 700         {   /* oc <= col[j] and oc > col[j-1] */
 701             --j;
 702             memmove(col, col+1, sizeof(*col) * j);
 703             col[j].term = term;
 704             col[j].oc = oc;
 705         }
 706     }
 707     zebra_strmap_it_destroy(it);
 708     return col;
 709 }
 710
 711 static ZEBRA_RES facet_fetch(ZebraHandle zh, const char *setname,
 712                              ODR odr,
 713                              const char *elemsetname,
 714                              const Odr_oid *input_format,
 715                              const Odr_oid **output_format,
 716                              char **rec_bufp, int *rec_lenp)
 717 {
 718     zint *pos_array;
 719     int i;
 720     int num_recs = 10; /* number of records to analyze */
 721     int max_chunks = 2;
 722     ZebraMetaRecord *poset;
 723     ZEBRA_RES ret = ZEBRA_OK;
 724     int *ord_array;
 725     WRBUF wr = wrbuf_alloc();
 726     int use_xml = 0;
 727     int no_ord = 0;
 728     struct index_spec *spec, *spec_list;
 729     int error;
 730
 731     res_get_int(zh->res, "facetNumRecs", &num_recs);
 732     res_get_int(zh->res, "facetMaxChunks", &max_chunks);
 733
 734     /* see if XML is required for response */
 735     if (oid_oidcmp(input_format, yaz_oid_recsyn_xml) == 0)
 736         use_xml = 1;
 737
 738     spec_list = parse_index_spec(elemsetname, odr_getmem(odr), &error);
 739
 740     if (!spec_list || error)
 741     {
 742         zebra_setError(
 743             zh,
 744             YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_,
 745             0);
 746         return ZEBRA_FAIL;
 747     }
 748
 749     for (spec = spec_list; spec; spec = spec->next)
 750     {
 751         if (!spec->index_type)
 752         {
 753             zebra_setError(
 754                 zh,
 755                 YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_,
 756                 0);
 757             return ZEBRA_FAIL;
 758         }
 759         no_ord++;
 760     }
 761
 762     ord_array = odr_malloc(odr, sizeof(*ord_array) * no_ord);
 763
 764     for (spec = spec_list, i = 0; spec; spec = spec->next, i++)
 765     {
 766         int ord = zebraExplain_lookup_attr_str(zh->reg->zei,
 767                                                zinfo_index_category_index,
 768                                                spec->index_type,
 769                                                spec->index_name);
 770         if (ord == -1)
 771         {
 772             zebra_setError(
 773                 zh,
 774                 YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_,
 775                 0);
 776             return ZEBRA_FAIL;
 777         }
 778         ord_array[i] = ord;
 779     }
 780     pos_array = (zint *) odr_malloc(odr, num_recs * sizeof(*pos_array));
 781     for (i = 0; i < num_recs; i++)
 782         pos_array[i] = i+1;
 783     poset = zebra_meta_records_create(zh, setname, num_recs, pos_array);
 784     if (!poset)
 785     {
 786         zebra_setError(zh, YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
 787                        setname);
 788         ret = ZEBRA_FAIL;
 789     }
 790     else
 791     {
 792         yaz_timing_t timing = yaz_timing_create();
 793         zebra_strmap_t *map_array
 794             = odr_malloc(odr, sizeof *map_array * no_ord);
 795         for (i = 0; i < no_ord; i++)
 796             map_array[i] = zebra_strmap_create();
 797
 798         for (i = 0; i < num_recs; i++)
 799         {
 800             int j;
 801             zint sysnos[MAX_SYSNOS_PER_RECORD];
 802             int no_sysnos = MAX_SYSNOS_PER_RECORD;
 803             if (!poset[i].sysno)
 804                 continue;
 805             ret = zebra_result_recid_to_sysno(zh,  setname,
 806                                               poset[i].sysno,
 807                                               sysnos, &no_sysnos);
 808             assert(no_sysnos > 0);
 809             yaz_log(YLOG_DEBUG, "Analyzing rec=%d ISAM sysno=" ZINT_FORMAT " chunks=%d",
 810                     i, poset[i].sysno, no_sysnos);
 811             for (j = 0; j < no_sysnos && j < max_chunks; j++)
 812             {
 813                 size_t slen;
 814                 const char *str;
 815                 struct it_key key_in;
 816                 Record rec = rec_get(zh->reg->records, sysnos[j]);
 817                 zebra_rec_keys_t keys = zebra_rec_keys_open();
 818                 zebra_rec_keys_set_buf(keys, rec->info[recInfo_delKeys],
 819                                        rec->size[recInfo_delKeys], 0);
 820
 821                 yaz_log(YLOG_DEBUG, "rec %d " ZINT_FORMAT " %s",
 822                         j, sysnos[j], zebra_rec_keys_empty(keys) ? "empty" : "non-empty");
 823                 if (zebra_rec_keys_rewind(keys))
 824                 {
 825                     while (zebra_rec_keys_read(keys, &str, &slen, &key_in))
 826                     {
 827                         int i;
 828                         struct index_spec *spec;
 829                         for (spec = spec_list, i = 0; i < no_ord;
 830                              i++, spec = spec->next)
 831                         {
 832                             int ord = CAST_ZINT_TO_INT(key_in.mem[0]);
 833                             if (ord == ord_array[i] &&
 834                                 str[0] != FIRST_IN_FIELD_CHAR)
 835                             {
 836                                 int *freq;
 837                                 zebra_strmap_t sm = map_array[i];
 838
 839                                 freq = zebra_strmap_lookup(sm, str, 0, 0);
 840                                 if (freq)
 841                                     (*freq)++;
 842                                 else
 843                                 {
 844                                     int v = 1;
 845                                     zebra_strmap_add(sm, str, &v, sizeof v);
 846                                 }
 847                             }
 848                         }
 849                     }
 850                 }
 851                 zebra_rec_keys_close(keys);
 852                 rec_free(&rec);
 853             }
 854         }
 855         yaz_timing_stop(timing);
 856         yaz_log(YLOG_LOG, "facet first phase real=%4.2f",
 857                 yaz_timing_get_real(timing));
 858         yaz_timing_start(timing);
 859         if (use_xml)
 860             wrbuf_puts(wr, "<facets>\n");
 861         for (spec = spec_list, i = 0; i < no_ord; i++, spec = spec->next)
 862         {
 863             int j;
 864             NMEM nmem = nmem_create();
 865             struct term_collect *col;
 866             int no_collect_terms = 20;
 867
 868             if (spec->extra)
 869                 no_collect_terms = atoi(spec->extra);
 870             if (no_collect_terms < 1)
 871                 no_collect_terms = 1;
 872             col = term_collect_create(map_array[i], no_collect_terms, nmem);
 873             term_collect_freq(zh, col, no_collect_terms, ord_array[i],
 874                               resultSetRef(zh, setname));
 875
 876             if (use_xml)
 877                 wrbuf_printf(wr, "  <facet type=\"%s\" index=\"%s\">\n",
 878                              spec->index_type, spec->index_name);
 879             else
 880                 wrbuf_printf(wr, "facet %s %s\n",
 881                              spec->index_type, spec->index_name);
 882             for (j = 0; j < no_collect_terms; j++)
 883             {
 884                 if (col[j].term)
 885                 {
 886                     char dst_buf[IT_MAX_WORD];
 887                     zebra_term_untrans(zh, spec->index_type, dst_buf, col[j].term);
 888                     if (use_xml)
 889                     {
 890                         wrbuf_printf(wr, "    <term coccur=\"%d\"", col[j].oc);
 891                         if (col[j].set_occur)
 892                             wrbuf_printf(wr, " occur=\"" ZINT_FORMAT "\"",
 893                                          col[j].set_occur);
 894                         wrbuf_printf(wr, ">");
 895                         wrbuf_xmlputs(wr, dst_buf);
 896                         wrbuf_printf(wr, "</term>\n");
 897                     }
 898                     else
 899                     {
 900                         wrbuf_printf(wr, "term %d", col[j].oc);
 901                         if (col[j].set_occur)
 902                             wrbuf_printf(wr, " " ZINT_FORMAT,
 903                                          col[j].set_occur);
 904                         wrbuf_printf(wr, ": %s\n", dst_buf);
 905                     }
 906                 }
 907             }
 908             if (use_xml)
 909                 wrbuf_puts(wr, "  </facet>\n");
 910             nmem_destroy(nmem);
 911         }
 912         if (use_xml)
 913             wrbuf_puts(wr, "</facets>\n");
 914         for (i = 0; i < no_ord; i++)
 915             zebra_strmap_destroy(map_array[i]);
 916         yaz_timing_stop(timing);
 917         yaz_log(YLOG_LOG, "facet second phase real=%4.2f",
 918                 yaz_timing_get_real(timing));
 919         yaz_timing_destroy(&timing);
 920     }
 921     *rec_bufp = odr_strdup(odr, wrbuf_cstr(wr));
 922     wrbuf_destroy(wr);
 923     *rec_lenp = strlen(*rec_bufp);
 924     *output_format = yaz_oid_recsyn_xml;
 925
 926     zebra_meta_records_destroy(zh, poset, num_recs);
 927     return ret;
 928 }
 929
 930 int zebra_special_fetch(ZebraHandle zh, const char *setname,
 931                         zint sysno, int score, ODR odr,
 932                         const char *elemsetname,
 933                         const Odr_oid *input_format,
 934                         const Odr_oid **output_format,
 935                         char **rec_bufp, int *rec_lenp)
 936 {
 937     Record rec;
 938
 939     /* set output variables before processing possible error states */
 940     /* *rec_lenp = 0; */
 941
 942     if (elemsetname && 0 == strncmp(elemsetname, "facet", 5))
 943     {
 944         return facet_fetch(zh, setname, odr,
 945                            elemsetname + 5,
 946                            input_format, output_format,
 947                            rec_bufp, rec_lenp);
 948     }
 949
 950     if (elemsetname && 0 == strcmp(elemsetname, "snippet"))
 951     {
 952         return snippet_fetch(zh, setname, sysno, odr,
 953                              elemsetname + 7,
 954                              input_format, output_format,
 955                              rec_bufp, rec_lenp);
 956     }
 957
 958     /* processing zebra::meta::sysno elemset without fetching binary data */
 959     if (elemsetname && 0 == strcmp(elemsetname, "meta::sysno"))
 960     {
 961         int ret = 0;
 962         WRBUF wrbuf = wrbuf_alloc();
 963         if (!oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
 964         {
 965             wrbuf_printf(wrbuf, ZINT_FORMAT, sysno);
 966             *output_format = input_format;
 967         }
 968         else if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
 969         {
 970             wrbuf_printf(wrbuf, ZEBRA_XML_HEADER_STR
 971                          " sysno=\"" ZINT_FORMAT "\"/>\n",
 972                          sysno);
 973             *output_format = input_format;
 974         }
 975         *rec_lenp = wrbuf_len(wrbuf);
 976         if (*rec_lenp)
 977             *rec_bufp = odr_strdup(odr, wrbuf_cstr(wrbuf));
 978         else
 979             ret = YAZ_BIB1_NO_SYNTAXES_AVAILABLE_FOR_THIS_REQUEST;
 980         wrbuf_destroy(wrbuf);
 981         return ret;
 982     }
 983
 984     /* processing special elementsetname zebra::index:: for sort elements */
 985     if (elemsetname && 0 == strncmp(elemsetname, "index", 5))
 986     {
 987         int ret = zebra_special_sort_fetch(zh, sysno, odr,
 988                                            elemsetname + 5,
 989                                            input_format, output_format,
 990                                            rec_bufp, rec_lenp);
 991         if (ret != -1)
 992             return ret;
 993         /* not a sort index so we continue to get the full record */
 994     }
 995
 996
 997     /* fetching binary record up for all other display elementsets */
 998     rec = rec_get(zh->reg->records, sysno);
 999     if (!rec)
1000     {
1001         yaz_log(YLOG_WARN, "rec_get fail on sysno=" ZINT_FORMAT, sysno);
1002         return YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
1003     }
1004
1005     /* processing special elementsetnames zebra::data */
1006     if (elemsetname && 0 == strcmp(elemsetname, "data"))
1007     {
1008         struct ZebraRecStream stream;
1009         RecordAttr *recordAttr = rec_init_attr(zh->reg->zei, rec);
1010         zebra_create_record_stream(zh, &rec, &stream);
1011         *output_format = input_format;
1012         *rec_lenp = recordAttr->recordSize;
1013         *rec_bufp = (char *) odr_malloc(odr, *rec_lenp);
1014         stream.readf(&stream, *rec_bufp, *rec_lenp);
1015         stream.destroy(&stream);
1016         rec_free(&rec);
1017         return 0;
1018     }
1019
1020     /* only accept XML and SUTRS requests from now */
1021     if (oid_oidcmp(input_format, yaz_oid_recsyn_xml)
1022         && oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
1023     {
1024         yaz_log(YLOG_WARN, "unsupported format for element set zebra::%s",
1025                 elemsetname);
1026         return YAZ_BIB1_NO_SYNTAXES_AVAILABLE_FOR_THIS_REQUEST;
1027     }
1028
1029
1030     /* processing special elementsetnames zebra::meta:: */
1031     if (elemsetname && 0 == strcmp(elemsetname, "meta"))
1032     {
1033         int ret = 0;
1034         WRBUF wrbuf = wrbuf_alloc();
1035         RecordAttr *recordAttr = rec_init_attr(zh->reg->zei, rec);
1036
1037         if (!oid_oidcmp(input_format, yaz_oid_recsyn_xml))
1038         {
1039             *output_format = input_format;
1040
1041             wrbuf_printf(wrbuf, ZEBRA_XML_HEADER_STR
1042                          " sysno=\"" ZINT_FORMAT "\"", sysno);
1043             retrieve_puts_attr(wrbuf, "base", rec->info[recInfo_databaseName]);
1044             retrieve_puts_attr(wrbuf, "file", rec->info[recInfo_filename]);
1045             retrieve_puts_attr(wrbuf, "type", rec->info[recInfo_fileType]);
1046             if (score >= 0)
1047                 retrieve_puts_attr_int(wrbuf, "score", score);
1048
1049             wrbuf_printf(wrbuf,
1050                          " rank=\"" ZINT_FORMAT "\""
1051                          " size=\"%i\""
1052                          " set=\"zebra::%s\"/>\n",
1053                          recordAttr->staticrank,
1054                          recordAttr->recordSize,
1055                          elemsetname);
1056         }
1057         else if (!oid_oidcmp(input_format, yaz_oid_recsyn_sutrs))
1058         {
1059             *output_format = input_format;
1060             wrbuf_printf(wrbuf, "sysno " ZINT_FORMAT "\n", sysno);
1061             retrieve_puts_str(wrbuf, "base", rec->info[recInfo_databaseName]);
1062             retrieve_puts_str(wrbuf, "file", rec->info[recInfo_filename]);
1063             retrieve_puts_str(wrbuf, "type", rec->info[recInfo_fileType]);
1064             if (score >= 0)
1065                 retrieve_puts_int(wrbuf, "score", score);
1066
1067             wrbuf_printf(wrbuf,
1068                          "rank " ZINT_FORMAT "\n"
1069                          "size %i\n"
1070                          "set zebra::%s\n",
1071                          recordAttr->staticrank,
1072                          recordAttr->recordSize,
1073                          elemsetname);
1074         }
1075         *rec_lenp = wrbuf_len(wrbuf);
1076         if (*rec_lenp)
1077             *rec_bufp = odr_strdup(odr, wrbuf_cstr(wrbuf));
1078         else
1079             ret = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
1080
1081         wrbuf_destroy(wrbuf);
1082         rec_free(&rec);
1083         return ret;
1084     }
1085
1086     /* processing special elementsetnames zebra::index:: */
1087     if (elemsetname && 0 == strncmp(elemsetname, "index", 5))
1088     {
1089         int ret = zebra_special_index_fetch(zh, sysno, odr, rec,
1090                                             elemsetname + 5,
1091                                             input_format, output_format,
1092                                             rec_bufp, rec_lenp);
1093
1094         rec_free(&rec);
1095         return ret;
1096     }
1097
1098     if (rec)
1099         rec_free(&rec);
1100     return YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_;
1101 }
1102
1103
1104 int zebra_record_fetch(ZebraHandle zh, const char *setname,
1105                        zint sysno, int score,
1106                        ODR odr,
1107                        const Odr_oid *input_format, Z_RecordComposition *comp,
1108                        const Odr_oid **output_format,
1109                        char **rec_bufp, int *rec_lenp, char **basenamep,
1110                        char **addinfo)
1111 {
1112     Record rec;
1113     char *fname, *file_type, *basename;
1114     const char *elemsetname;
1115     struct ZebraRecStream stream;
1116     RecordAttr *recordAttr;
1117     void *clientData;
1118     int return_code = 0;
1119     zint sysnos[MAX_SYSNOS_PER_RECORD];
1120     int no_sysnos = MAX_SYSNOS_PER_RECORD;
1121     ZEBRA_RES res;
1122
1123     res = zebra_result_recid_to_sysno(zh, setname, sysno, sysnos, &no_sysnos);
1124     if (res != ZEBRA_OK)
1125         return ZEBRA_FAIL;
1126
1127     sysno = sysnos[0];
1128     *basenamep = 0;
1129     *addinfo = 0;
1130     elemsetname = yaz_get_esn(comp);
1131
1132     /* processing zebra special elementset names of form 'zebra:: */
1133     if (elemsetname && 0 == strncmp(elemsetname, "zebra::", 7))
1134         return  zebra_special_fetch(zh, setname, sysno, score, odr,
1135                                     elemsetname + 7,
1136                                     input_format, output_format,
1137                                     rec_bufp, rec_lenp);
1138
1139
1140     /* processing all other element set names */
1141     rec = rec_get(zh->reg->records, sysno);
1142     if (!rec)
1143     {
1144         yaz_log(YLOG_WARN, "rec_get fail on sysno=" ZINT_FORMAT, sysno);
1145         *basenamep = 0;
1146         return YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
1147     }
1148
1149
1150     recordAttr = rec_init_attr(zh->reg->zei, rec);
1151
1152     file_type = rec->info[recInfo_fileType];
1153     fname = rec->info[recInfo_filename];
1154     basename = rec->info[recInfo_databaseName];
1155     *basenamep = (char *) odr_malloc(odr, strlen(basename)+1);
1156     strcpy(*basenamep, basename);
1157
1158     yaz_log(YLOG_DEBUG, "retrieve localno=" ZINT_FORMAT " score=%d",
1159             sysno, score);
1160
1161     return_code = zebra_create_record_stream(zh, &rec, &stream);
1162
1163     if (rec)
1164     {
1165         RecType rt;
1166         struct recRetrieveCtrl retrieveCtrl;
1167
1168         retrieveCtrl.stream = &stream;
1169         retrieveCtrl.fname = fname;
1170         retrieveCtrl.localno = sysno;
1171         retrieveCtrl.staticrank = recordAttr->staticrank;
1172         retrieveCtrl.score = score;
1173         retrieveCtrl.recordSize = recordAttr->recordSize;
1174         retrieveCtrl.odr = odr;
1175         retrieveCtrl.input_format = retrieveCtrl.output_format = input_format;
1176         retrieveCtrl.comp = comp;
1177         retrieveCtrl.encoding = zh->record_encoding;
1178         retrieveCtrl.diagnostic = 0;
1179         retrieveCtrl.addinfo = 0;
1180         retrieveCtrl.dh = zh->reg->dh;
1181         retrieveCtrl.res = zh->res;
1182         retrieveCtrl.rec_buf = 0;
1183         retrieveCtrl.rec_len = -1;
1184
1185         if (!(rt = recType_byName(zh->reg->recTypes, zh->res,
1186                                   file_type, &clientData)))
1187         {
1188             char addinfo_str[100];
1189
1190             sprintf(addinfo_str, "Could not handle record type %.40s",
1191                     file_type);
1192
1193             *addinfo = odr_strdup(odr, addinfo_str);
1194             return_code = YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS;
1195         }
1196         else
1197         {
1198             (*rt->retrieve)(clientData, &retrieveCtrl);
1199             return_code = retrieveCtrl.diagnostic;
1200
1201             *output_format = retrieveCtrl.output_format;
1202             *rec_bufp = (char *) retrieveCtrl.rec_buf;
1203             *rec_lenp = retrieveCtrl.rec_len;
1204             *addinfo = retrieveCtrl.addinfo;
1205         }
1206
1207         stream.destroy(&stream);
1208         rec_free(&rec);
1209     }
1210
1211     return return_code;
1212 }
1213
1214 /*
1215  * Local variables:
1216  * c-basic-offset: 4
1217  * indent-tabs-mode: nil
1218  * End:
1219  * vim: shiftwidth=4 tabstop=8 expandtab
1220  */
1221