index/zsets.c

   1 /* This file is part of the Zebra server.
   2    Copyright (C) 1994-2011 Index Data
   3
   4 Zebra is free software; you can redistribute it and/or modify it under
   5 the terms of the GNU General Public License as published by the Free
   6 Software Foundation; either version 2, or (at your option) any later
   7 version.
   8
   9 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 for more details.
  13
  14 You should have received a copy of the GNU General Public License
  15 along with this program; if not, write to the Free Software
  16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  17
  18 */
  19
  20
  21 #include <stdio.h>
  22 #include <assert.h>
  23 #ifdef WIN32
  24 #include <io.h>
  25 #else
  26 #include <unistd.h>
  27 #endif
  28
  29 #include "index.h"
  30 #include "rank.h"
  31 #include <yaz/diagbib1.h>
  32 #include <rset.h>
  33
  34 #define ZSET_SORT_MAX_LEVEL 10
  35
  36 struct zebra_set_term_entry {
  37     int reg_type;
  38     char *db;
  39     char *index_name;
  40     char *term;
  41 };
  42
  43 struct zebra_set {
  44     char *name;
  45     RSET rset;
  46     NMEM nmem;
  47     NMEM rset_nmem; /* for creating the rsets in */
  48     zint hits;
  49     int num_bases;
  50     const char **basenames;
  51     Z_RPNQuery *rpn;
  52     Z_SortKeySpecList *sortSpec;
  53     struct zset_sort_info *sort_info;
  54     struct zebra_set_term_entry *term_entries;
  55     int term_entries_max;
  56     struct zebra_set *next;
  57     int locked;
  58     int estimated_hit_count;
  59
  60     zint cache_position;  /* last position */
  61     RSFD cache_rfd;       /* rfd (NULL if not existing) */
  62     zint cache_psysno;    /* sysno for last position */
  63     zint approx_limit;    /* limit before we do approx */
  64 };
  65
  66 struct zset_sort_entry {
  67     zint sysno;
  68     int score;
  69 };
  70
  71 struct zset_sort_info {
  72     int max_entries;
  73     int num_entries;
  74     struct zset_sort_entry *all_entries;
  75     struct zset_sort_entry **entries;
  76 };
  77
  78 static int log_level_set=0;
  79 static int log_level_sort=0;
  80 static int log_level_searchhits=0;
  81 static int log_level_searchterms=0;
  82 static int log_level_resultsets=0;
  83
  84 static void loglevels(void)
  85 {
  86     if (log_level_set)
  87         return;
  88     log_level_sort = yaz_log_module_level("sorting");
  89     log_level_searchhits = yaz_log_module_level("searchhits");
  90     log_level_searchterms = yaz_log_module_level("searchterms");
  91     log_level_resultsets = yaz_log_module_level("resultsets");
  92     log_level_set = 1;
  93 }
  94
  95
  96 static ZEBRA_RES resultSetSearch(ZebraHandle zh, NMEM nmem, NMEM rset_nmem,
  97                                  Z_RPNQuery *rpn, ZebraSet sset)
  98 {
  99     RSET rset = 0;
 100     Z_SortKeySpecList *sort_sequence;
 101     int sort_status, i;
 102     ZEBRA_RES res = ZEBRA_OK;
 103
 104     sort_sequence = (Z_SortKeySpecList *)
 105         nmem_malloc(nmem, sizeof(*sort_sequence));
 106     sort_sequence->num_specs = 10; /* FIXME - Hard-coded number */
 107     sort_sequence->specs = (Z_SortKeySpec **)
 108         nmem_malloc(nmem, sort_sequence->num_specs *
 109                     sizeof(*sort_sequence->specs));
 110     for (i = 0; i<sort_sequence->num_specs; i++)
 111         sort_sequence->specs[i] = 0;
 112
 113     rpn_get_top_approx_limit(zh, rpn->RPNStructure, &sset->approx_limit);
 114
 115     res = rpn_search_top(zh, rpn->RPNStructure, rpn->attributeSetId,
 116                          sset->approx_limit,
 117                          nmem, rset_nmem,
 118                          sort_sequence,
 119                          sset->num_bases, sset->basenames,
 120                          &rset);
 121     if (res != ZEBRA_OK)
 122     {
 123         sset->rset = 0;
 124         return res;
 125     }
 126     for (i = 0; sort_sequence->specs[i]; i++)
 127         ;
 128     sort_sequence->num_specs = i;
 129     rset_set_hits_limit(rset, sset->approx_limit);
 130
 131     if (!i)
 132     {
 133         res = resultSetRank(zh, sset, rset, rset_nmem);
 134     }
 135     else
 136     {
 137         res = resultSetSortSingle(zh, nmem, sset, rset,
 138                                   sort_sequence, &sort_status);
 139     }
 140     sset->rset = rset;
 141     return res;
 142 }
 143
 144
 145 ZEBRA_RES resultSetAddRPN(ZebraHandle zh, NMEM m, Z_RPNQuery *rpn,
 146                           int num_bases, char **basenames,
 147                           const char *setname,
 148                           zint *hits, int *estimated_hit_count)
 149 {
 150     ZebraSet zebraSet;
 151     int i;
 152     ZEBRA_RES res;
 153
 154     *hits = 0;
 155     *estimated_hit_count = 0;
 156
 157     zebraSet = resultSetAdd(zh, setname, 1);
 158     if (!zebraSet)
 159         return ZEBRA_FAIL;
 160     zebraSet->locked = 1;
 161     zebraSet->rpn = 0;
 162     zebraSet->nmem = m;
 163     zebraSet->rset_nmem = nmem_create();
 164
 165     zebraSet->num_bases = num_bases;
 166     zebraSet->basenames =
 167         nmem_malloc(zebraSet->nmem, num_bases * sizeof(*zebraSet->basenames));
 168     for (i = 0; i<num_bases; i++)
 169         zebraSet->basenames[i] = nmem_strdup(zebraSet->nmem, basenames[i]);
 170
 171     res = resultSetSearch(zh, zebraSet->nmem, zebraSet->rset_nmem,
 172                           rpn, zebraSet);
 173     *hits = zebraSet->hits;
 174     if (zebraSet->estimated_hit_count)
 175         *estimated_hit_count = 1;
 176
 177     if (zebraSet->rset)
 178         zebraSet->rpn = rpn;
 179     zebraSet->locked = 0;
 180     if (!zebraSet->rset)
 181         return ZEBRA_FAIL;
 182     return res;
 183 }
 184
 185 void resultSetAddTerm(ZebraHandle zh, ZebraSet s, int reg_type,
 186                       const char *db, const char *index_name,
 187                       const char *term)
 188 {
 189     assert(zh); /* compiler shut up */
 190     if (!s->nmem)
 191         s->nmem = nmem_create();
 192     if (!s->term_entries)
 193     {
 194         int i;
 195         s->term_entries_max = 1000;
 196         s->term_entries =
 197             nmem_malloc(s->nmem, s->term_entries_max *
 198                         sizeof(*s->term_entries));
 199         for (i = 0; i < s->term_entries_max; i++)
 200             s->term_entries[i].term = 0;
 201     }
 202     if (s->hits < s->term_entries_max)
 203     {
 204         s->term_entries[s->hits].reg_type = reg_type;
 205         s->term_entries[s->hits].db = nmem_strdup(s->nmem, db);
 206         s->term_entries[s->hits].index_name = nmem_strdup(s->nmem, index_name);
 207         s->term_entries[s->hits].term = nmem_strdup(s->nmem, term);
 208     }
 209     (s->hits)++;
 210 }
 211
 212 ZebraSet resultSetAdd(ZebraHandle zh, const char *name, int ov)
 213 {
 214     ZebraSet s;
 215     int i;
 216
 217     for (s = zh->sets; s; s = s->next)
 218         if (!strcmp(s->name, name))
 219             break;
 220
 221     if (!log_level_set)
 222         loglevels();
 223     if (s)
 224     {
 225         yaz_log(log_level_resultsets, "updating result set %s", name);
 226         if (!ov || s->locked)
 227             return NULL;
 228         if (s->rset)
 229         {
 230             if (s->cache_rfd)
 231                 rset_close(s->cache_rfd);
 232             rset_delete(s->rset);
 233         }
 234         if (s->rset_nmem)
 235             nmem_destroy(s->rset_nmem);
 236         if (s->nmem)
 237             nmem_destroy(s->nmem);
 238     }
 239     else
 240     {
 241         const char *sort_max_str = zebra_get_resource(zh, "sortmax", "1000");
 242
 243         yaz_log(log_level_resultsets, "adding result set %s", name);
 244         s = (ZebraSet) xmalloc(sizeof(*s));
 245         s->next = zh->sets;
 246         zh->sets = s;
 247         s->name = xstrdup(name);
 248
 249         s->sort_info = (struct zset_sort_info *)
 250             xmalloc(sizeof(*s->sort_info));
 251         s->sort_info->max_entries = atoi(sort_max_str);
 252         if (s->sort_info->max_entries < 2)
 253             s->sort_info->max_entries = 2;
 254
 255         s->sort_info->entries = (struct zset_sort_entry **)
 256             xmalloc(sizeof(*s->sort_info->entries) *
 257                     s->sort_info->max_entries);
 258         s->sort_info->all_entries = (struct zset_sort_entry *)
 259             xmalloc(sizeof(*s->sort_info->all_entries) *
 260                     s->sort_info->max_entries);
 261         for (i = 0; i < s->sort_info->max_entries; i++)
 262             s->sort_info->entries[i] = s->sort_info->all_entries + i;
 263     }
 264     s->locked = 0;
 265     s->term_entries = 0;
 266     s->hits = 0;
 267     s->rset = 0;
 268     s->rset_nmem = 0;
 269     s->nmem = 0;
 270     s->rpn = 0;
 271     s->sortSpec = 0;
 272     s->cache_position = 0;
 273     s->cache_rfd = 0;
 274     s->approx_limit = zh->approx_limit;
 275     s->estimated_hit_count = 0;
 276     return s;
 277 }
 278
 279 ZebraSet resultSetGet(ZebraHandle zh, const char *name)
 280 {
 281     ZebraSet s;
 282
 283     for (s = zh->sets; s; s = s->next)
 284         if (!strcmp(s->name, name))
 285         {
 286             if (!s->term_entries && !s->rset && s->rpn)
 287             {
 288                 NMEM nmem = nmem_create();
 289                 yaz_log(log_level_resultsets, "research %s", name);
 290                 if (!s->rset_nmem)
 291                     s->rset_nmem = nmem_create();
 292                 resultSetSearch(zh, nmem, s->rset_nmem, s->rpn, s);
 293                 if (s->rset && s->sortSpec)
 294                 {
 295                     int sort_status;
 296                     yaz_log(log_level_resultsets, "resort %s", name);
 297                     resultSetSortSingle(zh, nmem, s, s->rset, s->sortSpec,
 298                                         &sort_status);
 299                 }
 300                 nmem_destroy(nmem);
 301             }
 302             return s;
 303         }
 304     return NULL;
 305 }
 306
 307 ZEBRA_RES resultSetGetBaseNames(ZebraHandle zh, const char *setname,
 308                                 const char ***basenames, int *num_bases)
 309 {
 310     ZebraSet sset = resultSetGet(zh, setname);
 311     if (!sset)
 312         return ZEBRA_FAIL;
 313     *basenames = sset->basenames;
 314     *num_bases = sset->num_bases;
 315     return ZEBRA_OK;
 316
 317 }
 318
 319 void resultSetInvalidate(ZebraHandle zh)
 320 {
 321     ZebraSet s = zh->sets;
 322
 323     yaz_log(log_level_resultsets, "invalidating result sets");
 324     for (; s; s = s->next)
 325     {
 326         if (s->rset)
 327         {
 328             if (s->cache_rfd)
 329                 rset_close(s->cache_rfd);
 330             rset_delete(s->rset);
 331         }
 332         s->rset = 0;
 333         s->cache_rfd = 0;
 334         s->cache_position = 0;
 335         if (s->rset_nmem)
 336             nmem_destroy(s->rset_nmem);
 337         s->rset_nmem=0;
 338     }
 339 }
 340
 341 void resultSetDestroy(ZebraHandle zh, int num, char **names,int *statuses)
 342 {
 343     ZebraSet * ss = &zh->sets;
 344     int i;
 345
 346     if (statuses)
 347         for (i = 0; i<num; i++)
 348             statuses[i] = Z_DeleteStatus_resultSetDidNotExist;
 349     while (*ss)
 350     {
 351         int i = -1;
 352         ZebraSet s = *ss;
 353         if (num >= 0)
 354         {
 355             for (i = 0; i<num; i++)
 356                 if (!strcmp(s->name, names[i]))
 357                 {
 358                     if (statuses)
 359                         statuses[i] = Z_DeleteStatus_success;
 360                     i = -1;
 361                     break;
 362                 }
 363         }
 364         if (i < 0)
 365         {
 366             *ss = s->next;
 367
 368             xfree(s->sort_info->all_entries);
 369             xfree(s->sort_info->entries);
 370             xfree(s->sort_info);
 371
 372             if (s->nmem)
 373                 nmem_destroy(s->nmem);
 374             if (s->rset)
 375             {
 376                 if (s->cache_rfd)
 377                     rset_close(s->cache_rfd);
 378                 rset_delete(s->rset);
 379             }
 380             if (s->rset_nmem)
 381                 nmem_destroy(s->rset_nmem);
 382             xfree(s->name);
 383             xfree(s);
 384         }
 385         else
 386             ss = &s->next;
 387     }
 388 }
 389
 390 ZebraMetaRecord *zebra_meta_records_create_range(ZebraHandle zh,
 391                                                  const char *name,
 392                                                  zint start, int num)
 393 {
 394     zint pos_small[10];
 395     zint *pos = pos_small;
 396     ZebraMetaRecord *mr;
 397     int i;
 398
 399     if (num > 10000 || num <= 0)
 400         return 0;
 401
 402     if (num > 10)
 403         pos = xmalloc(sizeof(*pos) * num);
 404
 405     for (i = 0; i<num; i++)
 406         pos[i] = start+i;
 407
 408     mr = zebra_meta_records_create(zh, name, num, pos);
 409
 410     if (num > 10)
 411         xfree(pos);
 412     return mr;
 413 }
 414
 415 ZebraMetaRecord *zebra_meta_records_create(ZebraHandle zh, const char *name,
 416                                            int num, zint *positions)
 417 {
 418     ZebraSet sset;
 419     ZebraMetaRecord *sr = 0;
 420     RSET rset;
 421     int i;
 422     struct zset_sort_info *sort_info;
 423     size_t sysno_mem_index = 0;
 424
 425     if (zh->m_staticrank)
 426         sysno_mem_index = 1;
 427
 428     if (!log_level_set)
 429         loglevels();
 430     if (!(sset = resultSetGet(zh, name)))
 431         return NULL;
 432     if (!(rset = sset->rset))
 433     {
 434         if (!sset->term_entries)
 435             return 0;
 436         sr = (ZebraMetaRecord *) xmalloc(sizeof(*sr) * num);
 437         for (i = 0; i<num; i++)
 438         {
 439             sr[i].sysno = 0;
 440             sr[i].score = -1;
 441             sr[i].term = 0;
 442             sr[i].db = 0;
 443
 444             if (positions[i] <= sset->term_entries_max)
 445             {
 446                 sr[i].term = sset->term_entries[positions[i]-1].term;
 447                 sr[i].db = sset->term_entries[positions[i]-1].db;
 448             }
 449         }
 450     }
 451     else
 452     {
 453         sr = (ZebraMetaRecord *) xmalloc(sizeof(*sr) * num);
 454         for (i = 0; i<num; i++)
 455         {
 456             sr[i].sysno = 0;
 457             sr[i].score = -1;
 458             sr[i].term = 0;
 459             sr[i].db = 0;
 460         }
 461         sort_info = sset->sort_info;
 462         if (sort_info)
 463         {
 464             zint position;
 465
 466             for (i = 0; i<num; i++)
 467             {
 468                 position = positions[i];
 469                 if (position > 0 && position <= sort_info->num_entries)
 470                 {
 471                     yaz_log(log_level_sort, "got pos=" ZINT_FORMAT
 472                             " (sorted)", position);
 473                     sr[i].sysno = sort_info->entries[position-1]->sysno;
 474                     sr[i].score = sort_info->entries[position-1]->score;
 475                 }
 476             }
 477         }
 478         /* did we really get all entries using sort ? */
 479         for (i = 0; i<num; i++)
 480         {
 481             if (!sr[i].sysno)
 482                 break;
 483         }
 484         if (i < num) /* nope, get the rest, unsorted - sorry */
 485         {
 486             zint position = 0;
 487             int num_i = 0;
 488             zint psysno = 0;
 489             RSFD rfd;
 490             struct it_key key;
 491
 492             if (sort_info)
 493                 position = sort_info->num_entries;
 494             while (num_i < num && positions[num_i] <= position)
 495                 num_i++;
 496
 497             if (sset->cache_rfd &&
 498                 num_i < num && positions[num_i] > sset->cache_position)
 499             {
 500                 position = sset->cache_position;
 501                 rfd = sset->cache_rfd;
 502                 psysno = sset->cache_psysno;
 503             }
 504             else
 505             {
 506                 if (sset->cache_rfd)
 507                     rset_close(sset->cache_rfd);
 508                 rfd = rset_open(rset, RSETF_READ);
 509             }
 510             while (num_i < num && rset_read(rfd, &key, 0))
 511             {
 512                 zint this_sys = key.mem[sysno_mem_index];
 513                 if (this_sys != psysno)
 514                 {
 515                     psysno = this_sys;
 516                     if (sort_info)
 517                     {
 518                         /* determine we alreay have this in our set */
 519                         for (i = sort_info->num_entries; --i >= 0; )
 520                             if (psysno == sort_info->entries[i]->sysno)
 521                                 break;
 522                         if (i >= 0)
 523                             continue;
 524                     }
 525                     position++;
 526                     assert(num_i < num);
 527                     if (position == positions[num_i])
 528                     {
 529                         sr[num_i].sysno = psysno;
 530                         yaz_log(log_level_sort, "got pos=" ZINT_FORMAT " (unsorted)", position);
 531                         sr[num_i].score = -1;
 532                         num_i++;
 533                     }
 534                 }
 535             }
 536             sset->cache_position = position;
 537             sset->cache_psysno = psysno;
 538             sset->cache_rfd = rfd;
 539         }
 540     }
 541     return sr;
 542 }
 543
 544 void zebra_meta_records_destroy(ZebraHandle zh, ZebraMetaRecord *records,
 545                                 int num)
 546 {
 547     assert(zh); /* compiler shut up about unused arg */
 548     xfree(records);
 549 }
 550
 551 struct sortKeyInfo {
 552     int relation;
 553     int *ord; /* array of ord for each database searched */
 554     int *numerical; /* array of ord for each database searched */
 555     const char *index_type;
 556 };
 557
 558 void resultSetInsertSort(ZebraHandle zh, ZebraSet sset,
 559                          int database_no,
 560                          struct sortKeyInfo *criteria, int num_criteria,
 561                          zint sysno,
 562                          char *cmp_buf[], char *tmp_cmp_buf[])
 563 {
 564     struct zset_sort_entry *new_entry = NULL;
 565     struct zset_sort_info *sort_info = sset->sort_info;
 566     int i, j;
 567     WRBUF w = wrbuf_alloc();
 568
 569     zebra_sort_sysno(zh->reg->sort_index, sysno);
 570     for (i = 0; i<num_criteria; i++)
 571     {
 572         char *this_entry_buf = tmp_cmp_buf[i];
 573         memset(this_entry_buf, '\0', SORT_IDX_ENTRYSIZE);
 574
 575         if (criteria[i].ord[database_no] != -1)
 576         {
 577             yaz_log(log_level_sort, "pre zebra_sort_type ord is %d",
 578                     criteria[i].ord[database_no]);
 579             zebra_sort_type(zh->reg->sort_index, criteria[i].ord[database_no]);
 580             wrbuf_rewind(w);
 581             if (zebra_sort_read(zh->reg->sort_index, 0, w))
 582             {
 583                 /* consider each sort entry and take lowest/highest one
 584                    of the one as sorting key depending on whether sort is
 585                    ascending/descending */
 586                 int off = 0;
 587                 while (off != wrbuf_len(w))
 588                 {
 589                     size_t l = strlen(wrbuf_buf(w)+off);
 590                     assert(off < wrbuf_len(w));
 591
 592                     if (l >= SORT_IDX_ENTRYSIZE)
 593                         l = SORT_IDX_ENTRYSIZE-1;
 594                     if ( (off == 0)
 595                          || (criteria[i].relation == 'A'
 596                              && strcmp(wrbuf_buf(w)+off, this_entry_buf) < 0)
 597                          || (criteria[i].relation == 'D'
 598                              && strcmp(wrbuf_buf(w)+off, this_entry_buf) > 0)
 599                         )
 600                     {
 601                         memcpy(this_entry_buf, wrbuf_buf(w)+off, l);
 602                         this_entry_buf[l] = '\0';
 603                     }
 604                     off += 1 + strlen(wrbuf_buf(w)+off);
 605                 }
 606             }
 607         }
 608         else
 609         {
 610             yaz_log(log_level_sort, "criteria[i].ord is -1 so not reading from sort index");
 611         }
 612     }
 613     wrbuf_destroy(w);
 614     i = sort_info->num_entries;
 615     while (--i >= 0)
 616     {
 617         int rel = 0;
 618         for (j = 0; j<num_criteria; j++)
 619         {
 620             char *this_entry_buf = tmp_cmp_buf[j];
 621             char *other_entry_buf =
 622                 cmp_buf[j] + i * SORT_IDX_ENTRYSIZE;
 623             if (criteria[j].numerical[database_no])
 624             {
 625                 char this_entry_org[1024];
 626                 char other_entry_org[1024];
 627                 double diff;
 628                 const char *index_type = criteria[j].index_type;
 629                 zebra_term_untrans(zh, index_type, this_entry_org,
 630                                    this_entry_buf);
 631                 zebra_term_untrans(zh, index_type, other_entry_org,
 632                                    other_entry_buf);
 633                 diff = atof(this_entry_org) - atof(other_entry_org);
 634
 635                 if (diff > 0.0)
 636                     rel = 1;
 637                 else if (diff < 0.0)
 638                     rel = -1;
 639                 else
 640                     rel = 0;
 641             }
 642             else
 643             {
 644                 rel = memcmp(this_entry_buf, other_entry_buf,
 645                              SORT_IDX_ENTRYSIZE);
 646             }
 647             /* when the compare is equal, continue to next criteria,
 648                else break out */
 649             if (rel)
 650                 break;
 651         }
 652         if (!rel)
 653             break;
 654         if (criteria[j].relation == 'A')
 655         {
 656             if (rel > 0)
 657                 break;
 658         }
 659         else if (criteria[j].relation == 'D')
 660         {
 661             if (rel < 0)
 662                 break;
 663         }
 664     }
 665     ++i;
 666     yaz_log(log_level_sort, "ok, we want to insert record at position %d",i);
 667     j = sort_info->max_entries;
 668     if (i == j){
 669         yaz_log(log_level_sort, "sort_info->max_entries reached (%d) abort sort",j);
 670         return;
 671     }
 672
 673     if (sort_info->num_entries == j)
 674         --j;
 675     else
 676         j = (sort_info->num_entries)++;
 677     new_entry = sort_info->entries[j];
 678     /* move up all higher entries (to make room) */
 679     while (j != i)
 680     {
 681         int k;
 682         for (k = 0; k<num_criteria; k++)
 683         {
 684             char *j_buf = cmp_buf[k] + j * SORT_IDX_ENTRYSIZE;
 685             char *j_1_buf = cmp_buf[k] + (j-1) * SORT_IDX_ENTRYSIZE;
 686             memcpy(j_buf, j_1_buf, SORT_IDX_ENTRYSIZE);
 687         }
 688         sort_info->entries[j] = sort_info->entries[j-1];
 689         --j;
 690     }
 691     /* and insert the new entry at the correct place */
 692     sort_info->entries[i] = new_entry;
 693     assert(new_entry);
 694     /* and add this to the compare buffer */
 695     for (i = 0; i<num_criteria; i++)
 696     {
 697         char *new_entry_buf = cmp_buf[i] + j * SORT_IDX_ENTRYSIZE;
 698         char *this_entry_buf = tmp_cmp_buf[i];
 699         memcpy(new_entry_buf, this_entry_buf, SORT_IDX_ENTRYSIZE);
 700     }
 701     new_entry->sysno = sysno;
 702     new_entry->score = -1;
 703 }
 704
 705 void resultSetInsertRank(ZebraHandle zh, struct zset_sort_info *sort_info,
 706                          zint sysno, int score, int relation)
 707 {
 708     struct zset_sort_entry *new_entry = NULL;
 709     int i, j;
 710     assert(zh); /* compiler shut up about unused arg */
 711
 712     i = sort_info->num_entries;
 713     while (--i >= 0)
 714     {
 715         int rel = 0;
 716
 717         rel = score - sort_info->entries[i]->score;
 718
 719         if (relation == 'D')
 720         {
 721             if (rel >= 0)
 722                 break;
 723         }
 724         else if (relation == 'A')
 725         {
 726             if (rel <= 0)
 727                 break;
 728         }
 729     }
 730     ++i;
 731     j = sort_info->max_entries;
 732     if (i == j)
 733         return;
 734
 735     if (sort_info->num_entries == j)
 736         --j;
 737     else
 738         j = (sort_info->num_entries)++;
 739
 740     new_entry = sort_info->entries[j];
 741     while (j != i)
 742     {
 743         sort_info->entries[j] = sort_info->entries[j-1];
 744         --j;
 745     }
 746     sort_info->entries[i] = new_entry;
 747     assert(new_entry);
 748     new_entry->sysno = sysno;
 749     new_entry->score = score;
 750 }
 751
 752 static Z_RPNQuery *copy_RPNQuery(Z_RPNQuery *src, NMEM nmem)
 753 {
 754     Z_RPNQuery *dst = 0;
 755     ODR encode = odr_createmem(ODR_ENCODE);
 756     ODR decode = odr_createmem(ODR_DECODE);
 757
 758     if (z_RPNQuery(encode, &src, 0, 0))
 759     {
 760         int len;
 761         char *buf = odr_getbuf(encode, &len, 0);
 762
 763         if (buf)
 764         {
 765             odr_setbuf(decode, buf, len, 0);
 766             z_RPNQuery(decode, &dst, 0, 0);
 767         }
 768     }
 769     nmem_transfer(nmem, decode->mem);
 770     odr_destroy(encode);
 771     odr_destroy(decode);
 772     return dst;
 773 }
 774
 775 static Z_SortKeySpecList *copy_SortKeySpecList(Z_SortKeySpecList *src, NMEM nmem)
 776 {
 777     Z_SortKeySpecList *dst = 0;
 778     ODR encode = odr_createmem(ODR_ENCODE);
 779     ODR decode = odr_createmem(ODR_DECODE);
 780
 781     if (z_SortKeySpecList(encode, &src, 0, 0))
 782     {
 783         int len;
 784         char *buf = odr_getbuf(encode, &len, 0);
 785
 786         if (buf)
 787         {
 788             odr_setbuf(decode, buf, len, 0);
 789             z_SortKeySpecList(decode, &dst, 0, 0);
 790         }
 791     }
 792     nmem_transfer(nmem, decode->mem);
 793     odr_destroy(encode);
 794     odr_destroy(decode);
 795     return dst;
 796 }
 797
 798 ZebraSet resultSetClone(ZebraHandle zh, const char *setname,
 799                         ZebraSet rset)
 800 {
 801     ZebraSet nset;
 802     int i;
 803
 804     nset = resultSetAdd(zh, setname, 1);
 805     if (!nset)
 806         return 0;
 807
 808     nset->nmem = nmem_create();
 809
 810     nset->num_bases = rset->num_bases;
 811     nset->basenames =
 812         nmem_malloc(nset->nmem, nset->num_bases * sizeof(*rset->basenames));
 813     for (i = 0; i<rset->num_bases; i++)
 814         nset->basenames[i] = nmem_strdup(nset->nmem, rset->basenames[i]);
 815
 816     if (rset->rset)
 817         nset->rset = rset_dup(rset->rset);
 818     if (rset->rpn)
 819         nset->rpn = copy_RPNQuery(rset->rpn, nset->nmem);
 820     return nset;
 821 }
 822
 823 ZEBRA_RES resultSetSort(ZebraHandle zh, NMEM nmem,
 824                         int num_input_setnames, const char **input_setnames,
 825                         const char *output_setname,
 826                         Z_SortKeySpecList *sort_sequence, int *sort_status)
 827 {
 828     ZebraSet sset;
 829     RSET rset;
 830
 831     if (num_input_setnames == 0)
 832     {
 833         zebra_setError(zh, YAZ_BIB1_NO_RESULT_SET_NAME_SUPPLIED_ON_SORT, 0);
 834         return ZEBRA_FAIL;
 835     }
 836     if (num_input_setnames > 1)
 837     {
 838         zebra_setError(zh, YAZ_BIB1_SORT_TOO_MANY_INPUT_RESULTS, 0);
 839         return ZEBRA_FAIL;
 840     }
 841     if (!log_level_set)
 842         loglevels();
 843     yaz_log(log_level_sort, "result set sort input=%s output=%s",
 844             *input_setnames, output_setname);
 845     sset = resultSetGet(zh, input_setnames[0]);
 846     if (!sset)
 847     {
 848         zebra_setError(zh, YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
 849                        input_setnames[0]);
 850         return ZEBRA_FAIL;
 851     }
 852     if (!(rset = sset->rset))
 853     {
 854         zebra_setError(zh, YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
 855                        input_setnames[0]);
 856         return ZEBRA_FAIL;
 857     }
 858     if (strcmp(output_setname, input_setnames[0]))
 859         sset = resultSetClone(zh, output_setname, sset);
 860     sset->sortSpec = copy_SortKeySpecList(sort_sequence, sset->nmem);
 861     return resultSetSortSingle(zh, nmem, sset, rset, sort_sequence,
 862                                sort_status);
 863 }
 864
 865 ZEBRA_RES resultSetSortSingle(ZebraHandle zh, NMEM nmem,
 866                               ZebraSet sset, RSET rset,
 867                               Z_SortKeySpecList *sort_sequence,
 868                               int *sort_status)
 869 {
 870     int i;
 871     int ib;
 872     int n = 0;
 873     zint kno = 0;
 874     zint psysno = 0;
 875     struct it_key key;
 876     struct sortKeyInfo sort_criteria[ZSET_SORT_MAX_LEVEL];
 877     char *cmp_buf[ZSET_SORT_MAX_LEVEL];
 878     char *tmp_cmp_buf[ZSET_SORT_MAX_LEVEL];
 879     int num_criteria;
 880     RSFD rfd;
 881     TERMID termid;
 882     TERMID *terms;
 883     int numTerms = 0;
 884     size_t sysno_mem_index = 0;
 885
 886     int numbases = zh->num_basenames;
 887     yaz_log(log_level_sort, "searching %d databases",numbases);
 888
 889     if (zh->m_staticrank)
 890         sysno_mem_index = 1;
 891
 892     assert(nmem); /* compiler shut up about unused param */
 893     sset->sort_info->num_entries = 0;
 894
 895     rset_getterms(rset, 0, 0, &n);
 896     terms = (TERMID *) nmem_malloc(nmem, sizeof(*terms)*n);
 897     rset_getterms(rset, terms, n, &numTerms);
 898
 899     sset->hits = 0;
 900     num_criteria = sort_sequence->num_specs;
 901     if (num_criteria > ZSET_SORT_MAX_LEVEL)
 902         num_criteria = ZSET_SORT_MAX_LEVEL;
 903     /* set up the search criteria */
 904     for (i = 0; i < num_criteria; i++)
 905     {
 906         Z_SortKeySpec *sks = sort_sequence->specs[i];
 907         Z_SortKey *sk;
 908
 909         sort_criteria[i].ord = (int *)
 910             nmem_malloc(nmem, sizeof(int)*numbases);
 911         sort_criteria[i].numerical = (int *)
 912             nmem_malloc(nmem, sizeof(int)*numbases);
 913
 914         /* initialize ord and numerical for each database */
 915         for (ib = 0; ib < numbases; ib++)
 916         {
 917             sort_criteria[i].ord[ib] = -1;
 918             sort_criteria[i].numerical[ib] = 0;
 919         }
 920
 921         if (sks->which == Z_SortKeySpec_missingValueData)
 922         {
 923             zebra_setError(zh, YAZ_BIB1_UNSUPP_MISSING_DATA_ACTION, 0);
 924             return ZEBRA_FAIL;
 925         }
 926         if (*sks->sortRelation == Z_SortKeySpec_ascending)
 927             sort_criteria[i].relation = 'A';
 928         else if (*sks->sortRelation == Z_SortKeySpec_descending)
 929             sort_criteria[i].relation = 'D';
 930         else
 931         {
 932             zebra_setError(zh, YAZ_BIB1_ILLEGAL_SORT_RELATION, 0);
 933             return ZEBRA_FAIL;
 934         }
 935         if (sks->sortElement->which == Z_SortElement_databaseSpecific)
 936         {
 937             zebra_setError(zh, YAZ_BIB1_DATABASE_SPECIFIC_SORT_UNSUPP, 0);
 938             return ZEBRA_FAIL;
 939         }
 940         else if (sks->sortElement->which != Z_SortElement_generic)
 941         {
 942             zebra_setError(zh, YAZ_BIB1_SORT_ILLEGAL_SORT, 0);
 943             return ZEBRA_FAIL;
 944         }
 945         sk = sks->sortElement->u.generic;
 946         switch (sk->which)
 947         {
 948         case Z_SortKey_sortField:
 949             yaz_log(log_level_sort, "key %d is of type sortField", i+1);
 950             for (ib = 0; ib < numbases; ib++)
 951             {
 952                 zebraExplain_curDatabase(zh->reg->zei, zh->basenames[ib]);
 953                 sort_criteria[i].numerical[ib] = 0;
 954                 sort_criteria[i].ord[ib] =
 955                     zebraExplain_lookup_attr_str(zh->reg->zei,
 956                                                  zinfo_index_category_sort,
 957                                                  0, sk->u.sortField);
 958                 if (sks->which != Z_SortKeySpec_null
 959                     && sort_criteria[i].ord[ib] == -1)
 960                 {
 961                     zebra_setError(zh,
 962                                    YAZ_BIB1_CANNOT_SORT_ACCORDING_TO_SEQUENCE, 0);
 963                     return ZEBRA_FAIL;
 964                 }
 965             }
 966             break;
 967         case Z_SortKey_elementSpec:
 968             yaz_log(log_level_sort, "key %d is of type elementSpec", i+1);
 969             zebra_setError(zh, YAZ_BIB1_CANNOT_SORT_ACCORDING_TO_SEQUENCE, 0);
 970             return ZEBRA_FAIL;
 971         case Z_SortKey_sortAttributes:
 972             yaz_log(log_level_sort, "key %d is of type sortAttributes", i+1);
 973             /* for every database we searched, get the sort index file
 974                id (ord) and its numerical indication and store them in
 975                the sort_criteria */
 976             for (ib = 0; ib < numbases; ib++)
 977             {
 978                 zebraExplain_curDatabase(zh->reg->zei, zh->basenames[ib]);
 979                 if (zebra_sort_get_ord(zh, sk->u.sortAttributes,
 980                                        &sort_criteria[i].ord[ib],
 981                                        &sort_criteria[i].numerical[ib]) !=
 982                     ZEBRA_OK && sks->which != Z_SortKeySpec_null)
 983                     return ZEBRA_FAIL;
 984             }
 985             break;
 986         }
 987         /* right now we look up the index type based on the first database
 988            if the index_type's can differ between the indexes of different
 989            databases (which i guess they can?) then we have to store the
 990            index types for each database, just like the ord and numerical */
 991         if (zebraExplain_lookup_ord(zh->reg->zei, sort_criteria[i].ord[0],
 992                                     &sort_criteria[i].index_type,
 993                                     0, 0))
 994         {
 995             zebra_setError(zh, YAZ_BIB1_CANNOT_SORT_ACCORDING_TO_SEQUENCE, 0);
 996             return ZEBRA_FAIL;
 997         }
 998     }
 999     /* allocate space for each cmpare buf + one extra for tmp comparison */
1000     /* cmp_buf is an array of array, the first dimension is the criteria and the second dimension are
1001        all other result entries to compare against. This is slowly filled when records are processed.
1002        tmp_cmp_buf is an array with a value of the current record for each criteria
1003     */
1004     for (i = 0; i<num_criteria; i++)
1005     {
1006         cmp_buf[i] = xmalloc(sset->sort_info->max_entries
1007                              * SORT_IDX_ENTRYSIZE);
1008         tmp_cmp_buf[i] = xmalloc(SORT_IDX_ENTRYSIZE);
1009     }
1010     rfd = rset_open(rset, RSETF_READ);
1011     while (rset_read(rfd, &key, &termid))
1012     {
1013         zint this_sys = key.mem[sysno_mem_index];
1014         if (log_level_searchhits)
1015             key_logdump_txt(log_level_searchhits, &key, termid->name);
1016         kno++;
1017         if (this_sys != psysno)
1018         {
1019             int database_no = 0;
1020             if ((sset->hits & 255) == 0 && zh->break_handler_func)
1021             {
1022                 if (zh->break_handler_func(zh->break_handler_data))
1023                 {
1024                     rset_set_hits_limit(rset, 0);
1025                     break;
1026                 }
1027             }
1028             (sset->hits)++;
1029             psysno = this_sys;
1030
1031             /* determine database from the term, but only bother if more than
1032                one database is in use*/
1033             if (numbases > 1 && termid->ol)
1034             {
1035                 const char *this_db = 0;
1036                 if (zebraExplain_lookup_ord(zh->reg->zei, termid->ol->ord,  0, &this_db, 0)
1037                     == 0 && this_db)
1038                 {
1039                     for (ib = 0; ib < numbases; ib++)
1040                         if (!strcmp(this_db, zh->basenames[ib]))
1041                             database_no = ib;
1042                 }
1043             }
1044 #if 0
1045             yaz_log(YLOG_LOG, "sysno=" ZINT_FORMAT " database_no=%d", this_sys,
1046                 database_no);
1047             ord_list_print(termid->ol);
1048 #endif
1049             resultSetInsertSort(zh, sset, database_no,
1050                                 sort_criteria, num_criteria, psysno, cmp_buf,
1051                                 tmp_cmp_buf);
1052         }
1053     }
1054     rset_close(rfd);
1055
1056     /* free the compare buffers */
1057     for (i = 0; i<num_criteria; i++)
1058     {
1059         xfree(cmp_buf[i]);
1060         xfree(tmp_cmp_buf[i]);
1061     }
1062
1063     yaz_log(log_level_sort, ZINT_FORMAT " keys, " ZINT_FORMAT " sysnos, sort",
1064             kno, sset->hits);
1065     for (i = 0; i < numTerms; i++)
1066         yaz_log(log_level_sort, "term=\"%s\" type=%s count=" ZINT_FORMAT,
1067                 terms[i]->name, terms[i]->flags, terms[i]->rset->hits_count);
1068     *sort_status = Z_SortResponse_success;
1069     return ZEBRA_OK;
1070 }
1071
1072 RSET resultSetRef(ZebraHandle zh, const char *resultSetId)
1073 {
1074     ZebraSet s;
1075
1076     if ((s = resultSetGet(zh, resultSetId)))
1077         return s->rset;
1078     return NULL;
1079 }
1080
1081 ZEBRA_RES resultSetRank(ZebraHandle zh, ZebraSet zebraSet,
1082                         RSET rset, NMEM nmem)
1083 {
1084     struct it_key key;
1085     TERMID termid;
1086     TERMID *terms;
1087     zint kno = 0;
1088     int numTerms = 0;
1089     int n = 0;
1090     int i;
1091     ZebraRankClass rank_class;
1092     struct zset_sort_info *sort_info;
1093     const char *rank_handler_name = res_get_def(zh->res, "rank", "rank-1");
1094     size_t sysno_mem_index = 0;
1095
1096     if (zh->m_staticrank)
1097         sysno_mem_index = 1;
1098
1099     if (!log_level_set)
1100         loglevels();
1101     sort_info = zebraSet->sort_info;
1102     sort_info->num_entries = 0;
1103     zebraSet->hits = 0;
1104     zebraSet->estimated_hit_count = 0;
1105     rset_getterms(rset, 0, 0, &n);
1106     terms = (TERMID *) nmem_malloc(nmem, sizeof(*terms)*n);
1107     rset_getterms(rset, terms, n, &numTerms);
1108
1109     rank_class = zebraRankLookup(zh, rank_handler_name);
1110     if (!rank_class)
1111     {
1112         yaz_log(YLOG_WARN, "No such rank handler: %s", rank_handler_name);
1113         zebra_setError(zh, YAZ_BIB1_UNSUPP_SEARCH, "Cannot find rank handler");
1114         return ZEBRA_FAIL;
1115     }
1116     else
1117     {
1118         RSFD rfd = rset_open(rset, RSETF_READ);
1119         struct rank_control *rc = rank_class->control;
1120         int score;
1121         zint count = 0;
1122         void *handle = (*rc->begin) (zh->reg, rank_class->class_handle, rset,
1123                                      nmem, terms, numTerms);
1124         zint psysno = 0;  /* previous doc id / sys no */
1125         zint pstaticrank = 0; /* previous static rank */
1126         int stop_flag = 0;
1127         while (rset_read(rfd, &key, &termid))
1128         {
1129             zint this_sys = key.mem[sysno_mem_index];
1130
1131             zint seqno = key.mem[key.len-1];
1132             kno++;
1133             if (log_level_searchhits)
1134                 key_logdump_txt(log_level_searchhits, &key, termid->name);
1135             if (this_sys != psysno)
1136             {   /* new record .. */
1137                 if (!(rfd->counted_items & 255) && zh->break_handler_func)
1138                 {
1139                     if (zh->break_handler_func(zh->break_handler_data))
1140                     {
1141                         yaz_log(YLOG_LOG, "Aborted search");
1142                         stop_flag = 1;
1143                     }
1144                 }
1145                 if (rfd->counted_items > rset->hits_limit)
1146                     stop_flag = 1;
1147                 if (stop_flag)
1148                 {
1149                     zebraSet->estimated_hit_count = 1;
1150                     break;
1151                 }
1152                 if (psysno)
1153                 {   /* only if we did have a previous record */
1154                     score = (*rc->calc)(handle, psysno, pstaticrank,
1155                                         &stop_flag);
1156                     /* insert the hit. A=Ascending */
1157                     resultSetInsertRank(zh, sort_info, psysno, score, 'A');
1158                     count++;
1159                 }
1160                 psysno = this_sys;
1161                 if (zh->m_staticrank)
1162                     pstaticrank = key.mem[0];
1163             }
1164             (*rc->add)(handle, CAST_ZINT_TO_INT(seqno), termid);
1165         }
1166         /* no more items */
1167         if (psysno)
1168         {   /* we had - at least - one record */
1169             score = (*rc->calc)(handle, psysno, pstaticrank, &stop_flag);
1170             /* insert the hit. A=Ascending */
1171             resultSetInsertRank(zh, sort_info, psysno, score, 'A');
1172             count++;
1173         }
1174         (*rc->end)(zh->reg, handle);
1175         rset_close(rfd);
1176     }
1177     zebraSet->hits = rset->hits_count;
1178
1179     yaz_log(log_level_searchterms, ZINT_FORMAT " keys, "
1180             ZINT_FORMAT " sysnos, rank",  kno, zebraSet->hits);
1181     for (i = 0; i < numTerms; i++)
1182     {
1183         yaz_log(log_level_searchterms, "term=\"%s\" type=%s count="
1184                 ZINT_FORMAT,
1185                 terms[i]->name, terms[i]->flags, terms[i]->rset->hits_count);
1186     }
1187     return ZEBRA_OK;
1188 }
1189
1190 ZebraRankClass zebraRankLookup(ZebraHandle zh, const char *name)
1191 {
1192     ZebraRankClass p = zh->reg->rank_classes;
1193     while (p && strcmp(p->control->name, name))
1194         p = p->next;
1195     if (p && !p->init_flag)
1196     {
1197         if (p->control->create)
1198             p->class_handle = (*p->control->create)(zh);
1199         p->init_flag = 1;
1200     }
1201     return p;
1202 }
1203
1204 void zebraRankInstall(struct zebra_register *reg, struct rank_control *ctrl)
1205 {
1206     ZebraRankClass p = (ZebraRankClass) xmalloc(sizeof(*p));
1207     p->control = (struct rank_control *) xmalloc(sizeof(*p->control));
1208     memcpy(p->control, ctrl, sizeof(*p->control));
1209     p->control->name = xstrdup(ctrl->name);
1210     p->init_flag = 0;
1211     p->next = reg->rank_classes;
1212     reg->rank_classes = p;
1213 }
1214
1215 void zebraRankDestroy(struct zebra_register *reg)
1216 {
1217     ZebraRankClass p = reg->rank_classes;
1218     while (p)
1219     {
1220         ZebraRankClass p_next = p->next;
1221         if (p->init_flag && p->control->destroy)
1222             (*p->control->destroy)(reg, p->class_handle);
1223         xfree(p->control->name);
1224         xfree(p->control);
1225         xfree(p);
1226         p = p_next;
1227     }
1228     reg->rank_classes = NULL;
1229 }
1230
1231 static int trav_rset_for_termids(RSET rset, TERMID *termid_array,
1232                                  zint *hits_array, int *approx_array)
1233 {
1234     int no = 0;
1235     int i;
1236     for (i = 0; i<rset->no_children; i++)
1237         no += trav_rset_for_termids(rset->children[i],
1238                                     (termid_array ? termid_array + no : 0),
1239                                     (hits_array ? hits_array + no : 0),
1240                                     (approx_array ? approx_array + no : 0));
1241     if (rset->term)
1242     {
1243         if (termid_array)
1244             termid_array[no] = rset->term;
1245         if (hits_array)
1246             hits_array[no] = rset->hits_count;
1247         if (approx_array)
1248             approx_array[no] = rset->hits_approx;
1249 #if 0
1250         yaz_log(YLOG_LOG, "rset=%p term=%s limit=" ZINT_FORMAT
1251                 " count=" ZINT_FORMAT,
1252                 rset, rset->term->name, rset->hits_limit, rset->hits_count);
1253 #endif
1254         no++;
1255     }
1256     return no;
1257 }
1258
1259 ZEBRA_RES zebra_result_set_term_no(ZebraHandle zh, const char *setname,
1260                                    int *num_terms)
1261 {
1262     ZebraSet sset = resultSetGet(zh, setname);
1263     *num_terms = 0;
1264     if (sset)
1265     {
1266         *num_terms = trav_rset_for_termids(sset->rset, 0, 0, 0);
1267         return ZEBRA_OK;
1268     }
1269     return ZEBRA_FAIL;
1270 }
1271
1272 ZEBRA_RES zebra_result_set_term_info(ZebraHandle zh, const char *setname,
1273                                      int no, zint *count, int *approx,
1274                                      char *termbuf, size_t *termlen,
1275                                      const char **term_ref_id)
1276 {
1277     ZebraSet sset = resultSetGet(zh, setname);
1278     if (sset)
1279     {
1280         int num_terms = trav_rset_for_termids(sset->rset, 0, 0, 0);
1281         if (no >= 0 && no < num_terms)
1282         {
1283             TERMID *term_array = xmalloc(num_terms * sizeof(*term_array));
1284             zint *hits_array = xmalloc(num_terms * sizeof(*hits_array));
1285             int *approx_array = xmalloc(num_terms * sizeof(*approx_array));
1286
1287             trav_rset_for_termids(sset->rset, term_array,
1288                                   hits_array, approx_array);
1289
1290             if (count)
1291                 *count = hits_array[no];
1292             if (approx)
1293                 *approx = approx_array[no];
1294             if (termbuf)
1295             {
1296                 char *inbuf = term_array[no]->name;
1297                 size_t inleft = strlen(inbuf);
1298                 size_t outleft = *termlen - 1;
1299
1300                 if (zh->iconv_from_utf8 != 0)
1301                 {
1302                     char *outbuf = termbuf;
1303                     size_t ret;
1304
1305                     ret = yaz_iconv(zh->iconv_from_utf8, &inbuf, &inleft,
1306                                     &outbuf, &outleft);
1307                     if (ret == (size_t)(-1))
1308                         *termlen = 0;
1309                     else
1310                     {
1311                         yaz_iconv(zh->iconv_from_utf8, 0, 0,
1312                                   &outbuf, &outleft);
1313                         *termlen = outbuf - termbuf;
1314                     }
1315                 }
1316                 else
1317                 {
1318                     if (inleft > outleft)
1319                         inleft = outleft;
1320                     *termlen = inleft;
1321                     memcpy(termbuf, inbuf, *termlen);
1322                 }
1323                 termbuf[*termlen] = '\0';
1324             }
1325             if (term_ref_id)
1326                 *term_ref_id = term_array[no]->ref_id;
1327
1328             xfree(term_array);
1329             xfree(hits_array);
1330             xfree(approx_array);
1331             return ZEBRA_OK;
1332         }
1333     }
1334     return ZEBRA_FAIL;
1335 }
1336
1337 ZEBRA_RES zebra_snippets_hit_vector(ZebraHandle zh, const char *setname,
1338                                     zint sysno, zebra_snippets *snippets)
1339 {
1340     ZebraSet sset = resultSetGet(zh, setname);
1341     yaz_log(YLOG_DEBUG, "zebra_get_hit_vector setname=%s zysno=" ZINT_FORMAT,
1342             setname, sysno);
1343     if (!sset)
1344         return ZEBRA_FAIL;
1345     else
1346     {
1347         struct rset_key_control *kc = zebra_key_control_create(zh);
1348         NMEM nmem = nmem_create();
1349         struct it_key key;
1350         RSET rsets[2], rset_comb;
1351         RSET rset_temp = rset_create_temp(nmem, kc, kc->scope,
1352                                           res_get(zh->res, "setTmpDir"),0 );
1353
1354         TERMID termid;
1355         RSFD rsfd = rset_open(rset_temp, RSETF_WRITE);
1356
1357         key.mem[0] = sysno;
1358         key.mem[1] = 0;
1359         key.mem[2] = 0;
1360         key.mem[3] = 0;
1361         key.len = 2;
1362         rset_write(rsfd, &key);
1363         rset_close(rsfd);
1364
1365         rsets[0] = rset_temp;
1366         rsets[1] = rset_dup(sset->rset);
1367
1368         rset_comb = rset_create_and(nmem, kc, kc->scope, 2, rsets);
1369
1370         rsfd = rset_open(rset_comb, RSETF_READ);
1371
1372         while (rset_read(rsfd, &key, &termid))
1373         {
1374             if (termid)
1375             {
1376                 struct ord_list *ol;
1377                 for (ol = termid->ol; ol; ol = ol->next)
1378                 {
1379                     zebra_snippets_append(snippets, key.mem[key.len-1], 0,
1380                                           ol->ord, termid->name);
1381                 }
1382             }
1383         }
1384         rset_close(rsfd);
1385
1386         rset_delete(rset_comb);
1387         nmem_destroy(nmem);
1388         kc->dec(kc);
1389     }
1390     return ZEBRA_OK;
1391 }
1392
1393 static ZEBRA_RES zebra_recid_to_sysno(ZebraHandle zh,
1394                                       const char **basenames, int num_bases,
1395                                       zint recid,
1396                                       zint *sysnos, int *no_sysnos)
1397 {
1398     ZEBRA_RES res = ZEBRA_OK;
1399     int sysnos_offset = 0;
1400     int i;
1401
1402     if (!zh->reg->isamb || !zh->m_segment_indexing)
1403     {
1404         if (sysnos_offset < *no_sysnos)
1405             *sysnos = recid;
1406         sysnos_offset++;
1407     }
1408     else
1409     {
1410         for (i = 0; res == ZEBRA_OK && i < num_bases; i++)
1411         {
1412             const char *database = basenames[i];
1413             if (zebraExplain_curDatabase(zh->reg->zei, database) == 0)
1414             {
1415                 const char *index_type = "w";
1416                 const char *use_string = "_ALLRECORDS";
1417                 int ord;
1418                 zinfo_index_category_t cat = zinfo_index_category_alwaysmatches;
1419                 ord = zebraExplain_lookup_attr_str(zh->reg->zei, cat,
1420                                                    index_type, use_string);
1421                 if (ord != -1)
1422                 {
1423                     char ord_buf[32];
1424                     int ord_len = key_SU_encode(ord, ord_buf);
1425                     char *info;
1426
1427                     ord_buf[ord_len] = '\0';
1428
1429                     info = dict_lookup(zh->reg->dict, ord_buf);
1430                     if (info)
1431                     {
1432                         if (*info != sizeof(ISAM_P))
1433                         {
1434                             res = ZEBRA_FAIL;
1435                         }
1436                         else
1437                         {
1438                             ISAM_P isam_p;
1439                             ISAMB_PP pt;
1440                             struct it_key key_until, key_found;
1441                             int i = 0;
1442                             int r;
1443
1444                             memcpy(&isam_p, info+1, sizeof(ISAM_P));
1445
1446                             pt = isamb_pp_open(zh->reg->isamb, isam_p, 2);
1447                             if (!pt)
1448                                 res = ZEBRA_FAIL;
1449                             else
1450                             {
1451                                 key_until.mem[i++] = recid;
1452                                 key_until.mem[i++] = 0;  /* section_id */
1453                                 if (zh->m_segment_indexing)
1454                                     key_until.mem[i++] = 0; /* segment */
1455                                 key_until.mem[i++] = 0;
1456                                 key_until.len = i;
1457
1458                                 r = isamb_pp_forward(pt, &key_found, &key_until);
1459                                 while (r && key_found.mem[0] == recid)
1460                                 {
1461                                     if (sysnos_offset < *no_sysnos)
1462                                         sysnos[sysnos_offset++] =
1463                                             key_found.mem[key_found.len-1];
1464                                     r = isamb_pp_read(pt, &key_found);
1465                                 }
1466                                 isamb_pp_close(pt);
1467                             }
1468                         }
1469                     }
1470                 }
1471             }
1472         }
1473     }
1474     *no_sysnos = sysnos_offset;
1475     return res;
1476 }
1477
1478 ZEBRA_RES zebra_result_recid_to_sysno(ZebraHandle zh,
1479                                       const char *setname,
1480                                       zint recid,
1481                                       zint *sysnos, int *no_sysnos)
1482 {
1483     const char **basenames;
1484     int num_bases;
1485     ZEBRA_RES res;
1486
1487     res = resultSetGetBaseNames(zh, setname, &basenames, &num_bases);
1488     if (res != ZEBRA_OK)
1489         return ZEBRA_FAIL;
1490
1491     return zebra_recid_to_sysno(zh, basenames, num_bases,
1492                                 recid, sysnos, no_sysnos);
1493 }
1494
1495 void zebra_count_set(ZebraHandle zh, RSET rset, zint *count,
1496                      zint approx_limit)
1497 {
1498     zint psysno = 0;
1499     struct it_key key;
1500     RSFD rfd;
1501
1502     yaz_log(YLOG_DEBUG, "count_set");
1503
1504     rset->hits_limit = approx_limit;
1505
1506     *count = 0;
1507     rfd = rset_open(rset, RSETF_READ);
1508     while (rset_read(rfd, &key,0 /* never mind terms */))
1509     {
1510         if (key.mem[0] != psysno)
1511         {
1512             psysno = key.mem[0];
1513             if (rfd->counted_items >= rset->hits_limit)
1514                 break;
1515         }
1516     }
1517     rset_close(rfd);
1518     *count = rset->hits_count;
1519 }
1520
1521
1522 /*
1523  * Local variables:
1524  * c-basic-offset: 4
1525  * c-file-style: "Stroustrup"
1526  * indent-tabs-mode: nil
1527  * End:
1528  * vim: shiftwidth=4 tabstop=8 expandtab
1529  */
1530