index/zsets.c

   1 /* This file is part of the Zebra server.
   2    Copyright (C) 1995-2008 Index Data
   3
   4 Zebra is free software; you can redistribute it and/or modify it under
   5 the terms of the GNU General Public License as published by the Free
   6 Software Foundation; either version 2, or (at your option) any later
   7 version.
   8
   9 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 for more details.
  13
  14 You should have received a copy of the GNU General Public License
  15 along with this program; if not, write to the Free Software
  16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  17
  18 */
  19
  20
  21 #include <stdio.h>
  22 #include <assert.h>
  23 #ifdef WIN32
  24 #include <io.h>
  25 #else
  26 #include <unistd.h>
  27 #endif
  28
  29 #include "index.h"
  30 #include "rank.h"
  31 #include <yaz/diagbib1.h>
  32 #include <rset.h>
  33
  34 #define ZSET_SORT_MAX_LEVEL 10
  35
  36 struct zebra_set_term_entry {
  37     int reg_type;
  38     char *db;
  39     char *index_name;
  40     char *term;
  41 };
  42
  43 struct zebra_set {
  44     char *name;
  45     RSET rset;
  46     NMEM nmem;
  47     NMEM rset_nmem; /* for creating the rsets in */
  48     zint hits;
  49     int num_bases;
  50     const char **basenames;
  51     Z_RPNQuery *rpn;
  52     Z_SortKeySpecList *sortSpec;
  53     struct zset_sort_info *sort_info;
  54     struct zebra_set_term_entry *term_entries;
  55     int term_entries_max;
  56     struct zebra_set *next;
  57     int locked;
  58     int estimated_hit_count;
  59
  60     zint cache_position;  /* last position */
  61     RSFD cache_rfd;       /* rfd (NULL if not existing) */
  62     zint cache_psysno;    /* sysno for last position */
  63     zint approx_limit;    /* limit before we do approx */
  64 };
  65
  66 struct zset_sort_entry {
  67     zint sysno;
  68     int score;
  69 };
  70
  71 struct zset_sort_info {
  72     int max_entries;
  73     int num_entries;
  74     struct zset_sort_entry *all_entries;
  75     struct zset_sort_entry **entries;
  76 };
  77
  78 static int log_level_set=0;
  79 static int log_level_sort=0;
  80 static int log_level_searchhits=0;
  81 static int log_level_searchterms=0;
  82 static int log_level_resultsets=0;
  83
  84 static void loglevels(void)
  85 {
  86     if (log_level_set)
  87         return;
  88     log_level_sort = yaz_log_module_level("sorting");
  89     log_level_searchhits = yaz_log_module_level("searchhits");
  90     log_level_searchterms = yaz_log_module_level("searchterms");
  91     log_level_resultsets = yaz_log_module_level("resultsets");
  92     log_level_set = 1;
  93 }
  94
  95
  96 static ZEBRA_RES resultSetSearch(ZebraHandle zh, NMEM nmem, NMEM rset_nmem,
  97                                  Z_RPNQuery *rpn, ZebraSet sset)
  98 {
  99     RSET rset = 0;
 100     Z_SortKeySpecList *sort_sequence;
 101     int sort_status, i;
 102     ZEBRA_RES res = ZEBRA_OK;
 103
 104     sort_sequence = (Z_SortKeySpecList *)
 105         nmem_malloc(nmem, sizeof(*sort_sequence));
 106     sort_sequence->num_specs = 10; /* FIXME - Hard-coded number */
 107     sort_sequence->specs = (Z_SortKeySpec **)
 108         nmem_malloc(nmem, sort_sequence->num_specs *
 109                      sizeof(*sort_sequence->specs));
 110     for (i = 0; i<sort_sequence->num_specs; i++)
 111         sort_sequence->specs[i] = 0;
 112
 113     rpn_get_top_approx_limit(zh, rpn->RPNStructure, &sset->approx_limit);
 114
 115     res = rpn_search_top(zh, rpn->RPNStructure, rpn->attributeSetId,
 116                          nmem, rset_nmem,
 117                          sort_sequence,
 118                          sset->num_bases, sset->basenames,
 119                          &rset);
 120     if (res != ZEBRA_OK)
 121     {
 122         sset->rset = 0;
 123         return res;
 124     }
 125     for (i = 0; sort_sequence->specs[i]; i++)
 126         ;
 127     sort_sequence->num_specs = i;
 128     rset->hits_limit = sset->approx_limit;
 129     if (!i)
 130     {
 131         res = resultSetRank(zh, sset, rset, rset_nmem);
 132     }
 133     else
 134     {
 135         res = resultSetSortSingle(zh, nmem, sset, rset,
 136                                    sort_sequence, &sort_status);
 137     }
 138     sset->rset = rset;
 139     return res;
 140 }
 141
 142
 143 ZEBRA_RES resultSetAddRPN(ZebraHandle zh, NMEM m, Z_RPNQuery *rpn,
 144                           int num_bases, char **basenames,
 145                           const char *setname,
 146                           zint *hits, int *estimated_hit_count)
 147 {
 148     ZebraSet zebraSet;
 149     int i;
 150     ZEBRA_RES res;
 151
 152     *hits = 0;
 153     *estimated_hit_count = 0;
 154
 155     zebraSet = resultSetAdd(zh, setname, 1);
 156     if (!zebraSet)
 157         return ZEBRA_FAIL;
 158     zebraSet->locked = 1;
 159     zebraSet->rpn = 0;
 160     zebraSet->nmem = m;
 161     zebraSet->rset_nmem = nmem_create();
 162
 163     zebraSet->num_bases = num_bases;
 164     zebraSet->basenames =
 165         nmem_malloc(zebraSet->nmem, num_bases * sizeof(*zebraSet->basenames));
 166     for (i = 0; i<num_bases; i++)
 167         zebraSet->basenames[i] = nmem_strdup(zebraSet->nmem, basenames[i]);
 168
 169     res = resultSetSearch(zh, zebraSet->nmem, zebraSet->rset_nmem,
 170                           rpn, zebraSet);
 171     *hits = zebraSet->hits;
 172     if (zebraSet->estimated_hit_count)
 173         *estimated_hit_count = 1;
 174
 175     if (zebraSet->rset)
 176         zebraSet->rpn = rpn;
 177     zebraSet->locked = 0;
 178     if (!zebraSet->rset)
 179         return ZEBRA_FAIL;
 180     return res;
 181 }
 182
 183 void resultSetAddTerm(ZebraHandle zh, ZebraSet s, int reg_type,
 184                       const char *db, const char *index_name,
 185                       const char *term)
 186 {
 187     assert(zh); /* compiler shut up */
 188     if (!s->nmem)
 189         s->nmem = nmem_create();
 190     if (!s->term_entries)
 191     {
 192         int i;
 193         s->term_entries_max = 1000;
 194         s->term_entries =
 195             nmem_malloc(s->nmem, s->term_entries_max *
 196                          sizeof(*s->term_entries));
 197         for (i = 0; i < s->term_entries_max; i++)
 198             s->term_entries[i].term = 0;
 199     }
 200     if (s->hits < s->term_entries_max)
 201     {
 202         s->term_entries[s->hits].reg_type = reg_type;
 203         s->term_entries[s->hits].db = nmem_strdup(s->nmem, db);
 204         s->term_entries[s->hits].index_name = nmem_strdup(s->nmem, index_name);
 205         s->term_entries[s->hits].term = nmem_strdup(s->nmem, term);
 206     }
 207     (s->hits)++;
 208 }
 209
 210 ZebraSet resultSetAdd(ZebraHandle zh, const char *name, int ov)
 211 {
 212     ZebraSet s;
 213     int i;
 214
 215     for (s = zh->sets; s; s = s->next)
 216         if (!strcmp(s->name, name))
 217             break;
 218
 219     if (!log_level_set)
 220         loglevels();
 221     if (s)
 222     {
 223         yaz_log(log_level_resultsets, "updating result set %s", name);
 224         if (!ov || s->locked)
 225             return NULL;
 226         if (s->rset)
 227         {
 228             if (s->cache_rfd)
 229                 rset_close(s->cache_rfd);
 230             rset_delete(s->rset);
 231         }
 232         if (s->rset_nmem)
 233             nmem_destroy(s->rset_nmem);
 234         if (s->nmem)
 235             nmem_destroy(s->nmem);
 236     }
 237     else
 238     {
 239         const char *sort_max_str = zebra_get_resource(zh, "sortmax", "1000");
 240
 241         yaz_log(log_level_resultsets, "adding result set %s", name);
 242         s = (ZebraSet) xmalloc(sizeof(*s));
 243         s->next = zh->sets;
 244         zh->sets = s;
 245         s->name = xstrdup(name);
 246
 247         s->sort_info = (struct zset_sort_info *)
 248             xmalloc(sizeof(*s->sort_info));
 249         s->sort_info->max_entries = atoi(sort_max_str);
 250         if (s->sort_info->max_entries < 2)
 251             s->sort_info->max_entries = 2;
 252
 253         s->sort_info->entries = (struct zset_sort_entry **)
 254             xmalloc(sizeof(*s->sort_info->entries) *
 255                      s->sort_info->max_entries);
 256         s->sort_info->all_entries = (struct zset_sort_entry *)
 257             xmalloc(sizeof(*s->sort_info->all_entries) *
 258                      s->sort_info->max_entries);
 259         for (i = 0; i < s->sort_info->max_entries; i++)
 260             s->sort_info->entries[i] = s->sort_info->all_entries + i;
 261     }
 262     s->locked = 0;
 263     s->term_entries = 0;
 264     s->hits = 0;
 265     s->rset = 0;
 266     s->rset_nmem = 0;
 267     s->nmem = 0;
 268     s->rpn = 0;
 269     s->sortSpec = 0;
 270     s->cache_position = 0;
 271     s->cache_rfd = 0;
 272     s->approx_limit = zh->approx_limit;
 273     s->estimated_hit_count = 0;
 274     return s;
 275 }
 276
 277 ZebraSet resultSetGet(ZebraHandle zh, const char *name)
 278 {
 279     ZebraSet s;
 280
 281     for (s = zh->sets; s; s = s->next)
 282         if (!strcmp(s->name, name))
 283         {
 284             if (!s->term_entries && !s->rset && s->rpn)
 285             {
 286                 NMEM nmem = nmem_create();
 287                 yaz_log(log_level_resultsets, "research %s", name);
 288                 if (!s->rset_nmem)
 289                     s->rset_nmem = nmem_create();
 290                 resultSetSearch(zh, nmem, s->rset_nmem, s->rpn, s);
 291                 if (s->rset && s->sortSpec)
 292                 {
 293                     int sort_status;
 294                     yaz_log(log_level_resultsets, "resort %s", name);
 295                     resultSetSortSingle(zh, nmem, s, s->rset, s->sortSpec,
 296                                          &sort_status);
 297                 }
 298                 nmem_destroy(nmem);
 299             }
 300             return s;
 301         }
 302     return NULL;
 303 }
 304
 305 ZEBRA_RES resultSetGetBaseNames(ZebraHandle zh, const char *setname,
 306                                 const char ***basenames, int *num_bases)
 307 {
 308     ZebraSet sset = resultSetGet(zh, setname);
 309     if (!sset)
 310         return ZEBRA_FAIL;
 311     *basenames = sset->basenames;
 312     *num_bases = sset->num_bases;
 313     return ZEBRA_OK;
 314
 315 }
 316
 317 void resultSetInvalidate(ZebraHandle zh)
 318 {
 319     ZebraSet s = zh->sets;
 320
 321     yaz_log(log_level_resultsets, "invalidating result sets");
 322     for (; s; s = s->next)
 323     {
 324         if (s->rset)
 325         {
 326             if (s->cache_rfd)
 327                 rset_close(s->cache_rfd);
 328             rset_delete(s->rset);
 329         }
 330         s->rset = 0;
 331         s->cache_rfd = 0;
 332         s->cache_position = 0;
 333         if (s->rset_nmem)
 334             nmem_destroy(s->rset_nmem);
 335         s->rset_nmem=0;
 336     }
 337 }
 338
 339 void resultSetDestroy(ZebraHandle zh, int num, char **names,int *statuses)
 340 {
 341     ZebraSet * ss = &zh->sets;
 342     int i;
 343
 344     if (statuses)
 345         for (i = 0; i<num; i++)
 346             statuses[i] = Z_DeleteStatus_resultSetDidNotExist;
 347     while (*ss)
 348     {
 349         int i = -1;
 350         ZebraSet s = *ss;
 351         if (num >= 0)
 352         {
 353             for (i = 0; i<num; i++)
 354                 if (!strcmp(s->name, names[i]))
 355                 {
 356                     if (statuses)
 357                         statuses[i] = Z_DeleteStatus_success;
 358                     i = -1;
 359                     break;
 360                 }
 361         }
 362         if (i < 0)
 363         {
 364             *ss = s->next;
 365
 366             xfree(s->sort_info->all_entries);
 367             xfree(s->sort_info->entries);
 368             xfree(s->sort_info);
 369
 370             if (s->nmem)
 371                 nmem_destroy(s->nmem);
 372             if (s->rset)
 373             {
 374                 if (s->cache_rfd)
 375                     rset_close(s->cache_rfd);
 376                 rset_delete(s->rset);
 377             }
 378             if (s->rset_nmem)
 379                 nmem_destroy(s->rset_nmem);
 380             xfree(s->name);
 381             xfree(s);
 382         }
 383         else
 384             ss = &s->next;
 385     }
 386 }
 387
 388 ZebraMetaRecord *zebra_meta_records_create_range(ZebraHandle zh,
 389                                                  const char *name,
 390                                                  zint start, int num)
 391 {
 392     zint pos_small[10];
 393     zint *pos = pos_small;
 394     ZebraMetaRecord *mr;
 395     int i;
 396
 397     if (num > 10000 || num <= 0)
 398         return 0;
 399
 400     if (num > 10)
 401         pos = xmalloc(sizeof(*pos) * num);
 402
 403     for (i = 0; i<num; i++)
 404         pos[i] = start+i;
 405
 406     mr = zebra_meta_records_create(zh, name, num, pos);
 407
 408     if (num > 10)
 409         xfree(pos);
 410     return mr;
 411 }
 412
 413 ZebraMetaRecord *zebra_meta_records_create(ZebraHandle zh, const char *name,
 414                                            int num, zint *positions)
 415 {
 416     ZebraSet sset;
 417     ZebraMetaRecord *sr = 0;
 418     RSET rset;
 419     int i;
 420     struct zset_sort_info *sort_info;
 421     size_t sysno_mem_index = 0;
 422
 423     if (zh->m_staticrank)
 424         sysno_mem_index = 1;
 425
 426     if (!log_level_set)
 427         loglevels();
 428     if (!(sset = resultSetGet(zh, name)))
 429         return NULL;
 430     if (!(rset = sset->rset))
 431     {
 432         if (!sset->term_entries)
 433             return 0;
 434         sr = (ZebraMetaRecord *) xmalloc(sizeof(*sr) * num);
 435         for (i = 0; i<num; i++)
 436         {
 437             sr[i].sysno = 0;
 438             sr[i].score = -1;
 439             sr[i].term = 0;
 440             sr[i].db = 0;
 441
 442             if (positions[i] <= sset->term_entries_max)
 443             {
 444                 sr[i].term = sset->term_entries[positions[i]-1].term;
 445                 sr[i].db = sset->term_entries[positions[i]-1].db;
 446             }
 447         }
 448     }
 449     else
 450     {
 451         sr = (ZebraMetaRecord *) xmalloc(sizeof(*sr) * num);
 452         for (i = 0; i<num; i++)
 453         {
 454             sr[i].sysno = 0;
 455             sr[i].score = -1;
 456             sr[i].term = 0;
 457             sr[i].db = 0;
 458         }
 459         sort_info = sset->sort_info;
 460         if (sort_info)
 461         {
 462             zint position;
 463
 464             for (i = 0; i<num; i++)
 465             {
 466                 position = positions[i];
 467                 if (position > 0 && position <= sort_info->num_entries)
 468                 {
 469                     yaz_log(log_level_sort, "got pos=" ZINT_FORMAT
 470                             " (sorted)", position);
 471                     sr[i].sysno = sort_info->entries[position-1]->sysno;
 472                     sr[i].score = sort_info->entries[position-1]->score;
 473                 }
 474             }
 475         }
 476         /* did we really get all entries using sort ? */
 477         for (i = 0; i<num; i++)
 478         {
 479             if (!sr[i].sysno)
 480                 break;
 481         }
 482         if (i < num) /* nope, get the rest, unsorted - sorry */
 483         {
 484             zint position = 0;
 485             int num_i = 0;
 486             zint psysno = 0;
 487             RSFD rfd;
 488             struct it_key key;
 489
 490             if (sort_info)
 491                 position = sort_info->num_entries;
 492             while (num_i < num && positions[num_i] <= position)
 493                 num_i++;
 494
 495             if (sset->cache_rfd &&
 496                 num_i < num && positions[num_i] > sset->cache_position)
 497             {
 498                 position = sset->cache_position;
 499                 rfd = sset->cache_rfd;
 500                 psysno = sset->cache_psysno;
 501             }
 502             else
 503             {
 504                 if (sset->cache_rfd)
 505                     rset_close(sset->cache_rfd);
 506                 rfd = rset_open(rset, RSETF_READ);
 507             }
 508             while (num_i < num && rset_read(rfd, &key, 0))
 509             {
 510                 zint this_sys = key.mem[sysno_mem_index];
 511                 if (this_sys != psysno)
 512                 {
 513                     psysno = this_sys;
 514                     if (sort_info)
 515                     {
 516                         /* determine we alreay have this in our set */
 517                         for (i = sort_info->num_entries; --i >= 0; )
 518                             if (psysno == sort_info->entries[i]->sysno)
 519                                 break;
 520                         if (i >= 0)
 521                             continue;
 522                     }
 523                     position++;
 524                     assert(num_i < num);
 525                     if (position == positions[num_i])
 526                     {
 527                         sr[num_i].sysno = psysno;
 528                         yaz_log(log_level_sort, "got pos=" ZINT_FORMAT " (unsorted)", position);
 529                         sr[num_i].score = -1;
 530                         num_i++;
 531                     }
 532                 }
 533             }
 534             sset->cache_position = position;
 535             sset->cache_psysno = psysno;
 536             sset->cache_rfd = rfd;
 537         }
 538     }
 539     return sr;
 540 }
 541
 542 void zebra_meta_records_destroy(ZebraHandle zh, ZebraMetaRecord *records,
 543                                  int num)
 544 {
 545     assert(zh); /* compiler shut up about unused arg */
 546     xfree(records);
 547 }
 548
 549 struct sortKeyInfo {
 550     int relation;
 551     int ord;
 552     int numerical;
 553     const char *index_type;
 554 };
 555
 556 void resultSetInsertSort(ZebraHandle zh, ZebraSet sset,
 557                          struct sortKeyInfo *criteria, int num_criteria,
 558                          zint sysno,
 559                          char *cmp_buf[], char *tmp_cmp_buf[])
 560 {
 561     struct zset_sort_entry *new_entry = NULL;
 562     struct zset_sort_info *sort_info = sset->sort_info;
 563     int i, j;
 564
 565     zebra_sort_sysno(zh->reg->sort_index, sysno);
 566     for (i = 0; i<num_criteria; i++)
 567     {
 568         char *this_entry_buf = tmp_cmp_buf[i];
 569         memset(this_entry_buf, '\0', SORT_IDX_ENTRYSIZE);
 570         if (criteria[i].ord != -1)
 571         {
 572             zebra_sort_type(zh->reg->sort_index, criteria[i].ord);
 573             zebra_sort_read(zh->reg->sort_index, this_entry_buf);
 574         }
 575     }
 576     i = sort_info->num_entries;
 577     while (--i >= 0)
 578     {
 579         int rel = 0;
 580         for (j = 0; j<num_criteria; j++)
 581         {
 582             char *this_entry_buf = tmp_cmp_buf[j];
 583             char *other_entry_buf =
 584                 cmp_buf[j] + i * SORT_IDX_ENTRYSIZE;
 585             if (criteria[j].numerical)
 586             {
 587                 char this_entry_org[1024];
 588                 char other_entry_org[1024];
 589                 double diff;
 590                 const char *index_type = criteria[j].index_type;
 591                 zebra_term_untrans(zh, index_type, this_entry_org,
 592                                    this_entry_buf);
 593                 zebra_term_untrans(zh, index_type, other_entry_org,
 594                                    other_entry_buf);
 595                 diff = atof(this_entry_org) - atof(other_entry_org);
 596
 597                 if (diff > 0.0)
 598                     rel = 1;
 599                 else if (diff < 0.0)
 600                     rel = -1;
 601                 else
 602                     rel = 0;
 603             }
 604             else
 605             {
 606                 rel = memcmp(this_entry_buf, other_entry_buf,
 607                              SORT_IDX_ENTRYSIZE);
 608             }
 609             if (rel)
 610                 break;
 611         }
 612         if (!rel)
 613             break;
 614         if (criteria[j].relation == 'A')
 615         {
 616             if (rel > 0)
 617                 break;
 618         }
 619         else if (criteria[j].relation == 'D')
 620         {
 621             if (rel < 0)
 622                 break;
 623         }
 624     }
 625     ++i;
 626     j = sort_info->max_entries;
 627     if (i == j)
 628         return;
 629
 630     if (sort_info->num_entries == j)
 631         --j;
 632     else
 633         j = (sort_info->num_entries)++;
 634     new_entry = sort_info->entries[j];
 635     while (j != i)
 636     {
 637         int k;
 638         for (k = 0; k<num_criteria; k++)
 639         {
 640             char *j_buf = cmp_buf[k] + j * SORT_IDX_ENTRYSIZE;
 641             char *j_1_buf = cmp_buf[k] + (j-1) * SORT_IDX_ENTRYSIZE;
 642             memcpy(j_buf, j_1_buf, SORT_IDX_ENTRYSIZE);
 643         }
 644         sort_info->entries[j] = sort_info->entries[j-1];
 645         --j;
 646     }
 647     sort_info->entries[i] = new_entry;
 648     assert(new_entry);
 649     for (i = 0; i<num_criteria; i++)
 650     {
 651         char *new_entry_buf = cmp_buf[i] + j * SORT_IDX_ENTRYSIZE;
 652         char *this_entry_buf = tmp_cmp_buf[i];
 653         memcpy(new_entry_buf, this_entry_buf, SORT_IDX_ENTRYSIZE);
 654     }
 655     new_entry->sysno = sysno;
 656     new_entry->score = -1;
 657 }
 658
 659 void resultSetInsertRank(ZebraHandle zh, struct zset_sort_info *sort_info,
 660                          zint sysno, int score, int relation)
 661 {
 662     struct zset_sort_entry *new_entry = NULL;
 663     int i, j;
 664     assert(zh); /* compiler shut up about unused arg */
 665
 666     i = sort_info->num_entries;
 667     while (--i >= 0)
 668     {
 669         int rel = 0;
 670
 671         rel = score - sort_info->entries[i]->score;
 672
 673         if (relation == 'D')
 674         {
 675             if (rel >= 0)
 676                 break;
 677         }
 678         else if (relation == 'A')
 679         {
 680             if (rel <= 0)
 681                 break;
 682         }
 683     }
 684     ++i;
 685     j = sort_info->max_entries;
 686     if (i == j)
 687         return;
 688
 689     if (sort_info->num_entries == j)
 690         --j;
 691     else
 692         j = (sort_info->num_entries)++;
 693
 694     new_entry = sort_info->entries[j];
 695     while (j != i)
 696     {
 697         sort_info->entries[j] = sort_info->entries[j-1];
 698         --j;
 699     }
 700     sort_info->entries[i] = new_entry;
 701     assert(new_entry);
 702     new_entry->sysno = sysno;
 703     new_entry->score = score;
 704 }
 705
 706 static Z_RPNQuery *copy_RPNQuery(Z_RPNQuery *src, NMEM nmem)
 707 {
 708     Z_RPNQuery *dst = 0;
 709     ODR encode = odr_createmem(ODR_ENCODE);
 710     ODR decode = odr_createmem(ODR_DECODE);
 711
 712     if (z_RPNQuery(encode, &src, 0, 0))
 713     {
 714         int len;
 715         char *buf = odr_getbuf(encode, &len, 0);
 716
 717         if (buf)
 718         {
 719             odr_setbuf(decode, buf, len, 0);
 720             z_RPNQuery(decode, &dst, 0, 0);
 721         }
 722     }
 723     nmem_transfer(nmem, decode->mem);
 724     odr_destroy(encode);
 725     odr_destroy(decode);
 726     return dst;
 727 }
 728
 729 static Z_SortKeySpecList *copy_SortKeySpecList(Z_SortKeySpecList *src, NMEM nmem)
 730 {
 731     Z_SortKeySpecList *dst = 0;
 732     ODR encode = odr_createmem(ODR_ENCODE);
 733     ODR decode = odr_createmem(ODR_DECODE);
 734
 735     if (z_SortKeySpecList(encode, &src, 0, 0))
 736     {
 737         int len;
 738         char *buf = odr_getbuf(encode, &len, 0);
 739
 740         if (buf)
 741         {
 742             odr_setbuf(decode, buf, len, 0);
 743             z_SortKeySpecList(decode, &dst, 0, 0);
 744         }
 745     }
 746     nmem_transfer(nmem, decode->mem);
 747     odr_destroy(encode);
 748     odr_destroy(decode);
 749     return dst;
 750 }
 751
 752 ZebraSet resultSetClone(ZebraHandle zh, const char *setname,
 753                         ZebraSet rset)
 754 {
 755     ZebraSet nset;
 756     int i;
 757
 758     nset = resultSetAdd(zh, setname, 1);
 759     if (!nset)
 760         return 0;
 761
 762     nset->nmem = nmem_create();
 763
 764     nset->num_bases = rset->num_bases;
 765     nset->basenames =
 766         nmem_malloc(nset->nmem, nset->num_bases * sizeof(*rset->basenames));
 767     for (i = 0; i<rset->num_bases; i++)
 768         nset->basenames[i] = nmem_strdup(nset->nmem, rset->basenames[i]);
 769
 770     if (rset->rset)
 771         nset->rset = rset_dup(rset->rset);
 772     if (rset->rpn)
 773         nset->rpn = copy_RPNQuery(rset->rpn, nset->nmem);
 774     return nset;
 775 }
 776
 777 ZEBRA_RES resultSetSort(ZebraHandle zh, NMEM nmem,
 778                         int num_input_setnames, const char **input_setnames,
 779                         const char *output_setname,
 780                         Z_SortKeySpecList *sort_sequence, int *sort_status)
 781 {
 782     ZebraSet sset;
 783     RSET rset;
 784
 785     if (num_input_setnames == 0)
 786     {
 787         zebra_setError(zh, YAZ_BIB1_NO_RESULT_SET_NAME_SUPPLIED_ON_SORT, 0);
 788         return ZEBRA_FAIL;
 789     }
 790     if (num_input_setnames > 1)
 791     {
 792         zebra_setError(zh, YAZ_BIB1_SORT_TOO_MANY_INPUT_RESULTS, 0);
 793         return ZEBRA_FAIL;
 794     }
 795     if (!log_level_set)
 796         loglevels();
 797     yaz_log(log_level_sort, "result set sort input=%s output=%s",
 798           *input_setnames, output_setname);
 799     sset = resultSetGet(zh, input_setnames[0]);
 800     if (!sset)
 801     {
 802         zebra_setError(zh, YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
 803                        input_setnames[0]);
 804         return ZEBRA_FAIL;
 805     }
 806     if (!(rset = sset->rset))
 807     {
 808         zebra_setError(zh, YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
 809                        input_setnames[0]);
 810         return ZEBRA_FAIL;
 811     }
 812     if (strcmp(output_setname, input_setnames[0]))
 813         sset = resultSetClone(zh, output_setname, sset);
 814     sset->sortSpec = copy_SortKeySpecList(sort_sequence, sset->nmem);
 815     return resultSetSortSingle (zh, nmem, sset, rset, sort_sequence,
 816                                 sort_status);
 817 }
 818
 819 ZEBRA_RES resultSetSortSingle(ZebraHandle zh, NMEM nmem,
 820                               ZebraSet sset, RSET rset,
 821                               Z_SortKeySpecList *sort_sequence,
 822                               int *sort_status)
 823 {
 824     int i;
 825     int n = 0;
 826     zint kno = 0;
 827     zint psysno = 0;
 828     struct it_key key;
 829     struct sortKeyInfo sort_criteria[ZSET_SORT_MAX_LEVEL];
 830     char *cmp_buf[ZSET_SORT_MAX_LEVEL];
 831     char *tmp_cmp_buf[ZSET_SORT_MAX_LEVEL];
 832     int num_criteria;
 833     RSFD rfd;
 834     TERMID termid;
 835     TERMID *terms;
 836     int numTerms = 0;
 837     size_t sysno_mem_index = 0;
 838
 839     if (zh->m_staticrank)
 840         sysno_mem_index = 1;
 841
 842     assert(nmem); /* compiler shut up about unused param */
 843     sset->sort_info->num_entries = 0;
 844
 845     rset_getterms(rset, 0, 0, &n);
 846     terms = (TERMID *) nmem_malloc(nmem, sizeof(*terms)*n);
 847     rset_getterms(rset, terms, n, &numTerms);
 848
 849     sset->hits = 0;
 850     num_criteria = sort_sequence->num_specs;
 851     if (num_criteria > ZSET_SORT_MAX_LEVEL)
 852         num_criteria = ZSET_SORT_MAX_LEVEL;
 853     for (i = 0; i < num_criteria; i++)
 854     {
 855         Z_SortKeySpec *sks = sort_sequence->specs[i];
 856         Z_SortKey *sk;
 857         ZEBRA_RES res;
 858
 859         sort_criteria[i].ord = -1;
 860         sort_criteria[i].numerical = 0;
 861
 862         if (sks->which == Z_SortKeySpec_missingValueData)
 863         {
 864             zebra_setError(zh, YAZ_BIB1_UNSUPP_MISSING_DATA_ACTION, 0);
 865             return ZEBRA_FAIL;
 866         }
 867         if (*sks->sortRelation == Z_SortKeySpec_ascending)
 868             sort_criteria[i].relation = 'A';
 869         else if (*sks->sortRelation == Z_SortKeySpec_descending)
 870             sort_criteria[i].relation = 'D';
 871         else
 872         {
 873             zebra_setError(zh, YAZ_BIB1_ILLEGAL_SORT_RELATION, 0);
 874             return ZEBRA_FAIL;
 875         }
 876         if (sks->sortElement->which == Z_SortElement_databaseSpecific)
 877         {
 878             zebra_setError(zh, YAZ_BIB1_DATABASE_SPECIFIC_SORT_UNSUPP, 0);
 879             return ZEBRA_FAIL;
 880         }
 881         else if (sks->sortElement->which != Z_SortElement_generic)
 882         {
 883             zebra_setError(zh, YAZ_BIB1_SORT_ILLEGAL_SORT, 0);
 884             return ZEBRA_FAIL;
 885         }
 886         sk = sks->sortElement->u.generic;
 887         switch (sk->which)
 888         {
 889         case Z_SortKey_sortField:
 890             yaz_log(log_level_sort, "key %d is of type sortField",
 891                     i+1);
 892             sort_criteria[i].numerical = 0;
 893             sort_criteria[i].ord =
 894                 zebraExplain_lookup_attr_str(zh->reg->zei,
 895                                              zinfo_index_category_sort,
 896                                              0, sk->u.sortField);
 897             if (sks->which != Z_SortKeySpec_null
 898                 && sort_criteria[i].ord == -1)
 899             {
 900                 zebra_setError(zh,
 901                                YAZ_BIB1_CANNOT_SORT_ACCORDING_TO_SEQUENCE, 0);
 902                 return ZEBRA_FAIL;
 903             }
 904             break;
 905         case Z_SortKey_elementSpec:
 906             yaz_log(log_level_sort, "key %d is of type elementSpec",
 907                     i+1);
 908             zebra_setError(zh, YAZ_BIB1_CANNOT_SORT_ACCORDING_TO_SEQUENCE, 0);
 909             return ZEBRA_FAIL;
 910         case Z_SortKey_sortAttributes:
 911             yaz_log(log_level_sort, "key %d is of type sortAttributes", i+1);
 912             res = zebra_sort_get_ord(zh, sk->u.sortAttributes,
 913
 914                                      &sort_criteria[i].ord,
 915                                      &sort_criteria[i].numerical);
 916             if (sks->which != Z_SortKeySpec_null && res != ZEBRA_OK)
 917                 return ZEBRA_FAIL;
 918             break;
 919         }
 920         if (zebraExplain_lookup_ord(zh->reg->zei, sort_criteria[i].ord,
 921                                     &sort_criteria[i].index_type,
 922                                     0, 0))
 923         {
 924             zebra_setError(zh, YAZ_BIB1_CANNOT_SORT_ACCORDING_TO_SEQUENCE, 0);
 925             return ZEBRA_FAIL;
 926         }
 927     }
 928     /* allocate space for each cmpare buf + one extra for tmp comparison */
 929     for (i = 0; i<num_criteria; i++)
 930     {
 931         cmp_buf[i] = xmalloc(sset->sort_info->max_entries
 932                              * SORT_IDX_ENTRYSIZE);
 933         tmp_cmp_buf[i] = xmalloc(SORT_IDX_ENTRYSIZE);
 934     }
 935     rfd = rset_open(rset, RSETF_READ);
 936     while (rset_read(rfd, &key, &termid))
 937     {
 938         zint this_sys = key.mem[sysno_mem_index];
 939         if (log_level_searchhits)
 940             key_logdump_txt(log_level_searchhits, &key, termid->name);
 941         kno++;
 942         if (this_sys != psysno)
 943         {
 944             if ((sset->hits & 255) == 0 && zh->break_handler_func)
 945             {
 946                 if (zh->break_handler_func(zh->break_handler_data))
 947                 {
 948                     rset_set_hits_limit(rset, 0);
 949                     break;
 950                 }
 951             }
 952             (sset->hits)++;
 953             psysno = this_sys;
 954             resultSetInsertSort(zh, sset,
 955                                 sort_criteria, num_criteria, psysno, cmp_buf,
 956                                 tmp_cmp_buf);
 957         }
 958     }
 959     rset_close(rfd);
 960
 961     for (i = 0; i<num_criteria; i++)
 962     {
 963         xfree(cmp_buf[i]);
 964         xfree(tmp_cmp_buf[i]);
 965     }
 966
 967     yaz_log(log_level_sort, ZINT_FORMAT " keys, " ZINT_FORMAT " sysnos, sort",
 968             kno, sset->hits);
 969     for (i = 0; i < numTerms; i++)
 970         yaz_log(log_level_sort, "term=\"%s\" type=%s count=" ZINT_FORMAT,
 971                  terms[i]->name, terms[i]->flags, terms[i]->rset->hits_count);
 972     *sort_status = Z_SortResponse_success;
 973     return ZEBRA_OK;
 974 }
 975
 976 RSET resultSetRef(ZebraHandle zh, const char *resultSetId)
 977 {
 978     ZebraSet s;
 979
 980     if ((s = resultSetGet(zh, resultSetId)))
 981         return s->rset;
 982     return NULL;
 983 }
 984
 985 ZEBRA_RES resultSetRank(ZebraHandle zh, ZebraSet zebraSet,
 986                         RSET rset, NMEM nmem)
 987 {
 988     struct it_key key;
 989     TERMID termid;
 990     TERMID *terms;
 991     zint kno = 0;
 992     int numTerms = 0;
 993     int n = 0;
 994     int i;
 995     ZebraRankClass rank_class;
 996     struct zset_sort_info *sort_info;
 997     const char *rank_handler_name = res_get_def(zh->res, "rank", "rank-1");
 998     size_t sysno_mem_index = 0;
 999
1000     if (zh->m_staticrank)
1001         sysno_mem_index = 1;
1002
1003     if (!log_level_set)
1004         loglevels();
1005     sort_info = zebraSet->sort_info;
1006     sort_info->num_entries = 0;
1007     zebraSet->hits = 0;
1008     zebraSet->estimated_hit_count = 0;
1009     rset_getterms(rset, 0, 0, &n);
1010     terms = (TERMID *) nmem_malloc(nmem, sizeof(*terms)*n);
1011     rset_getterms(rset, terms, n, &numTerms);
1012
1013     rank_class = zebraRankLookup(zh, rank_handler_name);
1014     if (!rank_class)
1015     {
1016         yaz_log(YLOG_WARN, "No such rank handler: %s", rank_handler_name);
1017         zebra_setError(zh, YAZ_BIB1_UNSUPP_SEARCH, "Cannot find rank handler");
1018         return ZEBRA_FAIL;
1019     }
1020     else
1021     {
1022         RSFD rfd = rset_open(rset, RSETF_READ);
1023         struct rank_control *rc = rank_class->control;
1024         int score;
1025         zint count = 0;
1026         void *handle = (*rc->begin) (zh->reg, rank_class->class_handle, rset,
1027                                      nmem, terms, numTerms);
1028         zint psysno = 0;  /* previous doc id / sys no */
1029         zint pstaticrank = 0; /* previous static rank */
1030         int stop_flag = 0;
1031         while (rset_read(rfd, &key, &termid))
1032         {
1033             zint this_sys = key.mem[sysno_mem_index];
1034
1035             zint seqno = key.mem[key.len-1];
1036             kno++;
1037             if (log_level_searchhits)
1038                 key_logdump_txt(log_level_searchhits, &key, termid->name);
1039             if (this_sys != psysno)
1040             {   /* new record .. */
1041                 if (!(rfd->counted_items & 255) && zh->break_handler_func)
1042                 {
1043                     if (zh->break_handler_func(zh->break_handler_data))
1044                     {
1045                         yaz_log(YLOG_LOG, "Aborted search");
1046                         stop_flag = 1;
1047                     }
1048                 }
1049                 if (rfd->counted_items > rset->hits_limit)
1050                     stop_flag = 1;
1051                 if (psysno)
1052                 {   /* only if we did have a previous record */
1053                     score = (*rc->calc)(handle, psysno, pstaticrank,
1054                                          &stop_flag);
1055                     /* insert the hit. A=Ascending */
1056                     resultSetInsertRank(zh, sort_info, psysno, score, 'A');
1057                     count++;
1058                 }
1059                 if (stop_flag)
1060                 {
1061                     zebraSet->estimated_hit_count = 1;
1062                     rset_set_hits_limit(rset, 0);
1063                     break;
1064                 }
1065                 psysno = this_sys;
1066                 if (zh->m_staticrank)
1067                     pstaticrank = key.mem[0];
1068             }
1069             (*rc->add)(handle, CAST_ZINT_TO_INT(seqno), termid);
1070         }
1071         /* no more items */
1072         if (psysno)
1073         {   /* we had - at least - one record */
1074             score = (*rc->calc)(handle, psysno, pstaticrank, &stop_flag);
1075             /* insert the hit. A=Ascending */
1076             resultSetInsertRank(zh, sort_info, psysno, score, 'A');
1077             count++;
1078         }
1079         (*rc->end)(zh->reg, handle);
1080         rset_close(rfd);
1081     }
1082     zebraSet->hits = rset->hits_count;
1083
1084     yaz_log(log_level_searchterms, ZINT_FORMAT " keys, "
1085             ZINT_FORMAT " sysnos, rank",  kno, zebraSet->hits);
1086     for (i = 0; i < numTerms; i++)
1087     {
1088         yaz_log(log_level_searchterms, "term=\"%s\" type=%s count="
1089                 ZINT_FORMAT,
1090                 terms[i]->name, terms[i]->flags, terms[i]->rset->hits_count);
1091     }
1092     return ZEBRA_OK;
1093 }
1094
1095 ZebraRankClass zebraRankLookup(ZebraHandle zh, const char *name)
1096 {
1097     ZebraRankClass p = zh->reg->rank_classes;
1098     while (p && strcmp(p->control->name, name))
1099         p = p->next;
1100     if (p && !p->init_flag)
1101     {
1102         if (p->control->create)
1103             p->class_handle = (*p->control->create)(zh);
1104         p->init_flag = 1;
1105     }
1106     return p;
1107 }
1108
1109 void zebraRankInstall(struct zebra_register *reg, struct rank_control *ctrl)
1110 {
1111     ZebraRankClass p = (ZebraRankClass) xmalloc(sizeof(*p));
1112     p->control = (struct rank_control *) xmalloc(sizeof(*p->control));
1113     memcpy(p->control, ctrl, sizeof(*p->control));
1114     p->control->name = xstrdup(ctrl->name);
1115     p->init_flag = 0;
1116     p->next = reg->rank_classes;
1117     reg->rank_classes = p;
1118 }
1119
1120 void zebraRankDestroy(struct zebra_register *reg)
1121 {
1122     ZebraRankClass p = reg->rank_classes;
1123     while (p)
1124     {
1125         ZebraRankClass p_next = p->next;
1126         if (p->init_flag && p->control->destroy)
1127             (*p->control->destroy)(reg, p->class_handle);
1128         xfree(p->control->name);
1129         xfree(p->control);
1130         xfree(p);
1131         p = p_next;
1132     }
1133     reg->rank_classes = NULL;
1134 }
1135
1136 static int trav_rset_for_termids(RSET rset, TERMID *termid_array,
1137                                  zint *hits_array, int *approx_array)
1138 {
1139     int no = 0;
1140     int i;
1141     for (i = 0; i<rset->no_children; i++)
1142         no += trav_rset_for_termids(rset->children[i],
1143                                     (termid_array ? termid_array + no : 0),
1144                                     (hits_array ? hits_array + no : 0),
1145                                     (approx_array ? approx_array + no : 0));
1146     if (rset->term)
1147     {
1148         if (termid_array)
1149             termid_array[no] = rset->term;
1150         if (hits_array)
1151             hits_array[no] = rset->hits_count;
1152         if (approx_array)
1153             approx_array[no] = rset->hits_approx;
1154 #if 0
1155         yaz_log(YLOG_LOG, "rset=%p term=%s limit=" ZINT_FORMAT
1156                 " count=" ZINT_FORMAT,
1157                 rset, rset->term->name, rset->hits_limit, rset->hits_count);
1158 #endif
1159         no++;
1160     }
1161     return no;
1162 }
1163
1164 ZEBRA_RES zebra_result_set_term_no(ZebraHandle zh, const char *setname,
1165                                    int *num_terms)
1166 {
1167     ZebraSet sset = resultSetGet(zh, setname);
1168     *num_terms = 0;
1169     if (sset)
1170     {
1171         *num_terms = trav_rset_for_termids(sset->rset, 0, 0, 0);
1172         return ZEBRA_OK;
1173     }
1174     return ZEBRA_FAIL;
1175 }
1176
1177 ZEBRA_RES zebra_result_set_term_info(ZebraHandle zh, const char *setname,
1178                                      int no, zint *count, int *approx,
1179                                      char *termbuf, size_t *termlen,
1180                                      const char **term_ref_id)
1181 {
1182     ZebraSet sset = resultSetGet(zh, setname);
1183     if (sset)
1184     {
1185         int num_terms = trav_rset_for_termids(sset->rset, 0, 0, 0);
1186         if (no >= 0 && no < num_terms)
1187         {
1188             TERMID *term_array = xmalloc(num_terms * sizeof(*term_array));
1189             zint *hits_array = xmalloc(num_terms * sizeof(*hits_array));
1190             int *approx_array = xmalloc(num_terms * sizeof(*approx_array));
1191
1192             trav_rset_for_termids(sset->rset, term_array,
1193                                   hits_array, approx_array);
1194
1195             if (count)
1196                 *count = hits_array[no];
1197             if (approx)
1198                 *approx = approx_array[no];
1199             if (termbuf)
1200             {
1201                 char *inbuf = term_array[no]->name;
1202                 size_t inleft = strlen(inbuf);
1203                 size_t outleft = *termlen - 1;
1204
1205                 if (zh->iconv_from_utf8 != 0)
1206                 {
1207                     char *outbuf = termbuf;
1208                     size_t ret;
1209
1210                     ret = yaz_iconv(zh->iconv_from_utf8, &inbuf, &inleft,
1211                                     &outbuf, &outleft);
1212                     if (ret == (size_t)(-1))
1213                         *termlen = 0;
1214                     else
1215                     {
1216                         yaz_iconv(zh->iconv_from_utf8, 0, 0,
1217                                   &outbuf, &outleft);
1218                         *termlen = outbuf - termbuf;
1219                     }
1220                 }
1221                 else
1222                 {
1223                     if (inleft > outleft)
1224                         inleft = outleft;
1225                     *termlen = inleft;
1226                     memcpy(termbuf, inbuf, *termlen);
1227                 }
1228                 termbuf[*termlen] = '\0';
1229             }
1230             if (term_ref_id)
1231                 *term_ref_id = term_array[no]->ref_id;
1232
1233             xfree(term_array);
1234             xfree(hits_array);
1235             xfree(approx_array);
1236             return ZEBRA_OK;
1237         }
1238     }
1239     return ZEBRA_FAIL;
1240 }
1241
1242 ZEBRA_RES zebra_snippets_hit_vector(ZebraHandle zh, const char *setname,
1243                                     zint sysno, zebra_snippets *snippets)
1244 {
1245     ZebraSet sset = resultSetGet(zh, setname);
1246     yaz_log(YLOG_DEBUG, "zebra_get_hit_vector setname=%s zysno=" ZINT_FORMAT,
1247             setname, sysno);
1248     if (!sset)
1249         return ZEBRA_FAIL;
1250     else
1251     {
1252         struct rset_key_control *kc = zebra_key_control_create(zh);
1253         NMEM nmem = nmem_create();
1254         struct it_key key;
1255         RSET rsets[2], rset_comb;
1256         RSET rset_temp = rset_create_temp(nmem, kc, kc->scope,
1257                                           res_get(zh->res, "setTmpDir"),0 );
1258
1259         TERMID termid;
1260         RSFD rsfd = rset_open(rset_temp, RSETF_WRITE);
1261
1262         key.mem[0] = sysno;
1263         key.mem[1] = 0;
1264         key.mem[2] = 0;
1265         key.mem[3] = 0;
1266         key.len = 2;
1267         rset_write(rsfd, &key);
1268         rset_close(rsfd);
1269
1270         rsets[0] = rset_temp;
1271         rsets[1] = rset_dup(sset->rset);
1272
1273         rset_comb = rset_create_and(nmem, kc, kc->scope, 2, rsets);
1274
1275         rsfd = rset_open(rset_comb, RSETF_READ);
1276
1277         while (rset_read(rsfd, &key, &termid))
1278         {
1279             if (termid)
1280             {
1281                 struct ord_list *ol;
1282                 for (ol = termid->ol; ol; ol = ol->next)
1283                 {
1284                     zebra_snippets_append(snippets, key.mem[key.len-1], 0,
1285                                           ol->ord, termid->name);
1286                 }
1287             }
1288         }
1289         rset_close(rsfd);
1290
1291         rset_delete(rset_comb);
1292         nmem_destroy(nmem);
1293         kc->dec(kc);
1294     }
1295     return ZEBRA_OK;
1296 }
1297
1298 static ZEBRA_RES zebra_recid_to_sysno(ZebraHandle zh,
1299                                       const char **basenames, int num_bases,
1300                                       zint recid,
1301                                       zint *sysnos, int *no_sysnos)
1302 {
1303     ZEBRA_RES res = ZEBRA_OK;
1304     int sysnos_offset = 0;
1305     int i;
1306
1307     if (!zh->reg->isamb || !zh->m_segment_indexing)
1308     {
1309         if (sysnos_offset < *no_sysnos)
1310             *sysnos = recid;
1311         sysnos_offset++;
1312     }
1313     else
1314     {
1315         for (i = 0; res == ZEBRA_OK && i < num_bases; i++)
1316         {
1317             const char *database = basenames[i];
1318             if (zebraExplain_curDatabase(zh->reg->zei, database) == 0)
1319             {
1320                 const char *index_type = "w";
1321                 const char *use_string = "_ALLRECORDS";
1322                 int ord;
1323                 zinfo_index_category_t cat = zinfo_index_category_alwaysmatches;
1324                 ord = zebraExplain_lookup_attr_str(zh->reg->zei, cat,
1325                                                    index_type, use_string);
1326                 if (ord != -1)
1327                 {
1328                     char ord_buf[32];
1329                     int ord_len = key_SU_encode(ord, ord_buf);
1330                     char *info;
1331
1332                     ord_buf[ord_len] = '\0';
1333
1334                     info = dict_lookup(zh->reg->dict, ord_buf);
1335                     if (info)
1336                     {
1337                         if (*info != sizeof(ISAM_P))
1338                         {
1339                             res = ZEBRA_FAIL;
1340                         }
1341                         else
1342                         {
1343                             ISAM_P isam_p;
1344                             ISAMB_PP pt;
1345                             struct it_key key_until, key_found;
1346                             int i = 0;
1347                             int r;
1348
1349                             memcpy(&isam_p, info+1, sizeof(ISAM_P));
1350
1351                             pt = isamb_pp_open(zh->reg->isamb, isam_p, 2);
1352                             if (!pt)
1353                                 res = ZEBRA_FAIL;
1354                             else
1355                             {
1356                                 key_until.mem[i++] = recid;
1357                                 key_until.mem[i++] = 0;  /* section_id */
1358                                 if (zh->m_segment_indexing)
1359                                     key_until.mem[i++] = 0; /* segment */
1360                                 key_until.mem[i++] = 0;
1361                                 key_until.len = i;
1362
1363                                 r = isamb_pp_forward(pt, &key_found, &key_until);
1364                                 while (r && key_found.mem[0] == recid)
1365                                 {
1366                                     if (sysnos_offset < *no_sysnos)
1367                                         sysnos[sysnos_offset++] =
1368                                             key_found.mem[key_found.len-1];
1369                                     r = isamb_pp_read(pt, &key_found);
1370                                 }
1371                                 isamb_pp_close(pt);
1372                             }
1373                         }
1374                     }
1375                 }
1376             }
1377         }
1378     }
1379     *no_sysnos = sysnos_offset;
1380     return res;
1381 }
1382
1383 ZEBRA_RES zebra_result_recid_to_sysno(ZebraHandle zh,
1384                                       const char *setname,
1385                                       zint recid,
1386                                       zint *sysnos, int *no_sysnos)
1387 {
1388     const char **basenames;
1389     int num_bases;
1390     ZEBRA_RES res;
1391
1392     res = resultSetGetBaseNames(zh, setname, &basenames, &num_bases);
1393     if (res != ZEBRA_OK)
1394         return ZEBRA_FAIL;
1395
1396     return zebra_recid_to_sysno(zh, basenames, num_bases,
1397                                 recid, sysnos, no_sysnos);
1398 }
1399
1400 void zebra_count_set(ZebraHandle zh, RSET rset, zint *count,
1401                      zint approx_limit)
1402 {
1403     zint psysno = 0;
1404     struct it_key key;
1405     RSFD rfd;
1406
1407     yaz_log(YLOG_DEBUG, "count_set");
1408
1409     rset->hits_limit = approx_limit;
1410
1411     *count = 0;
1412     rfd = rset_open(rset, RSETF_READ);
1413     while (rset_read(rfd, &key,0 /* never mind terms */))
1414     {
1415         if (key.mem[0] != psysno)
1416         {
1417             psysno = key.mem[0];
1418             if (rfd->counted_items >= rset->hits_limit)
1419                 break;
1420         }
1421     }
1422     rset_close(rfd);
1423     *count = rset->hits_count;
1424 }
1425
1426
1427 /*
1428  * Local variables:
1429  * c-basic-offset: 4
1430  * indent-tabs-mode: nil
1431  * End:
1432  * vim: shiftwidth=4 tabstop=8 expandtab
1433  */
1434