index/zsets.c

   1 /* $Id: zsets.c,v 1.128 2007-12-03 11:49:11 adam Exp $
   2    Copyright (C) 1995-2007
   3    Index Data ApS
   4
   5 This file is part of the Zebra server.
   6
   7 Zebra is free software; you can redistribute it and/or modify it under
   8 the terms of the GNU General Public License as published by the Free
   9 Software Foundation; either version 2, or (at your option) any later
  10 version.
  11
  12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 for more details.
  16
  17 You should have received a copy of the GNU General Public License
  18 along with this program; if not, write to the Free Software
  19 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  20
  21 */
  22
  23
  24 #include <stdio.h>
  25 #include <assert.h>
  26 #ifdef WIN32
  27 #include <io.h>
  28 #else
  29 #include <unistd.h>
  30 #endif
  31
  32 #include "index.h"
  33 #include "rank.h"
  34 #include <yaz/diagbib1.h>
  35 #include <rset.h>
  36
  37 #define ZSET_SORT_MAX_LEVEL 10
  38
  39 struct zebra_set_term_entry {
  40     int reg_type;
  41     char *db;
  42     char *index_name;
  43     char *term;
  44 };
  45
  46 struct zebra_set {
  47     char *name;
  48     RSET rset;
  49     NMEM nmem;
  50     NMEM rset_nmem; /* for creating the rsets in */
  51     zint hits;
  52     int num_bases;
  53     const char **basenames;
  54     Z_RPNQuery *rpn;
  55     Z_SortKeySpecList *sortSpec;
  56     struct zset_sort_info *sort_info;
  57     struct zebra_set_term_entry *term_entries;
  58     int term_entries_max;
  59     struct zebra_set *next;
  60     int locked;
  61     int estimated_hit_count;
  62
  63     zint cache_position;  /* last position */
  64     RSFD cache_rfd;       /* rfd (NULL if not existing) */
  65     zint cache_psysno;    /* sysno for last position */
  66     zint approx_limit;    /* limit before we do approx */
  67 };
  68
  69 struct zset_sort_entry {
  70     zint sysno;
  71     int score;
  72 };
  73
  74 struct zset_sort_info {
  75     int max_entries;
  76     int num_entries;
  77     struct zset_sort_entry *all_entries;
  78     struct zset_sort_entry **entries;
  79 };
  80
  81 static int log_level_set=0;
  82 static int log_level_sort=0;
  83 static int log_level_searchhits=0;
  84 static int log_level_searchterms=0;
  85 static int log_level_resultsets=0;
  86
  87 static void loglevels(void)
  88 {
  89     if (log_level_set)
  90         return;
  91     log_level_sort = yaz_log_module_level("sorting");
  92     log_level_searchhits = yaz_log_module_level("searchhits");
  93     log_level_searchterms = yaz_log_module_level("searchterms");
  94     log_level_resultsets = yaz_log_module_level("resultsets");
  95     log_level_set = 1;
  96 }
  97
  98
  99 static ZEBRA_RES resultSetSearch(ZebraHandle zh, NMEM nmem, NMEM rset_nmem,
 100                                  Z_RPNQuery *rpn, ZebraSet sset)
 101 {
 102     RSET rset = 0;
 103     Z_SortKeySpecList *sort_sequence;
 104     int sort_status, i;
 105     ZEBRA_RES res = ZEBRA_OK;
 106
 107     sort_sequence = (Z_SortKeySpecList *)
 108         nmem_malloc(nmem, sizeof(*sort_sequence));
 109     sort_sequence->num_specs = 10; /* FIXME - Hard-coded number */
 110     sort_sequence->specs = (Z_SortKeySpec **)
 111         nmem_malloc(nmem, sort_sequence->num_specs *
 112                      sizeof(*sort_sequence->specs));
 113     for (i = 0; i<sort_sequence->num_specs; i++)
 114         sort_sequence->specs[i] = 0;
 115
 116     rpn_get_top_approx_limit(zh, rpn->RPNStructure, &sset->approx_limit);
 117
 118     res = rpn_search_top(zh, rpn->RPNStructure, rpn->attributeSetId,
 119                          nmem, rset_nmem,
 120                          sort_sequence,
 121                          sset->num_bases, sset->basenames,
 122                          &rset);
 123     if (res != ZEBRA_OK)
 124     {
 125         sset->rset = 0;
 126         return res;
 127     }
 128     for (i = 0; sort_sequence->specs[i]; i++)
 129         ;
 130     sort_sequence->num_specs = i;
 131     rset->hits_limit = sset->approx_limit;
 132     if (!i)
 133     {
 134         res = resultSetRank(zh, sset, rset, rset_nmem);
 135     }
 136     else
 137     {
 138         res = resultSetSortSingle(zh, nmem, sset, rset,
 139                                    sort_sequence, &sort_status);
 140     }
 141     sset->rset = rset;
 142     return res;
 143 }
 144
 145
 146 ZEBRA_RES resultSetAddRPN(ZebraHandle zh, NMEM m, Z_RPNQuery *rpn,
 147                           int num_bases, char **basenames,
 148                           const char *setname,
 149                           zint *hits, int *estimated_hit_count)
 150 {
 151     ZebraSet zebraSet;
 152     int i;
 153     ZEBRA_RES res;
 154
 155     *hits = 0;
 156     *estimated_hit_count = 0;
 157
 158     zebraSet = resultSetAdd(zh, setname, 1);
 159     if (!zebraSet)
 160         return ZEBRA_FAIL;
 161     zebraSet->locked = 1;
 162     zebraSet->rpn = 0;
 163     zebraSet->nmem = m;
 164     zebraSet->rset_nmem = nmem_create();
 165
 166     zebraSet->num_bases = num_bases;
 167     zebraSet->basenames =
 168         nmem_malloc(zebraSet->nmem, num_bases * sizeof(*zebraSet->basenames));
 169     for (i = 0; i<num_bases; i++)
 170         zebraSet->basenames[i] = nmem_strdup(zebraSet->nmem, basenames[i]);
 171
 172     res = resultSetSearch(zh, zebraSet->nmem, zebraSet->rset_nmem,
 173                           rpn, zebraSet);
 174     *hits = zebraSet->hits;
 175     if (zebraSet->estimated_hit_count)
 176         *estimated_hit_count = 1;
 177
 178     if (zebraSet->rset)
 179         zebraSet->rpn = rpn;
 180     zebraSet->locked = 0;
 181     if (!zebraSet->rset)
 182         return ZEBRA_FAIL;
 183     return res;
 184 }
 185
 186 void resultSetAddTerm(ZebraHandle zh, ZebraSet s, int reg_type,
 187                       const char *db, const char *index_name,
 188                       const char *term)
 189 {
 190     assert(zh); /* compiler shut up */
 191     if (!s->nmem)
 192         s->nmem = nmem_create();
 193     if (!s->term_entries)
 194     {
 195         int i;
 196         s->term_entries_max = 1000;
 197         s->term_entries =
 198             nmem_malloc(s->nmem, s->term_entries_max *
 199                          sizeof(*s->term_entries));
 200         for (i = 0; i < s->term_entries_max; i++)
 201             s->term_entries[i].term = 0;
 202     }
 203     if (s->hits < s->term_entries_max)
 204     {
 205         s->term_entries[s->hits].reg_type = reg_type;
 206         s->term_entries[s->hits].db = nmem_strdup(s->nmem, db);
 207         s->term_entries[s->hits].index_name = nmem_strdup(s->nmem, index_name);
 208         s->term_entries[s->hits].term = nmem_strdup(s->nmem, term);
 209     }
 210     (s->hits)++;
 211 }
 212
 213 ZebraSet resultSetAdd(ZebraHandle zh, const char *name, int ov)
 214 {
 215     ZebraSet s;
 216     int i;
 217
 218     for (s = zh->sets; s; s = s->next)
 219         if (!strcmp(s->name, name))
 220             break;
 221
 222     if (!log_level_set)
 223         loglevels();
 224     if (s)
 225     {
 226         yaz_log(log_level_resultsets, "updating result set %s", name);
 227         if (!ov || s->locked)
 228             return NULL;
 229         if (s->rset)
 230         {
 231             if (s->cache_rfd)
 232                 rset_close(s->cache_rfd);
 233             rset_delete(s->rset);
 234         }
 235         if (s->rset_nmem)
 236             nmem_destroy(s->rset_nmem);
 237         if (s->nmem)
 238             nmem_destroy(s->nmem);
 239     }
 240     else
 241     {
 242         const char *sort_max_str = zebra_get_resource(zh, "sortmax", "1000");
 243
 244         yaz_log(log_level_resultsets, "adding result set %s", name);
 245         s = (ZebraSet) xmalloc(sizeof(*s));
 246         s->next = zh->sets;
 247         zh->sets = s;
 248         s->name = xstrdup(name);
 249
 250         s->sort_info = (struct zset_sort_info *)
 251             xmalloc(sizeof(*s->sort_info));
 252         s->sort_info->max_entries = atoi(sort_max_str);
 253         if (s->sort_info->max_entries < 2)
 254             s->sort_info->max_entries = 2;
 255
 256         s->sort_info->entries = (struct zset_sort_entry **)
 257             xmalloc(sizeof(*s->sort_info->entries) *
 258                      s->sort_info->max_entries);
 259         s->sort_info->all_entries = (struct zset_sort_entry *)
 260             xmalloc(sizeof(*s->sort_info->all_entries) *
 261                      s->sort_info->max_entries);
 262         for (i = 0; i < s->sort_info->max_entries; i++)
 263             s->sort_info->entries[i] = s->sort_info->all_entries + i;
 264     }
 265     s->locked = 0;
 266     s->term_entries = 0;
 267     s->hits = 0;
 268     s->rset = 0;
 269     s->rset_nmem = 0;
 270     s->nmem = 0;
 271     s->rpn = 0;
 272     s->sortSpec = 0;
 273     s->cache_position = 0;
 274     s->cache_rfd = 0;
 275     s->approx_limit = zh->approx_limit;
 276     s->estimated_hit_count = 0;
 277     return s;
 278 }
 279
 280 ZebraSet resultSetGet(ZebraHandle zh, const char *name)
 281 {
 282     ZebraSet s;
 283
 284     for (s = zh->sets; s; s = s->next)
 285         if (!strcmp(s->name, name))
 286         {
 287             if (!s->term_entries && !s->rset && s->rpn)
 288             {
 289                 NMEM nmem = nmem_create();
 290                 yaz_log(log_level_resultsets, "research %s", name);
 291                 if (!s->rset_nmem)
 292                     s->rset_nmem = nmem_create();
 293                 resultSetSearch(zh, nmem, s->rset_nmem, s->rpn, s);
 294                 if (s->rset && s->sortSpec)
 295                 {
 296                     int sort_status;
 297                     yaz_log(log_level_resultsets, "resort %s", name);
 298                     resultSetSortSingle(zh, nmem, s, s->rset, s->sortSpec,
 299                                          &sort_status);
 300                 }
 301                 nmem_destroy(nmem);
 302             }
 303             return s;
 304         }
 305     return NULL;
 306 }
 307
 308 ZEBRA_RES resultSetGetBaseNames(ZebraHandle zh, const char *setname,
 309                                 const char ***basenames, int *num_bases)
 310 {
 311     ZebraSet sset = resultSetGet(zh, setname);
 312     if (!sset)
 313         return ZEBRA_FAIL;
 314     *basenames = sset->basenames;
 315     *num_bases = sset->num_bases;
 316     return ZEBRA_OK;
 317
 318 }
 319
 320 void resultSetInvalidate(ZebraHandle zh)
 321 {
 322     ZebraSet s = zh->sets;
 323
 324     yaz_log(log_level_resultsets, "invalidating result sets");
 325     for (; s; s = s->next)
 326     {
 327         if (s->rset)
 328         {
 329             if (s->cache_rfd)
 330                 rset_close(s->cache_rfd);
 331             rset_delete(s->rset);
 332         }
 333         s->rset = 0;
 334         s->cache_rfd = 0;
 335         s->cache_position = 0;
 336         if (s->rset_nmem)
 337             nmem_destroy(s->rset_nmem);
 338         s->rset_nmem=0;
 339     }
 340 }
 341
 342 void resultSetDestroy(ZebraHandle zh, int num, char **names,int *statuses)
 343 {
 344     ZebraSet * ss = &zh->sets;
 345     int i;
 346
 347     if (statuses)
 348         for (i = 0; i<num; i++)
 349             statuses[i] = Z_DeleteStatus_resultSetDidNotExist;
 350     while (*ss)
 351     {
 352         int i = -1;
 353         ZebraSet s = *ss;
 354         if (num >= 0)
 355         {
 356             for (i = 0; i<num; i++)
 357                 if (!strcmp(s->name, names[i]))
 358                 {
 359                     if (statuses)
 360                         statuses[i] = Z_DeleteStatus_success;
 361                     i = -1;
 362                     break;
 363                 }
 364         }
 365         if (i < 0)
 366         {
 367             *ss = s->next;
 368
 369             xfree(s->sort_info->all_entries);
 370             xfree(s->sort_info->entries);
 371             xfree(s->sort_info);
 372
 373             if (s->nmem)
 374                 nmem_destroy(s->nmem);
 375             if (s->rset)
 376             {
 377                 if (s->cache_rfd)
 378                     rset_close(s->cache_rfd);
 379                 rset_delete(s->rset);
 380             }
 381             if (s->rset_nmem)
 382                 nmem_destroy(s->rset_nmem);
 383             xfree(s->name);
 384             xfree(s);
 385         }
 386         else
 387             ss = &s->next;
 388     }
 389 }
 390
 391 ZebraMetaRecord *zebra_meta_records_create_range(ZebraHandle zh,
 392                                                  const char *name,
 393                                                  zint start, int num)
 394 {
 395     zint pos_small[10];
 396     zint *pos = pos_small;
 397     ZebraMetaRecord *mr;
 398     int i;
 399
 400     if (num > 10000 || num <= 0)
 401         return 0;
 402
 403     if (num > 10)
 404         pos = xmalloc(sizeof(*pos) * num);
 405
 406     for (i = 0; i<num; i++)
 407         pos[i] = start+i;
 408
 409     mr = zebra_meta_records_create(zh, name, num, pos);
 410
 411     if (num > 10)
 412         xfree(pos);
 413     return mr;
 414 }
 415
 416 ZebraMetaRecord *zebra_meta_records_create(ZebraHandle zh, const char *name,
 417                                            int num, zint *positions)
 418 {
 419     ZebraSet sset;
 420     ZebraMetaRecord *sr = 0;
 421     RSET rset;
 422     int i;
 423     struct zset_sort_info *sort_info;
 424     size_t sysno_mem_index = 0;
 425
 426     if (zh->m_staticrank)
 427         sysno_mem_index = 1;
 428
 429     if (!log_level_set)
 430         loglevels();
 431     if (!(sset = resultSetGet(zh, name)))
 432         return NULL;
 433     if (!(rset = sset->rset))
 434     {
 435         if (!sset->term_entries)
 436             return 0;
 437         sr = (ZebraMetaRecord *) xmalloc(sizeof(*sr) * num);
 438         for (i = 0; i<num; i++)
 439         {
 440             sr[i].sysno = 0;
 441             sr[i].score = -1;
 442             sr[i].term = 0;
 443             sr[i].db = 0;
 444
 445             if (positions[i] <= sset->term_entries_max)
 446             {
 447                 sr[i].term = sset->term_entries[positions[i]-1].term;
 448                 sr[i].db = sset->term_entries[positions[i]-1].db;
 449             }
 450         }
 451     }
 452     else
 453     {
 454         sr = (ZebraMetaRecord *) xmalloc(sizeof(*sr) * num);
 455         for (i = 0; i<num; i++)
 456         {
 457             sr[i].sysno = 0;
 458             sr[i].score = -1;
 459             sr[i].term = 0;
 460             sr[i].db = 0;
 461         }
 462         sort_info = sset->sort_info;
 463         if (sort_info)
 464         {
 465             zint position;
 466
 467             for (i = 0; i<num; i++)
 468             {
 469                 position = positions[i];
 470                 if (position > 0 && position <= sort_info->num_entries)
 471                 {
 472                     yaz_log(log_level_sort, "got pos=" ZINT_FORMAT
 473                             " (sorted)", position);
 474                     sr[i].sysno = sort_info->entries[position-1]->sysno;
 475                     sr[i].score = sort_info->entries[position-1]->score;
 476                 }
 477             }
 478         }
 479         /* did we really get all entries using sort ? */
 480         for (i = 0; i<num; i++)
 481         {
 482             if (!sr[i].sysno)
 483                 break;
 484         }
 485         if (i < num) /* nope, get the rest, unsorted - sorry */
 486         {
 487             zint position = 0;
 488             int num_i = 0;
 489             zint psysno = 0;
 490             RSFD rfd;
 491             struct it_key key;
 492
 493             if (sort_info)
 494                 position = sort_info->num_entries;
 495             while (num_i < num && positions[num_i] <= position)
 496                 num_i++;
 497
 498             if (sset->cache_rfd &&
 499                 num_i < num && positions[num_i] > sset->cache_position)
 500             {
 501                 position = sset->cache_position;
 502                 rfd = sset->cache_rfd;
 503                 psysno = sset->cache_psysno;
 504             }
 505             else
 506             {
 507                 if (sset->cache_rfd)
 508                     rset_close(sset->cache_rfd);
 509                 rfd = rset_open(rset, RSETF_READ);
 510             }
 511             while (num_i < num && rset_read(rfd, &key, 0))
 512             {
 513                 zint this_sys = key.mem[sysno_mem_index];
 514                 if (this_sys != psysno)
 515                 {
 516                     psysno = this_sys;
 517                     if (sort_info)
 518                     {
 519                         /* determine we alreay have this in our set */
 520                         for (i = sort_info->num_entries; --i >= 0; )
 521                             if (psysno == sort_info->entries[i]->sysno)
 522                                 break;
 523                         if (i >= 0)
 524                             continue;
 525                     }
 526                     position++;
 527                     assert(num_i < num);
 528                     if (position == positions[num_i])
 529                     {
 530                         sr[num_i].sysno = psysno;
 531                         yaz_log(log_level_sort, "got pos=" ZINT_FORMAT " (unsorted)", position);
 532                         sr[num_i].score = -1;
 533                         num_i++;
 534                     }
 535                 }
 536             }
 537             sset->cache_position = position;
 538             sset->cache_psysno = psysno;
 539             sset->cache_rfd = rfd;
 540         }
 541     }
 542     return sr;
 543 }
 544
 545 void zebra_meta_records_destroy(ZebraHandle zh, ZebraMetaRecord *records,
 546                                  int num)
 547 {
 548     assert(zh); /* compiler shut up about unused arg */
 549     xfree(records);
 550 }
 551
 552 struct sortKeyInfo {
 553     int relation;
 554     int ord;
 555     int numerical;
 556     const char *index_type;
 557 };
 558
 559 void resultSetInsertSort(ZebraHandle zh, ZebraSet sset,
 560                          struct sortKeyInfo *criteria, int num_criteria,
 561                          zint sysno,
 562                          char *cmp_buf[], char *tmp_cmp_buf[])
 563 {
 564     struct zset_sort_entry *new_entry = NULL;
 565     struct zset_sort_info *sort_info = sset->sort_info;
 566     int i, j;
 567
 568     zebra_sort_sysno(zh->reg->sort_index, sysno);
 569     for (i = 0; i<num_criteria; i++)
 570     {
 571         char *this_entry_buf = tmp_cmp_buf[i];
 572         memset(this_entry_buf, '\0', SORT_IDX_ENTRYSIZE);
 573         if (criteria[i].ord != -1)
 574         {
 575             zebra_sort_type(zh->reg->sort_index, criteria[i].ord);
 576             zebra_sort_read(zh->reg->sort_index, this_entry_buf);
 577         }
 578     }
 579     i = sort_info->num_entries;
 580     while (--i >= 0)
 581     {
 582         int rel = 0;
 583         for (j = 0; j<num_criteria; j++)
 584         {
 585             char *this_entry_buf = tmp_cmp_buf[j];
 586             char *other_entry_buf =
 587                 cmp_buf[j] + i * SORT_IDX_ENTRYSIZE;
 588             if (criteria[j].numerical)
 589             {
 590                 char this_entry_org[1024];
 591                 char other_entry_org[1024];
 592                 double diff;
 593                 const char *index_type = criteria[j].index_type;
 594                 zebra_term_untrans(zh, index_type, this_entry_org,
 595                                    this_entry_buf);
 596                 zebra_term_untrans(zh, index_type, other_entry_org,
 597                                    other_entry_buf);
 598                 diff = atof(this_entry_org) - atof(other_entry_org);
 599
 600                 if (diff > 0.0)
 601                     rel = 1;
 602                 else if (diff < 0.0)
 603                     rel = -1;
 604                 else
 605                     rel = 0;
 606             }
 607             else
 608             {
 609                 rel = memcmp(this_entry_buf, other_entry_buf,
 610                              SORT_IDX_ENTRYSIZE);
 611             }
 612             if (rel)
 613                 break;
 614         }
 615         if (!rel)
 616             break;
 617         if (criteria[j].relation == 'A')
 618         {
 619             if (rel > 0)
 620                 break;
 621         }
 622         else if (criteria[j].relation == 'D')
 623         {
 624             if (rel < 0)
 625                 break;
 626         }
 627     }
 628     ++i;
 629     j = sort_info->max_entries;
 630     if (i == j)
 631         return;
 632
 633     if (sort_info->num_entries == j)
 634         --j;
 635     else
 636         j = (sort_info->num_entries)++;
 637     new_entry = sort_info->entries[j];
 638     while (j != i)
 639     {
 640         int k;
 641         for (k = 0; k<num_criteria; k++)
 642         {
 643             char *j_buf = cmp_buf[k] + j * SORT_IDX_ENTRYSIZE;
 644             char *j_1_buf = cmp_buf[k] + (j-1) * SORT_IDX_ENTRYSIZE;
 645             memcpy(j_buf, j_1_buf, SORT_IDX_ENTRYSIZE);
 646         }
 647         sort_info->entries[j] = sort_info->entries[j-1];
 648         --j;
 649     }
 650     sort_info->entries[i] = new_entry;
 651     assert(new_entry);
 652     for (i = 0; i<num_criteria; i++)
 653     {
 654         char *new_entry_buf = cmp_buf[i] + j * SORT_IDX_ENTRYSIZE;
 655         char *this_entry_buf = tmp_cmp_buf[i];
 656         memcpy(new_entry_buf, this_entry_buf, SORT_IDX_ENTRYSIZE);
 657     }
 658     new_entry->sysno = sysno;
 659     new_entry->score = -1;
 660 }
 661
 662 void resultSetInsertRank(ZebraHandle zh, struct zset_sort_info *sort_info,
 663                          zint sysno, int score, int relation)
 664 {
 665     struct zset_sort_entry *new_entry = NULL;
 666     int i, j;
 667     assert(zh); /* compiler shut up about unused arg */
 668
 669     i = sort_info->num_entries;
 670     while (--i >= 0)
 671     {
 672         int rel = 0;
 673
 674         rel = score - sort_info->entries[i]->score;
 675
 676         if (relation == 'D')
 677         {
 678             if (rel >= 0)
 679                 break;
 680         }
 681         else if (relation == 'A')
 682         {
 683             if (rel <= 0)
 684                 break;
 685         }
 686     }
 687     ++i;
 688     j = sort_info->max_entries;
 689     if (i == j)
 690         return;
 691
 692     if (sort_info->num_entries == j)
 693         --j;
 694     else
 695         j = (sort_info->num_entries)++;
 696
 697     new_entry = sort_info->entries[j];
 698     while (j != i)
 699     {
 700         sort_info->entries[j] = sort_info->entries[j-1];
 701         --j;
 702     }
 703     sort_info->entries[i] = new_entry;
 704     assert(new_entry);
 705     new_entry->sysno = sysno;
 706     new_entry->score = score;
 707 }
 708
 709 static Z_RPNQuery *copy_RPNQuery(Z_RPNQuery *src, NMEM nmem)
 710 {
 711     Z_RPNQuery *dst = 0;
 712     ODR encode = odr_createmem(ODR_ENCODE);
 713     ODR decode = odr_createmem(ODR_DECODE);
 714
 715     if (z_RPNQuery(encode, &src, 0, 0))
 716     {
 717         int len;
 718         char *buf = odr_getbuf(encode, &len, 0);
 719
 720         if (buf)
 721         {
 722             odr_setbuf(decode, buf, len, 0);
 723             z_RPNQuery(decode, &dst, 0, 0);
 724         }
 725     }
 726     nmem_transfer(nmem, decode->mem);
 727     odr_destroy(encode);
 728     odr_destroy(decode);
 729     return dst;
 730 }
 731
 732 static Z_SortKeySpecList *copy_SortKeySpecList(Z_SortKeySpecList *src, NMEM nmem)
 733 {
 734     Z_SortKeySpecList *dst = 0;
 735     ODR encode = odr_createmem(ODR_ENCODE);
 736     ODR decode = odr_createmem(ODR_DECODE);
 737
 738     if (z_SortKeySpecList(encode, &src, 0, 0))
 739     {
 740         int len;
 741         char *buf = odr_getbuf(encode, &len, 0);
 742
 743         if (buf)
 744         {
 745             odr_setbuf(decode, buf, len, 0);
 746             z_SortKeySpecList(decode, &dst, 0, 0);
 747         }
 748     }
 749     nmem_transfer(nmem, decode->mem);
 750     odr_destroy(encode);
 751     odr_destroy(decode);
 752     return dst;
 753 }
 754
 755 ZebraSet resultSetClone(ZebraHandle zh, const char *setname,
 756                         ZebraSet rset)
 757 {
 758     ZebraSet nset;
 759     int i;
 760
 761     nset = resultSetAdd(zh, setname, 1);
 762     if (!nset)
 763         return 0;
 764
 765     nset->nmem = nmem_create();
 766
 767     nset->num_bases = rset->num_bases;
 768     nset->basenames =
 769         nmem_malloc(nset->nmem, nset->num_bases * sizeof(*rset->basenames));
 770     for (i = 0; i<rset->num_bases; i++)
 771         nset->basenames[i] = nmem_strdup(nset->nmem, rset->basenames[i]);
 772
 773     if (rset->rset)
 774         nset->rset = rset_dup(rset->rset);
 775     if (rset->rpn)
 776         nset->rpn = copy_RPNQuery(rset->rpn, nset->nmem);
 777     return nset;
 778 }
 779
 780 ZEBRA_RES resultSetSort(ZebraHandle zh, NMEM nmem,
 781                         int num_input_setnames, const char **input_setnames,
 782                         const char *output_setname,
 783                         Z_SortKeySpecList *sort_sequence, int *sort_status)
 784 {
 785     ZebraSet sset;
 786     RSET rset;
 787
 788     if (num_input_setnames == 0)
 789     {
 790         zebra_setError(zh, YAZ_BIB1_NO_RESULT_SET_NAME_SUPPLIED_ON_SORT, 0);
 791         return ZEBRA_FAIL;
 792     }
 793     if (num_input_setnames > 1)
 794     {
 795         zebra_setError(zh, YAZ_BIB1_SORT_TOO_MANY_INPUT_RESULTS, 0);
 796         return ZEBRA_FAIL;
 797     }
 798     if (!log_level_set)
 799         loglevels();
 800     yaz_log(log_level_sort, "result set sort input=%s output=%s",
 801           *input_setnames, output_setname);
 802     sset = resultSetGet(zh, input_setnames[0]);
 803     if (!sset)
 804     {
 805         zebra_setError(zh, YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
 806                        input_setnames[0]);
 807         return ZEBRA_FAIL;
 808     }
 809     if (!(rset = sset->rset))
 810     {
 811         zebra_setError(zh, YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
 812                        input_setnames[0]);
 813         return ZEBRA_FAIL;
 814     }
 815     if (strcmp(output_setname, input_setnames[0]))
 816         sset = resultSetClone(zh, output_setname, sset);
 817     sset->sortSpec = copy_SortKeySpecList(sort_sequence, sset->nmem);
 818     return resultSetSortSingle (zh, nmem, sset, rset, sort_sequence,
 819                                 sort_status);
 820 }
 821
 822 ZEBRA_RES resultSetSortSingle(ZebraHandle zh, NMEM nmem,
 823                               ZebraSet sset, RSET rset,
 824                               Z_SortKeySpecList *sort_sequence,
 825                               int *sort_status)
 826 {
 827     int i;
 828     int n = 0;
 829     zint kno = 0;
 830     zint psysno = 0;
 831     struct it_key key;
 832     struct sortKeyInfo sort_criteria[ZSET_SORT_MAX_LEVEL];
 833     char *cmp_buf[ZSET_SORT_MAX_LEVEL];
 834     char *tmp_cmp_buf[ZSET_SORT_MAX_LEVEL];
 835     int num_criteria;
 836     RSFD rfd;
 837     TERMID termid;
 838     TERMID *terms;
 839     int numTerms = 0;
 840     size_t sysno_mem_index = 0;
 841
 842     if (zh->m_staticrank)
 843         sysno_mem_index = 1;
 844
 845     assert(nmem); /* compiler shut up about unused param */
 846     sset->sort_info->num_entries = 0;
 847
 848     rset_getterms(rset, 0, 0, &n);
 849     terms = (TERMID *) nmem_malloc(nmem, sizeof(*terms)*n);
 850     rset_getterms(rset, terms, n, &numTerms);
 851
 852     sset->hits = 0;
 853     num_criteria = sort_sequence->num_specs;
 854     if (num_criteria > ZSET_SORT_MAX_LEVEL)
 855         num_criteria = ZSET_SORT_MAX_LEVEL;
 856     for (i = 0; i < num_criteria; i++)
 857     {
 858         Z_SortKeySpec *sks = sort_sequence->specs[i];
 859         Z_SortKey *sk;
 860         ZEBRA_RES res;
 861
 862         sort_criteria[i].ord = -1;
 863         sort_criteria[i].numerical = 0;
 864
 865         if (sks->which == Z_SortKeySpec_missingValueData)
 866         {
 867             zebra_setError(zh, YAZ_BIB1_UNSUPP_MISSING_DATA_ACTION, 0);
 868             return ZEBRA_FAIL;
 869         }
 870         if (*sks->sortRelation == Z_SortKeySpec_ascending)
 871             sort_criteria[i].relation = 'A';
 872         else if (*sks->sortRelation == Z_SortKeySpec_descending)
 873             sort_criteria[i].relation = 'D';
 874         else
 875         {
 876             zebra_setError(zh, YAZ_BIB1_ILLEGAL_SORT_RELATION, 0);
 877             return ZEBRA_FAIL;
 878         }
 879         if (sks->sortElement->which == Z_SortElement_databaseSpecific)
 880         {
 881             zebra_setError(zh, YAZ_BIB1_DATABASE_SPECIFIC_SORT_UNSUPP, 0);
 882             return ZEBRA_FAIL;
 883         }
 884         else if (sks->sortElement->which != Z_SortElement_generic)
 885         {
 886             zebra_setError(zh, YAZ_BIB1_SORT_ILLEGAL_SORT, 0);
 887             return ZEBRA_FAIL;
 888         }
 889         sk = sks->sortElement->u.generic;
 890         switch (sk->which)
 891         {
 892         case Z_SortKey_sortField:
 893             yaz_log(log_level_sort, "key %d is of type sortField",
 894                     i+1);
 895             sort_criteria[i].numerical = 0;
 896             sort_criteria[i].ord =
 897                 zebraExplain_lookup_attr_str(zh->reg->zei,
 898                                              zinfo_index_category_sort,
 899                                              0, sk->u.sortField);
 900             if (sks->which != Z_SortKeySpec_null
 901                 && sort_criteria[i].ord == -1)
 902             {
 903                 zebra_setError(zh,
 904                                YAZ_BIB1_CANNOT_SORT_ACCORDING_TO_SEQUENCE, 0);
 905                 return ZEBRA_FAIL;
 906             }
 907             break;
 908         case Z_SortKey_elementSpec:
 909             yaz_log(log_level_sort, "key %d is of type elementSpec",
 910                     i+1);
 911             zebra_setError(zh, YAZ_BIB1_CANNOT_SORT_ACCORDING_TO_SEQUENCE, 0);
 912             return ZEBRA_FAIL;
 913         case Z_SortKey_sortAttributes:
 914             yaz_log(log_level_sort, "key %d is of type sortAttributes", i+1);
 915             res = zebra_sort_get_ord(zh, sk->u.sortAttributes,
 916
 917                                      &sort_criteria[i].ord,
 918                                      &sort_criteria[i].numerical);
 919             if (sks->which != Z_SortKeySpec_null && res != ZEBRA_OK)
 920                 return ZEBRA_FAIL;
 921             break;
 922         }
 923         if (zebraExplain_lookup_ord(zh->reg->zei, sort_criteria[i].ord,
 924                                     &sort_criteria[i].index_type,
 925                                     0, 0))
 926         {
 927             zebra_setError(zh, YAZ_BIB1_CANNOT_SORT_ACCORDING_TO_SEQUENCE, 0);
 928             return ZEBRA_FAIL;
 929         }
 930     }
 931     /* allocate space for each cmpare buf + one extra for tmp comparison */
 932     for (i = 0; i<num_criteria; i++)
 933     {
 934         cmp_buf[i] = xmalloc(sset->sort_info->max_entries
 935                              * SORT_IDX_ENTRYSIZE);
 936         tmp_cmp_buf[i] = xmalloc(SORT_IDX_ENTRYSIZE);
 937     }
 938     rfd = rset_open(rset, RSETF_READ);
 939     while (rset_read(rfd, &key, &termid))
 940     {
 941         zint this_sys = key.mem[sysno_mem_index];
 942         if (log_level_searchhits)
 943             key_logdump_txt(log_level_searchhits, &key, termid->name);
 944         kno++;
 945         if (this_sys != psysno)
 946         {
 947             if ((sset->hits & 255) == 0 && zh->break_handler_func)
 948             {
 949                 if (zh->break_handler_func(zh->break_handler_data))
 950                 {
 951                     rset_set_hits_limit(rset, 0);
 952                     break;
 953                 }
 954             }
 955             (sset->hits)++;
 956             psysno = this_sys;
 957             resultSetInsertSort(zh, sset,
 958                                 sort_criteria, num_criteria, psysno, cmp_buf,
 959                                 tmp_cmp_buf);
 960         }
 961     }
 962     rset_close(rfd);
 963
 964     for (i = 0; i<num_criteria; i++)
 965     {
 966         xfree(cmp_buf[i]);
 967         xfree(tmp_cmp_buf[i]);
 968     }
 969
 970     yaz_log(log_level_sort, ZINT_FORMAT " keys, " ZINT_FORMAT " sysnos, sort",
 971             kno, sset->hits);
 972     for (i = 0; i < numTerms; i++)
 973         yaz_log(log_level_sort, "term=\"%s\" type=%s count=" ZINT_FORMAT,
 974                  terms[i]->name, terms[i]->flags, terms[i]->rset->hits_count);
 975     *sort_status = Z_SortResponse_success;
 976     return ZEBRA_OK;
 977 }
 978
 979 RSET resultSetRef(ZebraHandle zh, const char *resultSetId)
 980 {
 981     ZebraSet s;
 982
 983     if ((s = resultSetGet(zh, resultSetId)))
 984         return s->rset;
 985     return NULL;
 986 }
 987
 988 ZEBRA_RES resultSetRank(ZebraHandle zh, ZebraSet zebraSet,
 989                         RSET rset, NMEM nmem)
 990 {
 991     struct it_key key;
 992     TERMID termid;
 993     TERMID *terms;
 994     zint kno = 0;
 995     int numTerms = 0;
 996     int n = 0;
 997     int i;
 998     ZebraRankClass rank_class;
 999     struct zset_sort_info *sort_info;
1000     const char *rank_handler_name = res_get_def(zh->res, "rank", "rank-1");
1001     size_t sysno_mem_index = 0;
1002
1003     if (zh->m_staticrank)
1004         sysno_mem_index = 1;
1005
1006     if (!log_level_set)
1007         loglevels();
1008     sort_info = zebraSet->sort_info;
1009     sort_info->num_entries = 0;
1010     zebraSet->hits = 0;
1011     zebraSet->estimated_hit_count = 0;
1012     rset_getterms(rset, 0, 0, &n);
1013     terms = (TERMID *) nmem_malloc(nmem, sizeof(*terms)*n);
1014     rset_getterms(rset, terms, n, &numTerms);
1015
1016     rank_class = zebraRankLookup(zh, rank_handler_name);
1017     if (!rank_class)
1018     {
1019         yaz_log(YLOG_WARN, "No such rank handler: %s", rank_handler_name);
1020         zebra_setError(zh, YAZ_BIB1_UNSUPP_SEARCH, "Cannot find rank handler");
1021         return ZEBRA_FAIL;
1022     }
1023     else
1024     {
1025         RSFD rfd = rset_open(rset, RSETF_READ);
1026         struct rank_control *rc = rank_class->control;
1027         int score;
1028         zint count = 0;
1029         void *handle = (*rc->begin) (zh->reg, rank_class->class_handle, rset,
1030                                      nmem, terms, numTerms);
1031         zint psysno = 0;  /* previous doc id / sys no */
1032         zint pstaticrank = 0; /* previous static rank */
1033         int stop_flag = 0;
1034         while (rset_read(rfd, &key, &termid))
1035         {
1036             zint this_sys = key.mem[sysno_mem_index];
1037
1038             zint seqno = key.mem[key.len-1];
1039             kno++;
1040             if (log_level_searchhits)
1041                 key_logdump_txt(log_level_searchhits, &key, termid->name);
1042             if (this_sys != psysno)
1043             {   /* new record .. */
1044                 if (!(rfd->counted_items & 255) && zh->break_handler_func)
1045                 {
1046                     if (zh->break_handler_func(zh->break_handler_data))
1047                     {
1048                         yaz_log(YLOG_LOG, "Aborted search");
1049                         stop_flag = 1;
1050                     }
1051                 }
1052                 if (rfd->counted_items > rset->hits_limit)
1053                     stop_flag = 1;
1054                 if (psysno)
1055                 {   /* only if we did have a previous record */
1056                     score = (*rc->calc)(handle, psysno, pstaticrank,
1057                                          &stop_flag);
1058                     /* insert the hit. A=Ascending */
1059                     resultSetInsertRank(zh, sort_info, psysno, score, 'A');
1060                     count++;
1061                 }
1062                 if (stop_flag)
1063                 {
1064                     zebraSet->estimated_hit_count = 1;
1065                     rset_set_hits_limit(rset, 0);
1066                     break;
1067                 }
1068                 psysno = this_sys;
1069                 if (zh->m_staticrank)
1070                     pstaticrank = key.mem[0];
1071             }
1072             (*rc->add)(handle, CAST_ZINT_TO_INT(seqno), termid);
1073         }
1074         /* no more items */
1075         if (psysno)
1076         {   /* we had - at least - one record */
1077             score = (*rc->calc)(handle, psysno, pstaticrank, &stop_flag);
1078             /* insert the hit. A=Ascending */
1079             resultSetInsertRank(zh, sort_info, psysno, score, 'A');
1080             count++;
1081         }
1082         (*rc->end)(zh->reg, handle);
1083         rset_close(rfd);
1084     }
1085     zebraSet->hits = rset->hits_count;
1086
1087     yaz_log(log_level_searchterms, ZINT_FORMAT " keys, "
1088             ZINT_FORMAT " sysnos, rank",  kno, zebraSet->hits);
1089     for (i = 0; i < numTerms; i++)
1090     {
1091         yaz_log(log_level_searchterms, "term=\"%s\" type=%s count="
1092                 ZINT_FORMAT,
1093                 terms[i]->name, terms[i]->flags, terms[i]->rset->hits_count);
1094     }
1095     return ZEBRA_OK;
1096 }
1097
1098 ZebraRankClass zebraRankLookup(ZebraHandle zh, const char *name)
1099 {
1100     ZebraRankClass p = zh->reg->rank_classes;
1101     while (p && strcmp(p->control->name, name))
1102         p = p->next;
1103     if (p && !p->init_flag)
1104     {
1105         if (p->control->create)
1106             p->class_handle = (*p->control->create)(zh);
1107         p->init_flag = 1;
1108     }
1109     return p;
1110 }
1111
1112 void zebraRankInstall(struct zebra_register *reg, struct rank_control *ctrl)
1113 {
1114     ZebraRankClass p = (ZebraRankClass) xmalloc(sizeof(*p));
1115     p->control = (struct rank_control *) xmalloc(sizeof(*p->control));
1116     memcpy(p->control, ctrl, sizeof(*p->control));
1117     p->control->name = xstrdup(ctrl->name);
1118     p->init_flag = 0;
1119     p->next = reg->rank_classes;
1120     reg->rank_classes = p;
1121 }
1122
1123 void zebraRankDestroy(struct zebra_register *reg)
1124 {
1125     ZebraRankClass p = reg->rank_classes;
1126     while (p)
1127     {
1128         ZebraRankClass p_next = p->next;
1129         if (p->init_flag && p->control->destroy)
1130             (*p->control->destroy)(reg, p->class_handle);
1131         xfree(p->control->name);
1132         xfree(p->control);
1133         xfree(p);
1134         p = p_next;
1135     }
1136     reg->rank_classes = NULL;
1137 }
1138
1139 static int trav_rset_for_termids(RSET rset, TERMID *termid_array,
1140                                  zint *hits_array, int *approx_array)
1141 {
1142     int no = 0;
1143     int i;
1144     for (i = 0; i<rset->no_children; i++)
1145         no += trav_rset_for_termids(rset->children[i],
1146                                     (termid_array ? termid_array + no : 0),
1147                                     (hits_array ? hits_array + no : 0),
1148                                     (approx_array ? approx_array + no : 0));
1149     if (rset->term)
1150     {
1151         if (termid_array)
1152             termid_array[no] = rset->term;
1153         if (hits_array)
1154             hits_array[no] = rset->hits_count;
1155         if (approx_array)
1156             approx_array[no] = rset->hits_approx;
1157 #if 0
1158         yaz_log(YLOG_LOG, "rset=%p term=%s limit=" ZINT_FORMAT
1159                 " count=" ZINT_FORMAT,
1160                 rset, rset->term->name, rset->hits_limit, rset->hits_count);
1161 #endif
1162         no++;
1163     }
1164     return no;
1165 }
1166
1167 ZEBRA_RES zebra_result_set_term_no(ZebraHandle zh, const char *setname,
1168                                    int *num_terms)
1169 {
1170     ZebraSet sset = resultSetGet(zh, setname);
1171     *num_terms = 0;
1172     if (sset)
1173     {
1174         *num_terms = trav_rset_for_termids(sset->rset, 0, 0, 0);
1175         return ZEBRA_OK;
1176     }
1177     return ZEBRA_FAIL;
1178 }
1179
1180 ZEBRA_RES zebra_result_set_term_info(ZebraHandle zh, const char *setname,
1181                                      int no, zint *count, int *approx,
1182                                      char *termbuf, size_t *termlen,
1183                                      const char **term_ref_id)
1184 {
1185     ZebraSet sset = resultSetGet(zh, setname);
1186     if (sset)
1187     {
1188         int num_terms = trav_rset_for_termids(sset->rset, 0, 0, 0);
1189         if (no >= 0 && no < num_terms)
1190         {
1191             TERMID *term_array = xmalloc(num_terms * sizeof(*term_array));
1192             zint *hits_array = xmalloc(num_terms * sizeof(*hits_array));
1193             int *approx_array = xmalloc(num_terms * sizeof(*approx_array));
1194
1195             trav_rset_for_termids(sset->rset, term_array,
1196                                   hits_array, approx_array);
1197
1198             if (count)
1199                 *count = hits_array[no];
1200             if (approx)
1201                 *approx = approx_array[no];
1202             if (termbuf)
1203             {
1204                 char *inbuf = term_array[no]->name;
1205                 size_t inleft = strlen(inbuf);
1206                 size_t outleft = *termlen - 1;
1207
1208                 if (zh->iconv_from_utf8 != 0)
1209                 {
1210                     char *outbuf = termbuf;
1211                     size_t ret;
1212
1213                     ret = yaz_iconv(zh->iconv_from_utf8, &inbuf, &inleft,
1214                                     &outbuf, &outleft);
1215                     if (ret == (size_t)(-1))
1216                         *termlen = 0;
1217                     else
1218                     {
1219                         yaz_iconv(zh->iconv_from_utf8, 0, 0,
1220                                   &outbuf, &outleft);
1221                         *termlen = outbuf - termbuf;
1222                     }
1223                 }
1224                 else
1225                 {
1226                     if (inleft > outleft)
1227                         inleft = outleft;
1228                     *termlen = inleft;
1229                     memcpy(termbuf, inbuf, *termlen);
1230                 }
1231                 termbuf[*termlen] = '\0';
1232             }
1233             if (term_ref_id)
1234                 *term_ref_id = term_array[no]->ref_id;
1235
1236             xfree(term_array);
1237             xfree(hits_array);
1238             xfree(approx_array);
1239             return ZEBRA_OK;
1240         }
1241     }
1242     return ZEBRA_FAIL;
1243 }
1244
1245 ZEBRA_RES zebra_snippets_hit_vector(ZebraHandle zh, const char *setname,
1246                                     zint sysno, zebra_snippets *snippets)
1247 {
1248     ZebraSet sset = resultSetGet(zh, setname);
1249     yaz_log(YLOG_DEBUG, "zebra_get_hit_vector setname=%s zysno=" ZINT_FORMAT,
1250             setname, sysno);
1251     if (!sset)
1252         return ZEBRA_FAIL;
1253     else
1254     {
1255         struct rset_key_control *kc = zebra_key_control_create(zh);
1256         NMEM nmem = nmem_create();
1257         struct it_key key;
1258         RSET rsets[2], rset_comb;
1259         RSET rset_temp = rset_create_temp(nmem, kc, kc->scope,
1260                                           res_get(zh->res, "setTmpDir"),0 );
1261
1262         TERMID termid;
1263         RSFD rsfd = rset_open(rset_temp, RSETF_WRITE);
1264
1265         key.mem[0] = sysno;
1266         key.mem[1] = 0;
1267         key.mem[2] = 0;
1268         key.mem[3] = 0;
1269         key.len = 2;
1270         rset_write(rsfd, &key);
1271         rset_close(rsfd);
1272
1273         rsets[0] = rset_temp;
1274         rsets[1] = rset_dup(sset->rset);
1275
1276         rset_comb = rset_create_and(nmem, kc, kc->scope, 2, rsets);
1277
1278         rsfd = rset_open(rset_comb, RSETF_READ);
1279
1280         while (rset_read(rsfd, &key, &termid))
1281         {
1282             if (termid)
1283             {
1284                 struct ord_list *ol;
1285                 for (ol = termid->ol; ol; ol = ol->next)
1286                 {
1287                     zebra_snippets_append(snippets, key.mem[key.len-1], 0,
1288                                           ol->ord, termid->name);
1289                 }
1290             }
1291         }
1292         rset_close(rsfd);
1293
1294         rset_delete(rset_comb);
1295         nmem_destroy(nmem);
1296         kc->dec(kc);
1297     }
1298     return ZEBRA_OK;
1299 }
1300
1301 static ZEBRA_RES zebra_recid_to_sysno(ZebraHandle zh,
1302                                       const char **basenames, int num_bases,
1303                                       zint recid,
1304                                       zint *sysnos, int *no_sysnos)
1305 {
1306     ZEBRA_RES res = ZEBRA_OK;
1307     int sysnos_offset = 0;
1308     int i;
1309
1310     if (!zh->reg->isamb)
1311     {
1312         if (sysnos_offset < *no_sysnos)
1313             *sysnos = recid;
1314         sysnos_offset++;
1315     }
1316     else
1317     {
1318         for (i = 0; res == ZEBRA_OK && i < num_bases; i++)
1319         {
1320             const char *database = basenames[i];
1321             if (zebraExplain_curDatabase(zh->reg->zei, database) == 0)
1322             {
1323                 const char *index_type = "w";
1324                 const char *use_string = "_ALLRECORDS";
1325                 int ord;
1326                 zinfo_index_category_t cat = zinfo_index_category_alwaysmatches;
1327                 ord = zebraExplain_lookup_attr_str(zh->reg->zei, cat,
1328                                                    index_type, use_string);
1329                 if (ord != -1)
1330                 {
1331                     char ord_buf[32];
1332                     int ord_len = key_SU_encode(ord, ord_buf);
1333                     char *info;
1334
1335                     ord_buf[ord_len] = '\0';
1336
1337                     info = dict_lookup(zh->reg->dict, ord_buf);
1338                     if (info)
1339                     {
1340                         if (*info != sizeof(ISAM_P))
1341                         {
1342                             res = ZEBRA_FAIL;
1343                         }
1344                         else
1345                         {
1346                             ISAM_P isam_p;
1347                             ISAMB_PP pt;
1348                             struct it_key key_until, key_found;
1349                             int i = 0;
1350                             int r;
1351
1352                             memcpy(&isam_p, info+1, sizeof(ISAM_P));
1353
1354                             pt = isamb_pp_open(zh->reg->isamb, isam_p, 2);
1355                             if (!pt)
1356                                 res = ZEBRA_FAIL;
1357                             else
1358                             {
1359                                 key_until.mem[i++] = recid;
1360                                 key_until.mem[i++] = 0;  /* section_id */
1361                                 if (zh->m_segment_indexing)
1362                                     key_until.mem[i++] = 0; /* segment */
1363                                 key_until.mem[i++] = 0;
1364                                 key_until.len = i;
1365
1366                                 r = isamb_pp_forward(pt, &key_found, &key_until);
1367                                 while (r && key_found.mem[0] == recid)
1368                                 {
1369                                     if (sysnos_offset < *no_sysnos)
1370                                         sysnos[sysnos_offset] =
1371                                             key_found.mem[key_found.len-1];
1372
1373                                     yaz_log(YLOG_LOG,  "Found " ZINT_FORMAT,
1374                                             key_found.mem[key_found.len-1]);
1375                                     r = isamb_pp_read(pt, &key_found);
1376                                     sysnos_offset++;
1377                                 }
1378                                 isamb_pp_close(pt);
1379                             }
1380                         }
1381                     }
1382                 }
1383             }
1384         }
1385     }
1386     *no_sysnos = sysnos_offset;
1387     return res;
1388 }
1389
1390 ZEBRA_RES zebra_result_recid_to_sysno(ZebraHandle zh,
1391                                       const char *setname,
1392                                       zint recid,
1393                                       zint *sysnos, int *no_sysnos)
1394 {
1395     const char **basenames;
1396     int num_bases;
1397     ZEBRA_RES res;
1398
1399     res = resultSetGetBaseNames(zh, setname, &basenames, &num_bases);
1400     if (res != ZEBRA_OK)
1401         return ZEBRA_FAIL;
1402
1403     return zebra_recid_to_sysno(zh, basenames, num_bases,
1404                                 recid, sysnos, no_sysnos);
1405 }
1406
1407 void zebra_count_set(ZebraHandle zh, RSET rset, zint *count,
1408                      zint approx_limit)
1409 {
1410     zint psysno = 0;
1411     struct it_key key;
1412     RSFD rfd;
1413
1414     yaz_log(YLOG_DEBUG, "count_set");
1415
1416     rset->hits_limit = approx_limit;
1417
1418     *count = 0;
1419     rfd = rset_open(rset, RSETF_READ);
1420     while (rset_read(rfd, &key,0 /* never mind terms */))
1421     {
1422         if (key.mem[0] != psysno)
1423         {
1424             psysno = key.mem[0];
1425             if (rfd->counted_items >= rset->hits_limit)
1426                 break;
1427         }
1428     }
1429     rset_close(rfd);
1430     *count = rset->hits_count;
1431 }
1432
1433
1434 /*
1435  * Local variables:
1436  * c-basic-offset: 4
1437  * indent-tabs-mode: nil
1438  * End:
1439  * vim: shiftwidth=4 tabstop=8 expandtab
1440  */
1441