index/zsets.c

   1 /* This file is part of the Zebra server.
   2    Copyright (C) 1995-2008 Index Data
   3
   4 Zebra is free software; you can redistribute it and/or modify it under
   5 the terms of the GNU General Public License as published by the Free
   6 Software Foundation; either version 2, or (at your option) any later
   7 version.
   8
   9 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 for more details.
  13
  14 You should have received a copy of the GNU General Public License
  15 along with this program; if not, write to the Free Software
  16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  17
  18 */
  19
  20
  21 #include <stdio.h>
  22 #include <assert.h>
  23 #ifdef WIN32
  24 #include <io.h>
  25 #else
  26 #include <unistd.h>
  27 #endif
  28
  29 #include "index.h"
  30 #include "rank.h"
  31 #include <yaz/diagbib1.h>
  32 #include <rset.h>
  33
  34 #define ZSET_SORT_MAX_LEVEL 10
  35
  36 struct zebra_set_term_entry {
  37     int reg_type;
  38     char *db;
  39     char *index_name;
  40     char *term;
  41 };
  42
  43 struct zebra_set {
  44     char *name;
  45     RSET rset;
  46     NMEM nmem;
  47     NMEM rset_nmem; /* for creating the rsets in */
  48     zint hits;
  49     int num_bases;
  50     const char **basenames;
  51     Z_RPNQuery *rpn;
  52     Z_SortKeySpecList *sortSpec;
  53     struct zset_sort_info *sort_info;
  54     struct zebra_set_term_entry *term_entries;
  55     int term_entries_max;
  56     struct zebra_set *next;
  57     int locked;
  58     int estimated_hit_count;
  59
  60     zint cache_position;  /* last position */
  61     RSFD cache_rfd;       /* rfd (NULL if not existing) */
  62     zint cache_psysno;    /* sysno for last position */
  63     zint approx_limit;    /* limit before we do approx */
  64 };
  65
  66 struct zset_sort_entry {
  67     zint sysno;
  68     int score;
  69 };
  70
  71 struct zset_sort_info {
  72     int max_entries;
  73     int num_entries;
  74     struct zset_sort_entry *all_entries;
  75     struct zset_sort_entry **entries;
  76 };
  77
  78 static int log_level_set=0;
  79 static int log_level_sort=0;
  80 static int log_level_searchhits=0;
  81 static int log_level_searchterms=0;
  82 static int log_level_resultsets=0;
  83
  84 static void loglevels(void)
  85 {
  86     if (log_level_set)
  87         return;
  88     log_level_sort = yaz_log_module_level("sorting");
  89     log_level_searchhits = yaz_log_module_level("searchhits");
  90     log_level_searchterms = yaz_log_module_level("searchterms");
  91     log_level_resultsets = yaz_log_module_level("resultsets");
  92     log_level_set = 1;
  93 }
  94
  95
  96 static ZEBRA_RES resultSetSearch(ZebraHandle zh, NMEM nmem, NMEM rset_nmem,
  97                                  Z_RPNQuery *rpn, ZebraSet sset)
  98 {
  99     RSET rset = 0;
 100     Z_SortKeySpecList *sort_sequence;
 101     int sort_status, i;
 102     ZEBRA_RES res = ZEBRA_OK;
 103
 104     sort_sequence = (Z_SortKeySpecList *)
 105         nmem_malloc(nmem, sizeof(*sort_sequence));
 106     sort_sequence->num_specs = 10; /* FIXME - Hard-coded number */
 107     sort_sequence->specs = (Z_SortKeySpec **)
 108         nmem_malloc(nmem, sort_sequence->num_specs *
 109                     sizeof(*sort_sequence->specs));
 110     for (i = 0; i<sort_sequence->num_specs; i++)
 111         sort_sequence->specs[i] = 0;
 112
 113     rpn_get_top_approx_limit(zh, rpn->RPNStructure, &sset->approx_limit);
 114
 115     res = rpn_search_top(zh, rpn->RPNStructure, rpn->attributeSetId,
 116                          nmem, rset_nmem,
 117                          sort_sequence,
 118                          sset->num_bases, sset->basenames,
 119                          &rset);
 120     if (res != ZEBRA_OK)
 121     {
 122         sset->rset = 0;
 123         return res;
 124     }
 125     for (i = 0; sort_sequence->specs[i]; i++)
 126         ;
 127     sort_sequence->num_specs = i;
 128     rset->hits_limit = sset->approx_limit;
 129     if (!i)
 130     {
 131         res = resultSetRank(zh, sset, rset, rset_nmem);
 132     }
 133     else
 134     {
 135         res = resultSetSortSingle(zh, nmem, sset, rset,
 136                                   sort_sequence, &sort_status);
 137     }
 138     sset->rset = rset;
 139     return res;
 140 }
 141
 142
 143 ZEBRA_RES resultSetAddRPN(ZebraHandle zh, NMEM m, Z_RPNQuery *rpn,
 144                           int num_bases, char **basenames,
 145                           const char *setname,
 146                           zint *hits, int *estimated_hit_count)
 147 {
 148     ZebraSet zebraSet;
 149     int i;
 150     ZEBRA_RES res;
 151
 152     *hits = 0;
 153     *estimated_hit_count = 0;
 154
 155     zebraSet = resultSetAdd(zh, setname, 1);
 156     if (!zebraSet)
 157         return ZEBRA_FAIL;
 158     zebraSet->locked = 1;
 159     zebraSet->rpn = 0;
 160     zebraSet->nmem = m;
 161     zebraSet->rset_nmem = nmem_create();
 162
 163     zebraSet->num_bases = num_bases;
 164     zebraSet->basenames =
 165         nmem_malloc(zebraSet->nmem, num_bases * sizeof(*zebraSet->basenames));
 166     for (i = 0; i<num_bases; i++)
 167         zebraSet->basenames[i] = nmem_strdup(zebraSet->nmem, basenames[i]);
 168
 169     res = resultSetSearch(zh, zebraSet->nmem, zebraSet->rset_nmem,
 170                           rpn, zebraSet);
 171     *hits = zebraSet->hits;
 172     if (zebraSet->estimated_hit_count)
 173         *estimated_hit_count = 1;
 174
 175     if (zebraSet->rset)
 176         zebraSet->rpn = rpn;
 177     zebraSet->locked = 0;
 178     if (!zebraSet->rset)
 179         return ZEBRA_FAIL;
 180     return res;
 181 }
 182
 183 void resultSetAddTerm(ZebraHandle zh, ZebraSet s, int reg_type,
 184                       const char *db, const char *index_name,
 185                       const char *term)
 186 {
 187     assert(zh); /* compiler shut up */
 188     if (!s->nmem)
 189         s->nmem = nmem_create();
 190     if (!s->term_entries)
 191     {
 192         int i;
 193         s->term_entries_max = 1000;
 194         s->term_entries =
 195             nmem_malloc(s->nmem, s->term_entries_max *
 196                         sizeof(*s->term_entries));
 197         for (i = 0; i < s->term_entries_max; i++)
 198             s->term_entries[i].term = 0;
 199     }
 200     if (s->hits < s->term_entries_max)
 201     {
 202         s->term_entries[s->hits].reg_type = reg_type;
 203         s->term_entries[s->hits].db = nmem_strdup(s->nmem, db);
 204         s->term_entries[s->hits].index_name = nmem_strdup(s->nmem, index_name);
 205         s->term_entries[s->hits].term = nmem_strdup(s->nmem, term);
 206     }
 207     (s->hits)++;
 208 }
 209
 210 ZebraSet resultSetAdd(ZebraHandle zh, const char *name, int ov)
 211 {
 212     ZebraSet s;
 213     int i;
 214
 215     for (s = zh->sets; s; s = s->next)
 216         if (!strcmp(s->name, name))
 217             break;
 218
 219     if (!log_level_set)
 220         loglevels();
 221     if (s)
 222     {
 223         yaz_log(log_level_resultsets, "updating result set %s", name);
 224         if (!ov || s->locked)
 225             return NULL;
 226         if (s->rset)
 227         {
 228             if (s->cache_rfd)
 229                 rset_close(s->cache_rfd);
 230             rset_delete(s->rset);
 231         }
 232         if (s->rset_nmem)
 233             nmem_destroy(s->rset_nmem);
 234         if (s->nmem)
 235             nmem_destroy(s->nmem);
 236     }
 237     else
 238     {
 239         const char *sort_max_str = zebra_get_resource(zh, "sortmax", "1000");
 240
 241         yaz_log(log_level_resultsets, "adding result set %s", name);
 242         s = (ZebraSet) xmalloc(sizeof(*s));
 243         s->next = zh->sets;
 244         zh->sets = s;
 245         s->name = xstrdup(name);
 246
 247         s->sort_info = (struct zset_sort_info *)
 248             xmalloc(sizeof(*s->sort_info));
 249         s->sort_info->max_entries = atoi(sort_max_str);
 250         if (s->sort_info->max_entries < 2)
 251             s->sort_info->max_entries = 2;
 252
 253         s->sort_info->entries = (struct zset_sort_entry **)
 254             xmalloc(sizeof(*s->sort_info->entries) *
 255                     s->sort_info->max_entries);
 256         s->sort_info->all_entries = (struct zset_sort_entry *)
 257             xmalloc(sizeof(*s->sort_info->all_entries) *
 258                     s->sort_info->max_entries);
 259         for (i = 0; i < s->sort_info->max_entries; i++)
 260             s->sort_info->entries[i] = s->sort_info->all_entries + i;
 261     }
 262     s->locked = 0;
 263     s->term_entries = 0;
 264     s->hits = 0;
 265     s->rset = 0;
 266     s->rset_nmem = 0;
 267     s->nmem = 0;
 268     s->rpn = 0;
 269     s->sortSpec = 0;
 270     s->cache_position = 0;
 271     s->cache_rfd = 0;
 272     s->approx_limit = zh->approx_limit;
 273     s->estimated_hit_count = 0;
 274     return s;
 275 }
 276
 277 ZebraSet resultSetGet(ZebraHandle zh, const char *name)
 278 {
 279     ZebraSet s;
 280
 281     for (s = zh->sets; s; s = s->next)
 282         if (!strcmp(s->name, name))
 283         {
 284             if (!s->term_entries && !s->rset && s->rpn)
 285             {
 286                 NMEM nmem = nmem_create();
 287                 yaz_log(log_level_resultsets, "research %s", name);
 288                 if (!s->rset_nmem)
 289                     s->rset_nmem = nmem_create();
 290                 resultSetSearch(zh, nmem, s->rset_nmem, s->rpn, s);
 291                 if (s->rset && s->sortSpec)
 292                 {
 293                     int sort_status;
 294                     yaz_log(log_level_resultsets, "resort %s", name);
 295                     resultSetSortSingle(zh, nmem, s, s->rset, s->sortSpec,
 296                                         &sort_status);
 297                 }
 298                 nmem_destroy(nmem);
 299             }
 300             return s;
 301         }
 302     return NULL;
 303 }
 304
 305 ZEBRA_RES resultSetGetBaseNames(ZebraHandle zh, const char *setname,
 306                                 const char ***basenames, int *num_bases)
 307 {
 308     ZebraSet sset = resultSetGet(zh, setname);
 309     if (!sset)
 310         return ZEBRA_FAIL;
 311     *basenames = sset->basenames;
 312     *num_bases = sset->num_bases;
 313     return ZEBRA_OK;
 314
 315 }
 316
 317 void resultSetInvalidate(ZebraHandle zh)
 318 {
 319     ZebraSet s = zh->sets;
 320
 321     yaz_log(log_level_resultsets, "invalidating result sets");
 322     for (; s; s = s->next)
 323     {
 324         if (s->rset)
 325         {
 326             if (s->cache_rfd)
 327                 rset_close(s->cache_rfd);
 328             rset_delete(s->rset);
 329         }
 330         s->rset = 0;
 331         s->cache_rfd = 0;
 332         s->cache_position = 0;
 333         if (s->rset_nmem)
 334             nmem_destroy(s->rset_nmem);
 335         s->rset_nmem=0;
 336     }
 337 }
 338
 339 void resultSetDestroy(ZebraHandle zh, int num, char **names,int *statuses)
 340 {
 341     ZebraSet * ss = &zh->sets;
 342     int i;
 343
 344     if (statuses)
 345         for (i = 0; i<num; i++)
 346             statuses[i] = Z_DeleteStatus_resultSetDidNotExist;
 347     while (*ss)
 348     {
 349         int i = -1;
 350         ZebraSet s = *ss;
 351         if (num >= 0)
 352         {
 353             for (i = 0; i<num; i++)
 354                 if (!strcmp(s->name, names[i]))
 355                 {
 356                     if (statuses)
 357                         statuses[i] = Z_DeleteStatus_success;
 358                     i = -1;
 359                     break;
 360                 }
 361         }
 362         if (i < 0)
 363         {
 364             *ss = s->next;
 365
 366             xfree(s->sort_info->all_entries);
 367             xfree(s->sort_info->entries);
 368             xfree(s->sort_info);
 369
 370             if (s->nmem)
 371                 nmem_destroy(s->nmem);
 372             if (s->rset)
 373             {
 374                 if (s->cache_rfd)
 375                     rset_close(s->cache_rfd);
 376                 rset_delete(s->rset);
 377             }
 378             if (s->rset_nmem)
 379                 nmem_destroy(s->rset_nmem);
 380             xfree(s->name);
 381             xfree(s);
 382         }
 383         else
 384             ss = &s->next;
 385     }
 386 }
 387
 388 ZebraMetaRecord *zebra_meta_records_create_range(ZebraHandle zh,
 389                                                  const char *name,
 390                                                  zint start, int num)
 391 {
 392     zint pos_small[10];
 393     zint *pos = pos_small;
 394     ZebraMetaRecord *mr;
 395     int i;
 396
 397     if (num > 10000 || num <= 0)
 398         return 0;
 399
 400     if (num > 10)
 401         pos = xmalloc(sizeof(*pos) * num);
 402
 403     for (i = 0; i<num; i++)
 404         pos[i] = start+i;
 405
 406     mr = zebra_meta_records_create(zh, name, num, pos);
 407
 408     if (num > 10)
 409         xfree(pos);
 410     return mr;
 411 }
 412
 413 ZebraMetaRecord *zebra_meta_records_create(ZebraHandle zh, const char *name,
 414                                            int num, zint *positions)
 415 {
 416     ZebraSet sset;
 417     ZebraMetaRecord *sr = 0;
 418     RSET rset;
 419     int i;
 420     struct zset_sort_info *sort_info;
 421     size_t sysno_mem_index = 0;
 422
 423     if (zh->m_staticrank)
 424         sysno_mem_index = 1;
 425
 426     if (!log_level_set)
 427         loglevels();
 428     if (!(sset = resultSetGet(zh, name)))
 429         return NULL;
 430     if (!(rset = sset->rset))
 431     {
 432         if (!sset->term_entries)
 433             return 0;
 434         sr = (ZebraMetaRecord *) xmalloc(sizeof(*sr) * num);
 435         for (i = 0; i<num; i++)
 436         {
 437             sr[i].sysno = 0;
 438             sr[i].score = -1;
 439             sr[i].term = 0;
 440             sr[i].db = 0;
 441
 442             if (positions[i] <= sset->term_entries_max)
 443             {
 444                 sr[i].term = sset->term_entries[positions[i]-1].term;
 445                 sr[i].db = sset->term_entries[positions[i]-1].db;
 446             }
 447         }
 448     }
 449     else
 450     {
 451         sr = (ZebraMetaRecord *) xmalloc(sizeof(*sr) * num);
 452         for (i = 0; i<num; i++)
 453         {
 454             sr[i].sysno = 0;
 455             sr[i].score = -1;
 456             sr[i].term = 0;
 457             sr[i].db = 0;
 458         }
 459         sort_info = sset->sort_info;
 460         if (sort_info)
 461         {
 462             zint position;
 463
 464             for (i = 0; i<num; i++)
 465             {
 466                 position = positions[i];
 467                 if (position > 0 && position <= sort_info->num_entries)
 468                 {
 469                     yaz_log(log_level_sort, "got pos=" ZINT_FORMAT
 470                             " (sorted)", position);
 471                     sr[i].sysno = sort_info->entries[position-1]->sysno;
 472                     sr[i].score = sort_info->entries[position-1]->score;
 473                 }
 474             }
 475         }
 476         /* did we really get all entries using sort ? */
 477         for (i = 0; i<num; i++)
 478         {
 479             if (!sr[i].sysno)
 480                 break;
 481         }
 482         if (i < num) /* nope, get the rest, unsorted - sorry */
 483         {
 484             zint position = 0;
 485             int num_i = 0;
 486             zint psysno = 0;
 487             RSFD rfd;
 488             struct it_key key;
 489
 490             if (sort_info)
 491                 position = sort_info->num_entries;
 492             while (num_i < num && positions[num_i] <= position)
 493                 num_i++;
 494
 495             if (sset->cache_rfd &&
 496                 num_i < num && positions[num_i] > sset->cache_position)
 497             {
 498                 position = sset->cache_position;
 499                 rfd = sset->cache_rfd;
 500                 psysno = sset->cache_psysno;
 501             }
 502             else
 503             {
 504                 if (sset->cache_rfd)
 505                     rset_close(sset->cache_rfd);
 506                 rfd = rset_open(rset, RSETF_READ);
 507             }
 508             while (num_i < num && rset_read(rfd, &key, 0))
 509             {
 510                 zint this_sys = key.mem[sysno_mem_index];
 511                 if (this_sys != psysno)
 512                 {
 513                     psysno = this_sys;
 514                     if (sort_info)
 515                     {
 516                         /* determine we alreay have this in our set */
 517                         for (i = sort_info->num_entries; --i >= 0; )
 518                             if (psysno == sort_info->entries[i]->sysno)
 519                                 break;
 520                         if (i >= 0)
 521                             continue;
 522                     }
 523                     position++;
 524                     assert(num_i < num);
 525                     if (position == positions[num_i])
 526                     {
 527                         sr[num_i].sysno = psysno;
 528                         yaz_log(log_level_sort, "got pos=" ZINT_FORMAT " (unsorted)", position);
 529                         sr[num_i].score = -1;
 530                         num_i++;
 531                     }
 532                 }
 533             }
 534             sset->cache_position = position;
 535             sset->cache_psysno = psysno;
 536             sset->cache_rfd = rfd;
 537         }
 538     }
 539     return sr;
 540 }
 541
 542 void zebra_meta_records_destroy(ZebraHandle zh, ZebraMetaRecord *records,
 543                                 int num)
 544 {
 545     assert(zh); /* compiler shut up about unused arg */
 546     xfree(records);
 547 }
 548
 549 struct sortKeyInfo {
 550     int relation;
 551     int *ord; /* array of ord for each database searched */
 552     int *numerical; /* array of ord for each database searched */
 553     const char *index_type;
 554 };
 555
 556 void resultSetInsertSort(ZebraHandle zh, ZebraSet sset,
 557                          int database_no,
 558                          struct sortKeyInfo *criteria, int num_criteria,
 559                          zint sysno,
 560                          char *cmp_buf[], char *tmp_cmp_buf[])
 561 {
 562     struct zset_sort_entry *new_entry = NULL;
 563     struct zset_sort_info *sort_info = sset->sort_info;
 564     int i, j;
 565     WRBUF w = wrbuf_alloc();
 566
 567     zebra_sort_sysno(zh->reg->sort_index, sysno);
 568     for (i = 0; i<num_criteria; i++)
 569     {
 570         char *this_entry_buf = tmp_cmp_buf[i];
 571         memset(this_entry_buf, '\0', SORT_IDX_ENTRYSIZE);
 572
 573         if (criteria[i].ord[database_no] != -1)
 574         {
 575             yaz_log(log_level_sort, "pre zebra_sort_type ord is %d",
 576                     criteria[i].ord[database_no]);
 577             zebra_sort_type(zh->reg->sort_index, criteria[i].ord[database_no]);
 578             wrbuf_rewind(w);
 579             zebra_sort_read(zh->reg->sort_index, w);
 580             memcpy(this_entry_buf, wrbuf_buf(w),
 581                    (wrbuf_len(w) >= SORT_IDX_ENTRYSIZE) ?
 582                    SORT_IDX_ENTRYSIZE : wrbuf_len(w));
 583         }
 584         else
 585         {
 586             yaz_log(log_level_sort, "criteria[i].ord is -1 so not reading from sort index");
 587         }
 588     }
 589     wrbuf_destroy(w);
 590     i = sort_info->num_entries;
 591     while (--i >= 0)
 592     {
 593         int rel = 0;
 594         for (j = 0; j<num_criteria; j++)
 595         {
 596             char *this_entry_buf = tmp_cmp_buf[j];
 597             char *other_entry_buf =
 598                 cmp_buf[j] + i * SORT_IDX_ENTRYSIZE;
 599             if (criteria[j].numerical[database_no])
 600             {
 601                 char this_entry_org[1024];
 602                 char other_entry_org[1024];
 603                 double diff;
 604                 /* when searching multiple databases, we use the index
 605                    type of the first one. So if they differ between
 606                    databases, we have a problem here we could store the
 607                    index_type for each database, but if we didn't find the
 608                    record in any sort index, then we still don't know to
 609                    which database it belongs. */
 610                 const char *index_type = criteria[j].index_type;
 611                 zebra_term_untrans(zh, index_type, this_entry_org,
 612                                    this_entry_buf);
 613                 zebra_term_untrans(zh, index_type, other_entry_org,
 614                                    other_entry_buf);
 615                 diff = atof(this_entry_org) - atof(other_entry_org);
 616
 617                 if (diff > 0.0)
 618                     rel = 1;
 619                 else if (diff < 0.0)
 620                     rel = -1;
 621                 else
 622                     rel = 0;
 623             }
 624             else
 625             {
 626                 rel = memcmp(this_entry_buf, other_entry_buf,
 627                              SORT_IDX_ENTRYSIZE);
 628             }
 629             /* when the compare is equal, continue to next criteria,
 630                else break out */
 631             if (rel)
 632                 break;
 633         }
 634         if (!rel)
 635             break;
 636         if (criteria[j].relation == 'A')
 637         {
 638             if (rel > 0)
 639                 break;
 640         }
 641         else if (criteria[j].relation == 'D')
 642         {
 643             if (rel < 0)
 644                 break;
 645         }
 646     }
 647     ++i;
 648     yaz_log(log_level_sort, "ok, we want to insert record at position %d",i);
 649     j = sort_info->max_entries;
 650     if (i == j){
 651         yaz_log(log_level_sort, "sort_info->max_entries reached (%d) abort sort",j);
 652         return;
 653     }
 654
 655     if (sort_info->num_entries == j)
 656         --j;
 657     else
 658         j = (sort_info->num_entries)++;
 659     new_entry = sort_info->entries[j];
 660     /* move up all higher entries (to make room) */
 661     while (j != i)
 662     {
 663         int k;
 664         for (k = 0; k<num_criteria; k++)
 665         {
 666             char *j_buf = cmp_buf[k] + j * SORT_IDX_ENTRYSIZE;
 667             char *j_1_buf = cmp_buf[k] + (j-1) * SORT_IDX_ENTRYSIZE;
 668             memcpy(j_buf, j_1_buf, SORT_IDX_ENTRYSIZE);
 669         }
 670         sort_info->entries[j] = sort_info->entries[j-1];
 671         --j;
 672     }
 673     /* and insert the new entry at the correct place */
 674     sort_info->entries[i] = new_entry;
 675     assert(new_entry);
 676     /* and add this to the compare buffer */
 677     for (i = 0; i<num_criteria; i++)
 678     {
 679         char *new_entry_buf = cmp_buf[i] + j * SORT_IDX_ENTRYSIZE;
 680         char *this_entry_buf = tmp_cmp_buf[i];
 681         memcpy(new_entry_buf, this_entry_buf, SORT_IDX_ENTRYSIZE);
 682     }
 683     new_entry->sysno = sysno;
 684     new_entry->score = -1;
 685 }
 686
 687 void resultSetInsertRank(ZebraHandle zh, struct zset_sort_info *sort_info,
 688                          zint sysno, int score, int relation)
 689 {
 690     struct zset_sort_entry *new_entry = NULL;
 691     int i, j;
 692     assert(zh); /* compiler shut up about unused arg */
 693
 694     i = sort_info->num_entries;
 695     while (--i >= 0)
 696     {
 697         int rel = 0;
 698
 699         rel = score - sort_info->entries[i]->score;
 700
 701         if (relation == 'D')
 702         {
 703             if (rel >= 0)
 704                 break;
 705         }
 706         else if (relation == 'A')
 707         {
 708             if (rel <= 0)
 709                 break;
 710         }
 711     }
 712     ++i;
 713     j = sort_info->max_entries;
 714     if (i == j)
 715         return;
 716
 717     if (sort_info->num_entries == j)
 718         --j;
 719     else
 720         j = (sort_info->num_entries)++;
 721
 722     new_entry = sort_info->entries[j];
 723     while (j != i)
 724     {
 725         sort_info->entries[j] = sort_info->entries[j-1];
 726         --j;
 727     }
 728     sort_info->entries[i] = new_entry;
 729     assert(new_entry);
 730     new_entry->sysno = sysno;
 731     new_entry->score = score;
 732 }
 733
 734 static Z_RPNQuery *copy_RPNQuery(Z_RPNQuery *src, NMEM nmem)
 735 {
 736     Z_RPNQuery *dst = 0;
 737     ODR encode = odr_createmem(ODR_ENCODE);
 738     ODR decode = odr_createmem(ODR_DECODE);
 739
 740     if (z_RPNQuery(encode, &src, 0, 0))
 741     {
 742         int len;
 743         char *buf = odr_getbuf(encode, &len, 0);
 744
 745         if (buf)
 746         {
 747             odr_setbuf(decode, buf, len, 0);
 748             z_RPNQuery(decode, &dst, 0, 0);
 749         }
 750     }
 751     nmem_transfer(nmem, decode->mem);
 752     odr_destroy(encode);
 753     odr_destroy(decode);
 754     return dst;
 755 }
 756
 757 static Z_SortKeySpecList *copy_SortKeySpecList(Z_SortKeySpecList *src, NMEM nmem)
 758 {
 759     Z_SortKeySpecList *dst = 0;
 760     ODR encode = odr_createmem(ODR_ENCODE);
 761     ODR decode = odr_createmem(ODR_DECODE);
 762
 763     if (z_SortKeySpecList(encode, &src, 0, 0))
 764     {
 765         int len;
 766         char *buf = odr_getbuf(encode, &len, 0);
 767
 768         if (buf)
 769         {
 770             odr_setbuf(decode, buf, len, 0);
 771             z_SortKeySpecList(decode, &dst, 0, 0);
 772         }
 773     }
 774     nmem_transfer(nmem, decode->mem);
 775     odr_destroy(encode);
 776     odr_destroy(decode);
 777     return dst;
 778 }
 779
 780 ZebraSet resultSetClone(ZebraHandle zh, const char *setname,
 781                         ZebraSet rset)
 782 {
 783     ZebraSet nset;
 784     int i;
 785
 786     nset = resultSetAdd(zh, setname, 1);
 787     if (!nset)
 788         return 0;
 789
 790     nset->nmem = nmem_create();
 791
 792     nset->num_bases = rset->num_bases;
 793     nset->basenames =
 794         nmem_malloc(nset->nmem, nset->num_bases * sizeof(*rset->basenames));
 795     for (i = 0; i<rset->num_bases; i++)
 796         nset->basenames[i] = nmem_strdup(nset->nmem, rset->basenames[i]);
 797
 798     if (rset->rset)
 799         nset->rset = rset_dup(rset->rset);
 800     if (rset->rpn)
 801         nset->rpn = copy_RPNQuery(rset->rpn, nset->nmem);
 802     return nset;
 803 }
 804
 805 ZEBRA_RES resultSetSort(ZebraHandle zh, NMEM nmem,
 806                         int num_input_setnames, const char **input_setnames,
 807                         const char *output_setname,
 808                         Z_SortKeySpecList *sort_sequence, int *sort_status)
 809 {
 810     ZebraSet sset;
 811     RSET rset;
 812
 813     if (num_input_setnames == 0)
 814     {
 815         zebra_setError(zh, YAZ_BIB1_NO_RESULT_SET_NAME_SUPPLIED_ON_SORT, 0);
 816         return ZEBRA_FAIL;
 817     }
 818     if (num_input_setnames > 1)
 819     {
 820         zebra_setError(zh, YAZ_BIB1_SORT_TOO_MANY_INPUT_RESULTS, 0);
 821         return ZEBRA_FAIL;
 822     }
 823     if (!log_level_set)
 824         loglevels();
 825     yaz_log(log_level_sort, "result set sort input=%s output=%s",
 826             *input_setnames, output_setname);
 827     sset = resultSetGet(zh, input_setnames[0]);
 828     if (!sset)
 829     {
 830         zebra_setError(zh, YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
 831                        input_setnames[0]);
 832         return ZEBRA_FAIL;
 833     }
 834     if (!(rset = sset->rset))
 835     {
 836         zebra_setError(zh, YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
 837                        input_setnames[0]);
 838         return ZEBRA_FAIL;
 839     }
 840     if (strcmp(output_setname, input_setnames[0]))
 841         sset = resultSetClone(zh, output_setname, sset);
 842     sset->sortSpec = copy_SortKeySpecList(sort_sequence, sset->nmem);
 843     return resultSetSortSingle(zh, nmem, sset, rset, sort_sequence,
 844                                sort_status);
 845 }
 846
 847 ZEBRA_RES resultSetSortSingle(ZebraHandle zh, NMEM nmem,
 848                               ZebraSet sset, RSET rset,
 849                               Z_SortKeySpecList *sort_sequence,
 850                               int *sort_status)
 851 {
 852     int i;
 853     int ib;
 854     int n = 0;
 855     zint kno = 0;
 856     zint psysno = 0;
 857     struct it_key key;
 858     struct sortKeyInfo sort_criteria[ZSET_SORT_MAX_LEVEL];
 859     char *cmp_buf[ZSET_SORT_MAX_LEVEL];
 860     char *tmp_cmp_buf[ZSET_SORT_MAX_LEVEL];
 861     int num_criteria;
 862     RSFD rfd;
 863     TERMID termid;
 864     TERMID *terms;
 865     int numTerms = 0;
 866     size_t sysno_mem_index = 0;
 867
 868     int numbases = zh->num_basenames;
 869     yaz_log(log_level_sort, "searching %d databases",numbases);
 870
 871     if (zh->m_staticrank)
 872         sysno_mem_index = 1;
 873
 874     assert(nmem); /* compiler shut up about unused param */
 875     sset->sort_info->num_entries = 0;
 876
 877     rset_getterms(rset, 0, 0, &n);
 878     terms = (TERMID *) nmem_malloc(nmem, sizeof(*terms)*n);
 879     rset_getterms(rset, terms, n, &numTerms);
 880
 881     sset->hits = 0;
 882     num_criteria = sort_sequence->num_specs;
 883     if (num_criteria > ZSET_SORT_MAX_LEVEL)
 884         num_criteria = ZSET_SORT_MAX_LEVEL;
 885     /* set up the search criteria */
 886     for (i = 0; i < num_criteria; i++)
 887     {
 888         Z_SortKeySpec *sks = sort_sequence->specs[i];
 889         Z_SortKey *sk;
 890         ZEBRA_RES res;
 891
 892         sort_criteria[i].ord = (int *)
 893             nmem_malloc(nmem, sizeof(int)*numbases);
 894         sort_criteria[i].numerical = (int *)
 895             nmem_malloc(nmem, sizeof(int)*numbases);
 896
 897         /* initialize ord and numerical for each database */
 898         for (ib = 0; ib < numbases; ib++)
 899         {
 900             sort_criteria[i].ord[ib] = -1;
 901             sort_criteria[i].numerical[ib] = 0;
 902         }
 903
 904         if (sks->which == Z_SortKeySpec_missingValueData)
 905         {
 906             zebra_setError(zh, YAZ_BIB1_UNSUPP_MISSING_DATA_ACTION, 0);
 907             return ZEBRA_FAIL;
 908         }
 909         if (*sks->sortRelation == Z_SortKeySpec_ascending)
 910             sort_criteria[i].relation = 'A';
 911         else if (*sks->sortRelation == Z_SortKeySpec_descending)
 912             sort_criteria[i].relation = 'D';
 913         else
 914         {
 915             zebra_setError(zh, YAZ_BIB1_ILLEGAL_SORT_RELATION, 0);
 916             return ZEBRA_FAIL;
 917         }
 918         if (sks->sortElement->which == Z_SortElement_databaseSpecific)
 919         {
 920             zebra_setError(zh, YAZ_BIB1_DATABASE_SPECIFIC_SORT_UNSUPP, 0);
 921             return ZEBRA_FAIL;
 922         }
 923         else if (sks->sortElement->which != Z_SortElement_generic)
 924         {
 925             zebra_setError(zh, YAZ_BIB1_SORT_ILLEGAL_SORT, 0);
 926             return ZEBRA_FAIL;
 927         }
 928         sk = sks->sortElement->u.generic;
 929         switch (sk->which)
 930         {
 931         case Z_SortKey_sortField:
 932             yaz_log(log_level_sort, "key %d is of type sortField", i+1);
 933             for (ib = 0; ib < numbases; ib++)
 934             {
 935                 zebraExplain_curDatabase(zh->reg->zei, zh->basenames[ib]);
 936                 sort_criteria[i].numerical[ib] = 0;
 937                 sort_criteria[i].ord[ib] =
 938                     zebraExplain_lookup_attr_str(zh->reg->zei,
 939                                                  zinfo_index_category_sort,
 940                                                  0, sk->u.sortField);
 941                 if (sks->which != Z_SortKeySpec_null
 942                     && sort_criteria[i].ord[ib] == -1)
 943                 {
 944                     zebra_setError(zh,
 945                                    YAZ_BIB1_CANNOT_SORT_ACCORDING_TO_SEQUENCE, 0);
 946                     return ZEBRA_FAIL;
 947                 }
 948             }
 949             break;
 950         case Z_SortKey_elementSpec:
 951             yaz_log(log_level_sort, "key %d is of type elementSpec", i+1);
 952             zebra_setError(zh, YAZ_BIB1_CANNOT_SORT_ACCORDING_TO_SEQUENCE, 0);
 953             return ZEBRA_FAIL;
 954         case Z_SortKey_sortAttributes:
 955             yaz_log(log_level_sort, "key %d is of type sortAttributes", i+1);
 956             /* for every database we searched, get the sort index file
 957                id (ord) and its numerical indication and store them in
 958                the sort_criteria */
 959             for (ib = 0; ib < numbases; ib++)
 960             {
 961                 zebraExplain_curDatabase(zh->reg->zei, zh->basenames[ib]);
 962                 res = zebra_sort_get_ord(zh, sk->u.sortAttributes,
 963                                          &sort_criteria[i].ord[ib],
 964                                          &sort_criteria[i].numerical[ib]);
 965             }
 966
 967             if (sks->which != Z_SortKeySpec_null && res != ZEBRA_OK)
 968                 return ZEBRA_FAIL;
 969             break;
 970         }
 971         /* right now we look up the index type based on the first database
 972            if the index_type's can differ between the indexes of different
 973            databases (which i guess they can?) then we have to store the
 974            index types for each database, just like the ord and numerical */
 975         if (zebraExplain_lookup_ord(zh->reg->zei, sort_criteria[i].ord[0],
 976                                     &sort_criteria[i].index_type,
 977                                     0, 0))
 978         {
 979             zebra_setError(zh, YAZ_BIB1_CANNOT_SORT_ACCORDING_TO_SEQUENCE, 0);
 980             return ZEBRA_FAIL;
 981         }
 982     }
 983     /* allocate space for each cmpare buf + one extra for tmp comparison */
 984     /* cmp_buf is an array of array, the first dimension is the criteria and the second dimension are
 985        all other result entries to compare against. This is slowly filled when records are processed.
 986        tmp_cmp_buf is an array with a value of the current record for each criteria
 987     */
 988     for (i = 0; i<num_criteria; i++)
 989     {
 990         cmp_buf[i] = xmalloc(sset->sort_info->max_entries
 991                              * SORT_IDX_ENTRYSIZE);
 992         tmp_cmp_buf[i] = xmalloc(SORT_IDX_ENTRYSIZE);
 993     }
 994     rfd = rset_open(rset, RSETF_READ);
 995     while (rset_read(rfd, &key, &termid))
 996     {
 997         zint this_sys = key.mem[sysno_mem_index];
 998         if (log_level_searchhits)
 999             key_logdump_txt(log_level_searchhits, &key, termid->name);
1000         kno++;
1001         if (this_sys != psysno)
1002         {
1003             int database_no = 0;
1004             if ((sset->hits & 255) == 0 && zh->break_handler_func)
1005             {
1006                 if (zh->break_handler_func(zh->break_handler_data))
1007                 {
1008                     rset_set_hits_limit(rset, 0);
1009                     break;
1010                 }
1011             }
1012             (sset->hits)++;
1013             psysno = this_sys;
1014
1015             /* determine database from the term, but only bother if more than
1016                one database is in use*/
1017             if (numbases > 1 && termid->ol)
1018             {
1019                 const char *this_db = 0;
1020                 if (zebraExplain_lookup_ord(zh->reg->zei, termid->ol->ord,  0, &this_db, 0)
1021                     == 0 && this_db)
1022                 {
1023                     for (ib = 0; ib < numbases; ib++)
1024                         if (!strcmp(this_db, zh->basenames[ib]))
1025                             database_no = ib;
1026                 }
1027             }
1028 #if 0
1029             yaz_log(YLOG_LOG, "sysno=" ZINT_FORMAT " database_no=%d", this_sys,
1030                 database_no);
1031             ord_list_print(termid->ol);
1032 #endif
1033             resultSetInsertSort(zh, sset, database_no,
1034                                 sort_criteria, num_criteria, psysno, cmp_buf,
1035                                 tmp_cmp_buf);
1036         }
1037     }
1038     rset_close(rfd);
1039
1040     /* free the compare buffers */
1041     for (i = 0; i<num_criteria; i++)
1042     {
1043         xfree(cmp_buf[i]);
1044         xfree(tmp_cmp_buf[i]);
1045     }
1046
1047     yaz_log(log_level_sort, ZINT_FORMAT " keys, " ZINT_FORMAT " sysnos, sort",
1048             kno, sset->hits);
1049     for (i = 0; i < numTerms; i++)
1050         yaz_log(log_level_sort, "term=\"%s\" type=%s count=" ZINT_FORMAT,
1051                 terms[i]->name, terms[i]->flags, terms[i]->rset->hits_count);
1052     *sort_status = Z_SortResponse_success;
1053     return ZEBRA_OK;
1054 }
1055
1056 RSET resultSetRef(ZebraHandle zh, const char *resultSetId)
1057 {
1058     ZebraSet s;
1059
1060     if ((s = resultSetGet(zh, resultSetId)))
1061         return s->rset;
1062     return NULL;
1063 }
1064
1065 ZEBRA_RES resultSetRank(ZebraHandle zh, ZebraSet zebraSet,
1066                         RSET rset, NMEM nmem)
1067 {
1068     struct it_key key;
1069     TERMID termid;
1070     TERMID *terms;
1071     zint kno = 0;
1072     int numTerms = 0;
1073     int n = 0;
1074     int i;
1075     ZebraRankClass rank_class;
1076     struct zset_sort_info *sort_info;
1077     const char *rank_handler_name = res_get_def(zh->res, "rank", "rank-1");
1078     size_t sysno_mem_index = 0;
1079
1080     if (zh->m_staticrank)
1081         sysno_mem_index = 1;
1082
1083     if (!log_level_set)
1084         loglevels();
1085     sort_info = zebraSet->sort_info;
1086     sort_info->num_entries = 0;
1087     zebraSet->hits = 0;
1088     zebraSet->estimated_hit_count = 0;
1089     rset_getterms(rset, 0, 0, &n);
1090     terms = (TERMID *) nmem_malloc(nmem, sizeof(*terms)*n);
1091     rset_getterms(rset, terms, n, &numTerms);
1092
1093     rank_class = zebraRankLookup(zh, rank_handler_name);
1094     if (!rank_class)
1095     {
1096         yaz_log(YLOG_WARN, "No such rank handler: %s", rank_handler_name);
1097         zebra_setError(zh, YAZ_BIB1_UNSUPP_SEARCH, "Cannot find rank handler");
1098         return ZEBRA_FAIL;
1099     }
1100     else
1101     {
1102         RSFD rfd = rset_open(rset, RSETF_READ);
1103         struct rank_control *rc = rank_class->control;
1104         int score;
1105         zint count = 0;
1106         void *handle = (*rc->begin) (zh->reg, rank_class->class_handle, rset,
1107                                      nmem, terms, numTerms);
1108         zint psysno = 0;  /* previous doc id / sys no */
1109         zint pstaticrank = 0; /* previous static rank */
1110         int stop_flag = 0;
1111         while (rset_read(rfd, &key, &termid))
1112         {
1113             zint this_sys = key.mem[sysno_mem_index];
1114
1115             zint seqno = key.mem[key.len-1];
1116             kno++;
1117             if (log_level_searchhits)
1118                 key_logdump_txt(log_level_searchhits, &key, termid->name);
1119             if (this_sys != psysno)
1120             {   /* new record .. */
1121                 if (!(rfd->counted_items & 255) && zh->break_handler_func)
1122                 {
1123                     if (zh->break_handler_func(zh->break_handler_data))
1124                     {
1125                         yaz_log(YLOG_LOG, "Aborted search");
1126                         stop_flag = 1;
1127                     }
1128                 }
1129                 if (rfd->counted_items > rset->hits_limit)
1130                     stop_flag = 1;
1131                 if (psysno)
1132                 {   /* only if we did have a previous record */
1133                     score = (*rc->calc)(handle, psysno, pstaticrank,
1134                                         &stop_flag);
1135                     /* insert the hit. A=Ascending */
1136                     resultSetInsertRank(zh, sort_info, psysno, score, 'A');
1137                     count++;
1138                 }
1139                 if (stop_flag)
1140                 {
1141                     zebraSet->estimated_hit_count = 1;
1142                     rset_set_hits_limit(rset, 0);
1143                     break;
1144                 }
1145                 psysno = this_sys;
1146                 if (zh->m_staticrank)
1147                     pstaticrank = key.mem[0];
1148             }
1149             (*rc->add)(handle, CAST_ZINT_TO_INT(seqno), termid);
1150         }
1151         /* no more items */
1152         if (psysno)
1153         {   /* we had - at least - one record */
1154             score = (*rc->calc)(handle, psysno, pstaticrank, &stop_flag);
1155             /* insert the hit. A=Ascending */
1156             resultSetInsertRank(zh, sort_info, psysno, score, 'A');
1157             count++;
1158         }
1159         (*rc->end)(zh->reg, handle);
1160         rset_close(rfd);
1161     }
1162     zebraSet->hits = rset->hits_count;
1163
1164     yaz_log(log_level_searchterms, ZINT_FORMAT " keys, "
1165             ZINT_FORMAT " sysnos, rank",  kno, zebraSet->hits);
1166     for (i = 0; i < numTerms; i++)
1167     {
1168         yaz_log(log_level_searchterms, "term=\"%s\" type=%s count="
1169                 ZINT_FORMAT,
1170                 terms[i]->name, terms[i]->flags, terms[i]->rset->hits_count);
1171     }
1172     return ZEBRA_OK;
1173 }
1174
1175 ZebraRankClass zebraRankLookup(ZebraHandle zh, const char *name)
1176 {
1177     ZebraRankClass p = zh->reg->rank_classes;
1178     while (p && strcmp(p->control->name, name))
1179         p = p->next;
1180     if (p && !p->init_flag)
1181     {
1182         if (p->control->create)
1183             p->class_handle = (*p->control->create)(zh);
1184         p->init_flag = 1;
1185     }
1186     return p;
1187 }
1188
1189 void zebraRankInstall(struct zebra_register *reg, struct rank_control *ctrl)
1190 {
1191     ZebraRankClass p = (ZebraRankClass) xmalloc(sizeof(*p));
1192     p->control = (struct rank_control *) xmalloc(sizeof(*p->control));
1193     memcpy(p->control, ctrl, sizeof(*p->control));
1194     p->control->name = xstrdup(ctrl->name);
1195     p->init_flag = 0;
1196     p->next = reg->rank_classes;
1197     reg->rank_classes = p;
1198 }
1199
1200 void zebraRankDestroy(struct zebra_register *reg)
1201 {
1202     ZebraRankClass p = reg->rank_classes;
1203     while (p)
1204     {
1205         ZebraRankClass p_next = p->next;
1206         if (p->init_flag && p->control->destroy)
1207             (*p->control->destroy)(reg, p->class_handle);
1208         xfree(p->control->name);
1209         xfree(p->control);
1210         xfree(p);
1211         p = p_next;
1212     }
1213     reg->rank_classes = NULL;
1214 }
1215
1216 static int trav_rset_for_termids(RSET rset, TERMID *termid_array,
1217                                  zint *hits_array, int *approx_array)
1218 {
1219     int no = 0;
1220     int i;
1221     for (i = 0; i<rset->no_children; i++)
1222         no += trav_rset_for_termids(rset->children[i],
1223                                     (termid_array ? termid_array + no : 0),
1224                                     (hits_array ? hits_array + no : 0),
1225                                     (approx_array ? approx_array + no : 0));
1226     if (rset->term)
1227     {
1228         if (termid_array)
1229             termid_array[no] = rset->term;
1230         if (hits_array)
1231             hits_array[no] = rset->hits_count;
1232         if (approx_array)
1233             approx_array[no] = rset->hits_approx;
1234 #if 0
1235         yaz_log(YLOG_LOG, "rset=%p term=%s limit=" ZINT_FORMAT
1236                 " count=" ZINT_FORMAT,
1237                 rset, rset->term->name, rset->hits_limit, rset->hits_count);
1238 #endif
1239         no++;
1240     }
1241     return no;
1242 }
1243
1244 ZEBRA_RES zebra_result_set_term_no(ZebraHandle zh, const char *setname,
1245                                    int *num_terms)
1246 {
1247     ZebraSet sset = resultSetGet(zh, setname);
1248     *num_terms = 0;
1249     if (sset)
1250     {
1251         *num_terms = trav_rset_for_termids(sset->rset, 0, 0, 0);
1252         return ZEBRA_OK;
1253     }
1254     return ZEBRA_FAIL;
1255 }
1256
1257 ZEBRA_RES zebra_result_set_term_info(ZebraHandle zh, const char *setname,
1258                                      int no, zint *count, int *approx,
1259                                      char *termbuf, size_t *termlen,
1260                                      const char **term_ref_id)
1261 {
1262     ZebraSet sset = resultSetGet(zh, setname);
1263     if (sset)
1264     {
1265         int num_terms = trav_rset_for_termids(sset->rset, 0, 0, 0);
1266         if (no >= 0 && no < num_terms)
1267         {
1268             TERMID *term_array = xmalloc(num_terms * sizeof(*term_array));
1269             zint *hits_array = xmalloc(num_terms * sizeof(*hits_array));
1270             int *approx_array = xmalloc(num_terms * sizeof(*approx_array));
1271
1272             trav_rset_for_termids(sset->rset, term_array,
1273                                   hits_array, approx_array);
1274
1275             if (count)
1276                 *count = hits_array[no];
1277             if (approx)
1278                 *approx = approx_array[no];
1279             if (termbuf)
1280             {
1281                 char *inbuf = term_array[no]->name;
1282                 size_t inleft = strlen(inbuf);
1283                 size_t outleft = *termlen - 1;
1284
1285                 if (zh->iconv_from_utf8 != 0)
1286                 {
1287                     char *outbuf = termbuf;
1288                     size_t ret;
1289
1290                     ret = yaz_iconv(zh->iconv_from_utf8, &inbuf, &inleft,
1291                                     &outbuf, &outleft);
1292                     if (ret == (size_t)(-1))
1293                         *termlen = 0;
1294                     else
1295                     {
1296                         yaz_iconv(zh->iconv_from_utf8, 0, 0,
1297                                   &outbuf, &outleft);
1298                         *termlen = outbuf - termbuf;
1299                     }
1300                 }
1301                 else
1302                 {
1303                     if (inleft > outleft)
1304                         inleft = outleft;
1305                     *termlen = inleft;
1306                     memcpy(termbuf, inbuf, *termlen);
1307                 }
1308                 termbuf[*termlen] = '\0';
1309             }
1310             if (term_ref_id)
1311                 *term_ref_id = term_array[no]->ref_id;
1312
1313             xfree(term_array);
1314             xfree(hits_array);
1315             xfree(approx_array);
1316             return ZEBRA_OK;
1317         }
1318     }
1319     return ZEBRA_FAIL;
1320 }
1321
1322 ZEBRA_RES zebra_snippets_hit_vector(ZebraHandle zh, const char *setname,
1323                                     zint sysno, zebra_snippets *snippets)
1324 {
1325     ZebraSet sset = resultSetGet(zh, setname);
1326     yaz_log(YLOG_DEBUG, "zebra_get_hit_vector setname=%s zysno=" ZINT_FORMAT,
1327             setname, sysno);
1328     if (!sset)
1329         return ZEBRA_FAIL;
1330     else
1331     {
1332         struct rset_key_control *kc = zebra_key_control_create(zh);
1333         NMEM nmem = nmem_create();
1334         struct it_key key;
1335         RSET rsets[2], rset_comb;
1336         RSET rset_temp = rset_create_temp(nmem, kc, kc->scope,
1337                                           res_get(zh->res, "setTmpDir"),0 );
1338
1339         TERMID termid;
1340         RSFD rsfd = rset_open(rset_temp, RSETF_WRITE);
1341
1342         key.mem[0] = sysno;
1343         key.mem[1] = 0;
1344         key.mem[2] = 0;
1345         key.mem[3] = 0;
1346         key.len = 2;
1347         rset_write(rsfd, &key);
1348         rset_close(rsfd);
1349
1350         rsets[0] = rset_temp;
1351         rsets[1] = rset_dup(sset->rset);
1352
1353         rset_comb = rset_create_and(nmem, kc, kc->scope, 2, rsets);
1354
1355         rsfd = rset_open(rset_comb, RSETF_READ);
1356
1357         while (rset_read(rsfd, &key, &termid))
1358         {
1359             if (termid)
1360             {
1361                 struct ord_list *ol;
1362                 for (ol = termid->ol; ol; ol = ol->next)
1363                 {
1364                     zebra_snippets_append(snippets, key.mem[key.len-1], 0,
1365                                           ol->ord, termid->name);
1366                 }
1367             }
1368         }
1369         rset_close(rsfd);
1370
1371         rset_delete(rset_comb);
1372         nmem_destroy(nmem);
1373         kc->dec(kc);
1374     }
1375     return ZEBRA_OK;
1376 }
1377
1378 static ZEBRA_RES zebra_recid_to_sysno(ZebraHandle zh,
1379                                       const char **basenames, int num_bases,
1380                                       zint recid,
1381                                       zint *sysnos, int *no_sysnos)
1382 {
1383     ZEBRA_RES res = ZEBRA_OK;
1384     int sysnos_offset = 0;
1385     int i;
1386
1387     if (!zh->reg->isamb || !zh->m_segment_indexing)
1388     {
1389         if (sysnos_offset < *no_sysnos)
1390             *sysnos = recid;
1391         sysnos_offset++;
1392     }
1393     else
1394     {
1395         for (i = 0; res == ZEBRA_OK && i < num_bases; i++)
1396         {
1397             const char *database = basenames[i];
1398             if (zebraExplain_curDatabase(zh->reg->zei, database) == 0)
1399             {
1400                 const char *index_type = "w";
1401                 const char *use_string = "_ALLRECORDS";
1402                 int ord;
1403                 zinfo_index_category_t cat = zinfo_index_category_alwaysmatches;
1404                 ord = zebraExplain_lookup_attr_str(zh->reg->zei, cat,
1405                                                    index_type, use_string);
1406                 if (ord != -1)
1407                 {
1408                     char ord_buf[32];
1409                     int ord_len = key_SU_encode(ord, ord_buf);
1410                     char *info;
1411
1412                     ord_buf[ord_len] = '\0';
1413
1414                     info = dict_lookup(zh->reg->dict, ord_buf);
1415                     if (info)
1416                     {
1417                         if (*info != sizeof(ISAM_P))
1418                         {
1419                             res = ZEBRA_FAIL;
1420                         }
1421                         else
1422                         {
1423                             ISAM_P isam_p;
1424                             ISAMB_PP pt;
1425                             struct it_key key_until, key_found;
1426                             int i = 0;
1427                             int r;
1428
1429                             memcpy(&isam_p, info+1, sizeof(ISAM_P));
1430
1431                             pt = isamb_pp_open(zh->reg->isamb, isam_p, 2);
1432                             if (!pt)
1433                                 res = ZEBRA_FAIL;
1434                             else
1435                             {
1436                                 key_until.mem[i++] = recid;
1437                                 key_until.mem[i++] = 0;  /* section_id */
1438                                 if (zh->m_segment_indexing)
1439                                     key_until.mem[i++] = 0; /* segment */
1440                                 key_until.mem[i++] = 0;
1441                                 key_until.len = i;
1442
1443                                 r = isamb_pp_forward(pt, &key_found, &key_until);
1444                                 while (r && key_found.mem[0] == recid)
1445                                 {
1446                                     if (sysnos_offset < *no_sysnos)
1447                                         sysnos[sysnos_offset++] =
1448                                             key_found.mem[key_found.len-1];
1449                                     r = isamb_pp_read(pt, &key_found);
1450                                 }
1451                                 isamb_pp_close(pt);
1452                             }
1453                         }
1454                     }
1455                 }
1456             }
1457         }
1458     }
1459     *no_sysnos = sysnos_offset;
1460     return res;
1461 }
1462
1463 ZEBRA_RES zebra_result_recid_to_sysno(ZebraHandle zh,
1464                                       const char *setname,
1465                                       zint recid,
1466                                       zint *sysnos, int *no_sysnos)
1467 {
1468     const char **basenames;
1469     int num_bases;
1470     ZEBRA_RES res;
1471
1472     res = resultSetGetBaseNames(zh, setname, &basenames, &num_bases);
1473     if (res != ZEBRA_OK)
1474         return ZEBRA_FAIL;
1475
1476     return zebra_recid_to_sysno(zh, basenames, num_bases,
1477                                 recid, sysnos, no_sysnos);
1478 }
1479
1480 void zebra_count_set(ZebraHandle zh, RSET rset, zint *count,
1481                      zint approx_limit)
1482 {
1483     zint psysno = 0;
1484     struct it_key key;
1485     RSFD rfd;
1486
1487     yaz_log(YLOG_DEBUG, "count_set");
1488
1489     rset->hits_limit = approx_limit;
1490
1491     *count = 0;
1492     rfd = rset_open(rset, RSETF_READ);
1493     while (rset_read(rfd, &key,0 /* never mind terms */))
1494     {
1495         if (key.mem[0] != psysno)
1496         {
1497             psysno = key.mem[0];
1498             if (rfd->counted_items >= rset->hits_limit)
1499                 break;
1500         }
1501     }
1502     rset_close(rfd);
1503     *count = rset->hits_count;
1504 }
1505
1506
1507 /*
1508  * Local variables:
1509  * c-basic-offset: 4
1510  * indent-tabs-mode: nil
1511  * End:
1512  * vim: shiftwidth=4 tabstop=8 expandtab
1513  */
1514