index/zsets.c

   1 /* This file is part of the Zebra server.
   2    Copyright (C) 1995-2008 Index Data
   3
   4 Zebra is free software; you can redistribute it and/or modify it under
   5 the terms of the GNU General Public License as published by the Free
   6 Software Foundation; either version 2, or (at your option) any later
   7 version.
   8
   9 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 for more details.
  13
  14 You should have received a copy of the GNU General Public License
  15 along with this program; if not, write to the Free Software
  16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  17
  18 */
  19
  20
  21 #include <stdio.h>
  22 #include <assert.h>
  23 #ifdef WIN32
  24 #include <io.h>
  25 #else
  26 #include <unistd.h>
  27 #endif
  28
  29 #include "index.h"
  30 #include "rank.h"
  31 #include <yaz/diagbib1.h>
  32 #include <rset.h>
  33
  34 #define ZSET_SORT_MAX_LEVEL 10
  35
  36 struct zebra_set_term_entry {
  37     int reg_type;
  38     char *db;
  39     char *index_name;
  40     char *term;
  41 };
  42
  43 struct zebra_set {
  44     char *name;
  45     RSET rset;
  46     NMEM nmem;
  47     NMEM rset_nmem; /* for creating the rsets in */
  48     zint hits;
  49     int num_bases;
  50     const char **basenames;
  51     Z_RPNQuery *rpn;
  52     Z_SortKeySpecList *sortSpec;
  53     struct zset_sort_info *sort_info;
  54     struct zebra_set_term_entry *term_entries;
  55     int term_entries_max;
  56     struct zebra_set *next;
  57     int locked;
  58     int estimated_hit_count;
  59
  60     zint cache_position;  /* last position */
  61     RSFD cache_rfd;       /* rfd (NULL if not existing) */
  62     zint cache_psysno;    /* sysno for last position */
  63     zint approx_limit;    /* limit before we do approx */
  64 };
  65
  66 struct zset_sort_entry {
  67     zint sysno;
  68     int score;
  69 };
  70
  71 struct zset_sort_info {
  72     int max_entries;
  73     int num_entries;
  74     struct zset_sort_entry *all_entries;
  75     struct zset_sort_entry **entries;
  76 };
  77
  78 static int log_level_set=0;
  79 static int log_level_sort=0;
  80 static int log_level_searchhits=0;
  81 static int log_level_searchterms=0;
  82 static int log_level_resultsets=0;
  83
  84 static void loglevels(void)
  85 {
  86     if (log_level_set)
  87         return;
  88     log_level_sort = yaz_log_module_level("sorting");
  89     log_level_searchhits = yaz_log_module_level("searchhits");
  90     log_level_searchterms = yaz_log_module_level("searchterms");
  91     log_level_resultsets = yaz_log_module_level("resultsets");
  92     log_level_set = 1;
  93 }
  94
  95
  96 static ZEBRA_RES resultSetSearch(ZebraHandle zh, NMEM nmem, NMEM rset_nmem,
  97                                  Z_RPNQuery *rpn, ZebraSet sset)
  98 {
  99     RSET rset = 0;
 100     Z_SortKeySpecList *sort_sequence;
 101     int sort_status, i;
 102     ZEBRA_RES res = ZEBRA_OK;
 103
 104     sort_sequence = (Z_SortKeySpecList *)
 105         nmem_malloc(nmem, sizeof(*sort_sequence));
 106     sort_sequence->num_specs = 10; /* FIXME - Hard-coded number */
 107     sort_sequence->specs = (Z_SortKeySpec **)
 108         nmem_malloc(nmem, sort_sequence->num_specs *
 109                     sizeof(*sort_sequence->specs));
 110     for (i = 0; i<sort_sequence->num_specs; i++)
 111         sort_sequence->specs[i] = 0;
 112
 113     rpn_get_top_approx_limit(zh, rpn->RPNStructure, &sset->approx_limit);
 114
 115     res = rpn_search_top(zh, rpn->RPNStructure, rpn->attributeSetId,
 116                          nmem, rset_nmem,
 117                          sort_sequence,
 118                          sset->num_bases, sset->basenames,
 119                          &rset);
 120     if (res != ZEBRA_OK)
 121     {
 122         sset->rset = 0;
 123         return res;
 124     }
 125     for (i = 0; sort_sequence->specs[i]; i++)
 126         ;
 127     sort_sequence->num_specs = i;
 128     rset->hits_limit = sset->approx_limit;
 129     if (!i)
 130     {
 131         res = resultSetRank(zh, sset, rset, rset_nmem);
 132     }
 133     else
 134     {
 135         res = resultSetSortSingle(zh, nmem, sset, rset,
 136                                   sort_sequence, &sort_status);
 137     }
 138     sset->rset = rset;
 139     return res;
 140 }
 141
 142
 143 ZEBRA_RES resultSetAddRPN(ZebraHandle zh, NMEM m, Z_RPNQuery *rpn,
 144                           int num_bases, char **basenames,
 145                           const char *setname,
 146                           zint *hits, int *estimated_hit_count)
 147 {
 148     ZebraSet zebraSet;
 149     int i;
 150     ZEBRA_RES res;
 151
 152     *hits = 0;
 153     *estimated_hit_count = 0;
 154
 155     zebraSet = resultSetAdd(zh, setname, 1);
 156     if (!zebraSet)
 157         return ZEBRA_FAIL;
 158     zebraSet->locked = 1;
 159     zebraSet->rpn = 0;
 160     zebraSet->nmem = m;
 161     zebraSet->rset_nmem = nmem_create();
 162
 163     zebraSet->num_bases = num_bases;
 164     zebraSet->basenames =
 165         nmem_malloc(zebraSet->nmem, num_bases * sizeof(*zebraSet->basenames));
 166     for (i = 0; i<num_bases; i++)
 167         zebraSet->basenames[i] = nmem_strdup(zebraSet->nmem, basenames[i]);
 168
 169     res = resultSetSearch(zh, zebraSet->nmem, zebraSet->rset_nmem,
 170                           rpn, zebraSet);
 171     *hits = zebraSet->hits;
 172     if (zebraSet->estimated_hit_count)
 173         *estimated_hit_count = 1;
 174
 175     if (zebraSet->rset)
 176         zebraSet->rpn = rpn;
 177     zebraSet->locked = 0;
 178     if (!zebraSet->rset)
 179         return ZEBRA_FAIL;
 180     return res;
 181 }
 182
 183 void resultSetAddTerm(ZebraHandle zh, ZebraSet s, int reg_type,
 184                       const char *db, const char *index_name,
 185                       const char *term)
 186 {
 187     assert(zh); /* compiler shut up */
 188     if (!s->nmem)
 189         s->nmem = nmem_create();
 190     if (!s->term_entries)
 191     {
 192         int i;
 193         s->term_entries_max = 1000;
 194         s->term_entries =
 195             nmem_malloc(s->nmem, s->term_entries_max *
 196                         sizeof(*s->term_entries));
 197         for (i = 0; i < s->term_entries_max; i++)
 198             s->term_entries[i].term = 0;
 199     }
 200     if (s->hits < s->term_entries_max)
 201     {
 202         s->term_entries[s->hits].reg_type = reg_type;
 203         s->term_entries[s->hits].db = nmem_strdup(s->nmem, db);
 204         s->term_entries[s->hits].index_name = nmem_strdup(s->nmem, index_name);
 205         s->term_entries[s->hits].term = nmem_strdup(s->nmem, term);
 206     }
 207     (s->hits)++;
 208 }
 209
 210 ZebraSet resultSetAdd(ZebraHandle zh, const char *name, int ov)
 211 {
 212     ZebraSet s;
 213     int i;
 214
 215     for (s = zh->sets; s; s = s->next)
 216         if (!strcmp(s->name, name))
 217             break;
 218
 219     if (!log_level_set)
 220         loglevels();
 221     if (s)
 222     {
 223         yaz_log(log_level_resultsets, "updating result set %s", name);
 224         if (!ov || s->locked)
 225             return NULL;
 226         if (s->rset)
 227         {
 228             if (s->cache_rfd)
 229                 rset_close(s->cache_rfd);
 230             rset_delete(s->rset);
 231         }
 232         if (s->rset_nmem)
 233             nmem_destroy(s->rset_nmem);
 234         if (s->nmem)
 235             nmem_destroy(s->nmem);
 236     }
 237     else
 238     {
 239         const char *sort_max_str = zebra_get_resource(zh, "sortmax", "1000");
 240
 241         yaz_log(log_level_resultsets, "adding result set %s", name);
 242         s = (ZebraSet) xmalloc(sizeof(*s));
 243         s->next = zh->sets;
 244         zh->sets = s;
 245         s->name = xstrdup(name);
 246
 247         s->sort_info = (struct zset_sort_info *)
 248             xmalloc(sizeof(*s->sort_info));
 249         s->sort_info->max_entries = atoi(sort_max_str);
 250         if (s->sort_info->max_entries < 2)
 251             s->sort_info->max_entries = 2;
 252
 253         s->sort_info->entries = (struct zset_sort_entry **)
 254             xmalloc(sizeof(*s->sort_info->entries) *
 255                     s->sort_info->max_entries);
 256         s->sort_info->all_entries = (struct zset_sort_entry *)
 257             xmalloc(sizeof(*s->sort_info->all_entries) *
 258                     s->sort_info->max_entries);
 259         for (i = 0; i < s->sort_info->max_entries; i++)
 260             s->sort_info->entries[i] = s->sort_info->all_entries + i;
 261     }
 262     s->locked = 0;
 263     s->term_entries = 0;
 264     s->hits = 0;
 265     s->rset = 0;
 266     s->rset_nmem = 0;
 267     s->nmem = 0;
 268     s->rpn = 0;
 269     s->sortSpec = 0;
 270     s->cache_position = 0;
 271     s->cache_rfd = 0;
 272     s->approx_limit = zh->approx_limit;
 273     s->estimated_hit_count = 0;
 274     return s;
 275 }
 276
 277 ZebraSet resultSetGet(ZebraHandle zh, const char *name)
 278 {
 279     ZebraSet s;
 280
 281     for (s = zh->sets; s; s = s->next)
 282         if (!strcmp(s->name, name))
 283         {
 284             if (!s->term_entries && !s->rset && s->rpn)
 285             {
 286                 NMEM nmem = nmem_create();
 287                 yaz_log(log_level_resultsets, "research %s", name);
 288                 if (!s->rset_nmem)
 289                     s->rset_nmem = nmem_create();
 290                 resultSetSearch(zh, nmem, s->rset_nmem, s->rpn, s);
 291                 if (s->rset && s->sortSpec)
 292                 {
 293                     int sort_status;
 294                     yaz_log(log_level_resultsets, "resort %s", name);
 295                     resultSetSortSingle(zh, nmem, s, s->rset, s->sortSpec,
 296                                         &sort_status);
 297                 }
 298                 nmem_destroy(nmem);
 299             }
 300             return s;
 301         }
 302     return NULL;
 303 }
 304
 305 ZEBRA_RES resultSetGetBaseNames(ZebraHandle zh, const char *setname,
 306                                 const char ***basenames, int *num_bases)
 307 {
 308     ZebraSet sset = resultSetGet(zh, setname);
 309     if (!sset)
 310         return ZEBRA_FAIL;
 311     *basenames = sset->basenames;
 312     *num_bases = sset->num_bases;
 313     return ZEBRA_OK;
 314
 315 }
 316
 317 void resultSetInvalidate(ZebraHandle zh)
 318 {
 319     ZebraSet s = zh->sets;
 320
 321     yaz_log(log_level_resultsets, "invalidating result sets");
 322     for (; s; s = s->next)
 323     {
 324         if (s->rset)
 325         {
 326             if (s->cache_rfd)
 327                 rset_close(s->cache_rfd);
 328             rset_delete(s->rset);
 329         }
 330         s->rset = 0;
 331         s->cache_rfd = 0;
 332         s->cache_position = 0;
 333         if (s->rset_nmem)
 334             nmem_destroy(s->rset_nmem);
 335         s->rset_nmem=0;
 336     }
 337 }
 338
 339 void resultSetDestroy(ZebraHandle zh, int num, char **names,int *statuses)
 340 {
 341     ZebraSet * ss = &zh->sets;
 342     int i;
 343
 344     if (statuses)
 345         for (i = 0; i<num; i++)
 346             statuses[i] = Z_DeleteStatus_resultSetDidNotExist;
 347     while (*ss)
 348     {
 349         int i = -1;
 350         ZebraSet s = *ss;
 351         if (num >= 0)
 352         {
 353             for (i = 0; i<num; i++)
 354                 if (!strcmp(s->name, names[i]))
 355                 {
 356                     if (statuses)
 357                         statuses[i] = Z_DeleteStatus_success;
 358                     i = -1;
 359                     break;
 360                 }
 361         }
 362         if (i < 0)
 363         {
 364             *ss = s->next;
 365
 366             xfree(s->sort_info->all_entries);
 367             xfree(s->sort_info->entries);
 368             xfree(s->sort_info);
 369
 370             if (s->nmem)
 371                 nmem_destroy(s->nmem);
 372             if (s->rset)
 373             {
 374                 if (s->cache_rfd)
 375                     rset_close(s->cache_rfd);
 376                 rset_delete(s->rset);
 377             }
 378             if (s->rset_nmem)
 379                 nmem_destroy(s->rset_nmem);
 380             xfree(s->name);
 381             xfree(s);
 382         }
 383         else
 384             ss = &s->next;
 385     }
 386 }
 387
 388 ZebraMetaRecord *zebra_meta_records_create_range(ZebraHandle zh,
 389                                                  const char *name,
 390                                                  zint start, int num)
 391 {
 392     zint pos_small[10];
 393     zint *pos = pos_small;
 394     ZebraMetaRecord *mr;
 395     int i;
 396
 397     if (num > 10000 || num <= 0)
 398         return 0;
 399
 400     if (num > 10)
 401         pos = xmalloc(sizeof(*pos) * num);
 402
 403     for (i = 0; i<num; i++)
 404         pos[i] = start+i;
 405
 406     mr = zebra_meta_records_create(zh, name, num, pos);
 407
 408     if (num > 10)
 409         xfree(pos);
 410     return mr;
 411 }
 412
 413 ZebraMetaRecord *zebra_meta_records_create(ZebraHandle zh, const char *name,
 414                                            int num, zint *positions)
 415 {
 416     ZebraSet sset;
 417     ZebraMetaRecord *sr = 0;
 418     RSET rset;
 419     int i;
 420     struct zset_sort_info *sort_info;
 421     size_t sysno_mem_index = 0;
 422
 423     if (zh->m_staticrank)
 424         sysno_mem_index = 1;
 425
 426     if (!log_level_set)
 427         loglevels();
 428     if (!(sset = resultSetGet(zh, name)))
 429         return NULL;
 430     if (!(rset = sset->rset))
 431     {
 432         if (!sset->term_entries)
 433             return 0;
 434         sr = (ZebraMetaRecord *) xmalloc(sizeof(*sr) * num);
 435         for (i = 0; i<num; i++)
 436         {
 437             sr[i].sysno = 0;
 438             sr[i].score = -1;
 439             sr[i].term = 0;
 440             sr[i].db = 0;
 441
 442             if (positions[i] <= sset->term_entries_max)
 443             {
 444                 sr[i].term = sset->term_entries[positions[i]-1].term;
 445                 sr[i].db = sset->term_entries[positions[i]-1].db;
 446             }
 447         }
 448     }
 449     else
 450     {
 451         sr = (ZebraMetaRecord *) xmalloc(sizeof(*sr) * num);
 452         for (i = 0; i<num; i++)
 453         {
 454             sr[i].sysno = 0;
 455             sr[i].score = -1;
 456             sr[i].term = 0;
 457             sr[i].db = 0;
 458         }
 459         sort_info = sset->sort_info;
 460         if (sort_info)
 461         {
 462             zint position;
 463
 464             for (i = 0; i<num; i++)
 465             {
 466                 position = positions[i];
 467                 if (position > 0 && position <= sort_info->num_entries)
 468                 {
 469                     yaz_log(log_level_sort, "got pos=" ZINT_FORMAT
 470                             " (sorted)", position);
 471                     sr[i].sysno = sort_info->entries[position-1]->sysno;
 472                     sr[i].score = sort_info->entries[position-1]->score;
 473                 }
 474             }
 475         }
 476         /* did we really get all entries using sort ? */
 477         for (i = 0; i<num; i++)
 478         {
 479             if (!sr[i].sysno)
 480                 break;
 481         }
 482         if (i < num) /* nope, get the rest, unsorted - sorry */
 483         {
 484             zint position = 0;
 485             int num_i = 0;
 486             zint psysno = 0;
 487             RSFD rfd;
 488             struct it_key key;
 489
 490             if (sort_info)
 491                 position = sort_info->num_entries;
 492             while (num_i < num && positions[num_i] <= position)
 493                 num_i++;
 494
 495             if (sset->cache_rfd &&
 496                 num_i < num && positions[num_i] > sset->cache_position)
 497             {
 498                 position = sset->cache_position;
 499                 rfd = sset->cache_rfd;
 500                 psysno = sset->cache_psysno;
 501             }
 502             else
 503             {
 504                 if (sset->cache_rfd)
 505                     rset_close(sset->cache_rfd);
 506                 rfd = rset_open(rset, RSETF_READ);
 507             }
 508             while (num_i < num && rset_read(rfd, &key, 0))
 509             {
 510                 zint this_sys = key.mem[sysno_mem_index];
 511                 if (this_sys != psysno)
 512                 {
 513                     psysno = this_sys;
 514                     if (sort_info)
 515                     {
 516                         /* determine we alreay have this in our set */
 517                         for (i = sort_info->num_entries; --i >= 0; )
 518                             if (psysno == sort_info->entries[i]->sysno)
 519                                 break;
 520                         if (i >= 0)
 521                             continue;
 522                     }
 523                     position++;
 524                     assert(num_i < num);
 525                     if (position == positions[num_i])
 526                     {
 527                         sr[num_i].sysno = psysno;
 528                         yaz_log(log_level_sort, "got pos=" ZINT_FORMAT " (unsorted)", position);
 529                         sr[num_i].score = -1;
 530                         num_i++;
 531                     }
 532                 }
 533             }
 534             sset->cache_position = position;
 535             sset->cache_psysno = psysno;
 536             sset->cache_rfd = rfd;
 537         }
 538     }
 539     return sr;
 540 }
 541
 542 void zebra_meta_records_destroy(ZebraHandle zh, ZebraMetaRecord *records,
 543                                 int num)
 544 {
 545     assert(zh); /* compiler shut up about unused arg */
 546     xfree(records);
 547 }
 548
 549 struct sortKeyInfo {
 550     int relation;
 551     int *ord; /* array of ord for each database searched */
 552     int *numerical; /* array of ord for each database searched */
 553     const char *index_type;
 554 };
 555
 556 void resultSetInsertSort(ZebraHandle zh, ZebraSet sset,
 557                          int database_no,
 558                          struct sortKeyInfo *criteria, int num_criteria,
 559                          zint sysno,
 560                          char *cmp_buf[], char *tmp_cmp_buf[])
 561 {
 562     struct zset_sort_entry *new_entry = NULL;
 563     struct zset_sort_info *sort_info = sset->sort_info;
 564     int i, j;
 565
 566     zebra_sort_sysno(zh->reg->sort_index, sysno);
 567     for (i = 0; i<num_criteria; i++)
 568     {
 569         char *this_entry_buf = tmp_cmp_buf[i];
 570         memset(this_entry_buf, '\0', SORT_IDX_ENTRYSIZE);
 571
 572         if (criteria[i].ord[database_no] != -1)
 573         {
 574             yaz_log(log_level_sort, "pre zebra_sort_type ord is %d",
 575                     criteria[i].ord[database_no]);
 576             zebra_sort_type(zh->reg->sort_index, criteria[i].ord[database_no]);
 577             zebra_sort_read(zh->reg->sort_index, this_entry_buf);
 578         }
 579         else
 580         {
 581             yaz_log(log_level_sort, "criteria[i].ord is -1 so not reading from sort index");
 582         }
 583     }
 584     i = sort_info->num_entries;
 585     while (--i >= 0)
 586     {
 587         int rel = 0;
 588         for (j = 0; j<num_criteria; j++)
 589         {
 590             char *this_entry_buf = tmp_cmp_buf[j];
 591             char *other_entry_buf =
 592                 cmp_buf[j] + i * SORT_IDX_ENTRYSIZE;
 593             if (criteria[j].numerical[database_no])
 594             {
 595                 char this_entry_org[1024];
 596                 char other_entry_org[1024];
 597                 double diff;
 598                 /* when searching multiple databases, we use the index
 599                    type of the first one. So if they differ between
 600                    databases, we have a problem here we could store the
 601                    index_type for each database, but if we didn't find the
 602                    record in any sort index, then we still don't know to
 603                    which database it belongs. */
 604                 const char *index_type = criteria[j].index_type;
 605                 zebra_term_untrans(zh, index_type, this_entry_org,
 606                                    this_entry_buf);
 607                 zebra_term_untrans(zh, index_type, other_entry_org,
 608                                    other_entry_buf);
 609                 diff = atof(this_entry_org) - atof(other_entry_org);
 610
 611                 if (diff > 0.0)
 612                     rel = 1;
 613                 else if (diff < 0.0)
 614                     rel = -1;
 615                 else
 616                     rel = 0;
 617             }
 618             else
 619             {
 620                 rel = memcmp(this_entry_buf, other_entry_buf,
 621                              SORT_IDX_ENTRYSIZE);
 622             }
 623             /* when the compare is equal, continue to next criteria,
 624                else break out */
 625             if (rel)
 626                 break;
 627         }
 628         if (!rel)
 629             break;
 630         if (criteria[j].relation == 'A')
 631         {
 632             if (rel > 0)
 633                 break;
 634         }
 635         else if (criteria[j].relation == 'D')
 636         {
 637             if (rel < 0)
 638                 break;
 639         }
 640     }
 641     ++i;
 642     yaz_log(log_level_sort, "ok, we want to insert record at position %d",i);
 643     j = sort_info->max_entries;
 644     if (i == j){
 645         yaz_log(log_level_sort, "sort_info->max_entries reached (%d) abort sort",j);
 646         return;
 647     }
 648
 649     if (sort_info->num_entries == j)
 650         --j;
 651     else
 652         j = (sort_info->num_entries)++;
 653     new_entry = sort_info->entries[j];
 654     /* move up all higher entries (to make room) */
 655     while (j != i)
 656     {
 657         int k;
 658         for (k = 0; k<num_criteria; k++)
 659         {
 660             char *j_buf = cmp_buf[k] + j * SORT_IDX_ENTRYSIZE;
 661             char *j_1_buf = cmp_buf[k] + (j-1) * SORT_IDX_ENTRYSIZE;
 662             memcpy(j_buf, j_1_buf, SORT_IDX_ENTRYSIZE);
 663         }
 664         sort_info->entries[j] = sort_info->entries[j-1];
 665         --j;
 666     }
 667     /* and insert the new entry at the correct place */
 668     sort_info->entries[i] = new_entry;
 669     assert(new_entry);
 670     /* and add this to the compare buffer */
 671     for (i = 0; i<num_criteria; i++)
 672     {
 673         char *new_entry_buf = cmp_buf[i] + j * SORT_IDX_ENTRYSIZE;
 674         char *this_entry_buf = tmp_cmp_buf[i];
 675         memcpy(new_entry_buf, this_entry_buf, SORT_IDX_ENTRYSIZE);
 676     }
 677     new_entry->sysno = sysno;
 678     new_entry->score = -1;
 679 }
 680
 681 void resultSetInsertRank(ZebraHandle zh, struct zset_sort_info *sort_info,
 682                          zint sysno, int score, int relation)
 683 {
 684     struct zset_sort_entry *new_entry = NULL;
 685     int i, j;
 686     assert(zh); /* compiler shut up about unused arg */
 687
 688     i = sort_info->num_entries;
 689     while (--i >= 0)
 690     {
 691         int rel = 0;
 692
 693         rel = score - sort_info->entries[i]->score;
 694
 695         if (relation == 'D')
 696         {
 697             if (rel >= 0)
 698                 break;
 699         }
 700         else if (relation == 'A')
 701         {
 702             if (rel <= 0)
 703                 break;
 704         }
 705     }
 706     ++i;
 707     j = sort_info->max_entries;
 708     if (i == j)
 709         return;
 710
 711     if (sort_info->num_entries == j)
 712         --j;
 713     else
 714         j = (sort_info->num_entries)++;
 715
 716     new_entry = sort_info->entries[j];
 717     while (j != i)
 718     {
 719         sort_info->entries[j] = sort_info->entries[j-1];
 720         --j;
 721     }
 722     sort_info->entries[i] = new_entry;
 723     assert(new_entry);
 724     new_entry->sysno = sysno;
 725     new_entry->score = score;
 726 }
 727
 728 static Z_RPNQuery *copy_RPNQuery(Z_RPNQuery *src, NMEM nmem)
 729 {
 730     Z_RPNQuery *dst = 0;
 731     ODR encode = odr_createmem(ODR_ENCODE);
 732     ODR decode = odr_createmem(ODR_DECODE);
 733
 734     if (z_RPNQuery(encode, &src, 0, 0))
 735     {
 736         int len;
 737         char *buf = odr_getbuf(encode, &len, 0);
 738
 739         if (buf)
 740         {
 741             odr_setbuf(decode, buf, len, 0);
 742             z_RPNQuery(decode, &dst, 0, 0);
 743         }
 744     }
 745     nmem_transfer(nmem, decode->mem);
 746     odr_destroy(encode);
 747     odr_destroy(decode);
 748     return dst;
 749 }
 750
 751 static Z_SortKeySpecList *copy_SortKeySpecList(Z_SortKeySpecList *src, NMEM nmem)
 752 {
 753     Z_SortKeySpecList *dst = 0;
 754     ODR encode = odr_createmem(ODR_ENCODE);
 755     ODR decode = odr_createmem(ODR_DECODE);
 756
 757     if (z_SortKeySpecList(encode, &src, 0, 0))
 758     {
 759         int len;
 760         char *buf = odr_getbuf(encode, &len, 0);
 761
 762         if (buf)
 763         {
 764             odr_setbuf(decode, buf, len, 0);
 765             z_SortKeySpecList(decode, &dst, 0, 0);
 766         }
 767     }
 768     nmem_transfer(nmem, decode->mem);
 769     odr_destroy(encode);
 770     odr_destroy(decode);
 771     return dst;
 772 }
 773
 774 ZebraSet resultSetClone(ZebraHandle zh, const char *setname,
 775                         ZebraSet rset)
 776 {
 777     ZebraSet nset;
 778     int i;
 779
 780     nset = resultSetAdd(zh, setname, 1);
 781     if (!nset)
 782         return 0;
 783
 784     nset->nmem = nmem_create();
 785
 786     nset->num_bases = rset->num_bases;
 787     nset->basenames =
 788         nmem_malloc(nset->nmem, nset->num_bases * sizeof(*rset->basenames));
 789     for (i = 0; i<rset->num_bases; i++)
 790         nset->basenames[i] = nmem_strdup(nset->nmem, rset->basenames[i]);
 791
 792     if (rset->rset)
 793         nset->rset = rset_dup(rset->rset);
 794     if (rset->rpn)
 795         nset->rpn = copy_RPNQuery(rset->rpn, nset->nmem);
 796     return nset;
 797 }
 798
 799 ZEBRA_RES resultSetSort(ZebraHandle zh, NMEM nmem,
 800                         int num_input_setnames, const char **input_setnames,
 801                         const char *output_setname,
 802                         Z_SortKeySpecList *sort_sequence, int *sort_status)
 803 {
 804     ZebraSet sset;
 805     RSET rset;
 806
 807     if (num_input_setnames == 0)
 808     {
 809         zebra_setError(zh, YAZ_BIB1_NO_RESULT_SET_NAME_SUPPLIED_ON_SORT, 0);
 810         return ZEBRA_FAIL;
 811     }
 812     if (num_input_setnames > 1)
 813     {
 814         zebra_setError(zh, YAZ_BIB1_SORT_TOO_MANY_INPUT_RESULTS, 0);
 815         return ZEBRA_FAIL;
 816     }
 817     if (!log_level_set)
 818         loglevels();
 819     yaz_log(log_level_sort, "result set sort input=%s output=%s",
 820             *input_setnames, output_setname);
 821     sset = resultSetGet(zh, input_setnames[0]);
 822     if (!sset)
 823     {
 824         zebra_setError(zh, YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
 825                        input_setnames[0]);
 826         return ZEBRA_FAIL;
 827     }
 828     if (!(rset = sset->rset))
 829     {
 830         zebra_setError(zh, YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
 831                        input_setnames[0]);
 832         return ZEBRA_FAIL;
 833     }
 834     if (strcmp(output_setname, input_setnames[0]))
 835         sset = resultSetClone(zh, output_setname, sset);
 836     sset->sortSpec = copy_SortKeySpecList(sort_sequence, sset->nmem);
 837     return resultSetSortSingle(zh, nmem, sset, rset, sort_sequence,
 838                                sort_status);
 839 }
 840
 841 ZEBRA_RES resultSetSortSingle(ZebraHandle zh, NMEM nmem,
 842                               ZebraSet sset, RSET rset,
 843                               Z_SortKeySpecList *sort_sequence,
 844                               int *sort_status)
 845 {
 846     int i;
 847     int ib;
 848     int n = 0;
 849     zint kno = 0;
 850     zint psysno = 0;
 851     struct it_key key;
 852     struct sortKeyInfo sort_criteria[ZSET_SORT_MAX_LEVEL];
 853     char *cmp_buf[ZSET_SORT_MAX_LEVEL];
 854     char *tmp_cmp_buf[ZSET_SORT_MAX_LEVEL];
 855     int num_criteria;
 856     RSFD rfd;
 857     TERMID termid;
 858     TERMID *terms;
 859     int numTerms = 0;
 860     size_t sysno_mem_index = 0;
 861
 862     int numbases = zh->num_basenames;
 863     yaz_log(log_level_sort, "searching %d databases",numbases);
 864
 865     if (zh->m_staticrank)
 866         sysno_mem_index = 1;
 867
 868     assert(nmem); /* compiler shut up about unused param */
 869     sset->sort_info->num_entries = 0;
 870
 871     rset_getterms(rset, 0, 0, &n);
 872     terms = (TERMID *) nmem_malloc(nmem, sizeof(*terms)*n);
 873     rset_getterms(rset, terms, n, &numTerms);
 874
 875     sset->hits = 0;
 876     num_criteria = sort_sequence->num_specs;
 877     if (num_criteria > ZSET_SORT_MAX_LEVEL)
 878         num_criteria = ZSET_SORT_MAX_LEVEL;
 879     /* set up the search criteria */
 880     for (i = 0; i < num_criteria; i++)
 881     {
 882         Z_SortKeySpec *sks = sort_sequence->specs[i];
 883         Z_SortKey *sk;
 884         ZEBRA_RES res;
 885
 886         sort_criteria[i].ord = (int *)
 887             nmem_malloc(nmem, sizeof(int)*numbases);
 888         sort_criteria[i].numerical = (int *)
 889             nmem_malloc(nmem, sizeof(int)*numbases);
 890
 891         /* initialize ord and numerical for each database */
 892         for (ib = 0; ib < numbases; ib++)
 893         {
 894             sort_criteria[i].ord[ib] = -1;
 895             sort_criteria[i].numerical[ib] = 0;
 896         }
 897
 898         if (sks->which == Z_SortKeySpec_missingValueData)
 899         {
 900             zebra_setError(zh, YAZ_BIB1_UNSUPP_MISSING_DATA_ACTION, 0);
 901             return ZEBRA_FAIL;
 902         }
 903         if (*sks->sortRelation == Z_SortKeySpec_ascending)
 904             sort_criteria[i].relation = 'A';
 905         else if (*sks->sortRelation == Z_SortKeySpec_descending)
 906             sort_criteria[i].relation = 'D';
 907         else
 908         {
 909             zebra_setError(zh, YAZ_BIB1_ILLEGAL_SORT_RELATION, 0);
 910             return ZEBRA_FAIL;
 911         }
 912         if (sks->sortElement->which == Z_SortElement_databaseSpecific)
 913         {
 914             zebra_setError(zh, YAZ_BIB1_DATABASE_SPECIFIC_SORT_UNSUPP, 0);
 915             return ZEBRA_FAIL;
 916         }
 917         else if (sks->sortElement->which != Z_SortElement_generic)
 918         {
 919             zebra_setError(zh, YAZ_BIB1_SORT_ILLEGAL_SORT, 0);
 920             return ZEBRA_FAIL;
 921         }
 922         sk = sks->sortElement->u.generic;
 923         switch (sk->which)
 924         {
 925         case Z_SortKey_sortField:
 926             yaz_log(log_level_sort, "key %d is of type sortField", i+1);
 927             for (ib = 0; ib < numbases; ib++)
 928             {
 929                 zebraExplain_curDatabase(zh->reg->zei, zh->basenames[ib]);
 930                 sort_criteria[i].numerical[ib] = 0;
 931                 sort_criteria[i].ord[ib] =
 932                     zebraExplain_lookup_attr_str(zh->reg->zei,
 933                                                  zinfo_index_category_sort,
 934                                                  0, sk->u.sortField);
 935                 if (sks->which != Z_SortKeySpec_null
 936                     && sort_criteria[i].ord[ib] == -1)
 937                 {
 938                     zebra_setError(zh,
 939                                    YAZ_BIB1_CANNOT_SORT_ACCORDING_TO_SEQUENCE, 0);
 940                     return ZEBRA_FAIL;
 941                 }
 942             }
 943             break;
 944         case Z_SortKey_elementSpec:
 945             yaz_log(log_level_sort, "key %d is of type elementSpec", i+1);
 946             zebra_setError(zh, YAZ_BIB1_CANNOT_SORT_ACCORDING_TO_SEQUENCE, 0);
 947             return ZEBRA_FAIL;
 948         case Z_SortKey_sortAttributes:
 949             yaz_log(log_level_sort, "key %d is of type sortAttributes", i+1);
 950             /* for every database we searched, get the sort index file
 951                id (ord) and its numerical indication and store them in
 952                the sort_criteria */
 953             for (ib = 0; ib < numbases; ib++)
 954             {
 955                 zebraExplain_curDatabase(zh->reg->zei, zh->basenames[ib]);
 956                 res = zebra_sort_get_ord(zh, sk->u.sortAttributes,
 957                                          &sort_criteria[i].ord[ib],
 958                                          &sort_criteria[i].numerical[ib]);
 959             }
 960
 961             if (sks->which != Z_SortKeySpec_null && res != ZEBRA_OK)
 962                 return ZEBRA_FAIL;
 963             break;
 964         }
 965         /* right now we look up the index type based on the first database
 966            if the index_type's can differ between the indexes of different
 967            databases (which i guess they can?) then we have to store the
 968            index types for each database, just like the ord and numerical */
 969         if (zebraExplain_lookup_ord(zh->reg->zei, sort_criteria[i].ord[0],
 970                                     &sort_criteria[i].index_type,
 971                                     0, 0))
 972         {
 973             zebra_setError(zh, YAZ_BIB1_CANNOT_SORT_ACCORDING_TO_SEQUENCE, 0);
 974             return ZEBRA_FAIL;
 975         }
 976     }
 977     /* allocate space for each cmpare buf + one extra for tmp comparison */
 978     /* cmp_buf is an array of array, the first dimension is the criteria and the second dimension are
 979        all other result entries to compare against. This is slowly filled when records are processed.
 980        tmp_cmp_buf is an array with a value of the current record for each criteria
 981     */
 982     for (i = 0; i<num_criteria; i++)
 983     {
 984         cmp_buf[i] = xmalloc(sset->sort_info->max_entries
 985                              * SORT_IDX_ENTRYSIZE);
 986         tmp_cmp_buf[i] = xmalloc(SORT_IDX_ENTRYSIZE);
 987     }
 988     rfd = rset_open(rset, RSETF_READ);
 989     while (rset_read(rfd, &key, &termid))
 990     {
 991         zint this_sys = key.mem[sysno_mem_index];
 992         if (log_level_searchhits)
 993             key_logdump_txt(log_level_searchhits, &key, termid->name);
 994         kno++;
 995         if (this_sys != psysno)
 996         {
 997             int database_no = 0;
 998             if ((sset->hits & 255) == 0 && zh->break_handler_func)
 999             {
1000                 if (zh->break_handler_func(zh->break_handler_data))
1001                 {
1002                     rset_set_hits_limit(rset, 0);
1003                     break;
1004                 }
1005             }
1006             (sset->hits)++;
1007             psysno = this_sys;
1008
1009             /* determine database from the term, but only bother if more than
1010                one database is in use*/
1011             if (numbases > 1 && termid->ol)
1012             {
1013                 const char *this_db = 0;
1014                 if (zebraExplain_lookup_ord(zh->reg->zei, termid->ol->ord,  0, &this_db, 0)
1015                     == 0 && this_db)
1016                 {
1017                     for (ib = 0; ib < numbases; ib++)
1018                         if (!strcmp(this_db, zh->basenames[ib]))
1019                             database_no = ib;
1020                 }
1021             }
1022 #if 0
1023             yaz_log(YLOG_LOG, "sysno=" ZINT_FORMAT " database_no=%d", this_sys,
1024                 database_no);
1025 #endif
1026             ord_list_print(termid->ol);
1027             resultSetInsertSort(zh, sset, database_no,
1028                                 sort_criteria, num_criteria, psysno, cmp_buf,
1029                                 tmp_cmp_buf);
1030         }
1031     }
1032     rset_close(rfd);
1033
1034     /* free the compare buffers */
1035     for (i = 0; i<num_criteria; i++)
1036     {
1037         xfree(cmp_buf[i]);
1038         xfree(tmp_cmp_buf[i]);
1039     }
1040
1041     yaz_log(log_level_sort, ZINT_FORMAT " keys, " ZINT_FORMAT " sysnos, sort",
1042             kno, sset->hits);
1043     for (i = 0; i < numTerms; i++)
1044         yaz_log(log_level_sort, "term=\"%s\" type=%s count=" ZINT_FORMAT,
1045                 terms[i]->name, terms[i]->flags, terms[i]->rset->hits_count);
1046     *sort_status = Z_SortResponse_success;
1047     return ZEBRA_OK;
1048 }
1049
1050 RSET resultSetRef(ZebraHandle zh, const char *resultSetId)
1051 {
1052     ZebraSet s;
1053
1054     if ((s = resultSetGet(zh, resultSetId)))
1055         return s->rset;
1056     return NULL;
1057 }
1058
1059 ZEBRA_RES resultSetRank(ZebraHandle zh, ZebraSet zebraSet,
1060                         RSET rset, NMEM nmem)
1061 {
1062     struct it_key key;
1063     TERMID termid;
1064     TERMID *terms;
1065     zint kno = 0;
1066     int numTerms = 0;
1067     int n = 0;
1068     int i;
1069     ZebraRankClass rank_class;
1070     struct zset_sort_info *sort_info;
1071     const char *rank_handler_name = res_get_def(zh->res, "rank", "rank-1");
1072     size_t sysno_mem_index = 0;
1073
1074     if (zh->m_staticrank)
1075         sysno_mem_index = 1;
1076
1077     if (!log_level_set)
1078         loglevels();
1079     sort_info = zebraSet->sort_info;
1080     sort_info->num_entries = 0;
1081     zebraSet->hits = 0;
1082     zebraSet->estimated_hit_count = 0;
1083     rset_getterms(rset, 0, 0, &n);
1084     terms = (TERMID *) nmem_malloc(nmem, sizeof(*terms)*n);
1085     rset_getterms(rset, terms, n, &numTerms);
1086
1087     rank_class = zebraRankLookup(zh, rank_handler_name);
1088     if (!rank_class)
1089     {
1090         yaz_log(YLOG_WARN, "No such rank handler: %s", rank_handler_name);
1091         zebra_setError(zh, YAZ_BIB1_UNSUPP_SEARCH, "Cannot find rank handler");
1092         return ZEBRA_FAIL;
1093     }
1094     else
1095     {
1096         RSFD rfd = rset_open(rset, RSETF_READ);
1097         struct rank_control *rc = rank_class->control;
1098         int score;
1099         zint count = 0;
1100         void *handle = (*rc->begin) (zh->reg, rank_class->class_handle, rset,
1101                                      nmem, terms, numTerms);
1102         zint psysno = 0;  /* previous doc id / sys no */
1103         zint pstaticrank = 0; /* previous static rank */
1104         int stop_flag = 0;
1105         while (rset_read(rfd, &key, &termid))
1106         {
1107             zint this_sys = key.mem[sysno_mem_index];
1108
1109             zint seqno = key.mem[key.len-1];
1110             kno++;
1111             if (log_level_searchhits)
1112                 key_logdump_txt(log_level_searchhits, &key, termid->name);
1113             if (this_sys != psysno)
1114             {   /* new record .. */
1115                 if (!(rfd->counted_items & 255) && zh->break_handler_func)
1116                 {
1117                     if (zh->break_handler_func(zh->break_handler_data))
1118                     {
1119                         yaz_log(YLOG_LOG, "Aborted search");
1120                         stop_flag = 1;
1121                     }
1122                 }
1123                 if (rfd->counted_items > rset->hits_limit)
1124                     stop_flag = 1;
1125                 if (psysno)
1126                 {   /* only if we did have a previous record */
1127                     score = (*rc->calc)(handle, psysno, pstaticrank,
1128                                         &stop_flag);
1129                     /* insert the hit. A=Ascending */
1130                     resultSetInsertRank(zh, sort_info, psysno, score, 'A');
1131                     count++;
1132                 }
1133                 if (stop_flag)
1134                 {
1135                     zebraSet->estimated_hit_count = 1;
1136                     rset_set_hits_limit(rset, 0);
1137                     break;
1138                 }
1139                 psysno = this_sys;
1140                 if (zh->m_staticrank)
1141                     pstaticrank = key.mem[0];
1142             }
1143             (*rc->add)(handle, CAST_ZINT_TO_INT(seqno), termid);
1144         }
1145         /* no more items */
1146         if (psysno)
1147         {   /* we had - at least - one record */
1148             score = (*rc->calc)(handle, psysno, pstaticrank, &stop_flag);
1149             /* insert the hit. A=Ascending */
1150             resultSetInsertRank(zh, sort_info, psysno, score, 'A');
1151             count++;
1152         }
1153         (*rc->end)(zh->reg, handle);
1154         rset_close(rfd);
1155     }
1156     zebraSet->hits = rset->hits_count;
1157
1158     yaz_log(log_level_searchterms, ZINT_FORMAT " keys, "
1159             ZINT_FORMAT " sysnos, rank",  kno, zebraSet->hits);
1160     for (i = 0; i < numTerms; i++)
1161     {
1162         yaz_log(log_level_searchterms, "term=\"%s\" type=%s count="
1163                 ZINT_FORMAT,
1164                 terms[i]->name, terms[i]->flags, terms[i]->rset->hits_count);
1165     }
1166     return ZEBRA_OK;
1167 }
1168
1169 ZebraRankClass zebraRankLookup(ZebraHandle zh, const char *name)
1170 {
1171     ZebraRankClass p = zh->reg->rank_classes;
1172     while (p && strcmp(p->control->name, name))
1173         p = p->next;
1174     if (p && !p->init_flag)
1175     {
1176         if (p->control->create)
1177             p->class_handle = (*p->control->create)(zh);
1178         p->init_flag = 1;
1179     }
1180     return p;
1181 }
1182
1183 void zebraRankInstall(struct zebra_register *reg, struct rank_control *ctrl)
1184 {
1185     ZebraRankClass p = (ZebraRankClass) xmalloc(sizeof(*p));
1186     p->control = (struct rank_control *) xmalloc(sizeof(*p->control));
1187     memcpy(p->control, ctrl, sizeof(*p->control));
1188     p->control->name = xstrdup(ctrl->name);
1189     p->init_flag = 0;
1190     p->next = reg->rank_classes;
1191     reg->rank_classes = p;
1192 }
1193
1194 void zebraRankDestroy(struct zebra_register *reg)
1195 {
1196     ZebraRankClass p = reg->rank_classes;
1197     while (p)
1198     {
1199         ZebraRankClass p_next = p->next;
1200         if (p->init_flag && p->control->destroy)
1201             (*p->control->destroy)(reg, p->class_handle);
1202         xfree(p->control->name);
1203         xfree(p->control);
1204         xfree(p);
1205         p = p_next;
1206     }
1207     reg->rank_classes = NULL;
1208 }
1209
1210 static int trav_rset_for_termids(RSET rset, TERMID *termid_array,
1211                                  zint *hits_array, int *approx_array)
1212 {
1213     int no = 0;
1214     int i;
1215     for (i = 0; i<rset->no_children; i++)
1216         no += trav_rset_for_termids(rset->children[i],
1217                                     (termid_array ? termid_array + no : 0),
1218                                     (hits_array ? hits_array + no : 0),
1219                                     (approx_array ? approx_array + no : 0));
1220     if (rset->term)
1221     {
1222         if (termid_array)
1223             termid_array[no] = rset->term;
1224         if (hits_array)
1225             hits_array[no] = rset->hits_count;
1226         if (approx_array)
1227             approx_array[no] = rset->hits_approx;
1228 #if 0
1229         yaz_log(YLOG_LOG, "rset=%p term=%s limit=" ZINT_FORMAT
1230                 " count=" ZINT_FORMAT,
1231                 rset, rset->term->name, rset->hits_limit, rset->hits_count);
1232 #endif
1233         no++;
1234     }
1235     return no;
1236 }
1237
1238 ZEBRA_RES zebra_result_set_term_no(ZebraHandle zh, const char *setname,
1239                                    int *num_terms)
1240 {
1241     ZebraSet sset = resultSetGet(zh, setname);
1242     *num_terms = 0;
1243     if (sset)
1244     {
1245         *num_terms = trav_rset_for_termids(sset->rset, 0, 0, 0);
1246         return ZEBRA_OK;
1247     }
1248     return ZEBRA_FAIL;
1249 }
1250
1251 ZEBRA_RES zebra_result_set_term_info(ZebraHandle zh, const char *setname,
1252                                      int no, zint *count, int *approx,
1253                                      char *termbuf, size_t *termlen,
1254                                      const char **term_ref_id)
1255 {
1256     ZebraSet sset = resultSetGet(zh, setname);
1257     if (sset)
1258     {
1259         int num_terms = trav_rset_for_termids(sset->rset, 0, 0, 0);
1260         if (no >= 0 && no < num_terms)
1261         {
1262             TERMID *term_array = xmalloc(num_terms * sizeof(*term_array));
1263             zint *hits_array = xmalloc(num_terms * sizeof(*hits_array));
1264             int *approx_array = xmalloc(num_terms * sizeof(*approx_array));
1265
1266             trav_rset_for_termids(sset->rset, term_array,
1267                                   hits_array, approx_array);
1268
1269             if (count)
1270                 *count = hits_array[no];
1271             if (approx)
1272                 *approx = approx_array[no];
1273             if (termbuf)
1274             {
1275                 char *inbuf = term_array[no]->name;
1276                 size_t inleft = strlen(inbuf);
1277                 size_t outleft = *termlen - 1;
1278
1279                 if (zh->iconv_from_utf8 != 0)
1280                 {
1281                     char *outbuf = termbuf;
1282                     size_t ret;
1283
1284                     ret = yaz_iconv(zh->iconv_from_utf8, &inbuf, &inleft,
1285                                     &outbuf, &outleft);
1286                     if (ret == (size_t)(-1))
1287                         *termlen = 0;
1288                     else
1289                     {
1290                         yaz_iconv(zh->iconv_from_utf8, 0, 0,
1291                                   &outbuf, &outleft);
1292                         *termlen = outbuf - termbuf;
1293                     }
1294                 }
1295                 else
1296                 {
1297                     if (inleft > outleft)
1298                         inleft = outleft;
1299                     *termlen = inleft;
1300                     memcpy(termbuf, inbuf, *termlen);
1301                 }
1302                 termbuf[*termlen] = '\0';
1303             }
1304             if (term_ref_id)
1305                 *term_ref_id = term_array[no]->ref_id;
1306
1307             xfree(term_array);
1308             xfree(hits_array);
1309             xfree(approx_array);
1310             return ZEBRA_OK;
1311         }
1312     }
1313     return ZEBRA_FAIL;
1314 }
1315
1316 ZEBRA_RES zebra_snippets_hit_vector(ZebraHandle zh, const char *setname,
1317                                     zint sysno, zebra_snippets *snippets)
1318 {
1319     ZebraSet sset = resultSetGet(zh, setname);
1320     yaz_log(YLOG_DEBUG, "zebra_get_hit_vector setname=%s zysno=" ZINT_FORMAT,
1321             setname, sysno);
1322     if (!sset)
1323         return ZEBRA_FAIL;
1324     else
1325     {
1326         struct rset_key_control *kc = zebra_key_control_create(zh);
1327         NMEM nmem = nmem_create();
1328         struct it_key key;
1329         RSET rsets[2], rset_comb;
1330         RSET rset_temp = rset_create_temp(nmem, kc, kc->scope,
1331                                           res_get(zh->res, "setTmpDir"),0 );
1332
1333         TERMID termid;
1334         RSFD rsfd = rset_open(rset_temp, RSETF_WRITE);
1335
1336         key.mem[0] = sysno;
1337         key.mem[1] = 0;
1338         key.mem[2] = 0;
1339         key.mem[3] = 0;
1340         key.len = 2;
1341         rset_write(rsfd, &key);
1342         rset_close(rsfd);
1343
1344         rsets[0] = rset_temp;
1345         rsets[1] = rset_dup(sset->rset);
1346
1347         rset_comb = rset_create_and(nmem, kc, kc->scope, 2, rsets);
1348
1349         rsfd = rset_open(rset_comb, RSETF_READ);
1350
1351         while (rset_read(rsfd, &key, &termid))
1352         {
1353             if (termid)
1354             {
1355                 struct ord_list *ol;
1356                 for (ol = termid->ol; ol; ol = ol->next)
1357                 {
1358                     zebra_snippets_append(snippets, key.mem[key.len-1], 0,
1359                                           ol->ord, termid->name);
1360                 }
1361             }
1362         }
1363         rset_close(rsfd);
1364
1365         rset_delete(rset_comb);
1366         nmem_destroy(nmem);
1367         kc->dec(kc);
1368     }
1369     return ZEBRA_OK;
1370 }
1371
1372 static ZEBRA_RES zebra_recid_to_sysno(ZebraHandle zh,
1373                                       const char **basenames, int num_bases,
1374                                       zint recid,
1375                                       zint *sysnos, int *no_sysnos)
1376 {
1377     ZEBRA_RES res = ZEBRA_OK;
1378     int sysnos_offset = 0;
1379     int i;
1380
1381     if (!zh->reg->isamb || !zh->m_segment_indexing)
1382     {
1383         if (sysnos_offset < *no_sysnos)
1384             *sysnos = recid;
1385         sysnos_offset++;
1386     }
1387     else
1388     {
1389         for (i = 0; res == ZEBRA_OK && i < num_bases; i++)
1390         {
1391             const char *database = basenames[i];
1392             if (zebraExplain_curDatabase(zh->reg->zei, database) == 0)
1393             {
1394                 const char *index_type = "w";
1395                 const char *use_string = "_ALLRECORDS";
1396                 int ord;
1397                 zinfo_index_category_t cat = zinfo_index_category_alwaysmatches;
1398                 ord = zebraExplain_lookup_attr_str(zh->reg->zei, cat,
1399                                                    index_type, use_string);
1400                 if (ord != -1)
1401                 {
1402                     char ord_buf[32];
1403                     int ord_len = key_SU_encode(ord, ord_buf);
1404                     char *info;
1405
1406                     ord_buf[ord_len] = '\0';
1407
1408                     info = dict_lookup(zh->reg->dict, ord_buf);
1409                     if (info)
1410                     {
1411                         if (*info != sizeof(ISAM_P))
1412                         {
1413                             res = ZEBRA_FAIL;
1414                         }
1415                         else
1416                         {
1417                             ISAM_P isam_p;
1418                             ISAMB_PP pt;
1419                             struct it_key key_until, key_found;
1420                             int i = 0;
1421                             int r;
1422
1423                             memcpy(&isam_p, info+1, sizeof(ISAM_P));
1424
1425                             pt = isamb_pp_open(zh->reg->isamb, isam_p, 2);
1426                             if (!pt)
1427                                 res = ZEBRA_FAIL;
1428                             else
1429                             {
1430                                 key_until.mem[i++] = recid;
1431                                 key_until.mem[i++] = 0;  /* section_id */
1432                                 if (zh->m_segment_indexing)
1433                                     key_until.mem[i++] = 0; /* segment */
1434                                 key_until.mem[i++] = 0;
1435                                 key_until.len = i;
1436
1437                                 r = isamb_pp_forward(pt, &key_found, &key_until);
1438                                 while (r && key_found.mem[0] == recid)
1439                                 {
1440                                     if (sysnos_offset < *no_sysnos)
1441                                         sysnos[sysnos_offset++] =
1442                                             key_found.mem[key_found.len-1];
1443                                     r = isamb_pp_read(pt, &key_found);
1444                                 }
1445                                 isamb_pp_close(pt);
1446                             }
1447                         }
1448                     }
1449                 }
1450             }
1451         }
1452     }
1453     *no_sysnos = sysnos_offset;
1454     return res;
1455 }
1456
1457 ZEBRA_RES zebra_result_recid_to_sysno(ZebraHandle zh,
1458                                       const char *setname,
1459                                       zint recid,
1460                                       zint *sysnos, int *no_sysnos)
1461 {
1462     const char **basenames;
1463     int num_bases;
1464     ZEBRA_RES res;
1465
1466     res = resultSetGetBaseNames(zh, setname, &basenames, &num_bases);
1467     if (res != ZEBRA_OK)
1468         return ZEBRA_FAIL;
1469
1470     return zebra_recid_to_sysno(zh, basenames, num_bases,
1471                                 recid, sysnos, no_sysnos);
1472 }
1473
1474 void zebra_count_set(ZebraHandle zh, RSET rset, zint *count,
1475                      zint approx_limit)
1476 {
1477     zint psysno = 0;
1478     struct it_key key;
1479     RSFD rfd;
1480
1481     yaz_log(YLOG_DEBUG, "count_set");
1482
1483     rset->hits_limit = approx_limit;
1484
1485     *count = 0;
1486     rfd = rset_open(rset, RSETF_READ);
1487     while (rset_read(rfd, &key,0 /* never mind terms */))
1488     {
1489         if (key.mem[0] != psysno)
1490         {
1491             psysno = key.mem[0];
1492             if (rfd->counted_items >= rset->hits_limit)
1493                 break;
1494         }
1495     }
1496     rset_close(rfd);
1497     *count = rset->hits_count;
1498 }
1499
1500
1501 /*
1502  * Local variables:
1503  * c-basic-offset: 4
1504  * indent-tabs-mode: nil
1505  * End:
1506  * vim: shiftwidth=4 tabstop=8 expandtab
1507  */
1508