index/zsets.c

   1 /* $Id: zsets.c,v 1.110 2006-08-14 10:40:15 adam Exp $
   2    Copyright (C) 1995-2006
   3    Index Data ApS
   4
   5 This file is part of the Zebra server.
   6
   7 Zebra is free software; you can redistribute it and/or modify it under
   8 the terms of the GNU General Public License as published by the Free
   9 Software Foundation; either version 2, or (at your option) any later
  10 version.
  11
  12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 for more details.
  16
  17 You should have received a copy of the GNU General Public License
  18 along with this program; if not, write to the Free Software
  19 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  20
  21 */
  22
  23
  24 #include <stdio.h>
  25 #include <assert.h>
  26 #ifdef WIN32
  27 #include <io.h>
  28 #else
  29 #include <unistd.h>
  30 #endif
  31
  32 #include "index.h"
  33 #include "rank.h"
  34 #include <yaz/diagbib1.h>
  35 #include <rset.h>
  36
  37 #define ZSET_SORT_MAX_LEVEL 10
  38
  39 struct zebra_set_term_entry {
  40     int reg_type;
  41     char *db;
  42     char *index_name;
  43     char *term;
  44 };
  45
  46 struct zebra_set {
  47     char *name;
  48     RSET rset;
  49     NMEM nmem;
  50     NMEM rset_nmem; /* for creating the rsets in */
  51     zint hits;
  52     int num_bases;
  53     char **basenames;
  54     Z_RPNQuery *rpn;
  55     Z_SortKeySpecList *sortSpec;
  56     struct zset_sort_info *sort_info;
  57     struct zebra_set_term_entry *term_entries;
  58     int term_entries_max;
  59     struct zebra_set *next;
  60     int locked;
  61
  62     zint cache_position;  /* last position */
  63     RSFD cache_rfd;       /* rfd (NULL if not existing) */
  64     zint cache_psysno;    /* sysno for last position */
  65     zint approx_limit;    /* limit before we do approx */
  66 };
  67
  68 struct zset_sort_entry {
  69     zint sysno;
  70     int score;
  71 };
  72
  73 struct zset_sort_info {
  74     int max_entries;
  75     int num_entries;
  76     struct zset_sort_entry *all_entries;
  77     struct zset_sort_entry **entries;
  78 };
  79
  80 static int log_level_set=0;
  81 static int log_level_sort=0;
  82 static int log_level_searchhits=0;
  83 static int log_level_searchterms=0;
  84 static int log_level_resultsets=0;
  85
  86 static void loglevels()
  87 {
  88     if (log_level_set)
  89         return;
  90     log_level_sort = yaz_log_module_level("sorting");
  91     log_level_searchhits = yaz_log_module_level("searchhits");
  92     log_level_searchterms = yaz_log_module_level("searchterms");
  93     log_level_resultsets = yaz_log_module_level("resultsets");
  94     log_level_set = 1;
  95 }
  96
  97 ZEBRA_RES resultSetSearch(ZebraHandle zh, NMEM nmem, NMEM rset_nmem,
  98                           Z_RPNQuery *rpn, ZebraSet sset)
  99 {
 100     RSET rset = 0;
 101     oident *attrset;
 102     Z_SortKeySpecList *sort_sequence;
 103     int sort_status, i;
 104     ZEBRA_RES res = ZEBRA_OK;
 105
 106     zh->hits = 0;
 107
 108     sort_sequence = (Z_SortKeySpecList *)
 109         nmem_malloc(nmem, sizeof(*sort_sequence));
 110     sort_sequence->num_specs = 10; /* FIXME - Hard-coded number */
 111     sort_sequence->specs = (Z_SortKeySpec **)
 112         nmem_malloc(nmem, sort_sequence->num_specs *
 113                      sizeof(*sort_sequence->specs));
 114     for (i = 0; i<sort_sequence->num_specs; i++)
 115         sort_sequence->specs[i] = 0;
 116
 117     attrset = oid_getentbyoid (rpn->attributeSetId);
 118     res = rpn_search_top(zh, rpn->RPNStructure, attrset->value,
 119                          nmem, rset_nmem,
 120                          sort_sequence,
 121                          sset->num_bases, sset->basenames,
 122                          &rset);
 123     if (res != ZEBRA_OK)
 124     {
 125         sset->rset = 0;
 126         return res;
 127     }
 128     for (i = 0; sort_sequence->specs[i]; i++)
 129         ;
 130     sort_sequence->num_specs = i;
 131     rset->hits_limit = sset->approx_limit;
 132     if (!i)
 133     {
 134         res = resultSetRank (zh, sset, rset, rset_nmem);
 135     }
 136     else
 137     {
 138         res = resultSetSortSingle (zh, nmem, sset, rset,
 139                                    sort_sequence, &sort_status);
 140     }
 141     sset->rset = rset;
 142     return res;
 143 }
 144
 145
 146 ZEBRA_RES resultSetAddRPN (ZebraHandle zh, NMEM m, Z_RPNQuery *rpn,
 147                            int num_bases, char **basenames,
 148                            const char *setname)
 149 {
 150     ZebraSet zebraSet;
 151     int i;
 152     ZEBRA_RES res;
 153
 154     zh->hits = 0;
 155
 156     zebraSet = resultSetAdd(zh, setname, 1);
 157     if (!zebraSet)
 158         return ZEBRA_FAIL;
 159     zebraSet->locked = 1;
 160     zebraSet->rpn = 0;
 161     zebraSet->nmem = m;
 162     zebraSet->rset_nmem = nmem_create();
 163
 164     zebraSet->num_bases = num_bases;
 165     zebraSet->basenames =
 166         nmem_malloc (zebraSet->nmem, num_bases * sizeof(*zebraSet->basenames));
 167     for (i = 0; i<num_bases; i++)
 168         zebraSet->basenames[i] = nmem_strdup(zebraSet->nmem, basenames[i]);
 169
 170     res = resultSetSearch(zh, zebraSet->nmem, zebraSet->rset_nmem,
 171                           rpn, zebraSet);
 172     zh->hits = zebraSet->hits;
 173     if (zebraSet->rset)
 174         zebraSet->rpn = rpn;
 175     zebraSet->locked = 0;
 176     if (!zebraSet->rset)
 177         return ZEBRA_FAIL;
 178     return res;
 179 }
 180
 181 void resultSetAddTerm(ZebraHandle zh, ZebraSet s, int reg_type,
 182                       const char *db, const char *index_name,
 183                       const char *term)
 184 {
 185     assert(zh); /* compiler shut up */
 186     if (!s->nmem)
 187         s->nmem = nmem_create ();
 188     if (!s->term_entries)
 189     {
 190         int i;
 191         s->term_entries_max = 1000;
 192         s->term_entries =
 193             nmem_malloc (s->nmem, s->term_entries_max *
 194                          sizeof(*s->term_entries));
 195         for (i = 0; i < s->term_entries_max; i++)
 196             s->term_entries[i].term = 0;
 197     }
 198     if (s->hits < s->term_entries_max)
 199     {
 200         s->term_entries[s->hits].reg_type = reg_type;
 201         s->term_entries[s->hits].db = nmem_strdup (s->nmem, db);
 202         s->term_entries[s->hits].index_name = nmem_strdup(s->nmem, index_name);
 203         s->term_entries[s->hits].term = nmem_strdup(s->nmem, term);
 204     }
 205     (s->hits)++;
 206 }
 207
 208 ZebraSet resultSetAdd(ZebraHandle zh, const char *name, int ov)
 209 {
 210     ZebraSet s;
 211     int i;
 212
 213     for (s = zh->sets; s; s = s->next)
 214         if (!strcmp (s->name, name))
 215             break;
 216
 217     if (!log_level_set)
 218         loglevels();
 219     if (s)
 220     {
 221         yaz_log(log_level_resultsets, "updating result set %s", name);
 222         if (!ov || s->locked)
 223             return NULL;
 224         if (s->rset)
 225         {
 226             if (s->cache_rfd)
 227                 rset_close(s->cache_rfd);
 228             rset_delete (s->rset);
 229         }
 230         if (s->rset_nmem)
 231             nmem_destroy (s->rset_nmem);
 232         if (s->nmem)
 233             nmem_destroy (s->nmem);
 234     }
 235     else
 236     {
 237         const char *sort_max_str = zebra_get_resource(zh, "sortmax", "1000");
 238
 239         yaz_log(log_level_resultsets, "adding result set %s", name);
 240         s = (ZebraSet) xmalloc (sizeof(*s));
 241         s->next = zh->sets;
 242         zh->sets = s;
 243         s->name = (char *) xmalloc (strlen(name)+1);
 244         strcpy (s->name, name);
 245
 246         s->sort_info = (struct zset_sort_info *)
 247             xmalloc (sizeof(*s->sort_info));
 248         s->sort_info->max_entries = atoi(sort_max_str);
 249         if (s->sort_info->max_entries < 2)
 250             s->sort_info->max_entries = 2;
 251
 252         s->sort_info->entries = (struct zset_sort_entry **)
 253             xmalloc (sizeof(*s->sort_info->entries) *
 254                      s->sort_info->max_entries);
 255         s->sort_info->all_entries = (struct zset_sort_entry *)
 256             xmalloc (sizeof(*s->sort_info->all_entries) *
 257                      s->sort_info->max_entries);
 258         for (i = 0; i < s->sort_info->max_entries; i++)
 259             s->sort_info->entries[i] = s->sort_info->all_entries + i;
 260     }
 261     s->locked = 0;
 262     s->term_entries = 0;
 263     s->hits = 0;
 264     s->rset = 0;
 265     s->rset_nmem = 0;
 266     s->nmem = 0;
 267     s->rpn = 0;
 268     s->sortSpec = 0;
 269     s->cache_position = 0;
 270     s->cache_rfd = 0;
 271     s->approx_limit = zh->approx_limit;
 272     return s;
 273 }
 274
 275 ZebraSet resultSetGet(ZebraHandle zh, const char *name)
 276 {
 277     ZebraSet s;
 278
 279     for (s = zh->sets; s; s = s->next)
 280         if (!strcmp (s->name, name))
 281         {
 282             if (!s->term_entries && !s->rset && s->rpn)
 283             {
 284                 NMEM nmem = nmem_create ();
 285                 yaz_log(log_level_resultsets, "research %s", name);
 286                 if (!s->rset_nmem)
 287                     s->rset_nmem=nmem_create();
 288                 resultSetSearch(zh, nmem, s->rset_nmem, s->rpn, s);
 289                 if (s->rset && s->sortSpec)
 290                 {
 291                     int sort_status;
 292                     yaz_log(log_level_resultsets, "resort %s", name);
 293                     resultSetSortSingle (zh, nmem, s, s->rset, s->sortSpec,
 294                                          &sort_status);
 295                 }
 296                 nmem_destroy (nmem);
 297             }
 298             return s;
 299         }
 300     return NULL;
 301 }
 302
 303 void resultSetInvalidate (ZebraHandle zh)
 304 {
 305     ZebraSet s = zh->sets;
 306
 307     yaz_log(log_level_resultsets, "invalidating result sets");
 308     for (; s; s = s->next)
 309     {
 310         if (s->rset)
 311         {
 312             if (s->cache_rfd)
 313                 rset_close(s->cache_rfd);
 314             rset_delete (s->rset);
 315         }
 316         s->rset = 0;
 317         s->cache_rfd = 0;
 318         s->cache_position = 0;
 319         if (s->rset_nmem)
 320             nmem_destroy(s->rset_nmem);
 321         s->rset_nmem=0;
 322     }
 323 }
 324
 325 void resultSetDestroy(ZebraHandle zh, int num, char **names,int *statuses)
 326 {
 327     ZebraSet * ss = &zh->sets;
 328     int i;
 329
 330     if (statuses)
 331         for (i = 0; i<num; i++)
 332             statuses[i] = Z_DeleteStatus_resultSetDidNotExist;
 333     while (*ss)
 334     {
 335         int i = -1;
 336         ZebraSet s = *ss;
 337         if (num >= 0)
 338         {
 339             for (i = 0; i<num; i++)
 340                 if (!strcmp (s->name, names[i]))
 341                 {
 342                     if (statuses)
 343                         statuses[i] = Z_DeleteStatus_success;
 344                     i = -1;
 345                     break;
 346                 }
 347         }
 348         if (i < 0)
 349         {
 350             *ss = s->next;
 351
 352             xfree (s->sort_info->all_entries);
 353             xfree (s->sort_info->entries);
 354             xfree (s->sort_info);
 355
 356             if (s->nmem)
 357                 nmem_destroy (s->nmem);
 358             if (s->rset)
 359             {
 360                 if (s->cache_rfd)
 361                     rset_close(s->cache_rfd);
 362                 rset_delete (s->rset);
 363             }
 364             if (s->rset_nmem)
 365                 nmem_destroy(s->rset_nmem);
 366             xfree (s->name);
 367             xfree (s);
 368         }
 369         else
 370             ss = &s->next;
 371     }
 372 }
 373
 374 ZebraMetaRecord *zebra_meta_records_create_range(ZebraHandle zh,
 375                                                  const char *name,
 376                                                  zint start, int num)
 377 {
 378     zint pos_small[10];
 379     zint *pos = pos_small;
 380     ZebraMetaRecord *mr;
 381     int i;
 382
 383     if (num > 10000 || num <= 0)
 384         return 0;
 385
 386     if (num > 10)
 387         pos = xmalloc(sizeof(*pos) * num);
 388
 389     for (i = 0; i<num; i++)
 390         pos[i] = start+i;
 391
 392     mr = zebra_meta_records_create(zh, name, num, pos);
 393
 394     if (num > 10)
 395         xfree(pos);
 396     return mr;
 397 }
 398
 399 ZebraMetaRecord *zebra_meta_records_create(ZebraHandle zh, const char *name,
 400                                            int num, zint *positions)
 401 {
 402     ZebraSet sset;
 403     ZebraMetaRecord *sr = 0;
 404     RSET rset;
 405     int i;
 406     struct zset_sort_info *sort_info;
 407     size_t sysno_mem_index = 0;
 408
 409     if (zh->m_staticrank)
 410         sysno_mem_index = 1;
 411
 412     if (!log_level_set)
 413         loglevels();
 414     if (!(sset = resultSetGet (zh, name)))
 415         return NULL;
 416     if (!(rset = sset->rset))
 417     {
 418         if (!sset->term_entries)
 419             return 0;
 420         sr = (ZebraMetaRecord *) xmalloc (sizeof(*sr) * num);
 421         for (i = 0; i<num; i++)
 422         {
 423             sr[i].sysno = 0;
 424             sr[i].score = -1;
 425             sr[i].term = 0;
 426             sr[i].db = 0;
 427
 428             if (positions[i] <= sset->term_entries_max)
 429             {
 430                 sr[i].term = sset->term_entries[positions[i]-1].term;
 431                 sr[i].db = sset->term_entries[positions[i]-1].db;
 432             }
 433         }
 434     }
 435     else
 436     {
 437         sr = (ZebraMetaRecord *) xmalloc (sizeof(*sr) * num);
 438         for (i = 0; i<num; i++)
 439         {
 440             sr[i].sysno = 0;
 441             sr[i].score = -1;
 442             sr[i].term = 0;
 443             sr[i].db = 0;
 444         }
 445         sort_info = sset->sort_info;
 446         if (sort_info)
 447         {
 448             zint position;
 449
 450             for (i = 0; i<num; i++)
 451             {
 452                 position = positions[i];
 453                 if (position > 0 && position <= sort_info->num_entries)
 454                 {
 455                     yaz_log(log_level_sort, "got pos=" ZINT_FORMAT
 456                             " (sorted)", position);
 457                     sr[i].sysno = sort_info->entries[position-1]->sysno;
 458                     sr[i].score = sort_info->entries[position-1]->score;
 459                 }
 460             }
 461         }
 462         /* did we really get all entries using sort ? */
 463         for (i = 0; i<num; i++)
 464         {
 465             if (!sr[i].sysno)
 466                 break;
 467         }
 468         if (i < num) /* nope, get the rest, unsorted - sorry */
 469         {
 470             zint position = 0;
 471             int num_i = 0;
 472             zint psysno = 0;
 473             RSFD rfd;
 474             struct it_key key;
 475
 476             if (sort_info)
 477                 position = sort_info->num_entries;
 478             while (num_i < num && positions[num_i] <= position)
 479                 num_i++;
 480
 481             if (sset->cache_rfd &&
 482                 num_i < num && positions[num_i] > sset->cache_position)
 483             {
 484                 position = sset->cache_position;
 485                 rfd = sset->cache_rfd;
 486                 psysno = sset->cache_psysno;
 487             }
 488             else
 489             {
 490                 if (sset->cache_rfd)
 491                     rset_close(sset->cache_rfd);
 492                 rfd = rset_open (rset, RSETF_READ);
 493             }
 494             while (num_i < num && rset_read (rfd, &key, 0))
 495             {
 496                 zint this_sys = key.mem[sysno_mem_index];
 497                 if (this_sys != psysno)
 498                 {
 499                     psysno = this_sys;
 500                     if (sort_info)
 501                     {
 502                         /* determine we alreay have this in our set */
 503                         for (i = sort_info->num_entries; --i >= 0; )
 504                             if (psysno == sort_info->entries[i]->sysno)
 505                                 break;
 506                         if (i >= 0)
 507                             continue;
 508                     }
 509                     position++;
 510                     assert (num_i < num);
 511                     if (position == positions[num_i])
 512                     {
 513                         sr[num_i].sysno = psysno;
 514                         yaz_log(log_level_sort, "got pos=" ZINT_FORMAT " (unsorted)", position);
 515                         sr[num_i].score = -1;
 516                         num_i++;
 517                     }
 518                 }
 519             }
 520             sset->cache_position = position;
 521             sset->cache_psysno = psysno;
 522             sset->cache_rfd = rfd;
 523         }
 524     }
 525     return sr;
 526 }
 527
 528 void zebra_meta_records_destroy (ZebraHandle zh, ZebraMetaRecord *records,
 529                                  int num)
 530 {
 531     assert(zh); /* compiler shut up about unused arg */
 532     xfree (records);
 533 }
 534
 535 struct sortKeyInfo {
 536     int relation;
 537 #if 0
 538     int attrUse;
 539 #else
 540     int ord;
 541 #endif
 542     int numerical;
 543 };
 544
 545 void resultSetInsertSort(ZebraHandle zh, ZebraSet sset,
 546                          struct sortKeyInfo *criteria, int num_criteria,
 547                          zint sysno,
 548                          char *cmp_buf[], char *tmp_cmp_buf[])
 549 {
 550     struct zset_sort_entry *new_entry = NULL;
 551     struct zset_sort_info *sort_info = sset->sort_info;
 552     int i, j;
 553
 554     sortIdx_sysno (zh->reg->sortIdx, sysno);
 555     for (i = 0; i<num_criteria; i++)
 556     {
 557         char *this_entry_buf = tmp_cmp_buf[i];
 558         memset(this_entry_buf, '\0', SORT_IDX_ENTRYSIZE);
 559         if (criteria[i].ord != -1)
 560         {
 561             sortIdx_type(zh->reg->sortIdx, criteria[i].ord);
 562             sortIdx_read(zh->reg->sortIdx, this_entry_buf);
 563         }
 564     }
 565     i = sort_info->num_entries;
 566     while (--i >= 0)
 567     {
 568         int rel = 0;
 569         for (j = 0; j<num_criteria; j++)
 570         {
 571             char *this_entry_buf = tmp_cmp_buf[j];
 572             char *other_entry_buf =
 573                 cmp_buf[j] + i * SORT_IDX_ENTRYSIZE;
 574             if (criteria[j].numerical)
 575             {
 576                 double diff = atof(this_entry_buf) - atof(other_entry_buf);
 577                 rel = 0;
 578                 if (diff > 0.0)
 579                     rel = 1;
 580                 else if (diff < 0.0)
 581                     rel = -1;
 582             }
 583             else
 584             {
 585                 rel = memcmp(this_entry_buf, other_entry_buf,
 586                              SORT_IDX_ENTRYSIZE);
 587             }
 588             if (rel)
 589                 break;
 590         }
 591         if (!rel)
 592             break;
 593         if (criteria[j].relation == 'A')
 594         {
 595             if (rel > 0)
 596                 break;
 597         }
 598         else if (criteria[j].relation == 'D')
 599         {
 600             if (rel < 0)
 601                 break;
 602         }
 603     }
 604     ++i;
 605     j = sort_info->max_entries;
 606     if (i == j)
 607         return;
 608
 609     if (sort_info->num_entries == j)
 610         --j;
 611     else
 612         j = (sort_info->num_entries)++;
 613     new_entry = sort_info->entries[j];
 614     while (j != i)
 615     {
 616         int k;
 617         for (k = 0; k<num_criteria; k++)
 618         {
 619             char *j_buf = cmp_buf[k] + j * SORT_IDX_ENTRYSIZE;
 620             char *j_1_buf = cmp_buf[k] + (j-1) * SORT_IDX_ENTRYSIZE;
 621             memcpy(j_buf, j_1_buf, SORT_IDX_ENTRYSIZE);
 622         }
 623         sort_info->entries[j] = sort_info->entries[j-1];
 624         --j;
 625     }
 626     sort_info->entries[i] = new_entry;
 627     assert (new_entry);
 628     for (i = 0; i<num_criteria; i++)
 629     {
 630         char *new_entry_buf = cmp_buf[i] + j * SORT_IDX_ENTRYSIZE;
 631         char *this_entry_buf = tmp_cmp_buf[i];
 632         memcpy(new_entry_buf, this_entry_buf, SORT_IDX_ENTRYSIZE);
 633     }
 634     new_entry->sysno = sysno;
 635     new_entry->score = -1;
 636 }
 637
 638 void resultSetInsertRank(ZebraHandle zh, struct zset_sort_info *sort_info,
 639                          zint sysno, int score, int relation)
 640 {
 641     struct zset_sort_entry *new_entry = NULL;
 642     int i, j;
 643     assert(zh); /* compiler shut up about unused arg */
 644
 645     i = sort_info->num_entries;
 646     while (--i >= 0)
 647     {
 648         int rel = 0;
 649
 650         rel = score - sort_info->entries[i]->score;
 651
 652         if (relation == 'D')
 653         {
 654             if (rel >= 0)
 655                 break;
 656         }
 657         else if (relation == 'A')
 658         {
 659             if (rel <= 0)
 660                 break;
 661         }
 662     }
 663     ++i;
 664     j = sort_info->max_entries;
 665     if (i == j)
 666         return;
 667
 668     if (sort_info->num_entries == j)
 669         --j;
 670     else
 671         j = (sort_info->num_entries)++;
 672
 673     new_entry = sort_info->entries[j];
 674     while (j != i)
 675     {
 676         sort_info->entries[j] = sort_info->entries[j-1];
 677         --j;
 678     }
 679     sort_info->entries[i] = new_entry;
 680     assert (new_entry);
 681     new_entry->sysno = sysno;
 682     new_entry->score = score;
 683 }
 684
 685 static Z_RPNQuery *copy_RPNQuery(Z_RPNQuery *src, NMEM nmem)
 686 {
 687     Z_RPNQuery *dst = 0;
 688     ODR encode = odr_createmem(ODR_ENCODE);
 689     ODR decode = odr_createmem(ODR_DECODE);
 690
 691     if (z_RPNQuery(encode, &src, 0, 0))
 692     {
 693         int len;
 694         char *buf = odr_getbuf(encode, &len, 0);
 695
 696         if (buf)
 697         {
 698             odr_setbuf(decode, buf, len, 0);
 699             z_RPNQuery(decode, &dst, 0, 0);
 700         }
 701     }
 702     nmem_transfer(nmem, decode->mem);
 703     odr_destroy(encode);
 704     odr_destroy(decode);
 705     return dst;
 706 }
 707
 708 static Z_SortKeySpecList *copy_SortKeySpecList(Z_SortKeySpecList *src, NMEM nmem)
 709 {
 710     Z_SortKeySpecList *dst = 0;
 711     ODR encode = odr_createmem(ODR_ENCODE);
 712     ODR decode = odr_createmem(ODR_DECODE);
 713
 714     if (z_SortKeySpecList(encode, &src, 0, 0))
 715     {
 716         int len;
 717         char *buf = odr_getbuf(encode, &len, 0);
 718
 719         if (buf)
 720         {
 721             odr_setbuf(decode, buf, len, 0);
 722             z_SortKeySpecList(decode, &dst, 0, 0);
 723         }
 724     }
 725     nmem_transfer(nmem, decode->mem);
 726     odr_destroy(encode);
 727     odr_destroy(decode);
 728     return dst;
 729 }
 730
 731 ZebraSet resultSetClone(ZebraHandle zh, const char *setname,
 732                         ZebraSet rset)
 733 {
 734     ZebraSet nset;
 735     int i;
 736
 737     nset = resultSetAdd(zh, setname, 1);
 738     if (!nset)
 739         return 0;
 740
 741     nset->nmem = nmem_create();
 742
 743     nset->num_bases = rset->num_bases;
 744     nset->basenames =
 745         nmem_malloc (nset->nmem, nset->num_bases * sizeof(*rset->basenames));
 746     for (i = 0; i<rset->num_bases; i++)
 747         nset->basenames[i] = nmem_strdup(nset->nmem, rset->basenames[i]);
 748
 749     if (rset->rset)
 750         nset->rset = rset_dup(rset->rset);
 751     if (rset->rpn)
 752         nset->rpn = copy_RPNQuery(rset->rpn, nset->nmem);
 753     return nset;
 754 }
 755
 756 ZEBRA_RES resultSetSort(ZebraHandle zh, NMEM nmem,
 757                         int num_input_setnames, const char **input_setnames,
 758                         const char *output_setname,
 759                         Z_SortKeySpecList *sort_sequence, int *sort_status)
 760 {
 761     ZebraSet sset;
 762     RSET rset;
 763
 764     if (num_input_setnames == 0)
 765     {
 766         zebra_setError(zh, YAZ_BIB1_NO_RESULT_SET_NAME_SUPPLIED_ON_SORT, 0);
 767         return ZEBRA_FAIL;
 768     }
 769     if (num_input_setnames > 1)
 770     {
 771         zebra_setError(zh, YAZ_BIB1_SORT_TOO_MANY_INPUT_RESULTS, 0);
 772         return ZEBRA_FAIL;
 773     }
 774     if (!log_level_set)
 775         loglevels();
 776     yaz_log(log_level_sort, "result set sort input=%s output=%s",
 777           *input_setnames, output_setname);
 778     sset = resultSetGet (zh, input_setnames[0]);
 779     if (!sset)
 780     {
 781         zebra_setError(zh, YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
 782                        input_setnames[0]);
 783         return ZEBRA_FAIL;
 784     }
 785     if (!(rset = sset->rset))
 786     {
 787         zebra_setError(zh, YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
 788                        input_setnames[0]);
 789         return ZEBRA_FAIL;
 790     }
 791     if (strcmp (output_setname, input_setnames[0]))
 792         sset = resultSetClone(zh, output_setname, sset);
 793     sset->sortSpec = copy_SortKeySpecList(sort_sequence, sset->nmem);
 794     return resultSetSortSingle (zh, nmem, sset, rset, sort_sequence,
 795                                 sort_status);
 796 }
 797
 798 ZEBRA_RES resultSetSortSingle(ZebraHandle zh, NMEM nmem,
 799                               ZebraSet sset, RSET rset,
 800                               Z_SortKeySpecList *sort_sequence,
 801                               int *sort_status)
 802 {
 803     int i;
 804     int n = 0;
 805     zint kno = 0;
 806     zint psysno = 0;
 807     struct it_key key;
 808     struct sortKeyInfo sort_criteria[ZSET_SORT_MAX_LEVEL];
 809     char *cmp_buf[ZSET_SORT_MAX_LEVEL];
 810     char *tmp_cmp_buf[ZSET_SORT_MAX_LEVEL];
 811     int num_criteria;
 812     RSFD rfd;
 813     TERMID termid;
 814     TERMID *terms;
 815     int numTerms = 0;
 816     size_t sysno_mem_index = 0;
 817
 818     if (zh->m_staticrank)
 819         sysno_mem_index = 1;
 820
 821     assert(nmem); /* compiler shut up about unused param */
 822     sset->sort_info->num_entries = 0;
 823
 824     rset_getterms(rset, 0, 0, &n);
 825     terms = (TERMID *) nmem_malloc(nmem, sizeof(*terms)*n);
 826     rset_getterms(rset, terms, n, &numTerms);
 827
 828     sset->hits = 0;
 829     num_criteria = sort_sequence->num_specs;
 830     if (num_criteria > ZSET_SORT_MAX_LEVEL)
 831         num_criteria = ZSET_SORT_MAX_LEVEL;
 832     for (i = 0; i < num_criteria; i++)
 833     {
 834         Z_SortKeySpec *sks = sort_sequence->specs[i];
 835         Z_SortKey *sk;
 836         ZEBRA_RES res;
 837
 838         sort_criteria[i].ord = -1;
 839         sort_criteria[i].numerical = 0;
 840
 841         if (sks->which == Z_SortKeySpec_missingValueData)
 842         {
 843             zebra_setError(zh, YAZ_BIB1_UNSUPP_MISSING_DATA_ACTION, 0);
 844             return ZEBRA_FAIL;
 845         }
 846         if (*sks->sortRelation == Z_SortKeySpec_ascending)
 847             sort_criteria[i].relation = 'A';
 848         else if (*sks->sortRelation == Z_SortKeySpec_descending)
 849             sort_criteria[i].relation = 'D';
 850         else
 851         {
 852             zebra_setError(zh, YAZ_BIB1_ILLEGAL_SORT_RELATION, 0);
 853             return ZEBRA_FAIL;
 854         }
 855         if (sks->sortElement->which == Z_SortElement_databaseSpecific)
 856         {
 857             zebra_setError(zh, YAZ_BIB1_DATABASE_SPECIFIC_SORT_UNSUPP, 0);
 858             return ZEBRA_FAIL;
 859         }
 860         else if (sks->sortElement->which != Z_SortElement_generic)
 861         {
 862             zebra_setError(zh, YAZ_BIB1_SORT_ILLEGAL_SORT, 0);
 863             return ZEBRA_FAIL;
 864         }
 865         sk = sks->sortElement->u.generic;
 866         switch (sk->which)
 867         {
 868         case Z_SortKey_sortField:
 869             yaz_log(log_level_sort, "key %d is of type sortField",
 870                     i+1);
 871             sort_criteria[i].numerical = 0;
 872             sort_criteria[i].ord =
 873                 zebraExplain_lookup_attr_str(zh->reg->zei,
 874                                              zinfo_index_category_sort,
 875                                              's',
 876                                              sk->u.sortField);
 877             if (sks->which != Z_SortKeySpec_null
 878                 && sort_criteria[i].ord == -1)
 879             {
 880                 zebra_setError(zh,
 881                                YAZ_BIB1_CANNOT_SORT_ACCORDING_TO_SEQUENCE, 0);
 882                 return ZEBRA_FAIL;
 883             }
 884             break;
 885         case Z_SortKey_elementSpec:
 886             yaz_log(log_level_sort, "key %d is of type elementSpec",
 887                     i+1);
 888             zebra_setError(zh, YAZ_BIB1_CANNOT_SORT_ACCORDING_TO_SEQUENCE, 0);
 889             return ZEBRA_FAIL;
 890         case Z_SortKey_sortAttributes:
 891             yaz_log(log_level_sort, "key %d is of type sortAttributes", i+1);
 892             res = zebra_sort_get_ord(zh, sk->u.sortAttributes,
 893                                      &sort_criteria[i].ord,
 894                                      &sort_criteria[i].numerical);
 895             if (sks->which != Z_SortKeySpec_null && res != ZEBRA_OK)
 896                 return ZEBRA_FAIL;
 897             break;
 898         }
 899     }
 900     /* allocate space for each cmpare buf + one extra for tmp comparison */
 901     for (i = 0; i<num_criteria; i++)
 902     {
 903         cmp_buf[i] = xmalloc(sset->sort_info->max_entries
 904                              * SORT_IDX_ENTRYSIZE);
 905         tmp_cmp_buf[i] = xmalloc(SORT_IDX_ENTRYSIZE);
 906     }
 907     rfd = rset_open (rset, RSETF_READ);
 908     while (rset_read (rfd, &key, &termid))
 909     {
 910         zint this_sys = key.mem[sysno_mem_index];
 911         if (log_level_searchhits)
 912             key_logdump_txt(log_level_searchhits, &key, termid->name);
 913         kno++;
 914         if (this_sys != psysno)
 915         {
 916             (sset->hits)++;
 917             psysno = this_sys;
 918             resultSetInsertSort(zh, sset,
 919                                 sort_criteria, num_criteria, psysno, cmp_buf,
 920                                 tmp_cmp_buf);
 921         }
 922     }
 923     rset_close (rfd);
 924
 925     for (i = 0; i<num_criteria; i++)
 926     {
 927         xfree(cmp_buf[i]);
 928         xfree(tmp_cmp_buf[i]);
 929     }
 930
 931     yaz_log(log_level_sort, ZINT_FORMAT " keys, " ZINT_FORMAT " sysnos, sort",
 932             kno, sset->hits);
 933     for (i = 0; i < numTerms; i++)
 934         yaz_log(log_level_sort, "term=\"%s\" type=%s count=" ZINT_FORMAT,
 935                  terms[i]->name, terms[i]->flags, terms[i]->rset->hits_count);
 936     *sort_status = Z_SortResponse_success;
 937     return ZEBRA_OK;
 938 }
 939
 940 RSET resultSetRef(ZebraHandle zh, const char *resultSetId)
 941 {
 942     ZebraSet s;
 943
 944     if ((s = resultSetGet (zh, resultSetId)))
 945         return s->rset;
 946     return NULL;
 947 }
 948
 949 ZEBRA_RES resultSetRank(ZebraHandle zh, ZebraSet zebraSet,
 950                         RSET rset, NMEM nmem)
 951 {
 952     struct it_key key;
 953     TERMID termid;
 954     TERMID *terms;
 955     zint kno = 0;
 956     int numTerms = 0;
 957     int n = 0;
 958     int i;
 959     ZebraRankClass rank_class;
 960     struct zset_sort_info *sort_info;
 961     const char *rank_handler_name = res_get_def(zh->res, "rank", "rank-1");
 962     size_t sysno_mem_index = 0;
 963
 964     if (zh->m_staticrank)
 965         sysno_mem_index = 1;
 966
 967     if (!log_level_set)
 968         loglevels();
 969     sort_info = zebraSet->sort_info;
 970     sort_info->num_entries = 0;
 971     zebraSet->hits = 0;
 972     rset_getterms(rset, 0, 0, &n);
 973     terms = (TERMID *) nmem_malloc(nmem, sizeof(*terms)*n);
 974     rset_getterms(rset, terms, n, &numTerms);
 975
 976
 977     rank_class = zebraRankLookup(zh, rank_handler_name);
 978     if (!rank_class)
 979     {
 980         yaz_log(YLOG_WARN, "No such rank handler: %s", rank_handler_name);
 981         zebra_setError(zh, YAZ_BIB1_UNSUPP_SEARCH, "Cannot find rank handler");
 982         return ZEBRA_FAIL;
 983     }
 984     else
 985     {
 986         RSFD rfd = rset_open(rset, RSETF_READ);
 987         struct rank_control *rc = rank_class->control;
 988         int score;
 989         zint count = 0;
 990
 991         void *handle =
 992             (*rc->begin) (zh->reg, rank_class->class_handle, rset, nmem,
 993                           terms, numTerms);
 994         zint psysno = 0;  /* previous doc id / sys no */
 995         zint pstaticrank = 0; /* previous static rank */
 996         int stop_flag = 0;
 997         while (rset_read(rfd, &key, &termid))
 998         {
 999             zint this_sys = key.mem[sysno_mem_index];
1000
1001             zint seqno = key.mem[key.len-1];
1002             kno++;
1003             if (log_level_searchhits)
1004                 key_logdump_txt(log_level_searchhits, &key, termid->name);
1005             if (this_sys != psysno)
1006             {   /* new record .. */
1007                 if (rfd->counted_items > rset->hits_limit)
1008                     break;
1009                 if (psysno)
1010                 {   /* only if we did have a previous record */
1011                     score = (*rc->calc) (handle, psysno, pstaticrank,
1012                                          &stop_flag);
1013                     /* insert the hit. A=Ascending */
1014                     resultSetInsertRank (zh, sort_info, psysno, score, 'A');
1015                     count++;
1016                     if (stop_flag)
1017                         break;
1018                 }
1019                 psysno = this_sys;
1020                 if (zh->m_staticrank)
1021                     pstaticrank = key.mem[0];
1022             }
1023             (*rc->add) (handle, CAST_ZINT_TO_INT(seqno), termid);
1024         }
1025         /* no more items */
1026         if (psysno)
1027         {   /* we had - at least - one record */
1028             score = (*rc->calc)(handle, psysno, pstaticrank, &stop_flag);
1029             /* insert the hit. A=Ascending */
1030             resultSetInsertRank(zh, sort_info, psysno, score, 'A');
1031             count++;
1032         }
1033         (*rc->end) (zh->reg, handle);
1034         rset_close (rfd);
1035     }
1036     zebraSet->hits = rset->hits_count;
1037
1038     yaz_log(log_level_searchterms, ZINT_FORMAT " keys, "
1039             ZINT_FORMAT " sysnos, rank",  kno, zebraSet->hits);
1040     for (i = 0; i < numTerms; i++)
1041     {
1042         yaz_log(log_level_searchterms, "term=\"%s\" type=%s count="
1043                 ZINT_FORMAT,
1044                 terms[i]->name, terms[i]->flags, terms[i]->rset->hits_count);
1045     }
1046     return ZEBRA_OK;
1047 }
1048
1049 ZebraRankClass zebraRankLookup(ZebraHandle zh, const char *name)
1050 {
1051     ZebraRankClass p = zh->reg->rank_classes;
1052     while (p && strcmp (p->control->name, name))
1053         p = p->next;
1054     if (p && !p->init_flag)
1055     {
1056         if (p->control->create)
1057             p->class_handle = (*p->control->create)(zh);
1058         p->init_flag = 1;
1059     }
1060     return p;
1061 }
1062
1063 void zebraRankInstall(struct zebra_register *reg, struct rank_control *ctrl)
1064 {
1065     ZebraRankClass p = (ZebraRankClass) xmalloc (sizeof(*p));
1066     p->control = (struct rank_control *) xmalloc (sizeof(*p->control));
1067     memcpy (p->control, ctrl, sizeof(*p->control));
1068     p->control->name = xstrdup (ctrl->name);
1069     p->init_flag = 0;
1070     p->next = reg->rank_classes;
1071     reg->rank_classes = p;
1072 }
1073
1074 void zebraRankDestroy(struct zebra_register *reg)
1075 {
1076     ZebraRankClass p = reg->rank_classes;
1077     while (p)
1078     {
1079         ZebraRankClass p_next = p->next;
1080         if (p->init_flag && p->control->destroy)
1081             (*p->control->destroy)(reg, p->class_handle);
1082         xfree(p->control->name);
1083         xfree(p->control);
1084         xfree(p);
1085         p = p_next;
1086     }
1087     reg->rank_classes = NULL;
1088 }
1089
1090 static int trav_rset_for_termids(RSET rset, TERMID *termid_array,
1091                                  zint *hits_array, int *approx_array)
1092 {
1093     int no = 0;
1094     int i;
1095     for (i = 0; i<rset->no_children; i++)
1096         no += trav_rset_for_termids(rset->children[i],
1097                                     (termid_array ? termid_array + no : 0),
1098                                     (hits_array ? hits_array + no : 0),
1099                                     (approx_array ? approx_array + no : 0));
1100     if (rset->term)
1101     {
1102         if (termid_array)
1103             termid_array[no] = rset->term;
1104         if (hits_array)
1105             hits_array[no] = rset->hits_count;
1106         if (approx_array)
1107             approx_array[no] = rset->hits_approx;
1108 #if 0
1109         yaz_log(YLOG_LOG, "rset=%p term=%s limit=" ZINT_FORMAT
1110                 " count=" ZINT_FORMAT,
1111                 rset, rset->term->name, rset->hits_limit, rset->hits_count);
1112 #endif
1113         no++;
1114     }
1115     return no;
1116 }
1117
1118 ZEBRA_RES zebra_result_set_term_no(ZebraHandle zh, const char *setname,
1119                                    int *num_terms)
1120 {
1121     ZebraSet sset = resultSetGet(zh, setname);
1122     *num_terms = 0;
1123     if (sset)
1124     {
1125         *num_terms = trav_rset_for_termids(sset->rset, 0, 0, 0);
1126         return ZEBRA_OK;
1127     }
1128     return ZEBRA_FAIL;
1129 }
1130
1131 ZEBRA_RES zebra_result_set_term_info(ZebraHandle zh, const char *setname,
1132                                      int no, zint *count, int *approx,
1133                                      char *termbuf, size_t *termlen,
1134                                      const char **term_ref_id)
1135 {
1136     ZebraSet sset = resultSetGet(zh, setname);
1137     if (sset)
1138     {
1139         int num_terms = trav_rset_for_termids(sset->rset, 0, 0, 0);
1140         if (no >= 0 && no < num_terms)
1141         {
1142             TERMID *term_array = xmalloc(num_terms * sizeof(*term_array));
1143             zint *hits_array = xmalloc(num_terms * sizeof(*hits_array));
1144             int *approx_array = xmalloc(num_terms * sizeof(*approx_array));
1145
1146             trav_rset_for_termids(sset->rset, term_array,
1147                                   hits_array, approx_array);
1148
1149             if (count)
1150                 *count = hits_array[no];
1151             if (approx)
1152                 *approx = approx_array[no];
1153             if (termbuf)
1154             {
1155                 char *inbuf = term_array[no]->name;
1156                 size_t inleft = strlen(inbuf);
1157                 size_t outleft = *termlen - 1;
1158
1159                 if (zh->iconv_from_utf8 != 0)
1160                 {
1161                     char *outbuf = termbuf;
1162                     size_t ret;
1163
1164                     ret = yaz_iconv(zh->iconv_from_utf8, &inbuf, &inleft,
1165                                     &outbuf, &outleft);
1166                     if (ret == (size_t)(-1))
1167                         *termlen = 0;
1168                     else
1169                         *termlen = outbuf - termbuf;
1170                 }
1171                 else
1172                 {
1173                     if (inleft > outleft)
1174                         inleft = outleft;
1175                     *termlen = inleft;
1176                     memcpy(termbuf, inbuf, *termlen);
1177                 }
1178                 termbuf[*termlen] = '\0';
1179             }
1180             if (term_ref_id)
1181                 *term_ref_id = term_array[no]->ref_id;
1182
1183             xfree(term_array);
1184             xfree(hits_array);
1185             xfree(approx_array);
1186             return ZEBRA_OK;
1187         }
1188     }
1189     return ZEBRA_FAIL;
1190 }
1191
1192 ZEBRA_RES zebra_snippets_hit_vector(ZebraHandle zh, const char *setname,
1193                                     zint sysno, zebra_snippets *snippets)
1194 {
1195     ZebraSet sset = resultSetGet(zh, setname);
1196     yaz_log(YLOG_DEBUG, "zebra_get_hit_vector setname=%s zysno=" ZINT_FORMAT,
1197             setname, sysno);
1198     if (!sset)
1199         return ZEBRA_FAIL;
1200     else
1201     {
1202         struct rset_key_control *kc = zebra_key_control_create(zh);
1203         NMEM nmem = nmem_create();
1204         struct it_key key;
1205         RSET rsets[2], rset_comb;
1206         RSET rset_temp = rset_create_temp(nmem, kc, kc->scope,
1207                                           res_get (zh->res, "setTmpDir"),0 );
1208
1209         TERMID termid;
1210         RSFD rsfd = rset_open(rset_temp, RSETF_WRITE);
1211
1212         key.mem[0] = sysno;
1213         key.mem[1] = 0;
1214         key.mem[2] = 0;
1215         key.mem[3] = 0;
1216         key.len = 2;
1217         rset_write (rsfd, &key);
1218         rset_close (rsfd);
1219
1220         rsets[0] = rset_temp;
1221         rsets[1] = rset_dup(sset->rset);
1222
1223         rset_comb = rset_create_and(nmem, kc, kc->scope, 2, rsets);
1224
1225         rsfd = rset_open(rset_comb, RSETF_READ);
1226
1227         while (rset_read(rsfd, &key, &termid))
1228         {
1229             if (termid)
1230             {
1231                 struct ord_list *ol;
1232                 for (ol = termid->ol; ol; ol = ol->next)
1233                 {
1234                     zebra_snippets_append(snippets, key.mem[key.len-1],
1235                                           ol->ord, termid->name);
1236                 }
1237             }
1238         }
1239         rset_close(rsfd);
1240
1241         rset_delete(rset_comb);
1242         nmem_destroy(nmem);
1243         kc->dec(kc);
1244     }
1245     return ZEBRA_OK;
1246 }
1247
1248 /*
1249  * Local variables:
1250  * c-basic-offset: 4
1251  * indent-tabs-mode: nil
1252  * End:
1253  * vim: shiftwidth=4 tabstop=8 expandtab
1254  */
1255