index/zsets.c

   1 /* $Id: zsets.c,v 1.111 2006-09-20 10:51:25 adam Exp $
   2    Copyright (C) 1995-2006
   3    Index Data ApS
   4
   5 This file is part of the Zebra server.
   6
   7 Zebra is free software; you can redistribute it and/or modify it under
   8 the terms of the GNU General Public License as published by the Free
   9 Software Foundation; either version 2, or (at your option) any later
  10 version.
  11
  12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
  13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 for more details.
  16
  17 You should have received a copy of the GNU General Public License
  18 along with this program; if not, write to the Free Software
  19 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  20
  21 */
  22
  23
  24 #include <stdio.h>
  25 #include <assert.h>
  26 #ifdef WIN32
  27 #include <io.h>
  28 #else
  29 #include <unistd.h>
  30 #endif
  31
  32 #include "index.h"
  33 #include "rank.h"
  34 #include <yaz/diagbib1.h>
  35 #include <rset.h>
  36
  37 #define ZSET_SORT_MAX_LEVEL 10
  38
  39 struct zebra_set_term_entry {
  40     int reg_type;
  41     char *db;
  42     char *index_name;
  43     char *term;
  44 };
  45
  46 struct zebra_set {
  47     char *name;
  48     RSET rset;
  49     NMEM nmem;
  50     NMEM rset_nmem; /* for creating the rsets in */
  51     zint hits;
  52     int num_bases;
  53     char **basenames;
  54     Z_RPNQuery *rpn;
  55     Z_SortKeySpecList *sortSpec;
  56     struct zset_sort_info *sort_info;
  57     struct zebra_set_term_entry *term_entries;
  58     int term_entries_max;
  59     struct zebra_set *next;
  60     int locked;
  61
  62     zint cache_position;  /* last position */
  63     RSFD cache_rfd;       /* rfd (NULL if not existing) */
  64     zint cache_psysno;    /* sysno for last position */
  65     zint approx_limit;    /* limit before we do approx */
  66 };
  67
  68 struct zset_sort_entry {
  69     zint sysno;
  70     int score;
  71 };
  72
  73 struct zset_sort_info {
  74     int max_entries;
  75     int num_entries;
  76     struct zset_sort_entry *all_entries;
  77     struct zset_sort_entry **entries;
  78 };
  79
  80 static int log_level_set=0;
  81 static int log_level_sort=0;
  82 static int log_level_searchhits=0;
  83 static int log_level_searchterms=0;
  84 static int log_level_resultsets=0;
  85
  86 static void loglevels()
  87 {
  88     if (log_level_set)
  89         return;
  90     log_level_sort = yaz_log_module_level("sorting");
  91     log_level_searchhits = yaz_log_module_level("searchhits");
  92     log_level_searchterms = yaz_log_module_level("searchterms");
  93     log_level_resultsets = yaz_log_module_level("resultsets");
  94     log_level_set = 1;
  95 }
  96
  97 ZEBRA_RES resultSetSearch(ZebraHandle zh, NMEM nmem, NMEM rset_nmem,
  98                           Z_RPNQuery *rpn, ZebraSet sset)
  99 {
 100     RSET rset = 0;
 101     oident *attrset;
 102     Z_SortKeySpecList *sort_sequence;
 103     int sort_status, i;
 104     ZEBRA_RES res = ZEBRA_OK;
 105
 106     zh->hits = 0;
 107
 108     sort_sequence = (Z_SortKeySpecList *)
 109         nmem_malloc(nmem, sizeof(*sort_sequence));
 110     sort_sequence->num_specs = 10; /* FIXME - Hard-coded number */
 111     sort_sequence->specs = (Z_SortKeySpec **)
 112         nmem_malloc(nmem, sort_sequence->num_specs *
 113                      sizeof(*sort_sequence->specs));
 114     for (i = 0; i<sort_sequence->num_specs; i++)
 115         sort_sequence->specs[i] = 0;
 116
 117     attrset = oid_getentbyoid (rpn->attributeSetId);
 118     res = rpn_search_top(zh, rpn->RPNStructure, attrset->value,
 119                          nmem, rset_nmem,
 120                          sort_sequence,
 121                          sset->num_bases, sset->basenames,
 122                          &rset);
 123     if (res != ZEBRA_OK)
 124     {
 125         sset->rset = 0;
 126         return res;
 127     }
 128     for (i = 0; sort_sequence->specs[i]; i++)
 129         ;
 130     sort_sequence->num_specs = i;
 131     rset->hits_limit = sset->approx_limit;
 132     if (!i)
 133     {
 134         res = resultSetRank (zh, sset, rset, rset_nmem);
 135     }
 136     else
 137     {
 138         res = resultSetSortSingle (zh, nmem, sset, rset,
 139                                    sort_sequence, &sort_status);
 140     }
 141     sset->rset = rset;
 142     return res;
 143 }
 144
 145
 146 ZEBRA_RES resultSetAddRPN (ZebraHandle zh, NMEM m, Z_RPNQuery *rpn,
 147                            int num_bases, char **basenames,
 148                            const char *setname)
 149 {
 150     ZebraSet zebraSet;
 151     int i;
 152     ZEBRA_RES res;
 153
 154     zh->hits = 0;
 155
 156     zebraSet = resultSetAdd(zh, setname, 1);
 157     if (!zebraSet)
 158         return ZEBRA_FAIL;
 159     zebraSet->locked = 1;
 160     zebraSet->rpn = 0;
 161     zebraSet->nmem = m;
 162     zebraSet->rset_nmem = nmem_create();
 163
 164     zebraSet->num_bases = num_bases;
 165     zebraSet->basenames =
 166         nmem_malloc (zebraSet->nmem, num_bases * sizeof(*zebraSet->basenames));
 167     for (i = 0; i<num_bases; i++)
 168         zebraSet->basenames[i] = nmem_strdup(zebraSet->nmem, basenames[i]);
 169
 170     res = resultSetSearch(zh, zebraSet->nmem, zebraSet->rset_nmem,
 171                           rpn, zebraSet);
 172     zh->hits = zebraSet->hits;
 173     if (zebraSet->rset)
 174         zebraSet->rpn = rpn;
 175     zebraSet->locked = 0;
 176     if (!zebraSet->rset)
 177         return ZEBRA_FAIL;
 178     return res;
 179 }
 180
 181 void resultSetAddTerm(ZebraHandle zh, ZebraSet s, int reg_type,
 182                       const char *db, const char *index_name,
 183                       const char *term)
 184 {
 185     assert(zh); /* compiler shut up */
 186     if (!s->nmem)
 187         s->nmem = nmem_create ();
 188     if (!s->term_entries)
 189     {
 190         int i;
 191         s->term_entries_max = 1000;
 192         s->term_entries =
 193             nmem_malloc (s->nmem, s->term_entries_max *
 194                          sizeof(*s->term_entries));
 195         for (i = 0; i < s->term_entries_max; i++)
 196             s->term_entries[i].term = 0;
 197     }
 198     if (s->hits < s->term_entries_max)
 199     {
 200         s->term_entries[s->hits].reg_type = reg_type;
 201         s->term_entries[s->hits].db = nmem_strdup (s->nmem, db);
 202         s->term_entries[s->hits].index_name = nmem_strdup(s->nmem, index_name);
 203         s->term_entries[s->hits].term = nmem_strdup(s->nmem, term);
 204     }
 205     (s->hits)++;
 206 }
 207
 208 ZebraSet resultSetAdd(ZebraHandle zh, const char *name, int ov)
 209 {
 210     ZebraSet s;
 211     int i;
 212
 213     for (s = zh->sets; s; s = s->next)
 214         if (!strcmp (s->name, name))
 215             break;
 216
 217     if (!log_level_set)
 218         loglevels();
 219     if (s)
 220     {
 221         yaz_log(log_level_resultsets, "updating result set %s", name);
 222         if (!ov || s->locked)
 223             return NULL;
 224         if (s->rset)
 225         {
 226             if (s->cache_rfd)
 227                 rset_close(s->cache_rfd);
 228             rset_delete (s->rset);
 229         }
 230         if (s->rset_nmem)
 231             nmem_destroy (s->rset_nmem);
 232         if (s->nmem)
 233             nmem_destroy (s->nmem);
 234     }
 235     else
 236     {
 237         const char *sort_max_str = zebra_get_resource(zh, "sortmax", "1000");
 238
 239         yaz_log(log_level_resultsets, "adding result set %s", name);
 240         s = (ZebraSet) xmalloc (sizeof(*s));
 241         s->next = zh->sets;
 242         zh->sets = s;
 243         s->name = (char *) xmalloc (strlen(name)+1);
 244         strcpy (s->name, name);
 245
 246         s->sort_info = (struct zset_sort_info *)
 247             xmalloc (sizeof(*s->sort_info));
 248         s->sort_info->max_entries = atoi(sort_max_str);
 249         if (s->sort_info->max_entries < 2)
 250             s->sort_info->max_entries = 2;
 251
 252         s->sort_info->entries = (struct zset_sort_entry **)
 253             xmalloc (sizeof(*s->sort_info->entries) *
 254                      s->sort_info->max_entries);
 255         s->sort_info->all_entries = (struct zset_sort_entry *)
 256             xmalloc (sizeof(*s->sort_info->all_entries) *
 257                      s->sort_info->max_entries);
 258         for (i = 0; i < s->sort_info->max_entries; i++)
 259             s->sort_info->entries[i] = s->sort_info->all_entries + i;
 260     }
 261     s->locked = 0;
 262     s->term_entries = 0;
 263     s->hits = 0;
 264     s->rset = 0;
 265     s->rset_nmem = 0;
 266     s->nmem = 0;
 267     s->rpn = 0;
 268     s->sortSpec = 0;
 269     s->cache_position = 0;
 270     s->cache_rfd = 0;
 271     s->approx_limit = zh->approx_limit;
 272     return s;
 273 }
 274
 275 ZebraSet resultSetGet(ZebraHandle zh, const char *name)
 276 {
 277     ZebraSet s;
 278
 279     for (s = zh->sets; s; s = s->next)
 280         if (!strcmp (s->name, name))
 281         {
 282             if (!s->term_entries && !s->rset && s->rpn)
 283             {
 284                 NMEM nmem = nmem_create ();
 285                 yaz_log(log_level_resultsets, "research %s", name);
 286                 if (!s->rset_nmem)
 287                     s->rset_nmem=nmem_create();
 288                 resultSetSearch(zh, nmem, s->rset_nmem, s->rpn, s);
 289                 if (s->rset && s->sortSpec)
 290                 {
 291                     int sort_status;
 292                     yaz_log(log_level_resultsets, "resort %s", name);
 293                     resultSetSortSingle (zh, nmem, s, s->rset, s->sortSpec,
 294                                          &sort_status);
 295                 }
 296                 nmem_destroy (nmem);
 297             }
 298             return s;
 299         }
 300     return NULL;
 301 }
 302
 303 void resultSetInvalidate (ZebraHandle zh)
 304 {
 305     ZebraSet s = zh->sets;
 306
 307     yaz_log(log_level_resultsets, "invalidating result sets");
 308     for (; s; s = s->next)
 309     {
 310         if (s->rset)
 311         {
 312             if (s->cache_rfd)
 313                 rset_close(s->cache_rfd);
 314             rset_delete (s->rset);
 315         }
 316         s->rset = 0;
 317         s->cache_rfd = 0;
 318         s->cache_position = 0;
 319         if (s->rset_nmem)
 320             nmem_destroy(s->rset_nmem);
 321         s->rset_nmem=0;
 322     }
 323 }
 324
 325 void resultSetDestroy(ZebraHandle zh, int num, char **names,int *statuses)
 326 {
 327     ZebraSet * ss = &zh->sets;
 328     int i;
 329
 330     if (statuses)
 331         for (i = 0; i<num; i++)
 332             statuses[i] = Z_DeleteStatus_resultSetDidNotExist;
 333     while (*ss)
 334     {
 335         int i = -1;
 336         ZebraSet s = *ss;
 337         if (num >= 0)
 338         {
 339             for (i = 0; i<num; i++)
 340                 if (!strcmp (s->name, names[i]))
 341                 {
 342                     if (statuses)
 343                         statuses[i] = Z_DeleteStatus_success;
 344                     i = -1;
 345                     break;
 346                 }
 347         }
 348         if (i < 0)
 349         {
 350             *ss = s->next;
 351
 352             xfree (s->sort_info->all_entries);
 353             xfree (s->sort_info->entries);
 354             xfree (s->sort_info);
 355
 356             if (s->nmem)
 357                 nmem_destroy (s->nmem);
 358             if (s->rset)
 359             {
 360                 if (s->cache_rfd)
 361                     rset_close(s->cache_rfd);
 362                 rset_delete (s->rset);
 363             }
 364             if (s->rset_nmem)
 365                 nmem_destroy(s->rset_nmem);
 366             xfree (s->name);
 367             xfree (s);
 368         }
 369         else
 370             ss = &s->next;
 371     }
 372 }
 373
 374 ZebraMetaRecord *zebra_meta_records_create_range(ZebraHandle zh,
 375                                                  const char *name,
 376                                                  zint start, int num)
 377 {
 378     zint pos_small[10];
 379     zint *pos = pos_small;
 380     ZebraMetaRecord *mr;
 381     int i;
 382
 383     if (num > 10000 || num <= 0)
 384         return 0;
 385
 386     if (num > 10)
 387         pos = xmalloc(sizeof(*pos) * num);
 388
 389     for (i = 0; i<num; i++)
 390         pos[i] = start+i;
 391
 392     mr = zebra_meta_records_create(zh, name, num, pos);
 393
 394     if (num > 10)
 395         xfree(pos);
 396     return mr;
 397 }
 398
 399 ZebraMetaRecord *zebra_meta_records_create(ZebraHandle zh, const char *name,
 400                                            int num, zint *positions)
 401 {
 402     ZebraSet sset;
 403     ZebraMetaRecord *sr = 0;
 404     RSET rset;
 405     int i;
 406     struct zset_sort_info *sort_info;
 407     size_t sysno_mem_index = 0;
 408
 409     if (zh->m_staticrank)
 410         sysno_mem_index = 1;
 411
 412     if (!log_level_set)
 413         loglevels();
 414     if (!(sset = resultSetGet (zh, name)))
 415         return NULL;
 416     if (!(rset = sset->rset))
 417     {
 418         if (!sset->term_entries)
 419             return 0;
 420         sr = (ZebraMetaRecord *) xmalloc (sizeof(*sr) * num);
 421         for (i = 0; i<num; i++)
 422         {
 423             sr[i].sysno = 0;
 424             sr[i].score = -1;
 425             sr[i].term = 0;
 426             sr[i].db = 0;
 427
 428             if (positions[i] <= sset->term_entries_max)
 429             {
 430                 sr[i].term = sset->term_entries[positions[i]-1].term;
 431                 sr[i].db = sset->term_entries[positions[i]-1].db;
 432             }
 433         }
 434     }
 435     else
 436     {
 437         sr = (ZebraMetaRecord *) xmalloc (sizeof(*sr) * num);
 438         for (i = 0; i<num; i++)
 439         {
 440             sr[i].sysno = 0;
 441             sr[i].score = -1;
 442             sr[i].term = 0;
 443             sr[i].db = 0;
 444         }
 445         sort_info = sset->sort_info;
 446         if (sort_info)
 447         {
 448             zint position;
 449
 450             for (i = 0; i<num; i++)
 451             {
 452                 position = positions[i];
 453                 if (position > 0 && position <= sort_info->num_entries)
 454                 {
 455                     yaz_log(log_level_sort, "got pos=" ZINT_FORMAT
 456                             " (sorted)", position);
 457                     sr[i].sysno = sort_info->entries[position-1]->sysno;
 458                     sr[i].score = sort_info->entries[position-1]->score;
 459                 }
 460             }
 461         }
 462         /* did we really get all entries using sort ? */
 463         for (i = 0; i<num; i++)
 464         {
 465             if (!sr[i].sysno)
 466                 break;
 467         }
 468         if (i < num) /* nope, get the rest, unsorted - sorry */
 469         {
 470             zint position = 0;
 471             int num_i = 0;
 472             zint psysno = 0;
 473             RSFD rfd;
 474             struct it_key key;
 475
 476             if (sort_info)
 477                 position = sort_info->num_entries;
 478             while (num_i < num && positions[num_i] <= position)
 479                 num_i++;
 480
 481             if (sset->cache_rfd &&
 482                 num_i < num && positions[num_i] > sset->cache_position)
 483             {
 484                 position = sset->cache_position;
 485                 rfd = sset->cache_rfd;
 486                 psysno = sset->cache_psysno;
 487             }
 488             else
 489             {
 490                 if (sset->cache_rfd)
 491                     rset_close(sset->cache_rfd);
 492                 rfd = rset_open (rset, RSETF_READ);
 493             }
 494             while (num_i < num && rset_read (rfd, &key, 0))
 495             {
 496                 zint this_sys = key.mem[sysno_mem_index];
 497                 if (this_sys != psysno)
 498                 {
 499                     psysno = this_sys;
 500                     if (sort_info)
 501                     {
 502                         /* determine we alreay have this in our set */
 503                         for (i = sort_info->num_entries; --i >= 0; )
 504                             if (psysno == sort_info->entries[i]->sysno)
 505                                 break;
 506                         if (i >= 0)
 507                             continue;
 508                     }
 509                     position++;
 510                     assert (num_i < num);
 511                     if (position == positions[num_i])
 512                     {
 513                         sr[num_i].sysno = psysno;
 514                         yaz_log(log_level_sort, "got pos=" ZINT_FORMAT " (unsorted)", position);
 515                         sr[num_i].score = -1;
 516                         num_i++;
 517                     }
 518                 }
 519             }
 520             sset->cache_position = position;
 521             sset->cache_psysno = psysno;
 522             sset->cache_rfd = rfd;
 523         }
 524     }
 525     return sr;
 526 }
 527
 528 void zebra_meta_records_destroy (ZebraHandle zh, ZebraMetaRecord *records,
 529                                  int num)
 530 {
 531     assert(zh); /* compiler shut up about unused arg */
 532     xfree (records);
 533 }
 534
 535 struct sortKeyInfo {
 536     int relation;
 537     int ord;
 538     int numerical;
 539     int index_type;
 540 };
 541
 542 void resultSetInsertSort(ZebraHandle zh, ZebraSet sset,
 543                          struct sortKeyInfo *criteria, int num_criteria,
 544                          zint sysno,
 545                          char *cmp_buf[], char *tmp_cmp_buf[])
 546 {
 547     struct zset_sort_entry *new_entry = NULL;
 548     struct zset_sort_info *sort_info = sset->sort_info;
 549     int i, j;
 550
 551     sortIdx_sysno (zh->reg->sortIdx, sysno);
 552     for (i = 0; i<num_criteria; i++)
 553     {
 554         char *this_entry_buf = tmp_cmp_buf[i];
 555         memset(this_entry_buf, '\0', SORT_IDX_ENTRYSIZE);
 556         if (criteria[i].ord != -1)
 557         {
 558             sortIdx_type(zh->reg->sortIdx, criteria[i].ord);
 559             sortIdx_read(zh->reg->sortIdx, this_entry_buf);
 560         }
 561     }
 562     i = sort_info->num_entries;
 563     while (--i >= 0)
 564     {
 565         int rel = 0;
 566         for (j = 0; j<num_criteria; j++)
 567         {
 568             char *this_entry_buf = tmp_cmp_buf[j];
 569             char *other_entry_buf =
 570                 cmp_buf[j] + i * SORT_IDX_ENTRYSIZE;
 571             if (criteria[j].numerical)
 572             {
 573                 char this_entry_org[1024];
 574                 char other_entry_org[1024];
 575                 double diff;
 576                 int index_type = criteria[j].index_type;
 577                 zebra_term_untrans(zh, index_type, this_entry_org,
 578                                    this_entry_buf);
 579                 zebra_term_untrans(zh, index_type, other_entry_org,
 580                                    other_entry_buf);
 581                 diff = atof(this_entry_org) - atof(other_entry_org);
 582
 583                 if (diff > 0.0)
 584                     rel = 1;
 585                 else if (diff < 0.0)
 586                     rel = -1;
 587                 else
 588                     rel = 0;
 589             }
 590             else
 591             {
 592                 rel = memcmp(this_entry_buf, other_entry_buf,
 593                              SORT_IDX_ENTRYSIZE);
 594             }
 595             if (rel)
 596                 break;
 597         }
 598         if (!rel)
 599             break;
 600         if (criteria[j].relation == 'A')
 601         {
 602             if (rel > 0)
 603                 break;
 604         }
 605         else if (criteria[j].relation == 'D')
 606         {
 607             if (rel < 0)
 608                 break;
 609         }
 610     }
 611     ++i;
 612     j = sort_info->max_entries;
 613     if (i == j)
 614         return;
 615
 616     if (sort_info->num_entries == j)
 617         --j;
 618     else
 619         j = (sort_info->num_entries)++;
 620     new_entry = sort_info->entries[j];
 621     while (j != i)
 622     {
 623         int k;
 624         for (k = 0; k<num_criteria; k++)
 625         {
 626             char *j_buf = cmp_buf[k] + j * SORT_IDX_ENTRYSIZE;
 627             char *j_1_buf = cmp_buf[k] + (j-1) * SORT_IDX_ENTRYSIZE;
 628             memcpy(j_buf, j_1_buf, SORT_IDX_ENTRYSIZE);
 629         }
 630         sort_info->entries[j] = sort_info->entries[j-1];
 631         --j;
 632     }
 633     sort_info->entries[i] = new_entry;
 634     assert (new_entry);
 635     for (i = 0; i<num_criteria; i++)
 636     {
 637         char *new_entry_buf = cmp_buf[i] + j * SORT_IDX_ENTRYSIZE;
 638         char *this_entry_buf = tmp_cmp_buf[i];
 639         memcpy(new_entry_buf, this_entry_buf, SORT_IDX_ENTRYSIZE);
 640     }
 641     new_entry->sysno = sysno;
 642     new_entry->score = -1;
 643 }
 644
 645 void resultSetInsertRank(ZebraHandle zh, struct zset_sort_info *sort_info,
 646                          zint sysno, int score, int relation)
 647 {
 648     struct zset_sort_entry *new_entry = NULL;
 649     int i, j;
 650     assert(zh); /* compiler shut up about unused arg */
 651
 652     i = sort_info->num_entries;
 653     while (--i >= 0)
 654     {
 655         int rel = 0;
 656
 657         rel = score - sort_info->entries[i]->score;
 658
 659         if (relation == 'D')
 660         {
 661             if (rel >= 0)
 662                 break;
 663         }
 664         else if (relation == 'A')
 665         {
 666             if (rel <= 0)
 667                 break;
 668         }
 669     }
 670     ++i;
 671     j = sort_info->max_entries;
 672     if (i == j)
 673         return;
 674
 675     if (sort_info->num_entries == j)
 676         --j;
 677     else
 678         j = (sort_info->num_entries)++;
 679
 680     new_entry = sort_info->entries[j];
 681     while (j != i)
 682     {
 683         sort_info->entries[j] = sort_info->entries[j-1];
 684         --j;
 685     }
 686     sort_info->entries[i] = new_entry;
 687     assert (new_entry);
 688     new_entry->sysno = sysno;
 689     new_entry->score = score;
 690 }
 691
 692 static Z_RPNQuery *copy_RPNQuery(Z_RPNQuery *src, NMEM nmem)
 693 {
 694     Z_RPNQuery *dst = 0;
 695     ODR encode = odr_createmem(ODR_ENCODE);
 696     ODR decode = odr_createmem(ODR_DECODE);
 697
 698     if (z_RPNQuery(encode, &src, 0, 0))
 699     {
 700         int len;
 701         char *buf = odr_getbuf(encode, &len, 0);
 702
 703         if (buf)
 704         {
 705             odr_setbuf(decode, buf, len, 0);
 706             z_RPNQuery(decode, &dst, 0, 0);
 707         }
 708     }
 709     nmem_transfer(nmem, decode->mem);
 710     odr_destroy(encode);
 711     odr_destroy(decode);
 712     return dst;
 713 }
 714
 715 static Z_SortKeySpecList *copy_SortKeySpecList(Z_SortKeySpecList *src, NMEM nmem)
 716 {
 717     Z_SortKeySpecList *dst = 0;
 718     ODR encode = odr_createmem(ODR_ENCODE);
 719     ODR decode = odr_createmem(ODR_DECODE);
 720
 721     if (z_SortKeySpecList(encode, &src, 0, 0))
 722     {
 723         int len;
 724         char *buf = odr_getbuf(encode, &len, 0);
 725
 726         if (buf)
 727         {
 728             odr_setbuf(decode, buf, len, 0);
 729             z_SortKeySpecList(decode, &dst, 0, 0);
 730         }
 731     }
 732     nmem_transfer(nmem, decode->mem);
 733     odr_destroy(encode);
 734     odr_destroy(decode);
 735     return dst;
 736 }
 737
 738 ZebraSet resultSetClone(ZebraHandle zh, const char *setname,
 739                         ZebraSet rset)
 740 {
 741     ZebraSet nset;
 742     int i;
 743
 744     nset = resultSetAdd(zh, setname, 1);
 745     if (!nset)
 746         return 0;
 747
 748     nset->nmem = nmem_create();
 749
 750     nset->num_bases = rset->num_bases;
 751     nset->basenames =
 752         nmem_malloc (nset->nmem, nset->num_bases * sizeof(*rset->basenames));
 753     for (i = 0; i<rset->num_bases; i++)
 754         nset->basenames[i] = nmem_strdup(nset->nmem, rset->basenames[i]);
 755
 756     if (rset->rset)
 757         nset->rset = rset_dup(rset->rset);
 758     if (rset->rpn)
 759         nset->rpn = copy_RPNQuery(rset->rpn, nset->nmem);
 760     return nset;
 761 }
 762
 763 ZEBRA_RES resultSetSort(ZebraHandle zh, NMEM nmem,
 764                         int num_input_setnames, const char **input_setnames,
 765                         const char *output_setname,
 766                         Z_SortKeySpecList *sort_sequence, int *sort_status)
 767 {
 768     ZebraSet sset;
 769     RSET rset;
 770
 771     if (num_input_setnames == 0)
 772     {
 773         zebra_setError(zh, YAZ_BIB1_NO_RESULT_SET_NAME_SUPPLIED_ON_SORT, 0);
 774         return ZEBRA_FAIL;
 775     }
 776     if (num_input_setnames > 1)
 777     {
 778         zebra_setError(zh, YAZ_BIB1_SORT_TOO_MANY_INPUT_RESULTS, 0);
 779         return ZEBRA_FAIL;
 780     }
 781     if (!log_level_set)
 782         loglevels();
 783     yaz_log(log_level_sort, "result set sort input=%s output=%s",
 784           *input_setnames, output_setname);
 785     sset = resultSetGet (zh, input_setnames[0]);
 786     if (!sset)
 787     {
 788         zebra_setError(zh, YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
 789                        input_setnames[0]);
 790         return ZEBRA_FAIL;
 791     }
 792     if (!(rset = sset->rset))
 793     {
 794         zebra_setError(zh, YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
 795                        input_setnames[0]);
 796         return ZEBRA_FAIL;
 797     }
 798     if (strcmp (output_setname, input_setnames[0]))
 799         sset = resultSetClone(zh, output_setname, sset);
 800     sset->sortSpec = copy_SortKeySpecList(sort_sequence, sset->nmem);
 801     return resultSetSortSingle (zh, nmem, sset, rset, sort_sequence,
 802                                 sort_status);
 803 }
 804
 805 ZEBRA_RES resultSetSortSingle(ZebraHandle zh, NMEM nmem,
 806                               ZebraSet sset, RSET rset,
 807                               Z_SortKeySpecList *sort_sequence,
 808                               int *sort_status)
 809 {
 810     int i;
 811     int n = 0;
 812     zint kno = 0;
 813     zint psysno = 0;
 814     struct it_key key;
 815     struct sortKeyInfo sort_criteria[ZSET_SORT_MAX_LEVEL];
 816     char *cmp_buf[ZSET_SORT_MAX_LEVEL];
 817     char *tmp_cmp_buf[ZSET_SORT_MAX_LEVEL];
 818     int num_criteria;
 819     RSFD rfd;
 820     TERMID termid;
 821     TERMID *terms;
 822     int numTerms = 0;
 823     size_t sysno_mem_index = 0;
 824
 825     if (zh->m_staticrank)
 826         sysno_mem_index = 1;
 827
 828     assert(nmem); /* compiler shut up about unused param */
 829     sset->sort_info->num_entries = 0;
 830
 831     rset_getterms(rset, 0, 0, &n);
 832     terms = (TERMID *) nmem_malloc(nmem, sizeof(*terms)*n);
 833     rset_getterms(rset, terms, n, &numTerms);
 834
 835     sset->hits = 0;
 836     num_criteria = sort_sequence->num_specs;
 837     if (num_criteria > ZSET_SORT_MAX_LEVEL)
 838         num_criteria = ZSET_SORT_MAX_LEVEL;
 839     for (i = 0; i < num_criteria; i++)
 840     {
 841         Z_SortKeySpec *sks = sort_sequence->specs[i];
 842         Z_SortKey *sk;
 843         ZEBRA_RES res;
 844
 845         sort_criteria[i].ord = -1;
 846         sort_criteria[i].numerical = 0;
 847
 848         if (sks->which == Z_SortKeySpec_missingValueData)
 849         {
 850             zebra_setError(zh, YAZ_BIB1_UNSUPP_MISSING_DATA_ACTION, 0);
 851             return ZEBRA_FAIL;
 852         }
 853         if (*sks->sortRelation == Z_SortKeySpec_ascending)
 854             sort_criteria[i].relation = 'A';
 855         else if (*sks->sortRelation == Z_SortKeySpec_descending)
 856             sort_criteria[i].relation = 'D';
 857         else
 858         {
 859             zebra_setError(zh, YAZ_BIB1_ILLEGAL_SORT_RELATION, 0);
 860             return ZEBRA_FAIL;
 861         }
 862         if (sks->sortElement->which == Z_SortElement_databaseSpecific)
 863         {
 864             zebra_setError(zh, YAZ_BIB1_DATABASE_SPECIFIC_SORT_UNSUPP, 0);
 865             return ZEBRA_FAIL;
 866         }
 867         else if (sks->sortElement->which != Z_SortElement_generic)
 868         {
 869             zebra_setError(zh, YAZ_BIB1_SORT_ILLEGAL_SORT, 0);
 870             return ZEBRA_FAIL;
 871         }
 872         sk = sks->sortElement->u.generic;
 873         switch (sk->which)
 874         {
 875         case Z_SortKey_sortField:
 876             yaz_log(log_level_sort, "key %d is of type sortField",
 877                     i+1);
 878             sort_criteria[i].numerical = 0;
 879             sort_criteria[i].ord =
 880                 zebraExplain_lookup_attr_str(zh->reg->zei,
 881                                              zinfo_index_category_sort,
 882                                              -1, sk->u.sortField);
 883             if (sks->which != Z_SortKeySpec_null
 884                 && sort_criteria[i].ord == -1)
 885             {
 886                 zebra_setError(zh,
 887                                YAZ_BIB1_CANNOT_SORT_ACCORDING_TO_SEQUENCE, 0);
 888                 return ZEBRA_FAIL;
 889             }
 890             break;
 891         case Z_SortKey_elementSpec:
 892             yaz_log(log_level_sort, "key %d is of type elementSpec",
 893                     i+1);
 894             zebra_setError(zh, YAZ_BIB1_CANNOT_SORT_ACCORDING_TO_SEQUENCE, 0);
 895             return ZEBRA_FAIL;
 896         case Z_SortKey_sortAttributes:
 897             yaz_log(log_level_sort, "key %d is of type sortAttributes", i+1);
 898             res = zebra_sort_get_ord(zh, sk->u.sortAttributes,
 899
 900                                      &sort_criteria[i].ord,
 901                                      &sort_criteria[i].numerical);
 902             if (sks->which != Z_SortKeySpec_null && res != ZEBRA_OK)
 903                 return ZEBRA_FAIL;
 904             break;
 905         }
 906         if (zebraExplain_lookup_ord(zh->reg->zei, sort_criteria[i].ord,
 907                                     &sort_criteria[i].index_type,
 908                                     0, 0))
 909         {
 910             zebra_setError(zh, YAZ_BIB1_CANNOT_SORT_ACCORDING_TO_SEQUENCE, 0);
 911             return ZEBRA_FAIL;
 912         }
 913     }
 914     /* allocate space for each cmpare buf + one extra for tmp comparison */
 915     for (i = 0; i<num_criteria; i++)
 916     {
 917         cmp_buf[i] = xmalloc(sset->sort_info->max_entries
 918                              * SORT_IDX_ENTRYSIZE);
 919         tmp_cmp_buf[i] = xmalloc(SORT_IDX_ENTRYSIZE);
 920     }
 921     rfd = rset_open (rset, RSETF_READ);
 922     while (rset_read (rfd, &key, &termid))
 923     {
 924         zint this_sys = key.mem[sysno_mem_index];
 925         if (log_level_searchhits)
 926             key_logdump_txt(log_level_searchhits, &key, termid->name);
 927         kno++;
 928         if (this_sys != psysno)
 929         {
 930             (sset->hits)++;
 931             psysno = this_sys;
 932             resultSetInsertSort(zh, sset,
 933                                 sort_criteria, num_criteria, psysno, cmp_buf,
 934                                 tmp_cmp_buf);
 935         }
 936     }
 937     rset_close (rfd);
 938
 939     for (i = 0; i<num_criteria; i++)
 940     {
 941         xfree(cmp_buf[i]);
 942         xfree(tmp_cmp_buf[i]);
 943     }
 944
 945     yaz_log(log_level_sort, ZINT_FORMAT " keys, " ZINT_FORMAT " sysnos, sort",
 946             kno, sset->hits);
 947     for (i = 0; i < numTerms; i++)
 948         yaz_log(log_level_sort, "term=\"%s\" type=%s count=" ZINT_FORMAT,
 949                  terms[i]->name, terms[i]->flags, terms[i]->rset->hits_count);
 950     *sort_status = Z_SortResponse_success;
 951     return ZEBRA_OK;
 952 }
 953
 954 RSET resultSetRef(ZebraHandle zh, const char *resultSetId)
 955 {
 956     ZebraSet s;
 957
 958     if ((s = resultSetGet (zh, resultSetId)))
 959         return s->rset;
 960     return NULL;
 961 }
 962
 963 ZEBRA_RES resultSetRank(ZebraHandle zh, ZebraSet zebraSet,
 964                         RSET rset, NMEM nmem)
 965 {
 966     struct it_key key;
 967     TERMID termid;
 968     TERMID *terms;
 969     zint kno = 0;
 970     int numTerms = 0;
 971     int n = 0;
 972     int i;
 973     ZebraRankClass rank_class;
 974     struct zset_sort_info *sort_info;
 975     const char *rank_handler_name = res_get_def(zh->res, "rank", "rank-1");
 976     size_t sysno_mem_index = 0;
 977
 978     if (zh->m_staticrank)
 979         sysno_mem_index = 1;
 980
 981     if (!log_level_set)
 982         loglevels();
 983     sort_info = zebraSet->sort_info;
 984     sort_info->num_entries = 0;
 985     zebraSet->hits = 0;
 986     rset_getterms(rset, 0, 0, &n);
 987     terms = (TERMID *) nmem_malloc(nmem, sizeof(*terms)*n);
 988     rset_getterms(rset, terms, n, &numTerms);
 989
 990
 991     rank_class = zebraRankLookup(zh, rank_handler_name);
 992     if (!rank_class)
 993     {
 994         yaz_log(YLOG_WARN, "No such rank handler: %s", rank_handler_name);
 995         zebra_setError(zh, YAZ_BIB1_UNSUPP_SEARCH, "Cannot find rank handler");
 996         return ZEBRA_FAIL;
 997     }
 998     else
 999     {
1000         RSFD rfd = rset_open(rset, RSETF_READ);
1001         struct rank_control *rc = rank_class->control;
1002         int score;
1003         zint count = 0;
1004
1005         void *handle =
1006             (*rc->begin) (zh->reg, rank_class->class_handle, rset, nmem,
1007                           terms, numTerms);
1008         zint psysno = 0;  /* previous doc id / sys no */
1009         zint pstaticrank = 0; /* previous static rank */
1010         int stop_flag = 0;
1011         while (rset_read(rfd, &key, &termid))
1012         {
1013             zint this_sys = key.mem[sysno_mem_index];
1014
1015             zint seqno = key.mem[key.len-1];
1016             kno++;
1017             if (log_level_searchhits)
1018                 key_logdump_txt(log_level_searchhits, &key, termid->name);
1019             if (this_sys != psysno)
1020             {   /* new record .. */
1021                 if (rfd->counted_items > rset->hits_limit)
1022                     break;
1023                 if (psysno)
1024                 {   /* only if we did have a previous record */
1025                     score = (*rc->calc) (handle, psysno, pstaticrank,
1026                                          &stop_flag);
1027                     /* insert the hit. A=Ascending */
1028                     resultSetInsertRank (zh, sort_info, psysno, score, 'A');
1029                     count++;
1030                     if (stop_flag)
1031                         break;
1032                 }
1033                 psysno = this_sys;
1034                 if (zh->m_staticrank)
1035                     pstaticrank = key.mem[0];
1036             }
1037             (*rc->add) (handle, CAST_ZINT_TO_INT(seqno), termid);
1038         }
1039         /* no more items */
1040         if (psysno)
1041         {   /* we had - at least - one record */
1042             score = (*rc->calc)(handle, psysno, pstaticrank, &stop_flag);
1043             /* insert the hit. A=Ascending */
1044             resultSetInsertRank(zh, sort_info, psysno, score, 'A');
1045             count++;
1046         }
1047         (*rc->end) (zh->reg, handle);
1048         rset_close (rfd);
1049     }
1050     zebraSet->hits = rset->hits_count;
1051
1052     yaz_log(log_level_searchterms, ZINT_FORMAT " keys, "
1053             ZINT_FORMAT " sysnos, rank",  kno, zebraSet->hits);
1054     for (i = 0; i < numTerms; i++)
1055     {
1056         yaz_log(log_level_searchterms, "term=\"%s\" type=%s count="
1057                 ZINT_FORMAT,
1058                 terms[i]->name, terms[i]->flags, terms[i]->rset->hits_count);
1059     }
1060     return ZEBRA_OK;
1061 }
1062
1063 ZebraRankClass zebraRankLookup(ZebraHandle zh, const char *name)
1064 {
1065     ZebraRankClass p = zh->reg->rank_classes;
1066     while (p && strcmp (p->control->name, name))
1067         p = p->next;
1068     if (p && !p->init_flag)
1069     {
1070         if (p->control->create)
1071             p->class_handle = (*p->control->create)(zh);
1072         p->init_flag = 1;
1073     }
1074     return p;
1075 }
1076
1077 void zebraRankInstall(struct zebra_register *reg, struct rank_control *ctrl)
1078 {
1079     ZebraRankClass p = (ZebraRankClass) xmalloc (sizeof(*p));
1080     p->control = (struct rank_control *) xmalloc (sizeof(*p->control));
1081     memcpy (p->control, ctrl, sizeof(*p->control));
1082     p->control->name = xstrdup (ctrl->name);
1083     p->init_flag = 0;
1084     p->next = reg->rank_classes;
1085     reg->rank_classes = p;
1086 }
1087
1088 void zebraRankDestroy(struct zebra_register *reg)
1089 {
1090     ZebraRankClass p = reg->rank_classes;
1091     while (p)
1092     {
1093         ZebraRankClass p_next = p->next;
1094         if (p->init_flag && p->control->destroy)
1095             (*p->control->destroy)(reg, p->class_handle);
1096         xfree(p->control->name);
1097         xfree(p->control);
1098         xfree(p);
1099         p = p_next;
1100     }
1101     reg->rank_classes = NULL;
1102 }
1103
1104 static int trav_rset_for_termids(RSET rset, TERMID *termid_array,
1105                                  zint *hits_array, int *approx_array)
1106 {
1107     int no = 0;
1108     int i;
1109     for (i = 0; i<rset->no_children; i++)
1110         no += trav_rset_for_termids(rset->children[i],
1111                                     (termid_array ? termid_array + no : 0),
1112                                     (hits_array ? hits_array + no : 0),
1113                                     (approx_array ? approx_array + no : 0));
1114     if (rset->term)
1115     {
1116         if (termid_array)
1117             termid_array[no] = rset->term;
1118         if (hits_array)
1119             hits_array[no] = rset->hits_count;
1120         if (approx_array)
1121             approx_array[no] = rset->hits_approx;
1122 #if 0
1123         yaz_log(YLOG_LOG, "rset=%p term=%s limit=" ZINT_FORMAT
1124                 " count=" ZINT_FORMAT,
1125                 rset, rset->term->name, rset->hits_limit, rset->hits_count);
1126 #endif
1127         no++;
1128     }
1129     return no;
1130 }
1131
1132 ZEBRA_RES zebra_result_set_term_no(ZebraHandle zh, const char *setname,
1133                                    int *num_terms)
1134 {
1135     ZebraSet sset = resultSetGet(zh, setname);
1136     *num_terms = 0;
1137     if (sset)
1138     {
1139         *num_terms = trav_rset_for_termids(sset->rset, 0, 0, 0);
1140         return ZEBRA_OK;
1141     }
1142     return ZEBRA_FAIL;
1143 }
1144
1145 ZEBRA_RES zebra_result_set_term_info(ZebraHandle zh, const char *setname,
1146                                      int no, zint *count, int *approx,
1147                                      char *termbuf, size_t *termlen,
1148                                      const char **term_ref_id)
1149 {
1150     ZebraSet sset = resultSetGet(zh, setname);
1151     if (sset)
1152     {
1153         int num_terms = trav_rset_for_termids(sset->rset, 0, 0, 0);
1154         if (no >= 0 && no < num_terms)
1155         {
1156             TERMID *term_array = xmalloc(num_terms * sizeof(*term_array));
1157             zint *hits_array = xmalloc(num_terms * sizeof(*hits_array));
1158             int *approx_array = xmalloc(num_terms * sizeof(*approx_array));
1159
1160             trav_rset_for_termids(sset->rset, term_array,
1161                                   hits_array, approx_array);
1162
1163             if (count)
1164                 *count = hits_array[no];
1165             if (approx)
1166                 *approx = approx_array[no];
1167             if (termbuf)
1168             {
1169                 char *inbuf = term_array[no]->name;
1170                 size_t inleft = strlen(inbuf);
1171                 size_t outleft = *termlen - 1;
1172
1173                 if (zh->iconv_from_utf8 != 0)
1174                 {
1175                     char *outbuf = termbuf;
1176                     size_t ret;
1177
1178                     ret = yaz_iconv(zh->iconv_from_utf8, &inbuf, &inleft,
1179                                     &outbuf, &outleft);
1180                     if (ret == (size_t)(-1))
1181                         *termlen = 0;
1182                     else
1183                         *termlen = outbuf - termbuf;
1184                 }
1185                 else
1186                 {
1187                     if (inleft > outleft)
1188                         inleft = outleft;
1189                     *termlen = inleft;
1190                     memcpy(termbuf, inbuf, *termlen);
1191                 }
1192                 termbuf[*termlen] = '\0';
1193             }
1194             if (term_ref_id)
1195                 *term_ref_id = term_array[no]->ref_id;
1196
1197             xfree(term_array);
1198             xfree(hits_array);
1199             xfree(approx_array);
1200             return ZEBRA_OK;
1201         }
1202     }
1203     return ZEBRA_FAIL;
1204 }
1205
1206 ZEBRA_RES zebra_snippets_hit_vector(ZebraHandle zh, const char *setname,
1207                                     zint sysno, zebra_snippets *snippets)
1208 {
1209     ZebraSet sset = resultSetGet(zh, setname);
1210     yaz_log(YLOG_DEBUG, "zebra_get_hit_vector setname=%s zysno=" ZINT_FORMAT,
1211             setname, sysno);
1212     if (!sset)
1213         return ZEBRA_FAIL;
1214     else
1215     {
1216         struct rset_key_control *kc = zebra_key_control_create(zh);
1217         NMEM nmem = nmem_create();
1218         struct it_key key;
1219         RSET rsets[2], rset_comb;
1220         RSET rset_temp = rset_create_temp(nmem, kc, kc->scope,
1221                                           res_get (zh->res, "setTmpDir"),0 );
1222
1223         TERMID termid;
1224         RSFD rsfd = rset_open(rset_temp, RSETF_WRITE);
1225
1226         key.mem[0] = sysno;
1227         key.mem[1] = 0;
1228         key.mem[2] = 0;
1229         key.mem[3] = 0;
1230         key.len = 2;
1231         rset_write (rsfd, &key);
1232         rset_close (rsfd);
1233
1234         rsets[0] = rset_temp;
1235         rsets[1] = rset_dup(sset->rset);
1236
1237         rset_comb = rset_create_and(nmem, kc, kc->scope, 2, rsets);
1238
1239         rsfd = rset_open(rset_comb, RSETF_READ);
1240
1241         while (rset_read(rsfd, &key, &termid))
1242         {
1243             if (termid)
1244             {
1245                 struct ord_list *ol;
1246                 for (ol = termid->ol; ol; ol = ol->next)
1247                 {
1248                     zebra_snippets_append(snippets, key.mem[key.len-1],
1249                                           ol->ord, termid->name);
1250                 }
1251             }
1252         }
1253         rset_close(rsfd);
1254
1255         rset_delete(rset_comb);
1256         nmem_destroy(nmem);
1257         kc->dec(kc);
1258     }
1259     return ZEBRA_OK;
1260 }
1261
1262 /*
1263  * Local variables:
1264  * c-basic-offset: 4
1265  * indent-tabs-mode: nil
1266  * End:
1267  * vim: shiftwidth=4 tabstop=8 expandtab
1268  */
1269