Changed the index type to a string everywhere.
[idzebra-moved-to-github.git] / index / zsets.c
1 /* $Id: zsets.c,v 1.124 2007-10-31 16:56:14 adam Exp $
2    Copyright (C) 1995-2007
3    Index Data ApS
4
5 This file is part of the Zebra server.
6
7 Zebra is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 2, or (at your option) any later
10 version.
11
12 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
15 for more details.
16
17 You should have received a copy of the GNU General Public License
18 along with this program; if not, write to the Free Software
19 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
20
21 */
22
23
24 #include <stdio.h>
25 #include <assert.h>
26 #ifdef WIN32
27 #include <io.h>
28 #else
29 #include <unistd.h>
30 #endif
31
32 #include "index.h"
33 #include "rank.h"
34 #include <yaz/diagbib1.h>
35 #include <rset.h>
36
37 #define ZSET_SORT_MAX_LEVEL 10
38
39 struct zebra_set_term_entry {
40     int reg_type;
41     char *db;
42     char *index_name;
43     char *term;
44 };
45
46 struct zebra_set {
47     char *name;
48     RSET rset;
49     NMEM nmem;
50     NMEM rset_nmem; /* for creating the rsets in */
51     zint hits;
52     int num_bases;
53     char **basenames;
54     Z_RPNQuery *rpn;
55     Z_SortKeySpecList *sortSpec;
56     struct zset_sort_info *sort_info;
57     struct zebra_set_term_entry *term_entries;
58     int term_entries_max;
59     struct zebra_set *next;
60     int locked;
61     int estimated_hit_count;
62
63     zint cache_position;  /* last position */
64     RSFD cache_rfd;       /* rfd (NULL if not existing) */
65     zint cache_psysno;    /* sysno for last position */
66     zint approx_limit;    /* limit before we do approx */
67 };
68
69 struct zset_sort_entry {
70     zint sysno;
71     int score;
72 };
73
74 struct zset_sort_info {
75     int max_entries;
76     int num_entries;
77     struct zset_sort_entry *all_entries;
78     struct zset_sort_entry **entries;
79 };
80
81 static int log_level_set=0;
82 static int log_level_sort=0;
83 static int log_level_searchhits=0;
84 static int log_level_searchterms=0;
85 static int log_level_resultsets=0;
86
87 static void loglevels(void)
88 {
89     if (log_level_set)
90         return;
91     log_level_sort = yaz_log_module_level("sorting");
92     log_level_searchhits = yaz_log_module_level("searchhits");
93     log_level_searchterms = yaz_log_module_level("searchterms");
94     log_level_resultsets = yaz_log_module_level("resultsets");
95     log_level_set = 1;
96 }
97
98
99 static ZEBRA_RES resultSetSearch(ZebraHandle zh, NMEM nmem, NMEM rset_nmem,
100                                  Z_RPNQuery *rpn, ZebraSet sset)
101 {
102     RSET rset = 0;
103     Z_SortKeySpecList *sort_sequence;
104     int sort_status, i;
105     ZEBRA_RES res = ZEBRA_OK;
106
107     sort_sequence = (Z_SortKeySpecList *)
108         nmem_malloc(nmem, sizeof(*sort_sequence));
109     sort_sequence->num_specs = 10; /* FIXME - Hard-coded number */
110     sort_sequence->specs = (Z_SortKeySpec **)
111         nmem_malloc(nmem, sort_sequence->num_specs *
112                      sizeof(*sort_sequence->specs));
113     for (i = 0; i<sort_sequence->num_specs; i++)
114         sort_sequence->specs[i] = 0;
115     
116     rpn_get_top_approx_limit(zh, rpn->RPNStructure, &sset->approx_limit);
117
118     res = rpn_search_top(zh, rpn->RPNStructure, rpn->attributeSetId,
119                          nmem, rset_nmem,
120                          sort_sequence,
121                          sset->num_bases, sset->basenames,
122                          &rset);
123     if (res != ZEBRA_OK)
124     {
125         sset->rset = 0;
126         return res;
127     }
128     for (i = 0; sort_sequence->specs[i]; i++)
129         ;
130     sort_sequence->num_specs = i;
131     rset->hits_limit = sset->approx_limit;
132     if (!i)
133     {
134         res = resultSetRank (zh, sset, rset, rset_nmem);
135     }
136     else
137     {
138         res = resultSetSortSingle (zh, nmem, sset, rset,
139                                    sort_sequence, &sort_status);
140     }
141     sset->rset = rset;
142     return res;
143 }
144
145
146 ZEBRA_RES resultSetAddRPN(ZebraHandle zh, NMEM m, Z_RPNQuery *rpn,
147                           int num_bases, char **basenames,
148                           const char *setname,
149                           zint *hits, int *estimated_hit_count)
150 {
151     ZebraSet zebraSet;
152     int i;
153     ZEBRA_RES res;
154
155     *hits = 0;
156     *estimated_hit_count = 0;
157
158     zebraSet = resultSetAdd(zh, setname, 1);
159     if (!zebraSet)
160         return ZEBRA_FAIL;
161     zebraSet->locked = 1;
162     zebraSet->rpn = 0;
163     zebraSet->nmem = m;
164     zebraSet->rset_nmem = nmem_create(); 
165
166     zebraSet->num_bases = num_bases;
167     zebraSet->basenames = 
168         nmem_malloc (zebraSet->nmem, num_bases * sizeof(*zebraSet->basenames));
169     for (i = 0; i<num_bases; i++)
170         zebraSet->basenames[i] = nmem_strdup(zebraSet->nmem, basenames[i]);
171
172     res = resultSetSearch(zh, zebraSet->nmem, zebraSet->rset_nmem,
173                           rpn, zebraSet);
174     *hits = zebraSet->hits;
175     if (zebraSet->estimated_hit_count)
176         *estimated_hit_count = 1;
177
178     if (zebraSet->rset)
179         zebraSet->rpn = rpn;
180     zebraSet->locked = 0;
181     if (!zebraSet->rset)
182         return ZEBRA_FAIL;
183     return res;
184 }
185
186 void resultSetAddTerm(ZebraHandle zh, ZebraSet s, int reg_type,
187                       const char *db, const char *index_name, 
188                       const char *term)
189 {
190     assert(zh); /* compiler shut up */
191     if (!s->nmem)
192         s->nmem = nmem_create ();
193     if (!s->term_entries)
194     {
195         int i;
196         s->term_entries_max = 1000;
197         s->term_entries =
198             nmem_malloc (s->nmem, s->term_entries_max * 
199                          sizeof(*s->term_entries));
200         for (i = 0; i < s->term_entries_max; i++)
201             s->term_entries[i].term = 0;
202     }
203     if (s->hits < s->term_entries_max)
204     {
205         s->term_entries[s->hits].reg_type = reg_type;
206         s->term_entries[s->hits].db = nmem_strdup (s->nmem, db);
207         s->term_entries[s->hits].index_name = nmem_strdup(s->nmem, index_name);
208         s->term_entries[s->hits].term = nmem_strdup(s->nmem, term);
209     }
210     (s->hits)++;
211 }
212
213 ZebraSet resultSetAdd(ZebraHandle zh, const char *name, int ov)
214 {
215     ZebraSet s;
216     int i;
217
218     for (s = zh->sets; s; s = s->next)
219         if (!strcmp (s->name, name))
220             break;
221     
222     if (!log_level_set)
223         loglevels();
224     if (s)
225     {
226         yaz_log(log_level_resultsets, "updating result set %s", name);
227         if (!ov || s->locked)
228             return NULL;
229         if (s->rset)
230         {
231             if (s->cache_rfd)
232                 rset_close(s->cache_rfd);
233             rset_delete (s->rset);
234         }
235         if (s->rset_nmem)
236             nmem_destroy (s->rset_nmem);
237         if (s->nmem)
238             nmem_destroy (s->nmem);
239     }
240     else
241     {
242         const char *sort_max_str = zebra_get_resource(zh, "sortmax", "1000");
243
244         yaz_log(log_level_resultsets, "adding result set %s", name);
245         s = (ZebraSet) xmalloc (sizeof(*s));
246         s->next = zh->sets;
247         zh->sets = s;
248         s->name = (char *) xmalloc (strlen(name)+1);
249         strcpy (s->name, name);
250
251         s->sort_info = (struct zset_sort_info *)
252             xmalloc (sizeof(*s->sort_info));
253         s->sort_info->max_entries = atoi(sort_max_str);
254         if (s->sort_info->max_entries < 2)
255             s->sort_info->max_entries = 2;
256
257         s->sort_info->entries = (struct zset_sort_entry **)
258             xmalloc (sizeof(*s->sort_info->entries) *
259                      s->sort_info->max_entries);
260         s->sort_info->all_entries = (struct zset_sort_entry *)
261             xmalloc (sizeof(*s->sort_info->all_entries) *
262                      s->sort_info->max_entries);
263         for (i = 0; i < s->sort_info->max_entries; i++)
264             s->sort_info->entries[i] = s->sort_info->all_entries + i;
265     }
266     s->locked = 0;
267     s->term_entries = 0;
268     s->hits = 0;
269     s->rset = 0;
270     s->rset_nmem = 0;
271     s->nmem = 0;
272     s->rpn = 0;
273     s->sortSpec = 0;
274     s->cache_position = 0;
275     s->cache_rfd = 0;
276     s->approx_limit = zh->approx_limit;
277     s->estimated_hit_count = 0;
278     return s;
279 }
280
281 ZebraSet resultSetGet(ZebraHandle zh, const char *name)
282 {
283     ZebraSet s;
284
285     for (s = zh->sets; s; s = s->next)
286         if (!strcmp (s->name, name))
287         {
288             if (!s->term_entries && !s->rset && s->rpn)
289             {
290                 NMEM nmem = nmem_create ();
291                 yaz_log(log_level_resultsets, "research %s", name);
292                 if (!s->rset_nmem)
293                     s->rset_nmem=nmem_create();
294                 resultSetSearch(zh, nmem, s->rset_nmem, s->rpn, s);
295                 if (s->rset && s->sortSpec)
296                 {
297                     int sort_status;
298                     yaz_log(log_level_resultsets, "resort %s", name);
299                     resultSetSortSingle (zh, nmem, s, s->rset, s->sortSpec,
300                                          &sort_status);
301                 }
302                 nmem_destroy (nmem);
303             }
304             return s;
305         }
306     return NULL;
307 }
308
309 void resultSetInvalidate (ZebraHandle zh)
310 {
311     ZebraSet s = zh->sets;
312     
313     yaz_log(log_level_resultsets, "invalidating result sets");
314     for (; s; s = s->next)
315     {
316         if (s->rset)
317         {
318             if (s->cache_rfd)
319                 rset_close(s->cache_rfd);
320             rset_delete (s->rset);
321         }
322         s->rset = 0;
323         s->cache_rfd = 0;
324         s->cache_position = 0;
325         if (s->rset_nmem)
326             nmem_destroy(s->rset_nmem);
327         s->rset_nmem=0;
328     }
329 }
330
331 void resultSetDestroy(ZebraHandle zh, int num, char **names,int *statuses)
332 {
333     ZebraSet * ss = &zh->sets;
334     int i;
335     
336     if (statuses)
337         for (i = 0; i<num; i++)
338             statuses[i] = Z_DeleteStatus_resultSetDidNotExist;
339     while (*ss)
340     {
341         int i = -1;
342         ZebraSet s = *ss;
343         if (num >= 0)
344         {
345             for (i = 0; i<num; i++)
346                 if (!strcmp (s->name, names[i]))
347                 {
348                     if (statuses)
349                         statuses[i] = Z_DeleteStatus_success;
350                     i = -1;
351                     break;
352                 }
353         }
354         if (i < 0)
355         {
356             *ss = s->next;
357             
358             xfree (s->sort_info->all_entries);
359             xfree (s->sort_info->entries);
360             xfree (s->sort_info);
361             
362             if (s->nmem)
363                 nmem_destroy (s->nmem);
364             if (s->rset)
365             {
366                 if (s->cache_rfd)
367                     rset_close(s->cache_rfd);
368                 rset_delete (s->rset);
369             }
370             if (s->rset_nmem)
371                 nmem_destroy(s->rset_nmem);
372             xfree (s->name);
373             xfree (s);
374         }
375         else
376             ss = &s->next;
377     }
378 }
379
380 ZebraMetaRecord *zebra_meta_records_create_range(ZebraHandle zh,
381                                                  const char *name, 
382                                                  zint start, int num)
383 {
384     zint pos_small[10];
385     zint *pos = pos_small;
386     ZebraMetaRecord *mr;
387     int i;
388
389     if (num > 10000 || num <= 0)
390         return 0;
391
392     if (num > 10)
393         pos = xmalloc(sizeof(*pos) * num);
394     
395     for (i = 0; i<num; i++)
396         pos[i] = start+i;
397
398     mr = zebra_meta_records_create(zh, name, num, pos);
399     
400     if (num > 10)
401         xfree(pos);
402     return mr;
403 }
404
405 ZebraMetaRecord *zebra_meta_records_create(ZebraHandle zh, const char *name, 
406                                            int num, zint *positions)
407 {
408     ZebraSet sset;
409     ZebraMetaRecord *sr = 0;
410     RSET rset;
411     int i;
412     struct zset_sort_info *sort_info;
413     size_t sysno_mem_index = 0;
414
415     if (zh->m_staticrank)
416         sysno_mem_index = 1;
417
418     if (!log_level_set)
419         loglevels();
420     if (!(sset = resultSetGet (zh, name)))
421         return NULL;
422     if (!(rset = sset->rset))
423     {
424         if (!sset->term_entries)
425             return 0;
426         sr = (ZebraMetaRecord *) xmalloc (sizeof(*sr) * num);
427         for (i = 0; i<num; i++)
428         {
429             sr[i].sysno = 0;
430             sr[i].score = -1;
431             sr[i].term = 0;
432             sr[i].db = 0;
433
434             if (positions[i] <= sset->term_entries_max)
435             {
436                 sr[i].term = sset->term_entries[positions[i]-1].term;
437                 sr[i].db = sset->term_entries[positions[i]-1].db;
438             }
439         }
440     }
441     else
442     {
443         sr = (ZebraMetaRecord *) xmalloc (sizeof(*sr) * num);
444         for (i = 0; i<num; i++)
445         {
446             sr[i].sysno = 0;
447             sr[i].score = -1;
448             sr[i].term = 0;
449             sr[i].db = 0;
450         }
451         sort_info = sset->sort_info;
452         if (sort_info)
453         {
454             zint position;
455             
456             for (i = 0; i<num; i++)
457             {
458                 position = positions[i];
459                 if (position > 0 && position <= sort_info->num_entries)
460                 {
461                     yaz_log(log_level_sort, "got pos=" ZINT_FORMAT
462                             " (sorted)", position);
463                     sr[i].sysno = sort_info->entries[position-1]->sysno;
464                     sr[i].score = sort_info->entries[position-1]->score;
465                 }
466             }
467         }
468         /* did we really get all entries using sort ? */
469         for (i = 0; i<num; i++)
470         {
471             if (!sr[i].sysno)
472                 break;
473         }
474         if (i < num) /* nope, get the rest, unsorted - sorry */
475         {
476             zint position = 0;
477             int num_i = 0;
478             zint psysno = 0;
479             RSFD rfd;
480             struct it_key key;
481             
482             if (sort_info)
483                 position = sort_info->num_entries;
484             while (num_i < num && positions[num_i] <= position)
485                 num_i++;
486             
487             if (sset->cache_rfd &&
488                 num_i < num && positions[num_i] > sset->cache_position)
489             {
490                 position = sset->cache_position;
491                 rfd = sset->cache_rfd;
492                 psysno = sset->cache_psysno;
493             }
494             else
495             {
496                 if (sset->cache_rfd)
497                     rset_close(sset->cache_rfd);
498                 rfd = rset_open (rset, RSETF_READ);
499             }
500             while (num_i < num && rset_read (rfd, &key, 0))
501             {
502                 zint this_sys = key.mem[sysno_mem_index];
503                 if (this_sys != psysno)
504                 {
505                     psysno = this_sys;
506                     if (sort_info)
507                     {
508                         /* determine we alreay have this in our set */
509                         for (i = sort_info->num_entries; --i >= 0; )
510                             if (psysno == sort_info->entries[i]->sysno)
511                                 break;
512                         if (i >= 0)
513                             continue;
514                     }
515                     position++;
516                     assert (num_i < num);
517                     if (position == positions[num_i])
518                     {
519                         sr[num_i].sysno = psysno;
520                         yaz_log(log_level_sort, "got pos=" ZINT_FORMAT " (unsorted)", position);
521                         sr[num_i].score = -1;
522                         num_i++;
523                     }
524                 }
525             }
526             sset->cache_position = position;
527             sset->cache_psysno = psysno;
528             sset->cache_rfd = rfd;
529         }
530     }
531     return sr;
532 }
533
534 void zebra_meta_records_destroy (ZebraHandle zh, ZebraMetaRecord *records,
535                                  int num)
536 {
537     assert(zh); /* compiler shut up about unused arg */
538     xfree (records);
539 }
540
541 struct sortKeyInfo {
542     int relation;
543     int ord;
544     int numerical;
545     const char *index_type;
546 };
547
548 void resultSetInsertSort(ZebraHandle zh, ZebraSet sset,
549                          struct sortKeyInfo *criteria, int num_criteria,
550                          zint sysno,
551                          char *cmp_buf[], char *tmp_cmp_buf[])
552 {
553     struct zset_sort_entry *new_entry = NULL;
554     struct zset_sort_info *sort_info = sset->sort_info;
555     int i, j;
556
557     zebra_sort_sysno(zh->reg->sort_index, sysno);
558     for (i = 0; i<num_criteria; i++)
559     {
560         char *this_entry_buf = tmp_cmp_buf[i];
561         memset(this_entry_buf, '\0', SORT_IDX_ENTRYSIZE);
562         if (criteria[i].ord != -1)
563         {
564             zebra_sort_type(zh->reg->sort_index, criteria[i].ord);
565             zebra_sort_read(zh->reg->sort_index, this_entry_buf);
566         }
567     }
568     i = sort_info->num_entries;
569     while (--i >= 0)
570     {
571         int rel = 0;
572         for (j = 0; j<num_criteria; j++)
573         {
574             char *this_entry_buf = tmp_cmp_buf[j];
575             char *other_entry_buf = 
576                 cmp_buf[j] + i * SORT_IDX_ENTRYSIZE;
577             if (criteria[j].numerical)
578             {
579                 char this_entry_org[1024];
580                 char other_entry_org[1024];
581                 double diff;
582                 const char *index_type = criteria[j].index_type;
583                 zebra_term_untrans(zh, index_type, this_entry_org,
584                                    this_entry_buf);
585                 zebra_term_untrans(zh, index_type, other_entry_org,
586                                    other_entry_buf);
587                 diff = atof(this_entry_org) - atof(other_entry_org);
588                 
589                 if (diff > 0.0)
590                     rel = 1;
591                 else if (diff < 0.0)
592                     rel = -1;
593                 else
594                     rel = 0;
595             }
596             else
597             {
598                 rel = memcmp(this_entry_buf, other_entry_buf,
599                              SORT_IDX_ENTRYSIZE);
600             }
601             if (rel)
602                 break;
603         }       
604         if (!rel)
605             break;
606         if (criteria[j].relation == 'A')
607         {
608             if (rel > 0)
609                 break;
610         }
611         else if (criteria[j].relation == 'D')
612         {
613             if (rel < 0)
614                 break;
615         }
616     }
617     ++i;
618     j = sort_info->max_entries;
619     if (i == j)
620         return;
621
622     if (sort_info->num_entries == j)
623         --j;
624     else
625         j = (sort_info->num_entries)++;
626     new_entry = sort_info->entries[j];
627     while (j != i)
628     {
629         int k;
630         for (k = 0; k<num_criteria; k++)
631         {
632             char *j_buf = cmp_buf[k] + j * SORT_IDX_ENTRYSIZE;
633             char *j_1_buf = cmp_buf[k] + (j-1) * SORT_IDX_ENTRYSIZE;
634             memcpy(j_buf, j_1_buf, SORT_IDX_ENTRYSIZE);
635         }
636         sort_info->entries[j] = sort_info->entries[j-1];
637         --j;
638     }
639     sort_info->entries[i] = new_entry;
640     assert (new_entry);
641     for (i = 0; i<num_criteria; i++)
642     {
643         char *new_entry_buf = cmp_buf[i] + j * SORT_IDX_ENTRYSIZE;
644         char *this_entry_buf = tmp_cmp_buf[i];
645         memcpy(new_entry_buf, this_entry_buf, SORT_IDX_ENTRYSIZE);
646     }
647     new_entry->sysno = sysno;
648     new_entry->score = -1;
649 }
650
651 void resultSetInsertRank(ZebraHandle zh, struct zset_sort_info *sort_info,
652                          zint sysno, int score, int relation)
653 {
654     struct zset_sort_entry *new_entry = NULL;
655     int i, j;
656     assert(zh); /* compiler shut up about unused arg */
657
658     i = sort_info->num_entries;
659     while (--i >= 0)
660     {
661         int rel = 0;
662
663         rel = score - sort_info->entries[i]->score;
664
665         if (relation == 'D')
666         {
667             if (rel >= 0)
668                 break;
669         }
670         else if (relation == 'A')
671         {
672             if (rel <= 0)
673                 break;
674         }
675     }
676     ++i;
677     j = sort_info->max_entries;
678     if (i == j)
679         return;
680
681     if (sort_info->num_entries == j)
682         --j;
683     else
684         j = (sort_info->num_entries)++;
685     
686     new_entry = sort_info->entries[j];
687     while (j != i)
688     {
689         sort_info->entries[j] = sort_info->entries[j-1];
690         --j;
691     }
692     sort_info->entries[i] = new_entry;
693     assert (new_entry);
694     new_entry->sysno = sysno;
695     new_entry->score = score;
696 }
697
698 static Z_RPNQuery *copy_RPNQuery(Z_RPNQuery *src, NMEM nmem)
699 {
700     Z_RPNQuery *dst = 0;
701     ODR encode = odr_createmem(ODR_ENCODE);
702     ODR decode = odr_createmem(ODR_DECODE);
703
704     if (z_RPNQuery(encode, &src, 0, 0))
705     {
706         int len;
707         char *buf = odr_getbuf(encode, &len, 0);
708
709         if (buf)
710         {
711             odr_setbuf(decode, buf, len, 0);
712             z_RPNQuery(decode, &dst, 0, 0);
713         }
714     }
715     nmem_transfer(nmem, decode->mem);
716     odr_destroy(encode);
717     odr_destroy(decode);
718     return dst;
719 }
720
721 static Z_SortKeySpecList *copy_SortKeySpecList(Z_SortKeySpecList *src, NMEM nmem)
722 {
723     Z_SortKeySpecList *dst = 0;
724     ODR encode = odr_createmem(ODR_ENCODE);
725     ODR decode = odr_createmem(ODR_DECODE);
726
727     if (z_SortKeySpecList(encode, &src, 0, 0))
728     {
729         int len;
730         char *buf = odr_getbuf(encode, &len, 0);
731
732         if (buf)
733         {
734             odr_setbuf(decode, buf, len, 0);
735             z_SortKeySpecList(decode, &dst, 0, 0);
736         }
737     }
738     nmem_transfer(nmem, decode->mem);
739     odr_destroy(encode);
740     odr_destroy(decode);
741     return dst;
742 }
743
744 ZebraSet resultSetClone(ZebraHandle zh, const char *setname,
745                         ZebraSet rset)
746 {
747     ZebraSet nset;
748     int i;
749
750     nset = resultSetAdd(zh, setname, 1);
751     if (!nset)
752         return 0;
753
754     nset->nmem = nmem_create();
755
756     nset->num_bases = rset->num_bases;
757     nset->basenames = 
758         nmem_malloc (nset->nmem, nset->num_bases * sizeof(*rset->basenames));
759     for (i = 0; i<rset->num_bases; i++)
760         nset->basenames[i] = nmem_strdup(nset->nmem, rset->basenames[i]);
761
762     if (rset->rset)
763         nset->rset = rset_dup(rset->rset);
764     if (rset->rpn)
765         nset->rpn = copy_RPNQuery(rset->rpn, nset->nmem);
766     return nset;
767 }
768
769 ZEBRA_RES resultSetSort(ZebraHandle zh, NMEM nmem,
770                         int num_input_setnames, const char **input_setnames,
771                         const char *output_setname,
772                         Z_SortKeySpecList *sort_sequence, int *sort_status)
773 {
774     ZebraSet sset;
775     RSET rset;
776
777     if (num_input_setnames == 0)
778     {
779         zebra_setError(zh, YAZ_BIB1_NO_RESULT_SET_NAME_SUPPLIED_ON_SORT, 0);
780         return ZEBRA_FAIL;
781     }
782     if (num_input_setnames > 1)
783     {
784         zebra_setError(zh, YAZ_BIB1_SORT_TOO_MANY_INPUT_RESULTS, 0);
785         return ZEBRA_FAIL;
786     }
787     if (!log_level_set)
788         loglevels();
789     yaz_log(log_level_sort, "result set sort input=%s output=%s",
790           *input_setnames, output_setname);
791     sset = resultSetGet (zh, input_setnames[0]);
792     if (!sset)
793     {
794         zebra_setError(zh, YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
795                        input_setnames[0]);
796         return ZEBRA_FAIL;
797     }
798     if (!(rset = sset->rset))
799     {
800         zebra_setError(zh, YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
801                        input_setnames[0]);
802         return ZEBRA_FAIL;
803     }
804     if (strcmp (output_setname, input_setnames[0]))
805         sset = resultSetClone(zh, output_setname, sset);
806     sset->sortSpec = copy_SortKeySpecList(sort_sequence, sset->nmem);
807     return resultSetSortSingle (zh, nmem, sset, rset, sort_sequence,
808                                 sort_status);
809 }
810
811 ZEBRA_RES resultSetSortSingle(ZebraHandle zh, NMEM nmem,
812                               ZebraSet sset, RSET rset,
813                               Z_SortKeySpecList *sort_sequence,
814                               int *sort_status)
815 {
816     int i;
817     int n = 0;
818     zint kno = 0;
819     zint psysno = 0;
820     struct it_key key;
821     struct sortKeyInfo sort_criteria[ZSET_SORT_MAX_LEVEL];
822     char *cmp_buf[ZSET_SORT_MAX_LEVEL];
823     char *tmp_cmp_buf[ZSET_SORT_MAX_LEVEL];
824     int num_criteria;
825     RSFD rfd;
826     TERMID termid;
827     TERMID *terms;
828     int numTerms = 0;
829     size_t sysno_mem_index = 0;
830
831     if (zh->m_staticrank)
832         sysno_mem_index = 1;
833
834     assert(nmem); /* compiler shut up about unused param */
835     sset->sort_info->num_entries = 0;
836
837     rset_getterms(rset, 0, 0, &n);
838     terms = (TERMID *) nmem_malloc(nmem, sizeof(*terms)*n);
839     rset_getterms(rset, terms, n, &numTerms);
840
841     sset->hits = 0;
842     num_criteria = sort_sequence->num_specs;
843     if (num_criteria > ZSET_SORT_MAX_LEVEL)
844         num_criteria = ZSET_SORT_MAX_LEVEL;
845     for (i = 0; i < num_criteria; i++)
846     {
847         Z_SortKeySpec *sks = sort_sequence->specs[i];
848         Z_SortKey *sk;
849         ZEBRA_RES res;
850
851         sort_criteria[i].ord = -1;
852         sort_criteria[i].numerical = 0;
853
854         if (sks->which == Z_SortKeySpec_missingValueData)
855         {
856             zebra_setError(zh, YAZ_BIB1_UNSUPP_MISSING_DATA_ACTION, 0);
857             return ZEBRA_FAIL;
858         }
859         if (*sks->sortRelation == Z_SortKeySpec_ascending)
860             sort_criteria[i].relation = 'A';
861         else if (*sks->sortRelation == Z_SortKeySpec_descending)
862             sort_criteria[i].relation = 'D';
863         else
864         {
865             zebra_setError(zh, YAZ_BIB1_ILLEGAL_SORT_RELATION, 0);
866             return ZEBRA_FAIL;
867         }
868         if (sks->sortElement->which == Z_SortElement_databaseSpecific)
869         {
870             zebra_setError(zh, YAZ_BIB1_DATABASE_SPECIFIC_SORT_UNSUPP, 0);
871             return ZEBRA_FAIL;
872         }
873         else if (sks->sortElement->which != Z_SortElement_generic)
874         {
875             zebra_setError(zh, YAZ_BIB1_SORT_ILLEGAL_SORT, 0);
876             return ZEBRA_FAIL;
877         }       
878         sk = sks->sortElement->u.generic;
879         switch (sk->which)
880         {
881         case Z_SortKey_sortField:
882             yaz_log(log_level_sort, "key %d is of type sortField",
883                     i+1);
884             sort_criteria[i].numerical = 0;
885             sort_criteria[i].ord = 
886                 zebraExplain_lookup_attr_str(zh->reg->zei,
887                                              zinfo_index_category_sort,
888                                              0, sk->u.sortField);
889             if (sks->which != Z_SortKeySpec_null
890                 && sort_criteria[i].ord == -1)
891             {
892                 zebra_setError(zh,
893                                YAZ_BIB1_CANNOT_SORT_ACCORDING_TO_SEQUENCE, 0);
894                 return ZEBRA_FAIL;
895             }
896             break;
897         case Z_SortKey_elementSpec:
898             yaz_log(log_level_sort, "key %d is of type elementSpec",
899                     i+1);
900             zebra_setError(zh, YAZ_BIB1_CANNOT_SORT_ACCORDING_TO_SEQUENCE, 0);
901             return ZEBRA_FAIL;
902         case Z_SortKey_sortAttributes:
903             yaz_log(log_level_sort, "key %d is of type sortAttributes", i+1);
904             res = zebra_sort_get_ord(zh, sk->u.sortAttributes,
905
906                                      &sort_criteria[i].ord,
907                                      &sort_criteria[i].numerical);
908             if (sks->which != Z_SortKeySpec_null && res != ZEBRA_OK)
909                 return ZEBRA_FAIL;
910             break;
911         }
912         if (zebraExplain_lookup_ord(zh->reg->zei, sort_criteria[i].ord,
913                                     &sort_criteria[i].index_type,
914                                     0, 0))
915         {
916             zebra_setError(zh, YAZ_BIB1_CANNOT_SORT_ACCORDING_TO_SEQUENCE, 0);
917             return ZEBRA_FAIL;
918         }
919     }
920     /* allocate space for each cmpare buf + one extra for tmp comparison */
921     for (i = 0; i<num_criteria; i++)
922     {
923         cmp_buf[i] = xmalloc(sset->sort_info->max_entries
924                              * SORT_IDX_ENTRYSIZE);
925         tmp_cmp_buf[i] = xmalloc(SORT_IDX_ENTRYSIZE);
926     }
927     rfd = rset_open (rset, RSETF_READ);
928     while (rset_read (rfd, &key, &termid))
929     {
930         zint this_sys = key.mem[sysno_mem_index];
931         if (log_level_searchhits)
932             key_logdump_txt(log_level_searchhits, &key, termid->name);
933         kno++;
934         if (this_sys != psysno)
935         {
936             if ((sset->hits & 255) == 0 && zh->break_handler_func)
937             {
938                 if (zh->break_handler_func(zh->break_handler_data))
939                 {
940                     rset_set_hits_limit(rset, 0);
941                     break;
942                 }
943             }
944             (sset->hits)++;
945             psysno = this_sys;
946             resultSetInsertSort(zh, sset,
947                                 sort_criteria, num_criteria, psysno, cmp_buf,
948                                 tmp_cmp_buf);
949         }
950     }
951     rset_close (rfd);
952
953     for (i = 0; i<num_criteria; i++)
954     {
955         xfree(cmp_buf[i]);
956         xfree(tmp_cmp_buf[i]);
957     }
958
959     yaz_log(log_level_sort, ZINT_FORMAT " keys, " ZINT_FORMAT " sysnos, sort",
960             kno, sset->hits);   
961     for (i = 0; i < numTerms; i++)
962         yaz_log(log_level_sort, "term=\"%s\" type=%s count=" ZINT_FORMAT,
963                  terms[i]->name, terms[i]->flags, terms[i]->rset->hits_count);
964     *sort_status = Z_SortResponse_success;
965     return ZEBRA_OK;
966 }
967
968 RSET resultSetRef(ZebraHandle zh, const char *resultSetId)
969 {
970     ZebraSet s;
971
972     if ((s = resultSetGet (zh, resultSetId)))
973         return s->rset;
974     return NULL;
975 }
976
977 ZEBRA_RES resultSetRank(ZebraHandle zh, ZebraSet zebraSet,
978                         RSET rset, NMEM nmem)
979 {
980     struct it_key key;
981     TERMID termid;
982     TERMID *terms;
983     zint kno = 0;
984     int numTerms = 0;
985     int n = 0;
986     int i;
987     ZebraRankClass rank_class;
988     struct zset_sort_info *sort_info;
989     const char *rank_handler_name = res_get_def(zh->res, "rank", "rank-1");
990     size_t sysno_mem_index = 0;
991
992     if (zh->m_staticrank)
993         sysno_mem_index = 1;
994
995     if (!log_level_set)
996         loglevels();
997     sort_info = zebraSet->sort_info;
998     sort_info->num_entries = 0;
999     zebraSet->hits = 0;
1000     zebraSet->estimated_hit_count = 0;
1001     rset_getterms(rset, 0, 0, &n);
1002     terms = (TERMID *) nmem_malloc(nmem, sizeof(*terms)*n);
1003     rset_getterms(rset, terms, n, &numTerms);
1004
1005     rank_class = zebraRankLookup(zh, rank_handler_name);
1006     if (!rank_class)
1007     {
1008         yaz_log(YLOG_WARN, "No such rank handler: %s", rank_handler_name);
1009         zebra_setError(zh, YAZ_BIB1_UNSUPP_SEARCH, "Cannot find rank handler");
1010         return ZEBRA_FAIL;
1011     }
1012     else
1013     {
1014         RSFD rfd = rset_open(rset, RSETF_READ);
1015         struct rank_control *rc = rank_class->control;
1016         int score;
1017         zint count = 0;
1018         void *handle = (*rc->begin) (zh->reg, rank_class->class_handle, rset,
1019                                      nmem, terms, numTerms);
1020         zint psysno = 0;  /* previous doc id / sys no */
1021         zint pstaticrank = 0; /* previous static rank */
1022         int stop_flag = 0;
1023         while (rset_read(rfd, &key, &termid))
1024         {
1025             zint this_sys = key.mem[sysno_mem_index];
1026
1027             zint seqno = key.mem[key.len-1];
1028             kno++;
1029             if (log_level_searchhits)
1030                 key_logdump_txt(log_level_searchhits, &key, termid->name);
1031             if (this_sys != psysno) 
1032             {   /* new record .. */
1033                 if (!(rfd->counted_items & 255) && zh->break_handler_func)
1034                 {
1035                     if (zh->break_handler_func(zh->break_handler_data))
1036                     {
1037                         yaz_log(YLOG_LOG, "Aborted search");
1038                         stop_flag = 1;
1039                     }
1040                 }
1041                 if (rfd->counted_items > rset->hits_limit)
1042                     stop_flag = 1;
1043                 if (psysno)
1044                 {   /* only if we did have a previous record */
1045                     score = (*rc->calc) (handle, psysno, pstaticrank,
1046                                          &stop_flag);
1047                     /* insert the hit. A=Ascending */
1048                     resultSetInsertRank (zh, sort_info, psysno, score, 'A');
1049                     count++;
1050                 }
1051                 if (stop_flag)
1052                 {
1053                     zebraSet->estimated_hit_count = 1;
1054                     rset_set_hits_limit(rset, 0);
1055                     break;
1056                 }
1057                 psysno = this_sys;
1058                 if (zh->m_staticrank)
1059                     pstaticrank = key.mem[0];
1060             }
1061             (*rc->add) (handle, CAST_ZINT_TO_INT(seqno), termid);
1062         }
1063         /* no more items */
1064         if (psysno)
1065         {   /* we had - at least - one record */
1066             score = (*rc->calc)(handle, psysno, pstaticrank, &stop_flag);
1067             /* insert the hit. A=Ascending */
1068             resultSetInsertRank(zh, sort_info, psysno, score, 'A');
1069             count++;
1070         }
1071         (*rc->end) (zh->reg, handle);
1072         rset_close (rfd);
1073     }
1074     zebraSet->hits = rset->hits_count;
1075
1076     yaz_log(log_level_searchterms, ZINT_FORMAT " keys, "
1077             ZINT_FORMAT " sysnos, rank",  kno, zebraSet->hits);
1078     for (i = 0; i < numTerms; i++)
1079     {
1080         yaz_log(log_level_searchterms, "term=\"%s\" type=%s count="
1081                 ZINT_FORMAT,
1082                 terms[i]->name, terms[i]->flags, terms[i]->rset->hits_count);
1083     }
1084     return ZEBRA_OK;
1085 }
1086
1087 ZebraRankClass zebraRankLookup(ZebraHandle zh, const char *name)
1088 {
1089     ZebraRankClass p = zh->reg->rank_classes;
1090     while (p && strcmp (p->control->name, name))
1091         p = p->next;
1092     if (p && !p->init_flag)
1093     {
1094         if (p->control->create)
1095             p->class_handle = (*p->control->create)(zh);
1096         p->init_flag = 1;
1097     }
1098     return p;
1099 }
1100
1101 void zebraRankInstall(struct zebra_register *reg, struct rank_control *ctrl)
1102 {
1103     ZebraRankClass p = (ZebraRankClass) xmalloc (sizeof(*p));
1104     p->control = (struct rank_control *) xmalloc (sizeof(*p->control));
1105     memcpy (p->control, ctrl, sizeof(*p->control));
1106     p->control->name = xstrdup (ctrl->name);
1107     p->init_flag = 0;
1108     p->next = reg->rank_classes;
1109     reg->rank_classes = p;
1110 }
1111
1112 void zebraRankDestroy(struct zebra_register *reg)
1113 {
1114     ZebraRankClass p = reg->rank_classes;
1115     while (p)
1116     {
1117         ZebraRankClass p_next = p->next;
1118         if (p->init_flag && p->control->destroy)
1119             (*p->control->destroy)(reg, p->class_handle);
1120         xfree(p->control->name);
1121         xfree(p->control);
1122         xfree(p);
1123         p = p_next;
1124     }
1125     reg->rank_classes = NULL;
1126 }
1127
1128 static int trav_rset_for_termids(RSET rset, TERMID *termid_array,
1129                                  zint *hits_array, int *approx_array)
1130 {
1131     int no = 0;
1132     int i;
1133     for (i = 0; i<rset->no_children; i++)
1134         no += trav_rset_for_termids(rset->children[i],
1135                                     (termid_array ? termid_array + no : 0),
1136                                     (hits_array ? hits_array + no : 0),
1137                                     (approx_array ? approx_array + no : 0));
1138     if (rset->term)
1139     {
1140         if (termid_array)
1141             termid_array[no] = rset->term;
1142         if (hits_array)
1143             hits_array[no] = rset->hits_count;
1144         if (approx_array)
1145             approx_array[no] = rset->hits_approx;
1146 #if 0
1147         yaz_log(YLOG_LOG, "rset=%p term=%s limit=" ZINT_FORMAT
1148                 " count=" ZINT_FORMAT,
1149                 rset, rset->term->name, rset->hits_limit, rset->hits_count);
1150 #endif
1151         no++;
1152     }
1153     return no;
1154 }
1155
1156 ZEBRA_RES zebra_result_set_term_no(ZebraHandle zh, const char *setname,
1157                                    int *num_terms)
1158 {
1159     ZebraSet sset = resultSetGet(zh, setname);
1160     *num_terms = 0;
1161     if (sset)
1162     {
1163         *num_terms = trav_rset_for_termids(sset->rset, 0, 0, 0);
1164         return ZEBRA_OK;
1165     }
1166     return ZEBRA_FAIL;
1167 }
1168
1169 ZEBRA_RES zebra_result_set_term_info(ZebraHandle zh, const char *setname,
1170                                      int no, zint *count, int *approx,
1171                                      char *termbuf, size_t *termlen,
1172                                      const char **term_ref_id)
1173 {
1174     ZebraSet sset = resultSetGet(zh, setname);
1175     if (sset)
1176     {
1177         int num_terms = trav_rset_for_termids(sset->rset, 0, 0, 0);
1178         if (no >= 0 && no < num_terms)
1179         {
1180             TERMID *term_array = xmalloc(num_terms * sizeof(*term_array));
1181             zint *hits_array = xmalloc(num_terms * sizeof(*hits_array));
1182             int *approx_array = xmalloc(num_terms * sizeof(*approx_array));
1183             
1184             trav_rset_for_termids(sset->rset, term_array,
1185                                   hits_array, approx_array);
1186
1187             if (count)
1188                 *count = hits_array[no];
1189             if (approx)
1190                 *approx = approx_array[no];
1191             if (termbuf)
1192             {
1193                 char *inbuf = term_array[no]->name;
1194                 size_t inleft = strlen(inbuf);
1195                 size_t outleft = *termlen - 1;
1196
1197                 if (zh->iconv_from_utf8 != 0)
1198                 {
1199                     char *outbuf = termbuf;
1200                     size_t ret;
1201                     
1202                     ret = yaz_iconv(zh->iconv_from_utf8, &inbuf, &inleft,
1203                                     &outbuf, &outleft);
1204                     if (ret == (size_t)(-1))
1205                         *termlen = 0;
1206                     else
1207                     {
1208                         yaz_iconv(zh->iconv_from_utf8, 0, 0, 
1209                                   &outbuf, &outleft);
1210                         *termlen = outbuf - termbuf;
1211                     }
1212                 }
1213                 else
1214                 {
1215                     if (inleft > outleft)
1216                         inleft = outleft;
1217                     *termlen = inleft;
1218                     memcpy(termbuf, inbuf, *termlen);
1219                 }
1220                 termbuf[*termlen] = '\0';
1221             }
1222             if (term_ref_id)
1223                 *term_ref_id = term_array[no]->ref_id;
1224
1225             xfree(term_array);
1226             xfree(hits_array);
1227             xfree(approx_array);
1228             return ZEBRA_OK;
1229         }
1230     }
1231     return ZEBRA_FAIL;
1232 }
1233
1234 ZEBRA_RES zebra_snippets_hit_vector(ZebraHandle zh, const char *setname,
1235                                     zint sysno, zebra_snippets *snippets)
1236 {
1237     ZebraSet sset = resultSetGet(zh, setname);
1238     yaz_log(YLOG_DEBUG, "zebra_get_hit_vector setname=%s zysno=" ZINT_FORMAT,
1239             setname, sysno);
1240     if (!sset)
1241         return ZEBRA_FAIL;
1242     else
1243     {
1244         struct rset_key_control *kc = zebra_key_control_create(zh);
1245         NMEM nmem = nmem_create();
1246         struct it_key key;
1247         RSET rsets[2], rset_comb;
1248         RSET rset_temp = rset_create_temp(nmem, kc, kc->scope, 
1249                                           res_get (zh->res, "setTmpDir"),0 );
1250         
1251         TERMID termid;
1252         RSFD rsfd = rset_open(rset_temp, RSETF_WRITE);
1253         
1254         key.mem[0] = sysno;
1255         key.mem[1] = 0;
1256         key.mem[2] = 0;
1257         key.mem[3] = 0;
1258         key.len = 2;
1259         rset_write (rsfd, &key);
1260         rset_close (rsfd);
1261
1262         rsets[0] = rset_temp;
1263         rsets[1] = rset_dup(sset->rset);
1264         
1265         rset_comb = rset_create_and(nmem, kc, kc->scope, 2, rsets);
1266
1267         rsfd = rset_open(rset_comb, RSETF_READ);
1268
1269         while (rset_read(rsfd, &key, &termid))
1270         {
1271             if (termid)
1272             {
1273                 struct ord_list *ol;
1274                 for (ol = termid->ol; ol; ol = ol->next)
1275                 {
1276                     zebra_snippets_append(snippets, key.mem[key.len-1], 0,
1277                                           ol->ord, termid->name);
1278                 }
1279             }
1280         }
1281         rset_close(rsfd);
1282         
1283         rset_delete(rset_comb);
1284         nmem_destroy(nmem);
1285         kc->dec(kc);
1286     }
1287     return ZEBRA_OK;
1288 }
1289
1290 /*
1291  * Local variables:
1292  * c-basic-offset: 4
1293  * indent-tabs-mode: nil
1294  * End:
1295  * vim: shiftwidth=4 tabstop=8 expandtab
1296  */
1297