X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=index%2Fzsets.c;h=556e9179bc2dcbe7905078da73b2308fe94b2a74;hb=c33ea56e3771c3b80ba66ef8fda3a09cad171ebb;hp=ed60386944b4fe31a85b342f06f3b3da3bc0b4e1;hpb=342c28e71fe5122d17d760a930107cab0266e9c6;p=idzebra-moved-to-github.git

diff --git a/index/zsets.c b/index/zsets.c
index ed60386..556e917 100644
--- a/index/zsets.c
+++ b/index/zsets.c
@@ -1,5 +1,5 @@
-/* $Id: zsets.c,v 1.93 2005-08-19 09:21:34 adam Exp $
-   Copyright (C) 1995-2005
+/* $Id: zsets.c,v 1.110 2006-08-14 10:40:15 adam Exp $
+   Copyright (C) 1995-2006
    Index Data ApS
 
 This file is part of the Zebra server.
@@ -15,9 +15,9 @@ FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 for more details.
 
 You should have received a copy of the GNU General Public License
-along with Zebra; see the file LICENSE.zebra.  If not, write to the
-Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
-02111-1307, USA.
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
 */
 
 
@@ -30,17 +30,16 @@ Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
 #endif
 
 #include "index.h"
+#include "rank.h"
 #include <yaz/diagbib1.h>
 #include <rset.h>
 
-#define SORT_IDX_ENTRYSIZE 64
-#define ZSET_SORT_MAX_LEVEL 3
+#define ZSET_SORT_MAX_LEVEL 10
 
 struct zebra_set_term_entry {
     int reg_type;
     char *db;
-    int set;
-    int use;
+    char *index_name;
     char *term;
 };
 
@@ -53,6 +52,7 @@ struct zebra_set {
     int num_bases;
     char **basenames;
     Z_RPNQuery *rpn;
+    Z_SortKeySpecList *sortSpec;
     struct zset_sort_info *sort_info;
     struct zebra_set_term_entry *term_entries;
     int term_entries_max;
@@ -68,7 +68,6 @@ struct zebra_set {
 struct zset_sort_entry {
     zint sysno;
     int score;
-    char buf[ZSET_SORT_MAX_LEVEL][SORT_IDX_ENTRYSIZE];
 };
 
 struct zset_sort_info {
@@ -179,9 +178,9 @@ ZEBRA_RES resultSetAddRPN (ZebraHandle zh, NMEM m, Z_RPNQuery *rpn,
     return res;
 }
 
-void resultSetAddTerm (ZebraHandle zh, ZebraSet s, int reg_type,
-                       const char *db, int set,
-                       int use, const char *term)
+void resultSetAddTerm(ZebraHandle zh, ZebraSet s, int reg_type,
+                      const char *db, const char *index_name, 
+                      const char *term)
 {
     assert(zh); /* compiler shut up */
     if (!s->nmem)
@@ -200,9 +199,8 @@ void resultSetAddTerm (ZebraHandle zh, ZebraSet s, int reg_type,
     {
         s->term_entries[s->hits].reg_type = reg_type;
         s->term_entries[s->hits].db = nmem_strdup (s->nmem, db);
-        s->term_entries[s->hits].set = set;
-        s->term_entries[s->hits].use = use;
-        s->term_entries[s->hits].term = nmem_strdup (s->nmem, term);
+        s->term_entries[s->hits].index_name = nmem_strdup(s->nmem, index_name);
+        s->term_entries[s->hits].term = nmem_strdup(s->nmem, term);
     }
     (s->hits)++;
 }
@@ -267,6 +265,7 @@ ZebraSet resultSetAdd(ZebraHandle zh, const char *name, int ov)
     s->rset_nmem = 0;
     s->nmem = 0;
     s->rpn = 0;
+    s->sortSpec = 0;
     s->cache_position = 0;
     s->cache_rfd = 0;
     s->approx_limit = zh->approx_limit;
@@ -287,6 +286,13 @@ ZebraSet resultSetGet(ZebraHandle zh, const char *name)
                 if (!s->rset_nmem)
                     s->rset_nmem=nmem_create();
 		resultSetSearch(zh, nmem, s->rset_nmem, s->rpn, s);
+		if (s->rset && s->sortSpec)
+		{
+		    int sort_status;
+		    yaz_log(log_level_resultsets, "resort %s", name);
+		    resultSetSortSingle (zh, nmem, s, s->rset, s->sortSpec,
+					 &sort_status);
+		}
                 nmem_destroy (nmem);
             }
             return s;
@@ -528,15 +534,19 @@ void zebra_meta_records_destroy (ZebraHandle zh, ZebraMetaRecord *records,
 
 struct sortKeyInfo {
     int relation;
+#if 0
     int attrUse;
+#else
+    int ord;
+#endif
     int numerical;
 };
 
-void resultSetInsertSort (ZebraHandle zh, ZebraSet sset,
-                          struct sortKeyInfo *criteria, int num_criteria,
-                          zint sysno)
+void resultSetInsertSort(ZebraHandle zh, ZebraSet sset,
+                         struct sortKeyInfo *criteria, int num_criteria,
+                         zint sysno,
+                         char *cmp_buf[], char *tmp_cmp_buf[])
 {
-    struct zset_sort_entry this_entry;
     struct zset_sort_entry *new_entry = NULL;
     struct zset_sort_info *sort_info = sset->sort_info;
     int i, j;
@@ -544,8 +554,13 @@ void resultSetInsertSort (ZebraHandle zh, ZebraSet sset,
     sortIdx_sysno (zh->reg->sortIdx, sysno);
     for (i = 0; i<num_criteria; i++)
     {
-        sortIdx_type (zh->reg->sortIdx, criteria[i].attrUse);
-        sortIdx_read (zh->reg->sortIdx, this_entry.buf[i]);
+        char *this_entry_buf = tmp_cmp_buf[i];
+        memset(this_entry_buf, '\0', SORT_IDX_ENTRYSIZE);
+        if (criteria[i].ord != -1)
+        {
+            sortIdx_type(zh->reg->sortIdx, criteria[i].ord);
+            sortIdx_read(zh->reg->sortIdx, this_entry_buf);
+        }
     }
     i = sort_info->num_entries;
     while (--i >= 0)
@@ -553,10 +568,12 @@ void resultSetInsertSort (ZebraHandle zh, ZebraSet sset,
         int rel = 0;
         for (j = 0; j<num_criteria; j++)
         {
+            char *this_entry_buf = tmp_cmp_buf[j];
+            char *other_entry_buf = 
+                cmp_buf[j] + i * SORT_IDX_ENTRYSIZE;
             if (criteria[j].numerical)
             {
-                double diff = atof(this_entry.buf[j]) -
-                              atof(sort_info->entries[i]->buf[j]);
+                double diff = atof(this_entry_buf) - atof(other_entry_buf);
                 rel = 0;
                 if (diff > 0.0)
                     rel = 1;
@@ -565,8 +582,8 @@ void resultSetInsertSort (ZebraHandle zh, ZebraSet sset,
             }
             else
             {
-                rel = memcmp (this_entry.buf[j], sort_info->entries[i]->buf[j],
-                          SORT_IDX_ENTRYSIZE);
+                rel = memcmp(this_entry_buf, other_entry_buf,
+                             SORT_IDX_ENTRYSIZE);
             }
             if (rel)
                 break;
@@ -596,19 +613,30 @@ void resultSetInsertSort (ZebraHandle zh, ZebraSet sset,
     new_entry = sort_info->entries[j];
     while (j != i)
     {
+        int k;
+        for (k = 0; k<num_criteria; k++)
+        {
+            char *j_buf = cmp_buf[k] + j * SORT_IDX_ENTRYSIZE;
+            char *j_1_buf = cmp_buf[k] + (j-1) * SORT_IDX_ENTRYSIZE;
+            memcpy(j_buf, j_1_buf, SORT_IDX_ENTRYSIZE);
+        }
         sort_info->entries[j] = sort_info->entries[j-1];
         --j;
     }
     sort_info->entries[i] = new_entry;
     assert (new_entry);
     for (i = 0; i<num_criteria; i++)
-        memcpy (new_entry->buf[i], this_entry.buf[i], SORT_IDX_ENTRYSIZE);
+    {
+        char *new_entry_buf = cmp_buf[i] + j * SORT_IDX_ENTRYSIZE;
+        char *this_entry_buf = tmp_cmp_buf[i];
+        memcpy(new_entry_buf, this_entry_buf, SORT_IDX_ENTRYSIZE);
+    }
     new_entry->sysno = sysno;
     new_entry->score = -1;
 }
 
-void resultSetInsertRank (ZebraHandle zh, struct zset_sort_info *sort_info,
-                          zint sysno, int score, int relation)
+void resultSetInsertRank(ZebraHandle zh, struct zset_sort_info *sort_info,
+                         zint sysno, int score, int relation)
 {
     struct zset_sort_entry *new_entry = NULL;
     int i, j;
@@ -654,6 +682,77 @@ void resultSetInsertRank (ZebraHandle zh, struct zset_sort_info *sort_info,
     new_entry->score = score;
 }
 
+static Z_RPNQuery *copy_RPNQuery(Z_RPNQuery *src, NMEM nmem)
+{
+    Z_RPNQuery *dst = 0;
+    ODR encode = odr_createmem(ODR_ENCODE);
+    ODR decode = odr_createmem(ODR_DECODE);
+
+    if (z_RPNQuery(encode, &src, 0, 0))
+    {
+	int len;
+	char *buf = odr_getbuf(encode, &len, 0);
+
+	if (buf)
+	{
+	    odr_setbuf(decode, buf, len, 0);
+	    z_RPNQuery(decode, &dst, 0, 0);
+	}
+    }
+    nmem_transfer(nmem, decode->mem);
+    odr_destroy(encode);
+    odr_destroy(decode);
+    return dst;
+}
+
+static Z_SortKeySpecList *copy_SortKeySpecList(Z_SortKeySpecList *src, NMEM nmem)
+{
+    Z_SortKeySpecList *dst = 0;
+    ODR encode = odr_createmem(ODR_ENCODE);
+    ODR decode = odr_createmem(ODR_DECODE);
+
+    if (z_SortKeySpecList(encode, &src, 0, 0))
+    {
+	int len;
+	char *buf = odr_getbuf(encode, &len, 0);
+
+	if (buf)
+	{
+	    odr_setbuf(decode, buf, len, 0);
+	    z_SortKeySpecList(decode, &dst, 0, 0);
+	}
+    }
+    nmem_transfer(nmem, decode->mem);
+    odr_destroy(encode);
+    odr_destroy(decode);
+    return dst;
+}
+
+ZebraSet resultSetClone(ZebraHandle zh, const char *setname,
+			ZebraSet rset)
+{
+    ZebraSet nset;
+    int i;
+
+    nset = resultSetAdd(zh, setname, 1);
+    if (!nset)
+	return 0;
+
+    nset->nmem = nmem_create();
+
+    nset->num_bases = rset->num_bases;
+    nset->basenames = 
+        nmem_malloc (nset->nmem, nset->num_bases * sizeof(*rset->basenames));
+    for (i = 0; i<rset->num_bases; i++)
+        nset->basenames[i] = nmem_strdup(nset->nmem, rset->basenames[i]);
+
+    if (rset->rset)
+	nset->rset = rset_dup(rset->rset);
+    if (rset->rpn)
+	nset->rpn = copy_RPNQuery(rset->rpn, nset->nmem);
+    return nset;
+}
+
 ZEBRA_RES resultSetSort(ZebraHandle zh, NMEM nmem,
 			int num_input_setnames, const char **input_setnames,
 			const char *output_setname,
@@ -690,11 +789,8 @@ ZEBRA_RES resultSetSort(ZebraHandle zh, NMEM nmem,
 	return ZEBRA_FAIL;
     }
     if (strcmp (output_setname, input_setnames[0]))
-    {
-        rset = rset_dup (rset);
-        sset = resultSetAdd (zh, output_setname, 1);
-        sset->rset = rset;
-    }
+	sset = resultSetClone(zh, output_setname, sset);
+    sset->sortSpec = copy_SortKeySpecList(sort_sequence, sset->nmem);
     return resultSetSortSingle (zh, nmem, sset, rset, sort_sequence,
 				sort_status);
 }
@@ -709,7 +805,9 @@ ZEBRA_RES resultSetSortSingle(ZebraHandle zh, NMEM nmem,
     zint kno = 0;
     zint psysno = 0;
     struct it_key key;
-    struct sortKeyInfo sort_criteria[3];
+    struct sortKeyInfo sort_criteria[ZSET_SORT_MAX_LEVEL];
+    char *cmp_buf[ZSET_SORT_MAX_LEVEL];
+    char *tmp_cmp_buf[ZSET_SORT_MAX_LEVEL];
     int num_criteria;
     RSFD rfd;
     TERMID termid;
@@ -720,7 +818,6 @@ ZEBRA_RES resultSetSortSingle(ZebraHandle zh, NMEM nmem,
     if (zh->m_staticrank)
 	sysno_mem_index = 1;
 
-
     assert(nmem); /* compiler shut up about unused param */
     sset->sort_info->num_entries = 0;
 
@@ -730,13 +827,22 @@ ZEBRA_RES resultSetSortSingle(ZebraHandle zh, NMEM nmem,
 
     sset->hits = 0;
     num_criteria = sort_sequence->num_specs;
-    if (num_criteria > 3)
-        num_criteria = 3;
+    if (num_criteria > ZSET_SORT_MAX_LEVEL)
+        num_criteria = ZSET_SORT_MAX_LEVEL;
     for (i = 0; i < num_criteria; i++)
     {
         Z_SortKeySpec *sks = sort_sequence->specs[i];
         Z_SortKey *sk;
+        ZEBRA_RES res;
 
+        sort_criteria[i].ord = -1;
+        sort_criteria[i].numerical = 0;
+
+        if (sks->which == Z_SortKeySpec_missingValueData)
+        {
+	    zebra_setError(zh, YAZ_BIB1_UNSUPP_MISSING_DATA_ACTION, 0);
+            return ZEBRA_FAIL;
+        }
         if (*sks->sortRelation == Z_SortKeySpec_ascending)
             sort_criteria[i].relation = 'A';
         else if (*sks->sortRelation == Z_SortKeySpec_descending)
@@ -762,8 +868,20 @@ ZEBRA_RES resultSetSortSingle(ZebraHandle zh, NMEM nmem,
         case Z_SortKey_sortField:
             yaz_log(log_level_sort, "key %d is of type sortField",
 		    i+1);
-            zebra_setError(zh, YAZ_BIB1_CANNOT_SORT_ACCORDING_TO_SEQUENCE, 0);
-            return ZEBRA_FAIL;
+            sort_criteria[i].numerical = 0;
+            sort_criteria[i].ord = 
+                zebraExplain_lookup_attr_str(zh->reg->zei,
+                                             zinfo_index_category_sort,
+                                             's',
+                                             sk->u.sortField);
+            if (sks->which != Z_SortKeySpec_null
+                && sort_criteria[i].ord == -1)
+            {
+                zebra_setError(zh,
+                               YAZ_BIB1_CANNOT_SORT_ACCORDING_TO_SEQUENCE, 0);
+                return ZEBRA_FAIL;
+            }
+            break;
         case Z_SortKey_elementSpec:
             yaz_log(log_level_sort, "key %d is of type elementSpec",
 		    i+1);
@@ -771,26 +889,21 @@ ZEBRA_RES resultSetSortSingle(ZebraHandle zh, NMEM nmem,
             return ZEBRA_FAIL;
         case Z_SortKey_sortAttributes:
             yaz_log(log_level_sort, "key %d is of type sortAttributes", i+1);
-            sort_criteria[i].attrUse =
-                zebra_maps_sort (zh->reg->zebra_maps,
-                                 sk->u.sortAttributes,
-                                 &sort_criteria[i].numerical);
-            yaz_log(log_level_sort, "use value = %d", sort_criteria[i].attrUse);
-            if (sort_criteria[i].attrUse == -1)
-            {
-		zebra_setError(
-		    zh, YAZ_BIB1_USE_ATTRIBUTE_REQUIRED_BUT_NOT_SUPPLIED, 0); 
-                return ZEBRA_FAIL;
-            }
-            if (sortIdx_type (zh->reg->sortIdx, sort_criteria[i].attrUse))
-            {
-		zebra_setError(
-		    zh, YAZ_BIB1_CANNOT_SORT_ACCORDING_TO_SEQUENCE, 0);
+            res = zebra_sort_get_ord(zh, sk->u.sortAttributes,
+                                     &sort_criteria[i].ord,
+                                     &sort_criteria[i].numerical);
+            if (sks->which != Z_SortKeySpec_null && res != ZEBRA_OK)
                 return ZEBRA_FAIL;
-            }
             break;
         }
     }
+    /* allocate space for each cmpare buf + one extra for tmp comparison */
+    for (i = 0; i<num_criteria; i++)
+    {
+        cmp_buf[i] = xmalloc(sset->sort_info->max_entries
+                             * SORT_IDX_ENTRYSIZE);
+        tmp_cmp_buf[i] = xmalloc(SORT_IDX_ENTRYSIZE);
+    }
     rfd = rset_open (rset, RSETF_READ);
     while (rset_read (rfd, &key, &termid))
     {
@@ -802,11 +915,19 @@ ZEBRA_RES resultSetSortSingle(ZebraHandle zh, NMEM nmem,
         {
             (sset->hits)++;
             psysno = this_sys;
-            resultSetInsertSort (zh, sset,
-                                 sort_criteria, num_criteria, psysno);
+            resultSetInsertSort(zh, sset,
+                                sort_criteria, num_criteria, psysno, cmp_buf,
+                                tmp_cmp_buf);
         }
     }
     rset_close (rfd);
+
+    for (i = 0; i<num_criteria; i++)
+    {
+        xfree(cmp_buf[i]);
+        xfree(tmp_cmp_buf[i]);
+    }
+
     yaz_log(log_level_sort, ZINT_FORMAT " keys, " ZINT_FORMAT " sysnos, sort",
 	    kno, sset->hits);   
     for (i = 0; i < numTerms; i++)
@@ -864,7 +985,7 @@ ZEBRA_RES resultSetRank(ZebraHandle zh, ZebraSet zebraSet,
     {
 	RSFD rfd = rset_open(rset, RSETF_READ);
 	struct rank_control *rc = rank_class->control;
-	double score;
+	int score;
 	zint count = 0;
 	
 	void *handle =
@@ -872,6 +993,7 @@ ZEBRA_RES resultSetRank(ZebraHandle zh, ZebraSet zebraSet,
 			  terms, numTerms);
 	zint psysno = 0;  /* previous doc id / sys no */
 	zint pstaticrank = 0; /* previous static rank */
+	int stop_flag = 0;
 	while (rset_read(rfd, &key, &termid))
 	{
 	    zint this_sys = key.mem[sysno_mem_index];
@@ -886,10 +1008,13 @@ ZEBRA_RES resultSetRank(ZebraHandle zh, ZebraSet zebraSet,
 		    break;
 		if (psysno)
 		{   /* only if we did have a previous record */
-		    score = (*rc->calc) (handle, psysno, pstaticrank);
+		    score = (*rc->calc) (handle, psysno, pstaticrank,
+					 &stop_flag);
 		    /* insert the hit. A=Ascending */
 		    resultSetInsertRank (zh, sort_info, psysno, score, 'A');
 		    count++;
+		    if (stop_flag)
+			break;
 		}
 		psysno = this_sys;
 		if (zh->m_staticrank)
@@ -900,7 +1025,7 @@ ZEBRA_RES resultSetRank(ZebraHandle zh, ZebraSet zebraSet,
 	/* no more items */
 	if (psysno)
 	{   /* we had - at least - one record */
-	    score = (*rc->calc)(handle, psysno, pstaticrank);
+	    score = (*rc->calc)(handle, psysno, pstaticrank, &stop_flag);
 	    /* insert the hit. A=Ascending */
 	    resultSetInsertRank(zh, sort_info, psysno, score, 'A');
 	    count++;
@@ -1068,7 +1193,7 @@ ZEBRA_RES zebra_snippets_hit_vector(ZebraHandle zh, const char *setname,
 				    zint sysno, zebra_snippets *snippets)
 {
     ZebraSet sset = resultSetGet(zh, setname);
-    yaz_log(YLOG_LOG, "zebra_get_hit_vector setname=%s zysno=" ZINT_FORMAT,
+    yaz_log(YLOG_DEBUG, "zebra_get_hit_vector setname=%s zysno=" ZINT_FORMAT,
 	    setname, sysno);
     if (!sset)
 	return ZEBRA_FAIL;
@@ -1078,8 +1203,8 @@ ZEBRA_RES zebra_snippets_hit_vector(ZebraHandle zh, const char *setname,
 	NMEM nmem = nmem_create();
 	struct it_key key;
 	RSET rsets[2], rset_comb;
-	RSET rset_temp = rstemp_create(nmem, kc, kc->scope, 
-				       res_get (zh->res, "setTmpDir"),0 );
+	RSET rset_temp = rset_create_temp(nmem, kc, kc->scope, 
+                                          res_get (zh->res, "setTmpDir"),0 );
 	
 	TERMID termid;
 	RSFD rsfd = rset_open(rset_temp, RSETF_WRITE);
@@ -1095,7 +1220,7 @@ ZEBRA_RES zebra_snippets_hit_vector(ZebraHandle zh, const char *setname,
 	rsets[0] = rset_temp;
 	rsets[1] = rset_dup(sset->rset);
 	
-	rset_comb = rsmulti_and_create(nmem, kc, kc->scope, 2, rsets);
+	rset_comb = rset_create_and(nmem, kc, kc->scope, 2, rsets);
 
 	rsfd = rset_open(rset_comb, RSETF_READ);
 
@@ -1107,7 +1232,6 @@ ZEBRA_RES zebra_snippets_hit_vector(ZebraHandle zh, const char *setname,
 		for (ol = termid->ol; ol; ol = ol->next)
 		{
 		    zebra_snippets_append(snippets, key.mem[key.len-1],
-					  termid->reg_type,
 					  ol->ord, termid->name);
 		}
 	    }
@@ -1116,7 +1240,16 @@ ZEBRA_RES zebra_snippets_hit_vector(ZebraHandle zh, const char *setname,
 	
 	rset_delete(rset_comb);
 	nmem_destroy(nmem);
+	kc->dec(kc);
     }
     return ZEBRA_OK;
 }
 
+/*
+ * Local variables:
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ * vim: shiftwidth=4 tabstop=8 expandtab
+ */
+