Estimating the term occurrences the same way as total hits, when estimatehits

[idzebra-moved-to-github.git] / index / zsets.c

diff --git a/index/zsets.c b/index/zsets.c
index a26ac40..929db7f 100644 (file)
--- a/index/zsets.c
+++ b/index/zsets.c
@@ -1,4 +1,4 @@
-/* $Id: zsets.c,v 1.50 2004-08-04 08:35:24 adam Exp $
+/* $Id: zsets.c,v 1.56 2004-08-19 14:47:06 heikki Exp $
    Copyright (C) 1995,1996,1997,1998,1999,2000,2001,2002,2003,2004
    Index Data Aps
 
@@ -46,7 +46,7 @@ struct zebra_set {
     char *name;
     RSET rset;
     NMEM nmem;
-    int hits;
+    zint hits;
     int num_bases;
     char **basenames;
     Z_RPNQuery *rpn;
@@ -111,6 +111,7 @@ void resultSetAddTerm (ZebraHandle zh, ZebraSet s, int reg_type,
                       const char *db, int set,
                       int use, const char *term)
 {
+    assert(zh); /* compiler shut up */
     if (!s->nmem)
        s->nmem = nmem_create ();
     if (!s->term_entries)
@@ -136,7 +137,7 @@ void resultSetAddTerm (ZebraHandle zh, ZebraSet s, int reg_type,
 
 
 int zebra_resultSetTerms (ZebraHandle zh, const char *setname, 
-                          int no, int *count, 
+                          int no, zint *count, 
                           int *type, char *out, size_t *len)
 {
     ZebraSet s = resultSetGet (zh, setname);
@@ -432,6 +433,7 @@ ZebraPosSet zebraPosSetCreate (ZebraHandle zh, const char *name,
 
 void zebraPosSetDestroy (ZebraHandle zh, ZebraPosSet records, int num)
 {
+    assert(zh); /* compiler shut up about unused arg */
     xfree (records);
 }
 
@@ -521,6 +523,7 @@ void resultSetInsertRank (ZebraHandle zh, struct zset_sort_info *sort_info,
 {
     struct zset_sort_entry *new_entry = NULL;
     int i, j;
+    assert(zh); /* compiler shut up about unused arg */
 
     i = sort_info->num_entries;
     while (--i >= 0)
@@ -617,6 +620,7 @@ void resultSetSortSingle (ZebraHandle zh, NMEM nmem,
     RSFD rfd;
 
     yaz_log (LOG_LOG, "resultSetSortSingle start");
+    assert(nmem); /* compiler shut up about unused param */
     sset->sort_info->num_entries = 0;
 
     sset->hits = 0;
@@ -697,7 +701,7 @@ void resultSetSortSingle (ZebraHandle zh, NMEM nmem,
     rset_close (rset, rfd);
 
     for (i = 0; i < rset->no_rset_terms; i++)
-       yaz_log (LOG_LOG, "term=\"%s\" nn=%d type=%s count=%d",
+       yaz_log (LOG_LOG, "term=\"%s\" nn=" ZINT_FORMAT " type=%s count=" ZINT_FORMAT,
                  rset->rset_terms[i]->name,
                  rset->rset_terms[i]->nn,
                  rset->rset_terms[i]->flags,
@@ -718,7 +722,7 @@ RSET resultSetRef (ZebraHandle zh, const char *resultSetId)
 
 void resultSetRank (ZebraHandle zh, ZebraSet zebraSet, RSET rset)
 {
-    int kno = 0;
+    zint kno = 0;
     struct it_key key;
     RSFD rfd;
     int term_index, i;
@@ -726,6 +730,10 @@ void resultSetRank (ZebraHandle zh, ZebraSet zebraSet, RSET rset)
     struct rank_control *rc;
     struct zset_sort_info *sort_info;
     const char *rank_handler_name = res_get_def(zh->res, "rank", "rank-1");
+    double cur,tot; 
+    zint est=-2; /* -2 not done, -1 can't do, >0 actual estimate*/
+    zint esthits;
+    double ratio;
 
     sort_info = zebraSet->sort_info;
     sort_info->num_entries = 0;
@@ -753,6 +761,8 @@ void resultSetRank (ZebraHandle zh, ZebraSet zebraSet, RSET rset)
        void *handle =
            (*rc->begin) (zh->reg, rank_class->class_handle, rset);
        (zebraSet->hits)++;
+    esthits=atoi(res_get_def(zh->res,"estimatehits","0"));
+    if (!esthits) est=-1; /* can not do */
        do
        {
 #if IT_KEY_NEW
@@ -770,8 +780,30 @@ void resultSetRank (ZebraHandle zh, ZebraSet zebraSet, RSET rset)
                psysno = this_sys;
            }
            (*rc->add) (handle, this_sys, term_index);
+        if ( (est==-2) && (zebraSet->hits==esthits))
+        { /* time to estimate the hits */
+            rset_pos(rset,rfd,&cur,&tot); 
+            if (tot>0) {
+                ratio=cur/tot;
+                est=(zint)(0.5+zebraSet->hits/ratio);
+                logf(LOG_LOG, "Estimating hits (%s) "
+                              "%0.1f->"ZINT_FORMAT
+                              "; %0.1f->"ZINT_FORMAT,
+                              rset->control->desc,
+                              cur, zebraSet->hits,
+                              tot,est);
+                i=0; /* round to 3 significant digits */
+                while (est>1000) {
+                    est/=10;
+                    i++;
+                }
+                while (i--) est*=10;
+                zebraSet->hits=est;
+            }
+        }
        }
-       while (rset_read (rset, rfd, &key, &term_index));
+       while (rset_read (rset, rfd, &key, &term_index) && (est<0) );
+           
        score = (*rc->calc) (handle, psysno);
        resultSetInsertRank (zh, sort_info, psysno, score, 'A');
        (*rc->end) (zh->reg, handle);
@@ -779,13 +811,20 @@ void resultSetRank (ZebraHandle zh, ZebraSet zebraSet, RSET rset)
     rset_close (rset, rfd);
 
     for (i = 0; i < rset->no_rset_terms; i++)
-       yaz_log (LOG_LOG, "term=\"%s\" nn=%d type=%s count=%d",
+    {
+        if (est>0)
+            rset->rset_terms[i]->count = 
+                est=(zint)(rset->rset_terms[i]->count/ratio);
+        yaz_log (LOG_LOG, "term=\"%s\" nn=" ZINT_FORMAT 
+                    " type=%s count=" ZINT_FORMAT,
                  rset->rset_terms[i]->name,
                  rset->rset_terms[i]->nn,
                  rset->rset_terms[i]->flags,
                  rset->rset_terms[i]->count);
+    }
     
-    yaz_log (LOG_LOG, "%d keys, %d distinct sysnos", kno, zebraSet->hits);
+    yaz_log (LOG_LOG, ZINT_FORMAT " keys, "ZINT_FORMAT" distinct sysnos", 
+                    kno, zebraSet->hits);
 }
 
 ZebraRankClass zebraRankLookup (ZebraHandle zh, const char *name)