From 80eb0a73f541530f383715296203036b3adbacc2 Mon Sep 17 00:00:00 2001 From: Sebastian Hammer Date: Tue, 11 Jun 1996 10:54:15 +0000 Subject: [PATCH] Relevance work --- index/zrpn.c | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++++------ rset/rsrel.c | 57 ++++++++++++++++++++++++++++++++++++++---------------- 2 files changed, 95 insertions(+), 23 deletions(-) diff --git a/index/zrpn.c b/index/zrpn.c index bc8410b..7ef2f44 100644 --- a/index/zrpn.c +++ b/index/zrpn.c @@ -4,7 +4,10 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: zrpn.c,v $ - * Revision 1.50 1996-06-07 08:51:53 adam + * Revision 1.51 1996-06-11 10:54:15 quinn + * Relevance work + * + * Revision 1.50 1996/06/07 08:51:53 adam * Bug fix: Character mapping was broken (introducued by last revision). * * Revision 1.49 1996/06/04 10:18:11 adam @@ -491,17 +494,25 @@ static RSET rset_trunc (ISAM isam, ISAM_P *isam_p, int no) return rset_trunc_r (isam, isam_p, 0, no, 100); } -struct grep_info { - ISAM_P *isam_p_buf; - int isam_p_size; - int isam_p_indx; -}; +#define TERM_COUNT + +struct grep_info { +#ifdef TERM_COUNT + int *term_no; +#endif + ISAM_P *isam_p_buf; + int isam_p_size; + int isam_p_indx; +}; static void add_isam_p (const char *info, struct grep_info *p) { if (p->isam_p_indx == p->isam_p_size) { ISAM_P *new_isam_p_buf; +#ifdef TERM_COUNT + int *new_term_no; +#endif p->isam_p_size = 2*p->isam_p_size + 100; new_isam_p_buf = xmalloc (sizeof(*new_isam_p_buf) * @@ -513,6 +524,18 @@ static void add_isam_p (const char *info, struct grep_info *p) xfree (p->isam_p_buf); } p->isam_p_buf = new_isam_p_buf; + +#ifdef TERM_COUNT + new_term_no = xmalloc (sizeof(*new_term_no) * + p->isam_p_size); + if (p->term_no) + { + memcpy (new_term_no, p->isam_p_buf, + p->isam_p_indx * sizeof(*p->term_no)); + xfree (p->term_no); + } + p->term_no = new_term_no; +#endif } assert (*info == sizeof(*p->isam_p_buf)); memcpy (p->isam_p_buf + p->isam_p_indx, info+1, sizeof(*p->isam_p_buf)); @@ -868,11 +891,13 @@ static RSET rpn_search_APT_relevance (ZServerInfo *zi, struct grep_info grep_info; char *p0 = termz; RSET result; + int term_index = 0; parms.key_size = sizeof(struct it_key); parms.max_rec = 100; parms.cmp = key_compare; parms.is = zi->wordIsam; + parms.no_terms = 0; if (zapt->term->which != Z_Term_general) { @@ -881,6 +906,9 @@ static RSET rpn_search_APT_relevance (ZServerInfo *zi, } trans_term (zi, zapt, termz); +#ifdef TERM_COUNT + grep_info.term_no = 0; +#endif grep_info.isam_p_indx = 0; grep_info.isam_p_size = 0; grep_info.isam_p_buf = NULL; @@ -914,13 +942,22 @@ static RSET rpn_search_APT_relevance (ZServerInfo *zi, if (field_term (zi, zapt, term_sub, 'w', attributeSet, &grep_info, num_bases, basenames)) return NULL; +#ifdef TERM_COUNT + for (; term_index < grep_info.isam_p_indx; term_index++) + grep_info.term_no[term_index] = parms.no_terms; + parms.no_terms++; +#endif } + parms.term_no = grep_info.term_no; parms.isam_positions = grep_info.isam_p_buf; parms.no_isam_positions = grep_info.isam_p_indx; if (grep_info.isam_p_indx > 0) result = rset_create (rset_kind_relevance, &parms); else result = rset_create (rset_kind_null, NULL); +#ifdef TERM_COUNT + xfree(grep_info.term_no); +#endif xfree (grep_info.isam_p_buf); return result; } @@ -942,6 +979,9 @@ static RSET rpn_search_APT_cphrase (ZServerInfo *zi, } trans_term (zi, zapt, termz); +#ifdef TERM_COUNT + grep_info.term_no = 0; +#endif grep_info.isam_p_indx = 0; grep_info.isam_p_size = 0; grep_info.isam_p_buf = NULL; @@ -960,6 +1000,9 @@ static RSET rpn_search_APT_cphrase (ZServerInfo *zi, else result = rset_trunc (zi->wordIsam, grep_info.isam_p_buf, grep_info.isam_p_indx); +#ifdef TERM_COUNT + xfree(grep_info.term_no); +#endif xfree (grep_info.isam_p_buf); return result; } @@ -1069,6 +1112,9 @@ static RSET rpn_search_APT_phrase (ZServerInfo *zi, } trans_term (zi, zapt, termz); +#ifdef TERM_COUNT + grep_info.term_no = 0; +#endif grep_info.isam_p_size = 0; grep_info.isam_p_buf = NULL; @@ -1123,6 +1169,9 @@ static RSET rpn_search_APT_phrase (ZServerInfo *zi, if (++rset_no >= sizeof(rset)/sizeof(*rset)) break; } +#ifdef TERM_COUNT + xfree(grep_info.term_no); +#endif xfree (grep_info.isam_p_buf); if (rset_no == 0) return rset_create (rset_kind_null, NULL); diff --git a/rset/rsrel.c b/rset/rsrel.c index 2f7af4c..16efb8e 100644 --- a/rset/rsrel.c +++ b/rset/rsrel.c @@ -4,7 +4,10 @@ * Sebastian Hammer, Adam Dickmeiss * * $Log: rsrel.c,v $ - * Revision 1.9 1995-12-11 09:15:26 adam + * Revision 1.10 1996-06-11 10:54:29 quinn + * Relevance work + * + * Revision 1.9 1995/12/11 09:15:26 adam * New set types: sand/sor/snot - ranked versions of and/or/not in * ranked/semi-ranked result sets. * Note: the snot not finished yet. @@ -138,14 +141,17 @@ static int qcomp (const void *p1, const void *p2) qsort_info->key_buf + i2*qsort_info->key_size); } +#define SCORE_SHOW 0.0 /* base score for showing up */ +#define SCORE_COOC 0.3 /* component dependent on co-oc */ +#define SCORE_DYN (1-(SCORE_SHOW+SCORE_COOC)) /* dynamic component of score */ + static void relevance (struct rset_rel_info *info, rset_relevance_parms *parms) { char **isam_buf; char *isam_tmp_buf; int *isam_r; - int *max_tf; + int *max_tf, *tf; ISPT *isam_pt; - double *wgt; int i; logf (LOG_DEBUG, "relevance"); @@ -153,31 +159,46 @@ static void relevance (struct rset_rel_info *info, rset_relevance_parms *parms) isam_r = xmalloc (sizeof (*isam_r) * parms->no_isam_positions); isam_pt = xmalloc (sizeof (*isam_pt) * parms->no_isam_positions); isam_tmp_buf = xmalloc (info->key_size); - max_tf = xmalloc (sizeof (*max_tf) * parms->no_isam_positions); - wgt = xmalloc (sizeof (*wgt) * parms->no_isam_positions); + max_tf = xmalloc (sizeof (*max_tf) * parms->no_terms); + tf = xmalloc (sizeof (*tf) * parms->no_terms); + for (i = 0; ino_terms; i++) + max_tf[i] = 0; for (i = 0; ino_isam_positions; i++) { isam_buf[i] = xmalloc (info->key_size); isam_pt[i] = is_position (parms->is, parms->isam_positions[i]); - max_tf [i] = is_numkeys (isam_pt[i]); + max_tf [parms->term_no[i]] = is_numkeys (isam_pt[i]); isam_r[i] = is_readkey (isam_pt[i], isam_buf[i]); logf (LOG_DEBUG, "max tf %d = %d", i, max_tf[i]); } while (1) { - int min = -1, i; + int min = -1, i, r; double score; + int co_oc, last_term; /* Number of co-occurrences */ + last_term = -1; /* find min with lowest sysno */ for (i = 0; ino_isam_positions; i++) + { if (isam_r[i] && - (min < 0 || (*parms->cmp)(isam_buf[i], isam_buf[min]) < 2)) + (min < 0 || (r = (*parms->cmp)(isam_buf[i], isam_buf[min])) < 2)) + { min = i; + co_oc = 1; + } + else if (!r && last_term != parms->term_no[i]) /* new occurrence */ + co_oc++; + last_term = parms->term_no[i]; + } + if (min < 0) break; memcpy (isam_tmp_buf, isam_buf[min], info->key_size); /* calculate for all with those sysno */ + for (i = 0; i < parms->no_terms; i++) + tf[i] = 0; for (i = 0; ino_isam_positions; i++) { int r; @@ -186,26 +207,28 @@ static void relevance (struct rset_rel_info *info, rset_relevance_parms *parms) r = (*parms->cmp)(isam_buf[i], isam_tmp_buf); else r = 2; +#if 0 if (r > 1 || r < -1) - wgt[i] = 0.0; - else + wgt[parms->term_no[i]] = 0.0; +#endif + if (r <= 1 && r >= -1) { - int tf = 0; do { - tf++; + tf[parms->term_no[i]]++; isam_r[i] = is_readkey (isam_pt[i], isam_buf[i]); } while (isam_r[i] && (*parms->cmp)(isam_buf[i], isam_tmp_buf) <= 1); - wgt[i] = 0.1+tf*0.9/max_tf[i]; } } /* calculate relevance value */ score = 0.0; - for (i = 0; ino_isam_positions; i++) - score += wgt[i]; + for (i = 0; ino_terms; i++) + if (tf[i]) + score += SCORE_SHOW + SCORE_COOC*co_oc/parms->no_terms + + SCORE_DYN*tf[i]/max_tf[i]; /* if value is in the top score, then save it - don't emit yet */ - add_rec (info, score/parms->no_isam_positions, isam_tmp_buf); + add_rec (info, score/parms->no_terms, isam_tmp_buf); } for (i = 0; ino_rec; i++) info->sysno_idx[i] = i; @@ -221,7 +244,7 @@ static void relevance (struct rset_rel_info *info, rset_relevance_parms *parms) xfree (isam_buf); xfree (isam_r); xfree (isam_pt); - xfree (wgt); + xfree(tf); } static void *r_create (const struct rset_control *sel, void *parms, -- 1.7.10.4