X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=src%2Frelevance.c;h=35b7d8312c040b677350d28a3485addcea460f7a;hb=c71d13caac58c9b17fccb3b00357db66066d4bb6;hp=7603a7a928c2422941e9f5340fe89b55d266cf51;hpb=beda9709478c02182ceadb5f0526c32d8986c039;p=pazpar2-moved-to-github.git diff --git a/src/relevance.c b/src/relevance.c index 7603a7a..35b7d83 100644 --- a/src/relevance.c +++ b/src/relevance.c @@ -1,5 +1,5 @@ /* This file is part of Pazpar2. - Copyright (C) 2006-2009 Index Data + Copyright (C) 2006-2010 Index Data Pazpar2 is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free @@ -21,18 +21,19 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA #include #endif +#include #include #include #include "relevance.h" -#include "pazpar2.h" +#include "session.h" struct relevance { int *doc_frequency_vec; int vec_len; struct word_entry *entries; - pp2_charset_t pct; + pp2_relevance_token_t prt; NMEM nmem; }; @@ -67,7 +68,8 @@ int word_entry_match(struct word_entry *entries, const char *norm_str) return 0; } -static struct word_entry *build_word_entries(pp2_charset_t pct, NMEM nmem, +static struct word_entry *build_word_entries(pp2_relevance_token_t prt, + NMEM nmem, const char **terms) { int termno = 1; /* >0 signals THERE is an entry */ @@ -76,34 +78,46 @@ static struct word_entry *build_word_entries(pp2_charset_t pct, NMEM nmem, for (; *p; p++) { - pp2_relevance_token_t prt = pp2_relevance_tokenize(pct, *p); const char *norm_str; + pp2_relevance_first(prt, *p, 0); while ((norm_str = pp2_relevance_token_next(prt))) add_word_entry(nmem, &entries, norm_str, termno); - - pp2_relevance_token_destroy(prt); - termno++; } return entries; } void relevance_countwords(struct relevance *r, struct record_cluster *cluster, - const char *words, int multiplier) + const char *words, int multiplier, const char *name) { - pp2_relevance_token_t prt = pp2_relevance_tokenize(r->pct, words); - + int *mult = cluster->term_frequency_vec_tmp; const char *norm_str; - - while ((norm_str = pp2_relevance_token_next(prt))) + int i, length = 0; + + pp2_relevance_first(r->prt, words, 0); + for (i = 1; i < r->vec_len; i++) + mult[i] = 0; + + while ((norm_str = pp2_relevance_token_next(r->prt))) { int res = word_entry_match(r->entries, norm_str); if (res) - cluster->term_frequency_vec[res] += multiplier; - cluster->term_frequency_vec[0]++; + { + assert(res < r->vec_len); + mult[res] += multiplier; + } + length++; + } + + for (i = 1; i < r->vec_len; i++) + { + if (length > 0) /* only add if non-empty */ + cluster->term_frequency_vecf[i] += (double) mult[i] / length; + cluster->term_frequency_vec[i] += mult[i]; } - pp2_relevance_token_destroy(prt); + + cluster->term_frequency_vec[0] += length; } struct relevance *relevance_create(pp2_charset_t pct, @@ -119,17 +133,44 @@ struct relevance *relevance_create(pp2_charset_t pct, res->doc_frequency_vec = nmem_malloc(nmem, res->vec_len * sizeof(int)); memset(res->doc_frequency_vec, 0, res->vec_len * sizeof(int)); res->nmem = nmem; - res->entries = build_word_entries(pct, nmem, terms); - res->pct = pct; + res->prt = pp2_relevance_tokenize(pct); + res->entries = build_word_entries(res->prt, nmem, terms); return res; } +void relevance_destroy(struct relevance **rp) +{ + if (*rp) + { + pp2_relevance_token_destroy((*rp)->prt); + *rp = 0; + } +} + void relevance_newrec(struct relevance *r, struct record_cluster *rec) { if (!rec->term_frequency_vec) { - rec->term_frequency_vec = nmem_malloc(r->nmem, r->vec_len * sizeof(int)); - memset(rec->term_frequency_vec, 0, r->vec_len * sizeof(int)); + int i; + + // term frequency [1,..] . [0] is total length of all fields + rec->term_frequency_vec = + nmem_malloc(r->nmem, + r->vec_len * sizeof(*rec->term_frequency_vec)); + for (i = 0; i < r->vec_len; i++) + rec->term_frequency_vec[i] = 0; + + // term frequency divided by length of field [1,...] + rec->term_frequency_vecf = + nmem_malloc(r->nmem, + r->vec_len * sizeof(*rec->term_frequency_vecf)); + for (i = 0; i < r->vec_len; i++) + rec->term_frequency_vecf[i] = 0.0; + + // for relevance_countwords (so we don't have to xmalloc/xfree) + rec->term_frequency_vec_tmp = + nmem_malloc(r->nmem, + r->vec_len * sizeof(*rec->term_frequency_vec_tmp)); } } @@ -151,7 +192,7 @@ void relevance_prepare_read(struct relevance *rel, struct reclist *reclist) int i; float *idfvec = xmalloc(rel->vec_len * sizeof(float)); - reclist_rewind(reclist); + reclist_enter(reclist); // Calculate document frequency vector for each term. for (i = 1; i < rel->vec_len; i++) { @@ -182,14 +223,22 @@ void relevance_prepare_read(struct relevance *rel, struct reclist *reclist) for (t = 1; t < rel->vec_len; t++) { float termfreq; - if (!rec->term_frequency_vec[0]) - break; - termfreq = (float) rec->term_frequency_vec[t] / rec->term_frequency_vec[0]; +#if 1 + termfreq = (float) rec->term_frequency_vecf[t]; +#else + if (rec->term_frequency_vec[0]) + { + termfreq = (float) + rec->term_frequency_vec[t] / rec->term_frequency_vec[0] ; + } + else + termfreq = 0.0; +#endif relevance += 100000 * (termfreq * idfvec[t] + 0.0000005); } - rec->relevance = relevance; + rec->relevance_score = relevance; } - reclist_rewind(reclist); + reclist_leave(reclist); xfree(idfvec); }