X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=src%2Frelevance.c;h=74f0e155f8628a7a40bb2b4b00317519f2796e38;hb=fb19bc317370b4f4b508088b034b4c2d19257042;hp=a002a7b631eec74f076d89b28821febe9e1f8e9c;hpb=446f32183265d59ee79e2859376c598fa24408e0;p=pazpar2-moved-to-github.git diff --git a/src/relevance.c b/src/relevance.c index a002a7b..74f0e15 100644 --- a/src/relevance.c +++ b/src/relevance.c @@ -1,5 +1,5 @@ /* This file is part of Pazpar2. - Copyright (C) 2006-2012 Index Data + Copyright (C) 2006-2013 Index Data Pazpar2 is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free @@ -28,16 +28,21 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA #include "relevance.h" #include "session.h" +#ifdef WIN32 +#define log2(x) (log(x)/log(2)) +#endif + struct relevance { int *doc_frequency_vec; int *term_frequency_vec_tmp; + int *term_pos; int vec_len; struct word_entry *entries; pp2_charset_token_t prt; int rank_cluster; - int follow_boost; - int lead_boost; + double follow_factor; + double lead_decay; int length_divide; NMEM nmem; }; @@ -46,13 +51,13 @@ struct word_entry { const char *norm_str; const char *display_str; int termno; - int follow_boost; char *ccl_field; struct word_entry *next; }; -static int word_entry_match(struct relevance *r, const char *norm_str, - const char *rank, int *mult) +static struct word_entry *word_entry_match(struct relevance *r, + const char *norm_str, + const char *rank, int *weight) { int i = 1; struct word_entry *entries = r->entries; @@ -60,11 +65,9 @@ static int word_entry_match(struct relevance *r, const char *norm_str, { if (*norm_str && !strcmp(norm_str, entries->norm_str)) { - int extra = r->follow_boost; - struct word_entry *e_follow = entries; const char *cp = 0; int no_read = 0; - sscanf(rank, "%d%n", mult, &no_read); + sscanf(rank, "%d%n", weight, &no_read); rank += no_read; while (*rank == ' ') rank++; @@ -72,75 +75,147 @@ static int word_entry_match(struct relevance *r, const char *norm_str, { if ((cp - rank) == strlen(entries->ccl_field) && memcmp(entries->ccl_field, rank, cp - rank) == 0) - *mult = atoi(cp + 1); + *weight = atoi(cp + 1); } - (*mult) += entries->follow_boost; - while ((e_follow = e_follow->next) != 0 && extra > 0) + return entries; + } + } + return 0; +} + +int relevance_snippet(struct relevance *r, + const char *words, const char *name, + WRBUF w_snippet) +{ + int no = 0; + const char *norm_str; + int highlight = 0; + + pp2_charset_token_first(r->prt, words, 0); + while ((norm_str = pp2_charset_token_next(r->prt))) + { + size_t org_start, org_len; + struct word_entry *entries = r->entries; + int i; + + pp2_get_org(r->prt, &org_start, &org_len); + for (; entries; entries = entries->next, i++) + { + if (*norm_str && !strcmp(norm_str, entries->norm_str)) + break; + } + if (entries) + { + if (!highlight) { - e_follow->follow_boost = extra--; + highlight = 1; + wrbuf_puts(w_snippet, ""); + no++; } - return entries->termno; } - entries->follow_boost = 0; + else + { + if (highlight) + { + highlight = 0; + wrbuf_puts(w_snippet, ""); + } + } + wrbuf_xmlputs_n(w_snippet, words + org_start, org_len); } - return 0; + if (highlight) + wrbuf_puts(w_snippet, ""); + if (no) + { + yaz_log(YLOG_DEBUG, "SNIPPET match: %s", wrbuf_cstr(w_snippet)); + } + return no; } void relevance_countwords(struct relevance *r, struct record_cluster *cluster, const char *words, const char *rank, const char *name) { - int *mult = r->term_frequency_vec_tmp; + int *w = r->term_frequency_vec_tmp; const char *norm_str; int i, length = 0; - int lead_mult = r->lead_boost; + double lead_decay = r->lead_decay; struct word_entry *e; - WRBUF w = cluster->relevance_explain1; + WRBUF wr = cluster->relevance_explain1; + int printed_about_field = 0; pp2_charset_token_first(r->prt, words, 0); for (e = r->entries, i = 1; i < r->vec_len; i++, e = e->next) { - mult[i] = 0; - e->follow_boost = 0; + w[i] = 0; + r->term_pos[i] = 0; } assert(rank); while ((norm_str = pp2_charset_token_next(r->prt))) { - int local_mult = 0; - int res = word_entry_match(r, norm_str, rank, &local_mult); - if (res) + int local_weight = 0; + e = word_entry_match(r, norm_str, rank, &local_weight); + if (e) { + int res = e->termno; + int j; + + if (!printed_about_field) + { + printed_about_field = 1; + wrbuf_printf(wr, "field=%s content=", name); + if (strlen(words) > 50) + { + wrbuf_xmlputs_n(wr, words, 49); + wrbuf_puts(wr, " ..."); + } + else + wrbuf_xmlputs(wr, words); + wrbuf_puts(wr, ";\n"); + } assert(res < r->vec_len); - mult[res] += local_mult + lead_mult; + w[res] += local_weight / (1 + log2(1 + lead_decay * length)); + wrbuf_printf(wr, "%s: w[%d] += w(%d) / " + "(1+log2(1+lead_decay(%f) * length(%d)));\n", + e->display_str, res, local_weight, lead_decay, length); + j = res - 1; + if (j > 0 && r->term_pos[j]) + { + int d = length + 1 - r->term_pos[j]; + wrbuf_printf(wr, "%s: w[%d] += w[%d](%d) * follow(%f) / " + "(1+log2(d(%d));\n", + e->display_str, res, res, w[res], + r->follow_factor, d); + w[res] += w[res] * r->follow_factor / (1 + log2(d)); + } + for (j = 0; j < r->vec_len; j++) + r->term_pos[j] = j < res ? 0 : length + 1; } - if (lead_mult > 0) - --lead_mult; length++; } for (e = r->entries, i = 1; i < r->vec_len; i++, e = e->next) { - if (length == 0 || mult[i] == 0) + if (length == 0 || w[i] == 0) continue; - wrbuf_printf(w, "%s: field=%s vecf[%d] += mult(%d)", - e->display_str, name, i, mult[i]); + wrbuf_printf(wr, "%s: tf[%d] += w[%d](%d)", e->display_str, i, i, w[i]); switch (r->length_divide) { case 0: - wrbuf_printf(w, ";\n"); - cluster->term_frequency_vecf[i] += (double) mult[i]; + cluster->term_frequency_vecf[i] += (double) w[i]; break; case 1: - wrbuf_printf(w, " / log2(1+length(%d));\n", length); + wrbuf_printf(wr, " / log2(1+length(%d))", length); cluster->term_frequency_vecf[i] += - (double) mult[i] / log2(1 + length); + (double) w[i] / log2(1 + length); break; case 2: - wrbuf_printf(w, " / length(%d);\n", length); - cluster->term_frequency_vecf[i] += (double) mult[i] / length; + wrbuf_printf(wr, " / length(%d)", length); + cluster->term_frequency_vecf[i] += (double) w[i] / length; } - cluster->term_frequency_vec[i] += mult[i]; + cluster->term_frequency_vec[i] += w[i]; + wrbuf_printf(wr, " (%f);\n", cluster->term_frequency_vecf[i]); } cluster->term_frequency_vec[0] += length; @@ -189,36 +264,47 @@ static void pull_terms(struct relevance *res, struct ccl_rpn_node *n) break; } } +void relevance_clear(struct relevance *r) +{ + if (r) + { + int i; + for (i = 0; i < r->vec_len; i++) + r->doc_frequency_vec[i] = 0; + } +} struct relevance *relevance_create_ccl(pp2_charset_fact_t pft, struct ccl_rpn_node *query, int rank_cluster, - int follow_boost, int lead_boost, + double follow_factor, double lead_decay, int length_divide) { NMEM nmem = nmem_create(); struct relevance *res = nmem_malloc(nmem, sizeof(*res)); - int i; res->nmem = nmem; res->entries = 0; res->vec_len = 1; res->rank_cluster = rank_cluster; - res->follow_boost = follow_boost; - res->lead_boost = lead_boost; + res->follow_factor = follow_factor; + res->lead_decay = lead_decay; res->length_divide = length_divide; res->prt = pp2_charset_token_create(pft, "relevance"); pull_terms(res, query); res->doc_frequency_vec = nmem_malloc(nmem, res->vec_len * sizeof(int)); - for (i = 0; i < res->vec_len; i++) - res->doc_frequency_vec[i] = 0; // worker array res->term_frequency_vec_tmp = nmem_malloc(res->nmem, res->vec_len * sizeof(*res->term_frequency_vec_tmp)); + + res->term_pos = + nmem_malloc(res->nmem, res->vec_len * sizeof(*res->term_pos)); + + relevance_clear(res); return res; } @@ -232,26 +318,35 @@ void relevance_destroy(struct relevance **rp) } } -void relevance_newrec(struct relevance *r, struct record_cluster *rec) +void relevance_mergerec(struct relevance *r, struct record_cluster *dst, + const struct record_cluster *src) { - if (!rec->term_frequency_vec) - { - int i; + int i; - // term frequency [1,..] . [0] is total length of all fields - rec->term_frequency_vec = - nmem_malloc(r->nmem, - r->vec_len * sizeof(*rec->term_frequency_vec)); - for (i = 0; i < r->vec_len; i++) - rec->term_frequency_vec[i] = 0; + for (i = 0; i < r->vec_len; i++) + dst->term_frequency_vec[i] += src->term_frequency_vec[i]; - // term frequency divided by length of field [1,...] - rec->term_frequency_vecf = - nmem_malloc(r->nmem, - r->vec_len * sizeof(*rec->term_frequency_vecf)); - for (i = 0; i < r->vec_len; i++) - rec->term_frequency_vecf[i] = 0.0; - } + for (i = 0; i < r->vec_len; i++) + dst->term_frequency_vecf[i] += src->term_frequency_vecf[i]; +} + +void relevance_newrec(struct relevance *r, struct record_cluster *rec) +{ + int i; + + // term frequency [1,..] . [0] is total length of all fields + rec->term_frequency_vec = + nmem_malloc(r->nmem, + r->vec_len * sizeof(*rec->term_frequency_vec)); + for (i = 0; i < r->vec_len; i++) + rec->term_frequency_vec[i] = 0; + + // term frequency divided by length of field [1,...] + rec->term_frequency_vecf = + nmem_malloc(r->nmem, + r->vec_len * sizeof(*rec->term_frequency_vecf)); + for (i = 0; i < r->vec_len; i++) + rec->term_frequency_vecf[i] = 0.0; } void relevance_donerecord(struct relevance *r, struct record_cluster *cluster) @@ -295,6 +390,7 @@ void relevance_prepare_read(struct relevance *rel, struct reclist *reclist) break; w = rec->relevance_explain2; wrbuf_rewind(w); + wrbuf_puts(w, "relevance = 0;\n"); for (i = 1; i < rel->vec_len; i++) { float termfreq = (float) rec->term_frequency_vecf[i]; @@ -303,7 +399,7 @@ void relevance_prepare_read(struct relevance *rel, struct reclist *reclist) wrbuf_printf(w, "idf[%d] = log(((1 + total(%d))/termoccur(%d));\n", i, rel->doc_frequency_vec[0], rel->doc_frequency_vec[i]); - wrbuf_printf(w, "%s: relevance += 100000 * vecf[%d](%f) * " + wrbuf_printf(w, "%s: relevance += 100000 * tf[%d](%f) * " "idf[%d](%f) (%d);\n", e->display_str, i, termfreq, i, idfvec[i], add); relevance += add;