X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=index%2Fzsets.c;h=08a2cf4d9257c61dbb0d7bb47a8b32ca229345fe;hb=6cb032fd2ece4742e3516e0941507143f677e5cd;hp=78b69d6ac9d36d33acc0c4f6c0d251c39fa797ea;hpb=daae305f8a0c450f4f220d82a509cfcebfff4c52;p=idzebra-moved-to-github.git diff --git a/index/zsets.c b/index/zsets.c index 78b69d6..08a2cf4 100644 --- a/index/zsets.c +++ b/index/zsets.c @@ -1,6 +1,6 @@ -/* $Id: zsets.c,v 1.70 2004-11-15 22:44:33 adam Exp $ - Copyright (C) 1995,1996,1997,1998,1999,2000,2001,2002,2003,2004 - Index Data Aps +/* $Id: zsets.c,v 1.82 2005-05-11 12:39:37 adam Exp $ + Copyright (C) 1995-2005 + Index Data ApS This file is part of the Zebra server. @@ -30,6 +30,7 @@ Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA #endif #include "index.h" +#include #include #define SORT_IDX_ENTRYSIZE 64 @@ -56,6 +57,10 @@ struct zebra_set { int term_entries_max; struct zebra_set *next; int locked; + + zint cache_position; /* last position */ + RSFD cache_rfd; /* rfd (NULL if not existing) */ + zint cache_psysno; /* sysno for last position */ }; struct zset_sort_entry { @@ -71,25 +76,88 @@ struct zset_sort_info { struct zset_sort_entry **entries; }; -ZebraSet resultSetAddRPN (ZebraHandle zh, NMEM m, - Z_RPNQuery *rpn, int num_bases, - char **basenames, - const char *setname) +static int log_level_set=0; +static int log_level_sort=0; +static int log_level_searchhits=0; +static int log_level_searchterms=0; +static int log_level_resultsets=0; + +static void loglevels() +{ + if (log_level_set) + return; + log_level_sort = yaz_log_module_level("sorting"); + log_level_searchhits = yaz_log_module_level("searchhits"); + log_level_searchterms = yaz_log_module_level("searchterms"); + log_level_resultsets = yaz_log_module_level("resultsets"); + log_level_set = 1; +} + +ZEBRA_RES resultSetSearch(ZebraHandle zh, NMEM nmem, NMEM rset_nmem, + Z_RPNQuery *rpn, ZebraSet sset) +{ + RSET rset = 0; + oident *attrset; + Z_SortKeySpecList *sort_sequence; + int sort_status, i; + ZEBRA_RES res = ZEBRA_OK; + + zh->hits = 0; + + sort_sequence = (Z_SortKeySpecList *) + nmem_malloc(nmem, sizeof(*sort_sequence)); + sort_sequence->num_specs = 10; /* FIXME - Hard-coded number */ + sort_sequence->specs = (Z_SortKeySpec **) + nmem_malloc(nmem, sort_sequence->num_specs * + sizeof(*sort_sequence->specs)); + for (i = 0; inum_specs; i++) + sort_sequence->specs[i] = 0; + + attrset = oid_getentbyoid (rpn->attributeSetId); + res = rpn_search_top(zh, rpn->RPNStructure, attrset->value, + nmem, rset_nmem, + sort_sequence, + sset->num_bases, sset->basenames, + &rset); + if (res != ZEBRA_OK) + { + sset->rset = 0; + return res; + } + for (i = 0; sort_sequence->specs[i]; i++) + ; + sort_sequence->num_specs = i; + if (!i) + { + res = resultSetRank (zh, sset, rset, rset_nmem); + } + else + { + res = resultSetSortSingle (zh, nmem, sset, rset, + sort_sequence, &sort_status); + } + sset->rset = rset; + return res; +} + + +ZEBRA_RES resultSetAddRPN (ZebraHandle zh, NMEM m, Z_RPNQuery *rpn, + int num_bases, char **basenames, + const char *setname) { ZebraSet zebraSet; int i; + ZEBRA_RES res; - zh->errCode = 0; - zh->errString = NULL; zh->hits = 0; zebraSet = resultSetAdd (zh, setname, 1); if (!zebraSet) - return 0; + return ZEBRA_FAIL; zebraSet->locked = 1; zebraSet->rpn = 0; zebraSet->nmem = m; - zebraSet->rset_nmem=nmem_create(); + zebraSet->rset_nmem = nmem_create(); zebraSet->num_bases = num_bases; zebraSet->basenames = @@ -97,16 +165,15 @@ ZebraSet resultSetAddRPN (ZebraHandle zh, NMEM m, for (i = 0; ibasenames[i] = nmem_strdup (zebraSet->nmem, basenames[i]); - - zebraSet->rset = rpn_search (zh, zebraSet->nmem, zebraSet->rset_nmem, - rpn, zebraSet->num_bases, - zebraSet->basenames, zebraSet->name, - zebraSet); + res = resultSetSearch(zh, zebraSet->nmem, zebraSet->rset_nmem, + rpn, zebraSet); zh->hits = zebraSet->hits; if (zebraSet->rset) zebraSet->rpn = rpn; zebraSet->locked = 0; - return zebraSet; + if (!zebraSet->rset) + return ZEBRA_FAIL; + return res; } void resultSetAddTerm (ZebraHandle zh, ZebraSet s, int reg_type, @@ -137,60 +204,6 @@ void resultSetAddTerm (ZebraHandle zh, ZebraSet s, int reg_type, (s->hits)++; } -#if 0 /* FIXME - Delete this, we don't count terms no more */ -int zebra_resultSetTerms (ZebraHandle zh, const char *setname, - int no, zint *count, - int *type, char *out, size_t *len) -{ - ZebraSet s = resultSetGet (zh, setname); - int no_max = 0; - - if (count) - *count = 0; - if (!s || !s->rset) - return 0; - no_max = s->rset->no_rset_terms; - if (no < 0 || no >= no_max) - return 0; - if (count) - *count = s->rset->rset_terms[no]->count; - if (type) - *type = s->rset->rset_terms[no]->type; - - if (out) - { - char *inbuf = s->rset->rset_terms[no]->name; - size_t inleft = strlen(inbuf); - size_t outleft = *len - 1; - int converted = 0; - - if (zh->iconv_from_utf8 != 0) - { - char *outbuf = out; - size_t ret; - - ret = yaz_iconv(zh->iconv_from_utf8, &inbuf, &inleft, - &outbuf, &outleft); - if (ret == (size_t)(-1)) - *len = 0; - else - *len = outbuf - out; - converted = 1; - } - if (!converted) - { - if (inleft > outleft) - inleft = outleft; - *len = inleft; - memcpy (out, inbuf, *len); - } - out[*len] = 0; - } - return no_max; -} - -#endif - ZebraSet resultSetAdd (ZebraHandle zh, const char *name, int ov) { ZebraSet s; @@ -199,13 +212,20 @@ ZebraSet resultSetAdd (ZebraHandle zh, const char *name, int ov) for (s = zh->sets; s; s = s->next) if (!strcmp (s->name, name)) break; + + if (!log_level_set) + loglevels(); if (s) { - yaz_log(LOG_DEBUG, "updating result set %s", name); + yaz_log(log_level_resultsets, "updating result set %s", name); if (!ov || s->locked) return NULL; if (s->rset) + { + if (s->cache_rfd) + rset_close(s->cache_rfd); rset_delete (s->rset); + } if (s->rset_nmem) nmem_destroy (s->rset_nmem); if (s->nmem) @@ -215,7 +235,7 @@ ZebraSet resultSetAdd (ZebraHandle zh, const char *name, int ov) { const char *sort_max_str = zebra_get_resource(zh, "sortmax", "1000"); - yaz_log(LOG_DEBUG, "adding result set %s", name); + yaz_log(log_level_resultsets, "adding result set %s", name); s = (ZebraSet) xmalloc (sizeof(*s)); s->next = zh->sets; zh->sets = s; @@ -241,9 +261,11 @@ ZebraSet resultSetAdd (ZebraHandle zh, const char *name, int ov) s->term_entries = 0; s->hits = 0; s->rset = 0; - s->rset_nmem=0; + s->rset_nmem = 0; s->nmem = 0; s->rpn = 0; + s->cache_position = 0; + s->cache_rfd = 0; return s; } @@ -257,12 +279,10 @@ ZebraSet resultSetGet (ZebraHandle zh, const char *name) if (!s->term_entries && !s->rset && s->rpn) { NMEM nmem = nmem_create (); - yaz_log(LOG_LOG, "research %s", name); + yaz_log(log_level_resultsets, "research %s", name); if (!s->rset_nmem) s->rset_nmem=nmem_create(); - s->rset = - rpn_search (zh, nmem, s->rset_nmem, s->rpn, s->num_bases, - s->basenames, s->name, s); + resultSetSearch(zh, nmem, s->rset_nmem, s->rpn, s); nmem_destroy (nmem); } return s; @@ -274,11 +294,18 @@ void resultSetInvalidate (ZebraHandle zh) { ZebraSet s = zh->sets; + yaz_log(log_level_resultsets, "invalidating result sets"); for (; s; s = s->next) { if (s->rset) + { + if (s->cache_rfd) + rset_close(s->cache_rfd); rset_delete (s->rset); + } s->rset = 0; + s->cache_rfd = 0; + s->cache_position = 0; if (s->rset_nmem) nmem_destroy(s->rset_nmem); s->rset_nmem=0; @@ -293,8 +320,6 @@ void resultSetDestroy (ZebraHandle zh, int num, char **names,int *statuses) if (statuses) for (i = 0; ierrCode = 0; - zh->errString = NULL; while (*ss) { int i = -1; @@ -321,7 +346,11 @@ void resultSetDestroy (ZebraHandle zh, int num, char **names,int *statuses) if (s->nmem) nmem_destroy (s->nmem); if (s->rset) + { + if (s->cache_rfd) + rset_close(s->cache_rfd); rset_delete (s->rset); + } if (s->rset_nmem) nmem_destroy(s->rset_nmem); xfree (s->name); @@ -332,22 +361,49 @@ void resultSetDestroy (ZebraHandle zh, int num, char **names,int *statuses) } } -ZebraPosSet zebraPosSetCreate (ZebraHandle zh, const char *name, - int num, int *positions) +ZebraMetaRecord *zebra_meta_records_create_range (ZebraHandle zh, + const char *name, + zint start, int num) +{ + zint pos_small[10]; + zint *pos = pos_small; + ZebraMetaRecord *mr; + int i; + + if (num > 10000 || num <= 0) + return 0; + + if (num > 10) + pos = xmalloc(sizeof(*pos) * num); + + for (i = 0; i 10) + xfree(pos); + return mr; +} + +ZebraMetaRecord *zebra_meta_records_create (ZebraHandle zh, const char *name, + int num, zint *positions) { ZebraSet sset; - ZebraPosSet sr = 0; + ZebraMetaRecord *sr = 0; RSET rset; int i; struct zset_sort_info *sort_info; + if (!log_level_set) + loglevels(); if (!(sset = resultSetGet (zh, name))) return NULL; if (!(rset = sset->rset)) { if (!sset->term_entries) return 0; - sr = (ZebraPosSet) xmalloc (sizeof(*sr) * num); + sr = (ZebraMetaRecord *) xmalloc (sizeof(*sr) * num); for (i = 0; isort_info; if (sort_info) { - int position; + zint position; for (i = 0; i 0 && position <= sort_info->num_entries) { - yaz_log(LOG_DEBUG, "got pos=%d (sorted)", position); + yaz_log(log_level_sort, "got pos=" ZINT_FORMAT + " (sorted)", position); sr[i].sysno = sort_info->entries[position-1]->sysno; sr[i].score = sort_info->entries[position-1]->score; } @@ -396,7 +453,7 @@ ZebraPosSet zebraPosSetCreate (ZebraHandle zh, const char *name, } if (i < num) /* nope, get the rest, unsorted - sorry */ { - int position = 0; + zint position = 0; int num_i = 0; zint psysno = 0; RSFD rfd; @@ -404,9 +461,22 @@ ZebraPosSet zebraPosSetCreate (ZebraHandle zh, const char *name, if (sort_info) position = sort_info->num_entries; - while (num_i < num && positions[num_i] < position) + while (num_i < num && positions[num_i] <= position) num_i++; - rfd = rset_open (rset, RSETF_READ); + + if (sset->cache_rfd && + num_i < num && positions[num_i] > sset->cache_position) + { + position = sset->cache_position; + rfd = sset->cache_rfd; + psysno = sset->cache_psysno; + } + else + { + if (sset->cache_rfd) + rset_close(sset->cache_rfd); + rfd = rset_open (rset, RSETF_READ); + } while (num_i < num && rset_read (rfd, &key, 0)) { zint this_sys = key.mem[0]; @@ -427,19 +497,22 @@ ZebraPosSet zebraPosSetCreate (ZebraHandle zh, const char *name, if (position == positions[num_i]) { sr[num_i].sysno = psysno; - yaz_log(LOG_DEBUG, "got pos=%d (unsorted)", position); + yaz_log(log_level_sort, "got pos=" ZINT_FORMAT " (unsorted)", position); sr[num_i].score = -1; num_i++; } } } - rset_close (rfd); + sset->cache_position = position; + sset->cache_psysno = psysno; + sset->cache_rfd = rfd; } } return sr; } -void zebraPosSetDestroy (ZebraHandle zh, ZebraPosSet records, int num) +void zebra_meta_records_destroy (ZebraHandle zh, ZebraMetaRecord *records, + int num) { assert(zh); /* compiler shut up about unused arg */ xfree (records); @@ -573,38 +646,40 @@ void resultSetInsertRank (ZebraHandle zh, struct zset_sort_info *sort_info, new_entry->score = score; } -void resultSetSort (ZebraHandle zh, NMEM nmem, - int num_input_setnames, const char **input_setnames, - const char *output_setname, - Z_SortKeySpecList *sort_sequence, int *sort_status) +ZEBRA_RES resultSetSort(ZebraHandle zh, NMEM nmem, + int num_input_setnames, const char **input_setnames, + const char *output_setname, + Z_SortKeySpecList *sort_sequence, int *sort_status) { ZebraSet sset; RSET rset; if (num_input_setnames == 0) { - zh->errCode = 208; - return ; + zebra_setError(zh, YAZ_BIB1_NO_RESULT_SET_NAME_SUPPLIED_ON_SORT, 0); + return ZEBRA_FAIL; } if (num_input_setnames > 1) { - zh->errCode = 230; - return; + zebra_setError(zh, YAZ_BIB1_SORT_TOO_MANY_INPUT_RESULTS, 0); + return ZEBRA_FAIL; } - yaz_log(LOG_DEBUG, "result set sort input=%s output=%s", + if (!log_level_set) + loglevels(); + yaz_log(log_level_sort, "result set sort input=%s output=%s", *input_setnames, output_setname); sset = resultSetGet (zh, input_setnames[0]); if (!sset) { - zh->errCode = 30; - zh->errString = nmem_strdup (nmem, input_setnames[0]); - return; + zebra_setError(zh, YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST, + input_setnames[0]); + return ZEBRA_FAIL; } if (!(rset = sset->rset)) { - zh->errCode = 30; - zh->errString = nmem_strdup (nmem, input_setnames[0]); - return; + zebra_setError(zh, YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST, + input_setnames[0]); + return ZEBRA_FAIL; } if (strcmp (output_setname, input_setnames[0])) { @@ -612,12 +687,14 @@ void resultSetSort (ZebraHandle zh, NMEM nmem, sset = resultSetAdd (zh, output_setname, 1); sset->rset = rset; } - resultSetSortSingle (zh, nmem, sset, rset, sort_sequence, sort_status); + return resultSetSortSingle (zh, nmem, sset, rset, sort_sequence, + sort_status); } -void resultSetSortSingle (ZebraHandle zh, NMEM nmem, - ZebraSet sset, RSET rset, - Z_SortKeySpecList *sort_sequence, int *sort_status) +ZEBRA_RES resultSetSortSingle(ZebraHandle zh, NMEM nmem, + ZebraSet sset, RSET rset, + Z_SortKeySpecList *sort_sequence, + int *sort_status) { int i; int n = 0; @@ -653,53 +730,57 @@ void resultSetSortSingle (ZebraHandle zh, NMEM nmem, sort_criteria[i].relation = 'D'; else { - zh->errCode = 214; - return; + zebra_setError(zh, YAZ_BIB1_ILLEGAL_SORT_RELATION, 0); + return ZEBRA_FAIL; } if (sks->sortElement->which == Z_SortElement_databaseSpecific) { - zh->errCode = 210; - return; + zebra_setError(zh, YAZ_BIB1_DATABASE_SPECIFIC_SORT_UNSUPP, 0); + return ZEBRA_FAIL; } else if (sks->sortElement->which != Z_SortElement_generic) { - zh->errCode = 237; - return; + zebra_setError(zh, YAZ_BIB1_SORT_ILLEGAL_SORT, 0); + return ZEBRA_FAIL; } sk = sks->sortElement->u.generic; switch (sk->which) { case Z_SortKey_sortField: - yaz_log(LOG_DEBUG, "Sort: key %d is of type sortField", i+1); - zh->errCode = 207; - return; + yaz_log(log_level_sort, "key %d is of type sortField", + i+1); + zebra_setError(zh, YAZ_BIB1_CANNOT_SORT_ACCORDING_TO_SEQUENCE, 0); + return ZEBRA_FAIL; case Z_SortKey_elementSpec: - yaz_log(LOG_DEBUG, "Sort: key %d is of type elementSpec", i+1); - zh->errCode = 207; - return; + yaz_log(log_level_sort, "key %d is of type elementSpec", + i+1); + zebra_setError(zh, YAZ_BIB1_CANNOT_SORT_ACCORDING_TO_SEQUENCE, 0); + return ZEBRA_FAIL; case Z_SortKey_sortAttributes: - yaz_log(LOG_DEBUG, "Sort: key %d is of type sortAttributes", i+1); + yaz_log(log_level_sort, "key %d is of type sortAttributes", i+1); sort_criteria[i].attrUse = zebra_maps_sort (zh->reg->zebra_maps, sk->u.sortAttributes, &sort_criteria[i].numerical); - yaz_log(LOG_DEBUG, "use value = %d", sort_criteria[i].attrUse); + yaz_log(log_level_sort, "use value = %d", sort_criteria[i].attrUse); if (sort_criteria[i].attrUse == -1) { - zh->errCode = 116; - return; + zebra_setError( + zh, YAZ_BIB1_USE_ATTRIBUTE_REQUIRED_BUT_NOT_SUPPLIED, 0); + return ZEBRA_FAIL; } if (sortIdx_type (zh->reg->sortIdx, sort_criteria[i].attrUse)) { - zh->errCode = 207; - return; + zebra_setError( + zh, YAZ_BIB1_CANNOT_SORT_ACCORDING_TO_SEQUENCE, 0); + return ZEBRA_FAIL; } break; } } rfd = rset_open (rset, RSETF_READ); + /* FIXME - pass a TERMID *, and use it for something below !! */ while (rset_read (rfd, &key, &termid)) - /* FIXME - pass a TERMID *, and use it for something below !! */ { zint this_sys = key.mem[0]; kno++; @@ -712,12 +793,13 @@ void resultSetSortSingle (ZebraHandle zh, NMEM nmem, } } rset_close (rfd); - yaz_log(LOG_DEBUG, ZINT_FORMAT " keys, " ZINT_FORMAT " sysnos, sort", + yaz_log(log_level_sort, ZINT_FORMAT " keys, " ZINT_FORMAT " sysnos, sort", kno, sset->hits); for (i = 0; i < numTerms; i++) - yaz_log(LOG_LOG, "term=\"%s\" type=%s count=" ZINT_FORMAT, + yaz_log(log_level_sort, "term=\"%s\" type=%s count=" ZINT_FORMAT, terms[i]->name, terms[i]->flags, rset_count(terms[i]->rset)); *sort_status = Z_SortResponse_success; + return ZEBRA_OK; } RSET resultSetRef (ZebraHandle zh, const char *resultSetId) @@ -729,7 +811,8 @@ RSET resultSetRef (ZebraHandle zh, const char *resultSetId) return NULL; } -void resultSetRank (ZebraHandle zh, ZebraSet zebraSet, RSET rset, NMEM nmem) +ZEBRA_RES resultSetRank (ZebraHandle zh, ZebraSet zebraSet, + RSET rset, NMEM nmem) { zint kno = 0; struct it_key key; @@ -748,10 +831,13 @@ void resultSetRank (ZebraHandle zh, ZebraSet zebraSet, RSET rset, NMEM nmem) zint esthits; double ratio; + if (!log_level_set) + loglevels(); sort_info = zebraSet->sort_info; sort_info->num_entries = 0; zebraSet->hits = 0; rset_getterms(rset, 0, 0, &n); + yaz_log(YLOG_LOG, "Got %d terms", n); terms = (TERMID *) nmem_malloc(nmem, sizeof(*terms)*n); rset_getterms(rset, terms, n, &numTerms); @@ -760,8 +846,9 @@ void resultSetRank (ZebraHandle zh, ZebraSet zebraSet, RSET rset, NMEM nmem) rank_class = zebraRankLookup (zh, rank_handler_name); if (!rank_class) { - yaz_log(LOG_WARN, "No such rank handler: %s", rank_handler_name); - return; + yaz_log(YLOG_WARN, "No such rank handler: %s", rank_handler_name); + zebra_setError(zh, YAZ_BIB1_UNSUPP_SEARCH, "Cannot find rank handler"); + return ZEBRA_FAIL; } rc = rank_class->control; @@ -779,8 +866,9 @@ void resultSetRank (ZebraHandle zh, ZebraSet zebraSet, RSET rset, NMEM nmem) do { zint this_sys = key.mem[0]; /* FIXME - assumes scope==2 */ - zint seqno = key.mem[1]; /* FIXME - assumes scope==2 */ + zint seqno = key.mem[key.len-1]; /* FIXME - assumes scope==2 */ kno++; + key_logdump_txt(log_level_searchhits,&key," Got hit"); if (this_sys != psysno) { score = (*rc->calc) (handle, psysno); @@ -789,7 +877,7 @@ void resultSetRank (ZebraHandle zh, ZebraSet zebraSet, RSET rset, NMEM nmem) (zebraSet->hits)++; psysno = this_sys; } - (*rc->add) (handle, seqno, termid); + (*rc->add) (handle, CAST_ZINT_TO_INT(seqno), termid); if ((est==-2) && (zebraSet->hits==esthits)) { /* time to estimate the hits */ @@ -797,7 +885,7 @@ void resultSetRank (ZebraHandle zh, ZebraSet zebraSet, RSET rset, NMEM nmem) if (tot>0) { ratio = cur/tot; est = (zint)(0.5+zebraSet->hits/ratio); - logf(LOG_LOG, "Estimating hits (%s) " + yaz_log(log_level_searchhits, "Estimating hits (%s) " "%0.1f->" ZINT_FORMAT "; %0.1f->" ZINT_FORMAT, rset->control->desc, @@ -820,14 +908,16 @@ void resultSetRank (ZebraHandle zh, ZebraSet zebraSet, RSET rset, NMEM nmem) (*rc->end) (zh->reg, handle); } rset_close (rfd); - - yaz_log(LOG_DEBUG, ZINT_FORMAT " keys, " ZINT_FORMAT " sysnos, rank", - kno, zebraSet->hits); + + yaz_log(log_level_searchterms, ZINT_FORMAT " keys, " + ZINT_FORMAT " sysnos, rank", kno, zebraSet->hits); for (i = 0; i < numTerms; i++) { - yaz_log(LOG_LOG, "term=\"%s\" type=%s count=" ZINT_FORMAT, - terms[i]->name, terms[i]->flags, rset_count(terms[i]->rset)); + yaz_log(log_level_searchterms, "term=\"%s\" type=%s count=" + ZINT_FORMAT, + terms[i]->name, terms[i]->flags, rset_count(terms[i]->rset)); } + return ZEBRA_OK; } ZebraRankClass zebraRankLookup (ZebraHandle zh, const char *name) @@ -863,9 +953,9 @@ void zebraRankDestroy (struct zebra_register *reg) ZebraRankClass p_next = p->next; if (p->init_flag && p->control->destroy) (*p->control->destroy)(reg, p->class_handle); - xfree (p->control->name); - xfree (p->control); - xfree (p); + xfree(p->control->name); + xfree(p->control); + xfree(p); p = p_next; } reg->rank_classes = NULL;