X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=index%2Fzsets.c;h=b24840029739595b2b1d6b8aae195d220157316f;hb=b5eb8cd7ae4134d915ed8b4aec25ab90c8bd0dcb;hp=d3fb047732d98372147043768ac307a0beaf9475;hpb=47ea1fc957c7b97bb30a26698f072109cae275e4;p=idzebra-moved-to-github.git diff --git a/index/zsets.c b/index/zsets.c index d3fb047..b248400 100644 --- a/index/zsets.c +++ b/index/zsets.c @@ -1,10 +1,26 @@ -/* - * Copyright (C) 1994-2002, Index Data - * All rights reserved. - * Sebastian Hammer, Adam Dickmeiss - * - * $Id: zsets.c,v 1.37 2002-07-25 13:06:43 adam Exp $ - */ +/* $Id: zsets.c,v 1.55 2004-08-10 08:19:15 heikki Exp $ + Copyright (C) 1995,1996,1997,1998,1999,2000,2001,2002,2003,2004 + Index Data Aps + +This file is part of the Zebra server. + +Zebra is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. + +Zebra is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with Zebra; see the file LICENSE.zebra. If not, write to the +Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA +02111-1307, USA. +*/ + + #include #include #ifdef WIN32 @@ -30,7 +46,7 @@ struct zebra_set { char *name; RSET rset; NMEM nmem; - int hits; + zint hits; int num_bases; char **basenames; Z_RPNQuery *rpn; @@ -42,7 +58,7 @@ struct zebra_set { }; struct zset_sort_entry { - int sysno; + zint sysno; int score; char buf[ZSET_SORT_MAX_LEVEL][SORT_IDX_ENTRYSIZE]; }; @@ -54,7 +70,7 @@ struct zset_sort_info { struct zset_sort_entry **entries; }; -ZebraSet resultSetAddRPN (ZebraHandle zh, ODR input, ODR output, +ZebraSet resultSetAddRPN (ZebraHandle zh, NMEM m, Z_RPNQuery *rpn, int num_bases, char **basenames, const char *setname) @@ -71,7 +87,7 @@ ZebraSet resultSetAddRPN (ZebraHandle zh, ODR input, ODR output, return 0; zebraSet->locked = 1; zebraSet->rpn = 0; - zebraSet->nmem = nmem_create (); + zebraSet->nmem = m; zebraSet->num_bases = num_bases; zebraSet->basenames = @@ -80,7 +96,7 @@ ZebraSet resultSetAddRPN (ZebraHandle zh, ODR input, ODR output, zebraSet->basenames[i] = nmem_strdup (zebraSet->nmem, basenames[i]); - zebraSet->rset = rpn_search (zh, output->mem, rpn, + zebraSet->rset = rpn_search (zh, zebraSet->nmem, rpn, zebraSet->num_bases, zebraSet->basenames, zebraSet->name, zebraSet); @@ -95,6 +111,7 @@ void resultSetAddTerm (ZebraHandle zh, ZebraSet s, int reg_type, const char *db, int set, int use, const char *term) { + assert(zh); /* compiler shut up */ if (!s->nmem) s->nmem = nmem_create (); if (!s->term_entries) @@ -120,7 +137,7 @@ void resultSetAddTerm (ZebraHandle zh, ZebraSet s, int reg_type, int zebra_resultSetTerms (ZebraHandle zh, const char *setname, - int no, int *count, + int no, zint *count, int *type, char *out, size_t *len) { ZebraSet s = resultSetGet (zh, setname); @@ -144,13 +161,13 @@ int zebra_resultSetTerms (ZebraHandle zh, const char *setname, size_t inleft = strlen(inbuf); size_t outleft = *len - 1; int converted = 0; -#if HAVE_ICONV_H - if (zh->iconv_from_utf8 != (iconv_t)(-1)) + + if (zh->iconv_from_utf8 != 0) { char *outbuf = out; size_t ret; - ret = iconv(zh->iconv_from_utf8, &inbuf, &inleft, + ret = yaz_iconv(zh->iconv_from_utf8, &inbuf, &inleft, &outbuf, &outleft); if (ret == (size_t)(-1)) *len = 0; @@ -158,7 +175,6 @@ int zebra_resultSetTerms (ZebraHandle zh, const char *setname, *len = outbuf - out; converted = 1; } -#endif if (!converted) { if (inleft > outleft) @@ -182,7 +198,7 @@ ZebraSet resultSetAdd (ZebraHandle zh, const char *name, int ov) break; if (s) { - logf (LOG_DEBUG, "updating result set %s", name); + yaz_log (LOG_DEBUG, "updating result set %s", name); if (!ov || s->locked) return NULL; if (s->rset) @@ -192,7 +208,9 @@ ZebraSet resultSetAdd (ZebraHandle zh, const char *name, int ov) } else { - logf (LOG_DEBUG, "adding result set %s", name); + const char *sort_max_str = zebra_get_resource(zh, "sortmax", "1000"); + + yaz_log (LOG_DEBUG, "adding result set %s", name); s = (ZebraSet) xmalloc (sizeof(*s)); s->next = zh->sets; zh->sets = s; @@ -201,7 +219,10 @@ ZebraSet resultSetAdd (ZebraHandle zh, const char *name, int ov) s->sort_info = (struct zset_sort_info *) xmalloc (sizeof(*s->sort_info)); - s->sort_info->max_entries = 1000; + s->sort_info->max_entries = atoi(sort_max_str); + if (s->sort_info->max_entries < 2) + s->sort_info->max_entries = 2; + s->sort_info->entries = (struct zset_sort_entry **) xmalloc (sizeof(*s->sort_info->entries) * s->sort_info->max_entries); @@ -348,7 +369,7 @@ ZebraPosSet zebraPosSetCreate (ZebraHandle zh, const char *name, position = positions[i]; if (position > 0 && position <= sort_info->num_entries) { - logf (LOG_DEBUG, "got pos=%d (sorted)", position); + yaz_log (LOG_DEBUG, "got pos=%d (sorted)", position); sr[i].sysno = sort_info->entries[position-1]->sysno; sr[i].score = sort_info->entries[position-1]->score; } @@ -364,7 +385,7 @@ ZebraPosSet zebraPosSetCreate (ZebraHandle zh, const char *name, { int position = 0; int num_i = 0; - int psysno = 0; + zint psysno = 0; int term_index; RSFD rfd; struct it_key key; @@ -376,9 +397,14 @@ ZebraPosSet zebraPosSetCreate (ZebraHandle zh, const char *name, rfd = rset_open (rset, RSETF_READ); while (num_i < num && rset_read (rset, rfd, &key, &term_index)) { - if (key.sysno != psysno) +#if IT_KEY_NEW + zint this_sys = key.mem[0]; +#else + zint this_sys = key.sysno; +#endif + if (this_sys != psysno) { - psysno = key.sysno; + psysno = this_sys; if (sort_info) { /* determine we alreay have this in our set */ @@ -393,7 +419,7 @@ ZebraPosSet zebraPosSetCreate (ZebraHandle zh, const char *name, if (position == positions[num_i]) { sr[num_i].sysno = psysno; - logf (LOG_DEBUG, "got pos=%d (unsorted)", position); + yaz_log (LOG_DEBUG, "got pos=%d (unsorted)", position); sr[num_i].score = -1; num_i++; } @@ -407,6 +433,7 @@ ZebraPosSet zebraPosSetCreate (ZebraHandle zh, const char *name, void zebraPosSetDestroy (ZebraHandle zh, ZebraPosSet records, int num) { + assert(zh); /* compiler shut up about unused arg */ xfree (records); } @@ -418,7 +445,7 @@ struct sortKeyInfo { void resultSetInsertSort (ZebraHandle zh, ZebraSet sset, struct sortKeyInfo *criteria, int num_criteria, - int sysno) + zint sysno) { struct zset_sort_entry this_entry; struct zset_sort_entry *new_entry = NULL; @@ -492,10 +519,11 @@ void resultSetInsertSort (ZebraHandle zh, ZebraSet sset, } void resultSetInsertRank (ZebraHandle zh, struct zset_sort_info *sort_info, - int sysno, int score, int relation) + zint sysno, int score, int relation) { struct zset_sort_entry *new_entry = NULL; int i, j; + assert(zh); /* compiler shut up about unused arg */ i = sort_info->num_entries; while (--i >= 0) @@ -555,7 +583,7 @@ void resultSetSort (ZebraHandle zh, NMEM nmem, zh->errCode = 230; return; } - logf (LOG_DEBUG, "result set sort input=%s output=%s", + yaz_log (LOG_DEBUG, "result set sort input=%s output=%s", *input_setnames, output_setname); sset = resultSetGet (zh, input_setnames[0]); if (!sset) @@ -583,14 +611,16 @@ void resultSetSortSingle (ZebraHandle zh, NMEM nmem, ZebraSet sset, RSET rset, Z_SortKeySpecList *sort_sequence, int *sort_status) { - int i, psysno = 0; + int i; + zint psysno = 0; struct it_key key; struct sortKeyInfo sort_criteria[3]; int num_criteria; int term_index; RSFD rfd; - logf (LOG_LOG, "resultSetSortSingle start"); + yaz_log (LOG_LOG, "resultSetSortSingle start"); + assert(nmem); /* compiler shut up about unused param */ sset->sort_info->num_entries = 0; sset->hits = 0; @@ -602,9 +632,9 @@ void resultSetSortSingle (ZebraHandle zh, NMEM nmem, Z_SortKeySpec *sks = sort_sequence->specs[i]; Z_SortKey *sk; - if (*sks->sortRelation == Z_SortRelation_ascending) + if (*sks->sortRelation == Z_SortKeySpec_ascending) sort_criteria[i].relation = 'A'; - else if (*sks->sortRelation == Z_SortRelation_descending) + else if (*sks->sortRelation == Z_SortKeySpec_descending) sort_criteria[i].relation = 'D'; else { @@ -625,20 +655,20 @@ void resultSetSortSingle (ZebraHandle zh, NMEM nmem, switch (sk->which) { case Z_SortKey_sortField: - logf (LOG_DEBUG, "Sort: key %d is of type sortField", i+1); + yaz_log (LOG_DEBUG, "Sort: key %d is of type sortField", i+1); zh->errCode = 207; return; case Z_SortKey_elementSpec: - logf (LOG_DEBUG, "Sort: key %d is of type elementSpec", i+1); + yaz_log (LOG_DEBUG, "Sort: key %d is of type elementSpec", i+1); zh->errCode = 207; return; case Z_SortKey_sortAttributes: - logf (LOG_DEBUG, "Sort: key %d is of type sortAttributes", i+1); + yaz_log (LOG_DEBUG, "Sort: key %d is of type sortAttributes", i+1); sort_criteria[i].attrUse = zebra_maps_sort (zh->reg->zebra_maps, sk->u.sortAttributes, &sort_criteria[i].numerical); - logf (LOG_DEBUG, "use value = %d", sort_criteria[i].attrUse); + yaz_log (LOG_DEBUG, "use value = %d", sort_criteria[i].attrUse); if (sort_criteria[i].attrUse == -1) { zh->errCode = 116; @@ -655,10 +685,15 @@ void resultSetSortSingle (ZebraHandle zh, NMEM nmem, rfd = rset_open (rset, RSETF_READ); while (rset_read (rset, rfd, &key, &term_index)) { - if (key.sysno != psysno) +#if IT_KEY_NEW + zint this_sys = key.mem[0]; +#else + zint this_sys = key.sysno; +#endif + if (this_sys != psysno) { (sset->hits)++; - psysno = key.sysno; + psysno = this_sys; resultSetInsertSort (zh, sset, sort_criteria, num_criteria, psysno); } @@ -666,17 +701,17 @@ void resultSetSortSingle (ZebraHandle zh, NMEM nmem, rset_close (rset, rfd); for (i = 0; i < rset->no_rset_terms; i++) - yaz_log (LOG_LOG, "term=\"%s\" nn=%d type=%s count=%d", + yaz_log (LOG_LOG, "term=\"%s\" nn=" ZINT_FORMAT " type=%s count=" ZINT_FORMAT, rset->rset_terms[i]->name, rset->rset_terms[i]->nn, rset->rset_terms[i]->flags, rset->rset_terms[i]->count); - *sort_status = Z_SortStatus_success; - logf (LOG_LOG, "resultSetSortSingle end"); + *sort_status = Z_SortResponse_success; + yaz_log (LOG_LOG, "resultSetSortSingle end"); } -RSET resultSetRef (ZebraHandle zh, Z_ResultSetId *resultSetId) +RSET resultSetRef (ZebraHandle zh, const char *resultSetId) { ZebraSet s; @@ -687,13 +722,17 @@ RSET resultSetRef (ZebraHandle zh, Z_ResultSetId *resultSetId) void resultSetRank (ZebraHandle zh, ZebraSet zebraSet, RSET rset) { - int kno = 0; + zint kno = 0; struct it_key key; RSFD rfd; int term_index, i; ZebraRankClass rank_class; struct rank_control *rc; struct zset_sort_info *sort_info; + const char *rank_handler_name = res_get_def(zh->res, "rank", "rank-1"); + double cur,tot; + zint est=-2; /* -2 not done, -1 can't do, >0 actual estimate*/ + zint esthits; sort_info = zebraSet->sort_info; sort_info->num_entries = 0; @@ -702,30 +741,69 @@ void resultSetRank (ZebraHandle zh, ZebraSet zebraSet, RSET rset) yaz_log (LOG_LOG, "resultSetRank"); - rank_class = zebraRankLookup (zh, "rank-1"); + rank_class = zebraRankLookup (zh, rank_handler_name); + if (!rank_class) + { + yaz_log (LOG_WARN, "No such rank handler: %s", rank_handler_name); + return; + } rc = rank_class->control; if (rset_read (rset, rfd, &key, &term_index)) { - int psysno = key.sysno; +#if IT_KEY_NEW + zint psysno = key.mem[0]; +#else + zint psysno = key.sysno; +#endif int score; void *handle = (*rc->begin) (zh->reg, rank_class->class_handle, rset); (zebraSet->hits)++; + esthits=atoi(res_get_def(zh->res,"estimatehits","0")); + if (!esthits) est=-1; /* can not do */ do { +#if IT_KEY_NEW + zint this_sys = key.mem[0]; +#else + zint this_sys = key.sysno; +#endif kno++; - if (key.sysno != psysno) + if (this_sys != psysno) { score = (*rc->calc) (handle, psysno); resultSetInsertRank (zh, sort_info, psysno, score, 'A'); (zebraSet->hits)++; - psysno = key.sysno; + psysno = this_sys; } - (*rc->add) (handle, key.seqno, term_index); + (*rc->add) (handle, this_sys, term_index); + if ( (est==-2) && (zebraSet->hits==esthits)) + { /* time to estimate the hits */ + double f; + rset_pos(rset,rfd,&cur,&tot); + if (tot>0) { + f=cur/tot; + est=(zint)(0.5+zebraSet->hits/f); + logf(LOG_LOG, "Estimating hits (%s) " + "%0.1f->"ZINT_FORMAT + "; %0.1f->"ZINT_FORMAT, + rset->control->desc, + cur, zebraSet->hits, + tot,est); + i=0; /* round to 3 significant digits */ + while (est>1000) { + est/=10; + i++; + } + while (i--) est*=10; + zebraSet->hits=est; + } + } } - while (rset_read (rset, rfd, &key, &term_index)); + while (rset_read (rset, rfd, &key, &term_index) && (est<0) ); + score = (*rc->calc) (handle, psysno); resultSetInsertRank (zh, sort_info, psysno, score, 'A'); (*rc->end) (zh->reg, handle); @@ -733,13 +811,14 @@ void resultSetRank (ZebraHandle zh, ZebraSet zebraSet, RSET rset) rset_close (rset, rfd); for (i = 0; i < rset->no_rset_terms; i++) - yaz_log (LOG_LOG, "term=\"%s\" nn=%d type=%s count=%d", + yaz_log (LOG_LOG, "term=\"%s\" nn=" ZINT_FORMAT " type=%s count=" ZINT_FORMAT, rset->rset_terms[i]->name, rset->rset_terms[i]->nn, rset->rset_terms[i]->flags, rset->rset_terms[i]->count); - yaz_log (LOG_LOG, "%d keys, %d distinct sysnos", kno, zebraSet->hits); + yaz_log (LOG_LOG, ZINT_FORMAT " keys, "ZINT_FORMAT" distinct sysnos", + kno, zebraSet->hits); } ZebraRankClass zebraRankLookup (ZebraHandle zh, const char *name) @@ -750,7 +829,7 @@ ZebraRankClass zebraRankLookup (ZebraHandle zh, const char *name) if (p && !p->init_flag) { if (p->control->create) - p->class_handle = (*p->control->create)(zh->reg); + p->class_handle = (*p->control->create)(zh); p->init_flag = 1; } return p;