X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=index%2Fzsets.c;h=1d24d9b8e932afc69e75ca78f464446d021c77e9;hb=2b99f08490d3bd763bb086693044a6db10c0bcf9;hp=dfb73e7446219fe7e5a0150f9a6a699bc95bde1a;hpb=8d3211326d448452d82a869143ed6d08d463954b;p=idzebra-moved-to-github.git diff --git a/index/zsets.c b/index/zsets.c index dfb73e7..1d24d9b 100644 --- a/index/zsets.c +++ b/index/zsets.c @@ -1,116 +1,26 @@ -/* - * Copyright (C) 1994-2000, Index Data - * All rights reserved. - * Sebastian Hammer, Adam Dickmeiss - * - * $Log: zsets.c,v $ - * Revision 1.31 2001-11-19 23:05:22 adam - * Added a few prototypes. - * - * Revision 1.30 2001/10/15 19:53:43 adam - * POSIX thread updates. First work on term sets. - * - * Revision 1.29 2001/01/22 10:42:56 adam - * Added numerical sort. - * - * Revision 1.28 2000/07/07 12:49:20 adam - * Optimized resultSetInsert{Rank,Sort}. - * - * Revision 1.27 2000/04/05 09:49:36 adam - * On Unix, zebra/z'mbol uses automake. - * - * Revision 1.26 2000/03/20 19:08:36 adam - * Added remote record import using Z39.50 extended services and Segment - * Requests. - * - * Revision 1.25 2000/03/15 15:00:31 adam - * First work on threaded version. - * - * Revision 1.24 1999/11/04 15:00:45 adam - * Implemented delete result set(s). - * - * Revision 1.23 1999/05/26 07:49:13 adam - * C++ compilation. - * - * Revision 1.22 1999/02/02 14:51:15 adam - * Updated WIN32 code specific sections. Changed header. - * - * Revision 1.21 1998/11/16 16:03:46 adam - * Moved loggin utilities to Yaz. Was implemented in file zlogs.c. - * - * Revision 1.20 1998/11/16 10:10:53 adam - * Fixed problem with zebraPosSetCreate that occurred when positions were - * less than 1. - * - * Revision 1.19 1998/09/22 10:48:22 adam - * Minor changes in search API. - * - * Revision 1.18 1998/09/22 10:03:45 adam - * Changed result sets to be persistent in the sense that they can - * be re-searched if needed. - * Fixed memory leak in rsm_or. - * - * Revision 1.17 1998/06/23 15:33:36 adam - * Added feature to specify sort criteria in query (type 7 specifies - * sort flags). - * - * Revision 1.16 1998/05/20 10:12:24 adam - * Implemented automatic EXPLAIN database maintenance. - * Modified Zebra to work with ASN.1 compiled version of YAZ. - * - * Revision 1.15 1998/03/05 08:45:14 adam - * New result set model and modular ranking system. Moved towards - * descent server API. System information stored as "SGML" records. - * - * Revision 1.14 1998/02/10 16:39:15 adam - * Minor change. - * - * Revision 1.13 1998/02/10 12:03:06 adam - * Implemented Sort. - * - * Revision 1.12 1997/09/25 14:57:36 adam - * Windows NT port. - * - * Revision 1.11 1996/12/23 15:30:46 adam - * Work on truncation. - * Bug fix: result sets weren't deleted after server shut down. - * - * Revision 1.10 1995/10/30 15:08:08 adam - * Bug fixes. - * - * Revision 1.9 1995/10/17 18:02:14 adam - * New feature: databases. Implemented as prefix to words in dictionary. - * - * Revision 1.8 1995/10/10 13:59:25 adam - * Function rset_open changed its wflag parameter to general flags. - * - * Revision 1.7 1995/10/06 14:38:01 adam - * New result set method: r_score. - * Local no (sysno) and score is transferred to retrieveCtrl. - * - * Revision 1.6 1995/09/28 09:19:49 adam - * xfree/xmalloc used everywhere. - * Extract/retrieve method seems to work for text records. - * - * Revision 1.5 1995/09/27 16:17:32 adam - * More work on retrieve. - * - * Revision 1.4 1995/09/07 13:58:36 adam - * New parameter: result-set file descriptor (RSFD) to support multiple - * positions within the same result-set. - * Boolean operators: and, or, not implemented. - * Result-set references. - * - * Revision 1.3 1995/09/06 16:11:19 adam - * Option: only one word key per file. - * - * Revision 1.2 1995/09/06 10:33:04 adam - * More work on present. Some log messages removed. - * - * Revision 1.1 1995/09/05 15:28:40 adam - * More work on search engine. - * - */ +/* $Id: zsets.c,v 1.54 2004-08-06 13:36:23 adam Exp $ + Copyright (C) 1995,1996,1997,1998,1999,2000,2001,2002,2003,2004 + Index Data Aps + +This file is part of the Zebra server. + +Zebra is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. + +Zebra is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with Zebra; see the file LICENSE.zebra. If not, write to the +Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA +02111-1307, USA. +*/ + + #include #include #ifdef WIN32 @@ -119,7 +29,7 @@ #include #endif -#include "zserver.h" +#include "index.h" #include #define SORT_IDX_ENTRYSIZE 64 @@ -148,7 +58,7 @@ struct zebra_set { }; struct zset_sort_entry { - int sysno; + zint sysno; int score; char buf[ZSET_SORT_MAX_LEVEL][SORT_IDX_ENTRYSIZE]; }; @@ -160,11 +70,13 @@ struct zset_sort_info { struct zset_sort_entry **entries; }; -ZebraSet resultSetAddRPN (ZebraHandle zh, ODR input, ODR output, - Z_RPNQuery *rpn, int num_bases, char **basenames, +ZebraSet resultSetAddRPN (ZebraHandle zh, NMEM m, + Z_RPNQuery *rpn, int num_bases, + char **basenames, const char *setname) { ZebraSet zebraSet; + int i; zh->errCode = 0; zh->errString = NULL; @@ -175,11 +87,16 @@ ZebraSet resultSetAddRPN (ZebraHandle zh, ODR input, ODR output, return 0; zebraSet->locked = 1; zebraSet->rpn = 0; + zebraSet->nmem = m; + zebraSet->num_bases = num_bases; - zebraSet->basenames = basenames; - zebraSet->nmem = odr_extract_mem (input); + zebraSet->basenames = + nmem_malloc (zebraSet->nmem, num_bases * sizeof(*zebraSet->basenames)); + for (i = 0; ibasenames[i] = nmem_strdup (zebraSet->nmem, basenames[i]); + - zebraSet->rset = rpn_search (zh, output->mem, rpn, + zebraSet->rset = rpn_search (zh, zebraSet->nmem, rpn, zebraSet->num_bases, zebraSet->basenames, zebraSet->name, zebraSet); @@ -217,6 +134,59 @@ void resultSetAddTerm (ZebraHandle zh, ZebraSet s, int reg_type, (s->hits)++; } + +int zebra_resultSetTerms (ZebraHandle zh, const char *setname, + int no, zint *count, + int *type, char *out, size_t *len) +{ + ZebraSet s = resultSetGet (zh, setname); + int no_max = 0; + + if (count) + *count = 0; + if (!s || !s->rset) + return 0; + no_max = s->rset->no_rset_terms; + if (no < 0 || no >= no_max) + return 0; + if (count) + *count = s->rset->rset_terms[no]->count; + if (type) + *type = s->rset->rset_terms[no]->type; + + if (out) + { + char *inbuf = s->rset->rset_terms[no]->name; + size_t inleft = strlen(inbuf); + size_t outleft = *len - 1; + int converted = 0; + + if (zh->iconv_from_utf8 != 0) + { + char *outbuf = out; + size_t ret; + + ret = yaz_iconv(zh->iconv_from_utf8, &inbuf, &inleft, + &outbuf, &outleft); + if (ret == (size_t)(-1)) + *len = 0; + else + *len = outbuf - out; + converted = 1; + } + if (!converted) + { + if (inleft > outleft) + inleft = outleft; + *len = inleft; + memcpy (out, inbuf, *len); + } + out[*len] = 0; + } + return no_max; +} + + ZebraSet resultSetAdd (ZebraHandle zh, const char *name, int ov) { ZebraSet s; @@ -227,7 +197,7 @@ ZebraSet resultSetAdd (ZebraHandle zh, const char *name, int ov) break; if (s) { - logf (LOG_DEBUG, "updating result set %s", name); + yaz_log (LOG_DEBUG, "updating result set %s", name); if (!ov || s->locked) return NULL; if (s->rset) @@ -237,7 +207,9 @@ ZebraSet resultSetAdd (ZebraHandle zh, const char *name, int ov) } else { - logf (LOG_DEBUG, "adding result set %s", name); + const char *sort_max_str = zebra_get_resource(zh, "sortmax", "1000"); + + yaz_log (LOG_DEBUG, "adding result set %s", name); s = (ZebraSet) xmalloc (sizeof(*s)); s->next = zh->sets; zh->sets = s; @@ -246,7 +218,10 @@ ZebraSet resultSetAdd (ZebraHandle zh, const char *name, int ov) s->sort_info = (struct zset_sort_info *) xmalloc (sizeof(*s->sort_info)); - s->sort_info->max_entries = 1000; + s->sort_info->max_entries = atoi(sort_max_str); + if (s->sort_info->max_entries < 2) + s->sort_info->max_entries = 2; + s->sort_info->entries = (struct zset_sort_entry **) xmalloc (sizeof(*s->sort_info->entries) * s->sort_info->max_entries); @@ -275,6 +250,7 @@ ZebraSet resultSetGet (ZebraHandle zh, const char *name) if (!s->term_entries && !s->rset && s->rpn) { NMEM nmem = nmem_create (); + yaz_log (LOG_LOG, "research %s", name); s->rset = rpn_search (zh, nmem, s->rpn, s->num_bases, s->basenames, s->name, s); @@ -285,9 +261,21 @@ ZebraSet resultSetGet (ZebraHandle zh, const char *name) return NULL; } +void resultSetInvalidate (ZebraHandle zh) +{ + ZebraSet s = zh->sets; + + for (; s; s = s->next) + { + if (s->rset) + rset_delete (s->rset); + s->rset = 0; + } +} + void resultSetDestroy (ZebraHandle zh, int num, char **names,int *statuses) { - ZebraSet *ss = &zh->sets; + ZebraSet * ss = &zh->sets; int i; if (statuses) @@ -348,8 +336,6 @@ ZebraPosSet zebraPosSetCreate (ZebraHandle zh, const char *name, sr = (ZebraPosSet) xmalloc (sizeof(*sr) * num); for (i = 0; iterm_entries; - sr[i].sysno = 0; sr[i].score = -1; sr[i].term = 0; @@ -382,7 +368,7 @@ ZebraPosSet zebraPosSetCreate (ZebraHandle zh, const char *name, position = positions[i]; if (position > 0 && position <= sort_info->num_entries) { - logf (LOG_DEBUG, "got pos=%d (sorted)", position); + yaz_log (LOG_DEBUG, "got pos=%d (sorted)", position); sr[i].sysno = sort_info->entries[position-1]->sysno; sr[i].score = sort_info->entries[position-1]->score; } @@ -398,7 +384,7 @@ ZebraPosSet zebraPosSetCreate (ZebraHandle zh, const char *name, { int position = 0; int num_i = 0; - int psysno = 0; + zint psysno = 0; int term_index; RSFD rfd; struct it_key key; @@ -410,9 +396,14 @@ ZebraPosSet zebraPosSetCreate (ZebraHandle zh, const char *name, rfd = rset_open (rset, RSETF_READ); while (num_i < num && rset_read (rset, rfd, &key, &term_index)) { - if (key.sysno != psysno) +#if IT_KEY_NEW + zint this_sys = key.mem[0]; +#else + zint this_sys = key.sysno; +#endif + if (this_sys != psysno) { - psysno = key.sysno; + psysno = this_sys; if (sort_info) { /* determine we alreay have this in our set */ @@ -427,7 +418,7 @@ ZebraPosSet zebraPosSetCreate (ZebraHandle zh, const char *name, if (position == positions[num_i]) { sr[num_i].sysno = psysno; - logf (LOG_DEBUG, "got pos=%d (unsorted)", position); + yaz_log (LOG_DEBUG, "got pos=%d (unsorted)", position); sr[num_i].score = -1; num_i++; } @@ -452,18 +443,18 @@ struct sortKeyInfo { void resultSetInsertSort (ZebraHandle zh, ZebraSet sset, struct sortKeyInfo *criteria, int num_criteria, - int sysno) + zint sysno) { struct zset_sort_entry this_entry; struct zset_sort_entry *new_entry = NULL; struct zset_sort_info *sort_info = sset->sort_info; int i, j; - sortIdx_sysno (zh->service->sortIdx, sysno); + sortIdx_sysno (zh->reg->sortIdx, sysno); for (i = 0; iservice->sortIdx, criteria[i].attrUse); - sortIdx_read (zh->service->sortIdx, this_entry.buf[i]); + sortIdx_type (zh->reg->sortIdx, criteria[i].attrUse); + sortIdx_read (zh->reg->sortIdx, this_entry.buf[i]); } i = sort_info->num_entries; while (--i >= 0) @@ -526,7 +517,7 @@ void resultSetInsertSort (ZebraHandle zh, ZebraSet sset, } void resultSetInsertRank (ZebraHandle zh, struct zset_sort_info *sort_info, - int sysno, int score, int relation) + zint sysno, int score, int relation) { struct zset_sort_entry *new_entry = NULL; int i, j; @@ -589,7 +580,7 @@ void resultSetSort (ZebraHandle zh, NMEM nmem, zh->errCode = 230; return; } - logf (LOG_DEBUG, "result set sort input=%s output=%s", + yaz_log (LOG_DEBUG, "result set sort input=%s output=%s", *input_setnames, output_setname); sset = resultSetGet (zh, input_setnames[0]); if (!sset) @@ -617,14 +608,15 @@ void resultSetSortSingle (ZebraHandle zh, NMEM nmem, ZebraSet sset, RSET rset, Z_SortKeySpecList *sort_sequence, int *sort_status) { - int i, psysno = 0; + int i; + zint psysno = 0; struct it_key key; struct sortKeyInfo sort_criteria[3]; int num_criteria; int term_index; RSFD rfd; - logf (LOG_DEBUG, "resultSetSortSingle start"); + yaz_log (LOG_LOG, "resultSetSortSingle start"); sset->sort_info->num_entries = 0; sset->hits = 0; @@ -636,9 +628,9 @@ void resultSetSortSingle (ZebraHandle zh, NMEM nmem, Z_SortKeySpec *sks = sort_sequence->specs[i]; Z_SortKey *sk; - if (*sks->sortRelation == Z_SortRelation_ascending) + if (*sks->sortRelation == Z_SortKeySpec_ascending) sort_criteria[i].relation = 'A'; - else if (*sks->sortRelation == Z_SortRelation_descending) + else if (*sks->sortRelation == Z_SortKeySpec_descending) sort_criteria[i].relation = 'D'; else { @@ -659,26 +651,26 @@ void resultSetSortSingle (ZebraHandle zh, NMEM nmem, switch (sk->which) { case Z_SortKey_sortField: - logf (LOG_DEBUG, "Sort: key %d is of type sortField", i+1); + yaz_log (LOG_DEBUG, "Sort: key %d is of type sortField", i+1); zh->errCode = 207; return; case Z_SortKey_elementSpec: - logf (LOG_DEBUG, "Sort: key %d is of type elementSpec", i+1); + yaz_log (LOG_DEBUG, "Sort: key %d is of type elementSpec", i+1); zh->errCode = 207; return; case Z_SortKey_sortAttributes: - logf (LOG_DEBUG, "Sort: key %d is of type sortAttributes", i+1); + yaz_log (LOG_DEBUG, "Sort: key %d is of type sortAttributes", i+1); sort_criteria[i].attrUse = - zebra_maps_sort (zh->service->zebra_maps, + zebra_maps_sort (zh->reg->zebra_maps, sk->u.sortAttributes, &sort_criteria[i].numerical); - logf (LOG_DEBUG, "use value = %d", sort_criteria[i].attrUse); + yaz_log (LOG_DEBUG, "use value = %d", sort_criteria[i].attrUse); if (sort_criteria[i].attrUse == -1) { zh->errCode = 116; return; } - if (sortIdx_type (zh->service->sortIdx, sort_criteria[i].attrUse)) + if (sortIdx_type (zh->reg->sortIdx, sort_criteria[i].attrUse)) { zh->errCode = 207; return; @@ -689,21 +681,33 @@ void resultSetSortSingle (ZebraHandle zh, NMEM nmem, rfd = rset_open (rset, RSETF_READ); while (rset_read (rset, rfd, &key, &term_index)) { - if (key.sysno != psysno) +#if IT_KEY_NEW + zint this_sys = key.mem[0]; +#else + zint this_sys = key.sysno; +#endif + if (this_sys != psysno) { (sset->hits)++; - psysno = key.sysno; + psysno = this_sys; resultSetInsertSort (zh, sset, sort_criteria, num_criteria, psysno); } } rset_close (rset, rfd); - *sort_status = Z_SortStatus_success; - logf (LOG_DEBUG, "resultSetSortSingle end"); + for (i = 0; i < rset->no_rset_terms; i++) + yaz_log (LOG_LOG, "term=\"%s\" nn=" ZINT_FORMAT " type=%s count=" ZINT_FORMAT, + rset->rset_terms[i]->name, + rset->rset_terms[i]->nn, + rset->rset_terms[i]->flags, + rset->rset_terms[i]->count); + + *sort_status = Z_SortResponse_success; + yaz_log (LOG_LOG, "resultSetSortSingle end"); } -RSET resultSetRef (ZebraHandle zh, Z_ResultSetId *resultSetId) +RSET resultSetRef (ZebraHandle zh, const char *resultSetId) { ZebraSet s; @@ -714,95 +718,137 @@ RSET resultSetRef (ZebraHandle zh, Z_ResultSetId *resultSetId) void resultSetRank (ZebraHandle zh, ZebraSet zebraSet, RSET rset) { - int kno = 0; + zint kno = 0; struct it_key key; RSFD rfd; int term_index, i; ZebraRankClass rank_class; struct rank_control *rc; struct zset_sort_info *sort_info; + const char *rank_handler_name = res_get_def(zh->res, "rank", "rank-1"); + double cur,tot; + zint est=-2; /* -2 not done, -1 can't do, >0 actual estimate*/ + zint esthits; sort_info = zebraSet->sort_info; sort_info->num_entries = 0; zebraSet->hits = 0; rfd = rset_open (rset, RSETF_READ); - logf (LOG_DEBUG, "resultSetRank"); - for (i = 0; i < rset->no_rset_terms; i++) - logf (LOG_DEBUG, "term=\"%s\" cnt=%d type=%s", - rset->rset_terms[i]->name, - rset->rset_terms[i]->nn, - rset->rset_terms[i]->flags); + yaz_log (LOG_LOG, "resultSetRank"); - rank_class = zebraRankLookup (zh, "rank-1"); + rank_class = zebraRankLookup (zh, rank_handler_name); + if (!rank_class) + { + yaz_log (LOG_WARN, "No such rank handler: %s", rank_handler_name); + return; + } rc = rank_class->control; if (rset_read (rset, rfd, &key, &term_index)) { - int psysno = key.sysno; +#if IT_KEY_NEW + zint psysno = key.mem[0]; +#else + zint psysno = key.sysno; +#endif int score; void *handle = - (*rc->begin) (zh, rank_class->class_handle, rset); + (*rc->begin) (zh->reg, rank_class->class_handle, rset); (zebraSet->hits)++; + esthits=atoi(res_get_def(zh->res,"estimatehits","0")); + if (!esthits) est=-1; /* can not do */ do { +#if IT_KEY_NEW + zint this_sys = key.mem[0]; +#else + zint this_sys = key.sysno; +#endif kno++; - if (key.sysno != psysno) + if (this_sys != psysno) { score = (*rc->calc) (handle, psysno); resultSetInsertRank (zh, sort_info, psysno, score, 'A'); (zebraSet->hits)++; - psysno = key.sysno; + psysno = this_sys; } - (*rc->add) (handle, key.seqno, term_index); + (*rc->add) (handle, this_sys, term_index); + if ( (est==-2) && (zebraSet->hits==esthits)) + { /* time to estimate the hits */ + float f; + rset_pos(rset,rfd,&cur,&tot); + if (tot>0) { + f=1.0*cur/tot; + est=(zint)(0.5+zebraSet->hits/f); + /* FIXME - round the guess to 3 digits */ + logf(LOG_LOG, "Estimating hits (%s) " + "%0.1f->%d" + "; %0.1f->"ZINT_FORMAT, + rset->control->desc, + cur, zebraSet->hits, + tot,est); + zebraSet->hits=est; + } + } } - while (rset_read (rset, rfd, &key, &term_index)); + while (rset_read (rset, rfd, &key, &term_index) && (est<0) ); + score = (*rc->calc) (handle, psysno); resultSetInsertRank (zh, sort_info, psysno, score, 'A'); - (*rc->end) (zh, handle); + (*rc->end) (zh->reg, handle); } rset_close (rset, rfd); - logf (LOG_DEBUG, "%d keys, %d distinct sysnos", kno, zebraSet->hits); + + for (i = 0; i < rset->no_rset_terms; i++) + yaz_log (LOG_LOG, "term=\"%s\" nn=" ZINT_FORMAT " type=%s count=" ZINT_FORMAT, + rset->rset_terms[i]->name, + rset->rset_terms[i]->nn, + rset->rset_terms[i]->flags, + rset->rset_terms[i]->count); + + yaz_log (LOG_LOG, ZINT_FORMAT " keys, %d distinct sysnos", + kno, zebraSet->hits); } ZebraRankClass zebraRankLookup (ZebraHandle zh, const char *name) { - ZebraRankClass p = zh->service->rank_classes; + ZebraRankClass p = zh->reg->rank_classes; while (p && strcmp (p->control->name, name)) p = p->next; if (p && !p->init_flag) { if (p->control->create) - p->class_handle = (*p->control->create)(zh->service); + p->class_handle = (*p->control->create)(zh); p->init_flag = 1; } return p; } -void zebraRankInstall (ZebraService zh, struct rank_control *ctrl) +void zebraRankInstall (struct zebra_register *reg, struct rank_control *ctrl) { ZebraRankClass p = (ZebraRankClass) xmalloc (sizeof(*p)); p->control = (struct rank_control *) xmalloc (sizeof(*p->control)); memcpy (p->control, ctrl, sizeof(*p->control)); p->control->name = xstrdup (ctrl->name); p->init_flag = 0; - p->next = zh->rank_classes; - zh->rank_classes = p; + p->next = reg->rank_classes; + reg->rank_classes = p; } -void zebraRankDestroy (ZebraService zh) +void zebraRankDestroy (struct zebra_register *reg) { - ZebraRankClass p = zh->rank_classes; + ZebraRankClass p = reg->rank_classes; while (p) { ZebraRankClass p_next = p->next; if (p->init_flag && p->control->destroy) - (*p->control->destroy)(zh, p->class_handle); + (*p->control->destroy)(reg, p->class_handle); xfree (p->control->name); xfree (p->control); xfree (p); p = p_next; } - zh->rank_classes = NULL; + reg->rank_classes = NULL; }