X-Git-Url: http://git.indexdata.com/?p=idzebra-moved-to-github.git;a=blobdiff_plain;f=index%2Fzsets.c;h=6db021b8f3fe4f1559dc83217ed900bb0b23d4b2;hp=ed5e1a6834d928ceeed1f25a8cd0dc1b572e21c5;hb=896c0427df9d8eff5de6a1735dcd992e067df844;hpb=4c1c4df443dc34e0a183de4760a2890396409bb8 diff --git a/index/zsets.c b/index/zsets.c index ed5e1a6..6db021b 100644 --- a/index/zsets.c +++ b/index/zsets.c @@ -1,96 +1,47 @@ -/* - * Copyright (C) 1994-1998, Index Data - * All rights reserved. - * Sebastian Hammer, Adam Dickmeiss - * - * $Log: zsets.c,v $ - * Revision 1.20 1998-11-16 10:10:53 adam - * Fixed problem with zebraPosSetCreate that occurred when positions were - * less than 1. - * - * Revision 1.19 1998/09/22 10:48:22 adam - * Minor changes in search API. - * - * Revision 1.18 1998/09/22 10:03:45 adam - * Changed result sets to be persistent in the sense that they can - * be re-searched if needed. - * Fixed memory leak in rsm_or. - * - * Revision 1.17 1998/06/23 15:33:36 adam - * Added feature to specify sort criteria in query (type 7 specifies - * sort flags). - * - * Revision 1.16 1998/05/20 10:12:24 adam - * Implemented automatic EXPLAIN database maintenance. - * Modified Zebra to work with ASN.1 compiled version of YAZ. - * - * Revision 1.15 1998/03/05 08:45:14 adam - * New result set model and modular ranking system. Moved towards - * descent server API. System information stored as "SGML" records. - * - * Revision 1.14 1998/02/10 16:39:15 adam - * Minor change. - * - * Revision 1.13 1998/02/10 12:03:06 adam - * Implemented Sort. - * - * Revision 1.12 1997/09/25 14:57:36 adam - * Windows NT port. - * - * Revision 1.11 1996/12/23 15:30:46 adam - * Work on truncation. - * Bug fix: result sets weren't deleted after server shut down. - * - * Revision 1.10 1995/10/30 15:08:08 adam - * Bug fixes. - * - * Revision 1.9 1995/10/17 18:02:14 adam - * New feature: databases. Implemented as prefix to words in dictionary. - * - * Revision 1.8 1995/10/10 13:59:25 adam - * Function rset_open changed its wflag parameter to general flags. - * - * Revision 1.7 1995/10/06 14:38:01 adam - * New result set method: r_score. - * Local no (sysno) and score is transferred to retrieveCtrl. - * - * Revision 1.6 1995/09/28 09:19:49 adam - * xfree/xmalloc used everywhere. - * Extract/retrieve method seems to work for text records. - * - * Revision 1.5 1995/09/27 16:17:32 adam - * More work on retrieve. - * - * Revision 1.4 1995/09/07 13:58:36 adam - * New parameter: result-set file descriptor (RSFD) to support multiple - * positions within the same result-set. - * Boolean operators: and, or, not implemented. - * Result-set references. - * - * Revision 1.3 1995/09/06 16:11:19 adam - * Option: only one word key per file. - * - * Revision 1.2 1995/09/06 10:33:04 adam - * More work on present. Some log messages removed. - * - * Revision 1.1 1995/09/05 15:28:40 adam - * More work on search engine. - * - */ +/* $Id: zsets.c,v 1.38 2002-08-02 19:26:56 adam Exp $ + Copyright (C) 1995,1996,1997,1998,1999,2000,2001,2002 + Index Data Aps + +This file is part of the Zebra server. + +Zebra is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. + +Zebra is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License +along with Zebra; see the file LICENSE.zebra. If not, write to the +Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA +02111-1307, USA. +*/ + + #include #include -#ifdef WINDOWS +#ifdef WIN32 #include #else #include #endif -#include "zserver.h" +#include "index.h" #include #define SORT_IDX_ENTRYSIZE 64 #define ZSET_SORT_MAX_LEVEL 3 +struct zebra_set_term_entry { + int reg_type; + char *db; + int set; + int use; + char *term; +}; struct zebra_set { char *name; RSET rset; @@ -100,7 +51,10 @@ struct zebra_set { char **basenames; Z_RPNQuery *rpn; struct zset_sort_info *sort_info; + struct zebra_set_term_entry *term_entries; + int term_entries_max; struct zebra_set *next; + int locked; }; struct zset_sort_entry { @@ -117,12 +71,12 @@ struct zset_sort_info { }; ZebraSet resultSetAddRPN (ZebraHandle zh, ODR input, ODR output, - Z_RPNQuery *rpn, int num_bases, char **basenames, + Z_RPNQuery *rpn, int num_bases, + char **basenames, const char *setname) { ZebraSet zebraSet; - - zlog_rpn (rpn); + int i; zh->errCode = 0; zh->errString = NULL; @@ -131,10 +85,16 @@ ZebraSet resultSetAddRPN (ZebraHandle zh, ODR input, ODR output, zebraSet = resultSetAdd (zh, setname, 1); if (!zebraSet) return 0; + zebraSet->locked = 1; zebraSet->rpn = 0; + zebraSet->nmem = nmem_create (); + zebraSet->num_bases = num_bases; - zebraSet->basenames = basenames; - zebraSet->nmem = odr_extract_mem (input); + zebraSet->basenames = + nmem_malloc (zebraSet->nmem, num_bases * sizeof(*zebraSet->basenames)); + for (i = 0; ibasenames[i] = nmem_strdup (zebraSet->nmem, basenames[i]); + zebraSet->rset = rpn_search (zh, output->mem, rpn, zebraSet->num_bases, @@ -143,9 +103,91 @@ ZebraSet resultSetAddRPN (ZebraHandle zh, ODR input, ODR output, zh->hits = zebraSet->hits; if (zebraSet->rset) zebraSet->rpn = rpn; + zebraSet->locked = 0; return zebraSet; } +void resultSetAddTerm (ZebraHandle zh, ZebraSet s, int reg_type, + const char *db, int set, + int use, const char *term) +{ + if (!s->nmem) + s->nmem = nmem_create (); + if (!s->term_entries) + { + int i; + s->term_entries_max = 1000; + s->term_entries = + nmem_malloc (s->nmem, s->term_entries_max * + sizeof(*s->term_entries)); + for (i = 0; i < s->term_entries_max; i++) + s->term_entries[i].term = 0; + } + if (s->hits < s->term_entries_max) + { + s->term_entries[s->hits].reg_type = reg_type; + s->term_entries[s->hits].db = nmem_strdup (s->nmem, db); + s->term_entries[s->hits].set = set; + s->term_entries[s->hits].use = use; + s->term_entries[s->hits].term = nmem_strdup (s->nmem, term); + } + (s->hits)++; +} + + +int zebra_resultSetTerms (ZebraHandle zh, const char *setname, + int no, int *count, + int *type, char *out, size_t *len) +{ + ZebraSet s = resultSetGet (zh, setname); + int no_max = 0; + + if (count) + *count = 0; + if (!s || !s->rset) + return 0; + no_max = s->rset->no_rset_terms; + if (no < 0 || no >= no_max) + return 0; + if (count) + *count = s->rset->rset_terms[no]->count; + if (type) + *type = s->rset->rset_terms[no]->type; + + if (out) + { + char *inbuf = s->rset->rset_terms[no]->name; + size_t inleft = strlen(inbuf); + size_t outleft = *len - 1; + int converted = 0; +#if HAVE_ICONV_H + if (zh->iconv_from_utf8 != (iconv_t)(-1)) + { + char *outbuf = out; + size_t ret; + + ret = iconv(zh->iconv_from_utf8, &inbuf, &inleft, + &outbuf, &outleft); + if (ret == (size_t)(-1)) + *len = 0; + else + *len = outbuf - out; + converted = 1; + } +#endif + if (!converted) + { + if (inleft > outleft) + inleft = outleft; + *len = inleft; + memcpy (out, inbuf, *len); + } + out[*len] = 0; + } + return no_max; +} + + ZebraSet resultSetAdd (ZebraHandle zh, const char *name, int ov) { ZebraSet s; @@ -157,7 +199,7 @@ ZebraSet resultSetAdd (ZebraHandle zh, const char *name, int ov) if (s) { logf (LOG_DEBUG, "updating result set %s", name); - if (!ov) + if (!ov || s->locked) return NULL; if (s->rset) rset_delete (s->rset); @@ -167,25 +209,30 @@ ZebraSet resultSetAdd (ZebraHandle zh, const char *name, int ov) else { logf (LOG_DEBUG, "adding result set %s", name); - s = xmalloc (sizeof(*s)); + s = (ZebraSet) xmalloc (sizeof(*s)); s->next = zh->sets; zh->sets = s; - s->name = xmalloc (strlen(name)+1); + s->name = (char *) xmalloc (strlen(name)+1); strcpy (s->name, name); - s->sort_info = xmalloc (sizeof(*s->sort_info)); + s->sort_info = (struct zset_sort_info *) + xmalloc (sizeof(*s->sort_info)); s->sort_info->max_entries = 1000; - s->sort_info->entries = + s->sort_info->entries = (struct zset_sort_entry **) xmalloc (sizeof(*s->sort_info->entries) * s->sort_info->max_entries); - s->sort_info->all_entries = + s->sort_info->all_entries = (struct zset_sort_entry *) xmalloc (sizeof(*s->sort_info->all_entries) * s->sort_info->max_entries); for (i = 0; i < s->sort_info->max_entries; i++) s->sort_info->entries[i] = s->sort_info->all_entries + i; } + s->locked = 0; + s->term_entries = 0; + s->hits = 0; s->rset = 0; - s->nmem = 0; + s->nmem = 0; + s->rpn = 0; return s; } @@ -196,47 +243,82 @@ ZebraSet resultSetGet (ZebraHandle zh, const char *name) for (s = zh->sets; s; s = s->next) if (!strcmp (s->name, name)) { - if (!s->rset && s->rpn) + if (!s->term_entries && !s->rset && s->rpn) { NMEM nmem = nmem_create (); + yaz_log (LOG_LOG, "research %s", name); s->rset = rpn_search (zh, nmem, s->rpn, s->num_bases, s->basenames, s->name, s); nmem_destroy (nmem); - } return s; } return NULL; } - -void resultSetDestroy (ZebraHandle zh) +void resultSetInvalidate (ZebraHandle zh) { - ZebraSet s, s1; - - for (s = zh->sets; s; s = s1) + ZebraSet s = zh->sets; + + for (; s; s = s->next) { - s1 = s->next; - - xfree (s->sort_info->all_entries); - xfree (s->sort_info->entries); - xfree (s->sort_info); + if (s->rset) + rset_delete (s->rset); + s->rset = 0; + } +} - if (s->nmem) - nmem_destroy (s->nmem); - rset_delete (s->rset); - xfree (s->name); - xfree (s); +void resultSetDestroy (ZebraHandle zh, int num, char **names,int *statuses) +{ + ZebraSet * ss = &zh->sets; + int i; + + if (statuses) + for (i = 0; ierrCode = 0; + zh->errString = NULL; + while (*ss) + { + int i = -1; + ZebraSet s = *ss; + if (num >= 0) + { + for (i = 0; iname, names[i])) + { + if (statuses) + statuses[i] = Z_DeleteStatus_success; + i = -1; + break; + } + } + if (i < 0) + { + *ss = s->next; + + xfree (s->sort_info->all_entries); + xfree (s->sort_info->entries); + xfree (s->sort_info); + + if (s->nmem) + nmem_destroy (s->nmem); + if (s->rset) + rset_delete (s->rset); + xfree (s->name); + xfree (s); + } + else + ss = &s->next; } - zh->sets = NULL; } ZebraPosSet zebraPosSetCreate (ZebraHandle zh, const char *name, int num, int *positions) { ZebraSet sset; - ZebraPosSet sr; + ZebraPosSet sr = 0; RSET rset; int i; struct zset_sort_info *sort_info; @@ -244,75 +326,97 @@ ZebraPosSet zebraPosSetCreate (ZebraHandle zh, const char *name, if (!(sset = resultSetGet (zh, name))) return NULL; if (!(rset = sset->rset)) - return NULL; - sr = xmalloc (sizeof(*sr) * num); - for (i = 0; isort_info; - if (sort_info) { - int position; - + if (!sset->term_entries) + return 0; + sr = (ZebraPosSet) xmalloc (sizeof(*sr) * num); for (i = 0; i 0 && position <= sort_info->num_entries) + sr[i].sysno = 0; + sr[i].score = -1; + sr[i].term = 0; + sr[i].db = 0; + + if (positions[i] <= sset->term_entries_max) { - logf (LOG_DEBUG, "got pos=%d (sorted)", position); - sr[i].sysno = sort_info->entries[position-1]->sysno; - sr[i].score = sort_info->entries[position-1]->score; + sr[i].term = sset->term_entries[positions[i]-1].term; + sr[i].db = sset->term_entries[positions[i]-1].db; } } } - /* did we really get all entries using sort ? */ - for (i = 0; isort_info; if (sort_info) - position = sort_info->num_entries; - while (num_i < num && positions[num_i] < position) - num_i++; - rfd = rset_open (rset, RSETF_READ); - while (num_i < num && rset_read (rset, rfd, &key, &term_index)) { - if (key.sysno != psysno) + int position; + + for (i = 0; i 0 && position <= sort_info->num_entries) { - /* determine we alreay have this in our set */ - for (i = sort_info->num_entries; --i >= 0; ) - if (psysno == sort_info->entries[i]->sysno) - break; - if (i >= 0) - continue; + logf (LOG_DEBUG, "got pos=%d (sorted)", position); + sr[i].sysno = sort_info->entries[position-1]->sysno; + sr[i].score = sort_info->entries[position-1]->score; } - position++; - assert (num_i < num); - if (position == positions[num_i]) + } + } + /* did we really get all entries using sort ? */ + for (i = 0; inum_entries; + while (num_i < num && positions[num_i] < position) + num_i++; + rfd = rset_open (rset, RSETF_READ); + while (num_i < num && rset_read (rset, rfd, &key, &term_index)) + { + if (key.sysno != psysno) { - sr[num_i].sysno = psysno; - logf (LOG_DEBUG, "got pos=%d (unsorted)", position); - sr[num_i].score = -1; - num_i++; + psysno = key.sysno; + if (sort_info) + { + /* determine we alreay have this in our set */ + for (i = sort_info->num_entries; --i >= 0; ) + if (psysno == sort_info->entries[i]->sysno) + break; + if (i >= 0) + continue; + } + position++; + assert (num_i < num); + if (position == positions[num_i]) + { + sr[num_i].sysno = psysno; + logf (LOG_DEBUG, "got pos=%d (unsorted)", position); + sr[num_i].score = -1; + num_i++; + } } } + rset_close (rset, rfd); } - rset_close (rset, rfd); } return sr; } @@ -322,13 +426,14 @@ void zebraPosSetDestroy (ZebraHandle zh, ZebraPosSet records, int num) xfree (records); } -struct sortKey { +struct sortKeyInfo { int relation; int attrUse; + int numerical; }; void resultSetInsertSort (ZebraHandle zh, ZebraSet sset, - struct sortKey *criteria, int num_criteria, + struct sortKeyInfo *criteria, int num_criteria, int sysno) { struct zset_sort_entry this_entry; @@ -336,11 +441,11 @@ void resultSetInsertSort (ZebraHandle zh, ZebraSet sset, struct zset_sort_info *sort_info = sset->sort_info; int i, j; - sortIdx_sysno (zh->sortIdx, sysno); + sortIdx_sysno (zh->reg->sortIdx, sysno); for (i = 0; isortIdx, criteria[i].attrUse); - sortIdx_read (zh->sortIdx, this_entry.buf[i]); + sortIdx_type (zh->reg->sortIdx, criteria[i].attrUse); + sortIdx_read (zh->reg->sortIdx, this_entry.buf[i]); } i = sort_info->num_entries; while (--i >= 0) @@ -348,8 +453,21 @@ void resultSetInsertSort (ZebraHandle zh, ZebraSet sset, int rel = 0; for (j = 0; jentries[i]->buf[j], + if (criteria[j].numerical) + { + double diff = atof(this_entry.buf[j]) - + atof(sort_info->entries[i]->buf[j]); + rel = 0; + if (diff > 0.0) + rel = 1; + else if (diff < 0.0) + rel = -1; + } + else + { + rel = memcmp (this_entry.buf[j], sort_info->entries[i]->buf[j], SORT_IDX_ENTRYSIZE); + } if (rel) break; } @@ -366,20 +484,23 @@ void resultSetInsertSort (ZebraHandle zh, ZebraSet sset, break; } } - j = sort_info->max_entries-1; + ++i; + j = sort_info->max_entries; if (i == j) return; - ++i; + + if (sort_info->num_entries == j) + --j; + else + j = (sort_info->num_entries)++; new_entry = sort_info->entries[j]; while (j != i) { sort_info->entries[j] = sort_info->entries[j-1]; --j; } - sort_info->entries[j] = new_entry; + sort_info->entries[i] = new_entry; assert (new_entry); - if (sort_info->num_entries != sort_info->max_entries) - (sort_info->num_entries)++; for (i = 0; ibuf[i], this_entry.buf[i], SORT_IDX_ENTRYSIZE); new_entry->sysno = sysno; @@ -410,20 +531,24 @@ void resultSetInsertRank (ZebraHandle zh, struct zset_sort_info *sort_info, break; } } - j = sort_info->max_entries-1; + ++i; + j = sort_info->max_entries; if (i == j) return; - ++i; + + if (sort_info->num_entries == j) + --j; + else + j = (sort_info->num_entries)++; + new_entry = sort_info->entries[j]; while (j != i) { sort_info->entries[j] = sort_info->entries[j-1]; --j; } - sort_info->entries[j] = new_entry; + sort_info->entries[i] = new_entry; assert (new_entry); - if (sort_info->num_entries != sort_info->max_entries) - (sort_info->num_entries)++; new_entry->sysno = sysno; new_entry->score = score; } @@ -476,12 +601,12 @@ void resultSetSortSingle (ZebraHandle zh, NMEM nmem, { int i, psysno = 0; struct it_key key; - struct sortKey sort_criteria[3]; + struct sortKeyInfo sort_criteria[3]; int num_criteria; int term_index; RSFD rfd; - logf (LOG_DEBUG, "resultSetSortSingle start"); + logf (LOG_LOG, "resultSetSortSingle start"); sset->sort_info->num_entries = 0; sset->hits = 0; @@ -526,14 +651,16 @@ void resultSetSortSingle (ZebraHandle zh, NMEM nmem, case Z_SortKey_sortAttributes: logf (LOG_DEBUG, "Sort: key %d is of type sortAttributes", i+1); sort_criteria[i].attrUse = - zebra_maps_sort (zh->zebra_maps, sk->u.sortAttributes); + zebra_maps_sort (zh->reg->zebra_maps, + sk->u.sortAttributes, + &sort_criteria[i].numerical); logf (LOG_DEBUG, "use value = %d", sort_criteria[i].attrUse); if (sort_criteria[i].attrUse == -1) { zh->errCode = 116; return; } - if (sortIdx_type (zh->sortIdx, sort_criteria[i].attrUse)) + if (sortIdx_type (zh->reg->sortIdx, sort_criteria[i].attrUse)) { zh->errCode = 207; return; @@ -554,9 +681,15 @@ void resultSetSortSingle (ZebraHandle zh, NMEM nmem, } rset_close (rset, rfd); - zh->errCode = 0; + for (i = 0; i < rset->no_rset_terms; i++) + yaz_log (LOG_LOG, "term=\"%s\" nn=%d type=%s count=%d", + rset->rset_terms[i]->name, + rset->rset_terms[i]->nn, + rset->rset_terms[i]->flags, + rset->rset_terms[i]->count); + *sort_status = Z_SortStatus_success; - logf (LOG_DEBUG, "resultSetSortSingle end"); + logf (LOG_LOG, "resultSetSortSingle end"); } RSET resultSetRef (ZebraHandle zh, Z_ResultSetId *resultSetId) @@ -583,12 +716,7 @@ void resultSetRank (ZebraHandle zh, ZebraSet zebraSet, RSET rset) zebraSet->hits = 0; rfd = rset_open (rset, RSETF_READ); - logf (LOG_DEBUG, "resultSetRank"); - for (i = 0; i < rset->no_rset_terms; i++) - logf (LOG_DEBUG, "term=\"%s\" cnt=%d type=%s", - rset->rset_terms[i]->name, - rset->rset_terms[i]->nn, - rset->rset_terms[i]->flags); + yaz_log (LOG_LOG, "resultSetRank"); rank_class = zebraRankLookup (zh, "rank-1"); rc = rank_class->control; @@ -598,7 +726,7 @@ void resultSetRank (ZebraHandle zh, ZebraSet zebraSet, RSET rset) int psysno = key.sysno; int score; void *handle = - (*rc->begin) (zh, rank_class->class_handle, rset); + (*rc->begin) (zh->reg, rank_class->class_handle, rset); (zebraSet->hits)++; do { @@ -616,49 +744,57 @@ void resultSetRank (ZebraHandle zh, ZebraSet zebraSet, RSET rset) while (rset_read (rset, rfd, &key, &term_index)); score = (*rc->calc) (handle, psysno); resultSetInsertRank (zh, sort_info, psysno, score, 'A'); - (*rc->end) (zh, handle); + (*rc->end) (zh->reg, handle); } rset_close (rset, rfd); - logf (LOG_DEBUG, "%d keys, %d distinct sysnos", kno, zebraSet->hits); + + for (i = 0; i < rset->no_rset_terms; i++) + yaz_log (LOG_LOG, "term=\"%s\" nn=%d type=%s count=%d", + rset->rset_terms[i]->name, + rset->rset_terms[i]->nn, + rset->rset_terms[i]->flags, + rset->rset_terms[i]->count); + + yaz_log (LOG_LOG, "%d keys, %d distinct sysnos", kno, zebraSet->hits); } ZebraRankClass zebraRankLookup (ZebraHandle zh, const char *name) { - ZebraRankClass p = zh->rank_classes; + ZebraRankClass p = zh->reg->rank_classes; while (p && strcmp (p->control->name, name)) p = p->next; if (p && !p->init_flag) { if (p->control->create) - p->class_handle = (*p->control->create)(zh); + p->class_handle = (*p->control->create)(zh->reg); p->init_flag = 1; } return p; } -void zebraRankInstall (ZebraHandle zh, struct rank_control *ctrl) +void zebraRankInstall (struct zebra_register *reg, struct rank_control *ctrl) { - ZebraRankClass p = xmalloc (sizeof(*p)); - p->control = xmalloc (sizeof(*p->control)); + ZebraRankClass p = (ZebraRankClass) xmalloc (sizeof(*p)); + p->control = (struct rank_control *) xmalloc (sizeof(*p->control)); memcpy (p->control, ctrl, sizeof(*p->control)); p->control->name = xstrdup (ctrl->name); p->init_flag = 0; - p->next = zh->rank_classes; - zh->rank_classes = p; + p->next = reg->rank_classes; + reg->rank_classes = p; } -void zebraRankDestroy (ZebraHandle zh) +void zebraRankDestroy (struct zebra_register *reg) { - ZebraRankClass p = zh->rank_classes; + ZebraRankClass p = reg->rank_classes; while (p) { ZebraRankClass p_next = p->next; if (p->init_flag && p->control->destroy) - (*p->control->destroy)(zh, p->class_handle); + (*p->control->destroy)(reg, p->class_handle); xfree (p->control->name); xfree (p->control); xfree (p); p = p_next; } - zh->rank_classes = NULL; + reg->rank_classes = NULL; }