-/* $Id: zsets.c,v 1.126 2007-11-30 12:19:09 adam Exp $
- Copyright (C) 1995-2007
- Index Data ApS
-
-This file is part of the Zebra server.
+/* This file is part of the Zebra server.
+ Copyright (C) 2004-2013 Index Data
Zebra is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free
*/
+#if HAVE_CONFIG_H
+#include <config.h>
+#endif
#include <stdio.h>
#include <assert.h>
#ifdef WIN32
sort_sequence->num_specs = 10; /* FIXME - Hard-coded number */
sort_sequence->specs = (Z_SortKeySpec **)
nmem_malloc(nmem, sort_sequence->num_specs *
- sizeof(*sort_sequence->specs));
+ sizeof(*sort_sequence->specs));
for (i = 0; i<sort_sequence->num_specs; i++)
sort_sequence->specs[i] = 0;
-
+
rpn_get_top_approx_limit(zh, rpn->RPNStructure, &sset->approx_limit);
res = rpn_search_top(zh, rpn->RPNStructure, rpn->attributeSetId,
+ sset->approx_limit,
nmem, rset_nmem,
sort_sequence,
sset->num_bases, sset->basenames,
for (i = 0; sort_sequence->specs[i]; i++)
;
sort_sequence->num_specs = i;
- rset->hits_limit = sset->approx_limit;
+ rset_set_hits_limit(rset, sset->approx_limit);
+
if (!i)
{
res = resultSetRank(zh, sset, rset, rset_nmem);
else
{
res = resultSetSortSingle(zh, nmem, sset, rset,
- sort_sequence, &sort_status);
+ sort_sequence, &sort_status);
}
sset->rset = rset;
return res;
zebraSet->locked = 1;
zebraSet->rpn = 0;
zebraSet->nmem = m;
- zebraSet->rset_nmem = nmem_create();
+ zebraSet->rset_nmem = nmem_create();
zebraSet->num_bases = num_bases;
- zebraSet->basenames =
+ zebraSet->basenames =
nmem_malloc(zebraSet->nmem, num_bases * sizeof(*zebraSet->basenames));
for (i = 0; i<num_bases; i++)
zebraSet->basenames[i] = nmem_strdup(zebraSet->nmem, basenames[i]);
}
void resultSetAddTerm(ZebraHandle zh, ZebraSet s, int reg_type,
- const char *db, const char *index_name,
+ const char *db, const char *index_name,
const char *term)
{
assert(zh); /* compiler shut up */
int i;
s->term_entries_max = 1000;
s->term_entries =
- nmem_malloc(s->nmem, s->term_entries_max *
- sizeof(*s->term_entries));
+ nmem_malloc(s->nmem, s->term_entries_max *
+ sizeof(*s->term_entries));
for (i = 0; i < s->term_entries_max; i++)
s->term_entries[i].term = 0;
}
for (s = zh->sets; s; s = s->next)
if (!strcmp(s->name, name))
break;
-
+
if (!log_level_set)
loglevels();
if (s)
s->sort_info->entries = (struct zset_sort_entry **)
xmalloc(sizeof(*s->sort_info->entries) *
- s->sort_info->max_entries);
+ s->sort_info->max_entries);
s->sort_info->all_entries = (struct zset_sort_entry *)
xmalloc(sizeof(*s->sort_info->all_entries) *
- s->sort_info->max_entries);
+ s->sort_info->max_entries);
for (i = 0; i < s->sort_info->max_entries; i++)
s->sort_info->entries[i] = s->sort_info->all_entries + i;
}
int sort_status;
yaz_log(log_level_resultsets, "resort %s", name);
resultSetSortSingle(zh, nmem, s, s->rset, s->sortSpec,
- &sort_status);
+ &sort_status);
}
nmem_destroy(nmem);
}
void resultSetInvalidate(ZebraHandle zh)
{
ZebraSet s = zh->sets;
-
+
yaz_log(log_level_resultsets, "invalidating result sets");
for (; s; s = s->next)
{
{
ZebraSet * ss = &zh->sets;
int i;
-
+
if (statuses)
for (i = 0; i<num; i++)
statuses[i] = Z_DeleteStatus_resultSetDidNotExist;
if (i < 0)
{
*ss = s->next;
-
+
xfree(s->sort_info->all_entries);
xfree(s->sort_info->entries);
xfree(s->sort_info);
-
+
if (s->nmem)
nmem_destroy(s->nmem);
if (s->rset)
}
ZebraMetaRecord *zebra_meta_records_create_range(ZebraHandle zh,
- const char *name,
+ const char *name,
zint start, int num)
{
zint pos_small[10];
if (num > 10)
pos = xmalloc(sizeof(*pos) * num);
-
+
for (i = 0; i<num; i++)
pos[i] = start+i;
mr = zebra_meta_records_create(zh, name, num, pos);
-
+
if (num > 10)
xfree(pos);
return mr;
}
-ZebraMetaRecord *zebra_meta_records_create(ZebraHandle zh, const char *name,
+ZebraMetaRecord *zebra_meta_records_create(ZebraHandle zh, const char *name,
int num, zint *positions)
{
ZebraSet sset;
if (sort_info)
{
zint position;
-
+
for (i = 0; i<num; i++)
{
position = positions[i];
zint psysno = 0;
RSFD rfd;
struct it_key key;
-
+
if (sort_info)
position = sort_info->num_entries;
while (num_i < num && positions[num_i] <= position)
num_i++;
-
+
if (sset->cache_rfd &&
num_i < num && positions[num_i] > sset->cache_position)
{
}
void zebra_meta_records_destroy(ZebraHandle zh, ZebraMetaRecord *records,
- int num)
+ int num)
{
assert(zh); /* compiler shut up about unused arg */
xfree(records);
struct sortKeyInfo {
int relation;
- int ord;
- int numerical;
+ int *ord; /* array of ord for each database searched */
+ int *numerical; /* array of ord for each database searched */
const char *index_type;
};
void resultSetInsertSort(ZebraHandle zh, ZebraSet sset,
+ int database_no,
struct sortKeyInfo *criteria, int num_criteria,
zint sysno,
char *cmp_buf[], char *tmp_cmp_buf[])
struct zset_sort_entry *new_entry = NULL;
struct zset_sort_info *sort_info = sset->sort_info;
int i, j;
+ WRBUF w = wrbuf_alloc();
zebra_sort_sysno(zh->reg->sort_index, sysno);
for (i = 0; i<num_criteria; i++)
{
char *this_entry_buf = tmp_cmp_buf[i];
memset(this_entry_buf, '\0', SORT_IDX_ENTRYSIZE);
- if (criteria[i].ord != -1)
+
+ if (criteria[i].ord[database_no] != -1)
+ {
+ yaz_log(log_level_sort, "pre zebra_sort_type ord is %d",
+ criteria[i].ord[database_no]);
+ zebra_sort_type(zh->reg->sort_index, criteria[i].ord[database_no]);
+ wrbuf_rewind(w);
+ if (zebra_sort_read(zh->reg->sort_index, 0, w))
+ {
+ /* consider each sort entry and take lowest/highest one
+ of the one as sorting key depending on whether sort is
+ ascending/descending */
+ int off = 0;
+ while (off != wrbuf_len(w))
+ {
+ size_t l = strlen(wrbuf_buf(w)+off);
+ assert(off < wrbuf_len(w));
+
+ if (l >= SORT_IDX_ENTRYSIZE)
+ l = SORT_IDX_ENTRYSIZE-1;
+ if ( (off == 0)
+ || (criteria[i].relation == 'A'
+ && strcmp(wrbuf_buf(w)+off, this_entry_buf) < 0)
+ || (criteria[i].relation == 'D'
+ && strcmp(wrbuf_buf(w)+off, this_entry_buf) > 0)
+ )
+ {
+ memcpy(this_entry_buf, wrbuf_buf(w)+off, l);
+ this_entry_buf[l] = '\0';
+ }
+ off += 1 + strlen(wrbuf_buf(w)+off);
+ }
+ }
+ }
+ else
{
- zebra_sort_type(zh->reg->sort_index, criteria[i].ord);
- zebra_sort_read(zh->reg->sort_index, this_entry_buf);
+ yaz_log(log_level_sort, "criteria[i].ord is -1 so not reading from sort index");
}
}
+ wrbuf_destroy(w);
i = sort_info->num_entries;
while (--i >= 0)
{
for (j = 0; j<num_criteria; j++)
{
char *this_entry_buf = tmp_cmp_buf[j];
- char *other_entry_buf =
+ char *other_entry_buf =
cmp_buf[j] + i * SORT_IDX_ENTRYSIZE;
- if (criteria[j].numerical)
+ if (criteria[j].numerical[database_no])
{
char this_entry_org[1024];
char other_entry_org[1024];
zebra_term_untrans(zh, index_type, other_entry_org,
other_entry_buf);
diff = atof(this_entry_org) - atof(other_entry_org);
-
+
if (diff > 0.0)
rel = 1;
else if (diff < 0.0)
rel = memcmp(this_entry_buf, other_entry_buf,
SORT_IDX_ENTRYSIZE);
}
+ /* when the compare is equal, continue to next criteria,
+ else break out */
if (rel)
break;
- }
+ }
if (!rel)
break;
if (criteria[j].relation == 'A')
}
}
++i;
+ yaz_log(log_level_sort, "ok, we want to insert record at position %d",i);
j = sort_info->max_entries;
- if (i == j)
+ if (i == j){
+ yaz_log(log_level_sort, "sort_info->max_entries reached (%d) abort sort",j);
return;
+ }
if (sort_info->num_entries == j)
--j;
else
j = (sort_info->num_entries)++;
new_entry = sort_info->entries[j];
+ /* move up all higher entries (to make room) */
while (j != i)
{
int k;
sort_info->entries[j] = sort_info->entries[j-1];
--j;
}
+ /* and insert the new entry at the correct place */
sort_info->entries[i] = new_entry;
assert(new_entry);
+ /* and add this to the compare buffer */
for (i = 0; i<num_criteria; i++)
{
char *new_entry_buf = cmp_buf[i] + j * SORT_IDX_ENTRYSIZE;
--j;
else
j = (sort_info->num_entries)++;
-
+
new_entry = sort_info->entries[j];
while (j != i)
{
nset->nmem = nmem_create();
nset->num_bases = rset->num_bases;
- nset->basenames =
+ nset->basenames =
nmem_malloc(nset->nmem, nset->num_bases * sizeof(*rset->basenames));
for (i = 0; i<rset->num_bases; i++)
nset->basenames[i] = nmem_strdup(nset->nmem, rset->basenames[i]);
if (!log_level_set)
loglevels();
yaz_log(log_level_sort, "result set sort input=%s output=%s",
- *input_setnames, output_setname);
+ *input_setnames, output_setname);
sset = resultSetGet(zh, input_setnames[0]);
if (!sset)
{
if (strcmp(output_setname, input_setnames[0]))
sset = resultSetClone(zh, output_setname, sset);
sset->sortSpec = copy_SortKeySpecList(sort_sequence, sset->nmem);
- return resultSetSortSingle (zh, nmem, sset, rset, sort_sequence,
- sort_status);
+ return resultSetSortSingle(zh, nmem, sset, rset, sort_sequence,
+ sort_status);
}
ZEBRA_RES resultSetSortSingle(ZebraHandle zh, NMEM nmem,
int *sort_status)
{
int i;
+ int ib;
int n = 0;
zint kno = 0;
zint psysno = 0;
int numTerms = 0;
size_t sysno_mem_index = 0;
+ int numbases = zh->num_basenames;
+ yaz_log(log_level_sort, "searching %d databases",numbases);
+
if (zh->m_staticrank)
sysno_mem_index = 1;
num_criteria = sort_sequence->num_specs;
if (num_criteria > ZSET_SORT_MAX_LEVEL)
num_criteria = ZSET_SORT_MAX_LEVEL;
+ /* set up the search criteria */
for (i = 0; i < num_criteria; i++)
{
Z_SortKeySpec *sks = sort_sequence->specs[i];
Z_SortKey *sk;
- ZEBRA_RES res;
- sort_criteria[i].ord = -1;
- sort_criteria[i].numerical = 0;
+ sort_criteria[i].ord = (int *)
+ nmem_malloc(nmem, sizeof(int)*numbases);
+ sort_criteria[i].numerical = (int *)
+ nmem_malloc(nmem, sizeof(int)*numbases);
+
+ /* initialize ord and numerical for each database */
+ for (ib = 0; ib < numbases; ib++)
+ {
+ sort_criteria[i].ord[ib] = -1;
+ sort_criteria[i].numerical[ib] = 0;
+ }
if (sks->which == Z_SortKeySpec_missingValueData)
{
{
zebra_setError(zh, YAZ_BIB1_SORT_ILLEGAL_SORT, 0);
return ZEBRA_FAIL;
- }
+ }
sk = sks->sortElement->u.generic;
switch (sk->which)
{
case Z_SortKey_sortField:
- yaz_log(log_level_sort, "key %d is of type sortField",
- i+1);
- sort_criteria[i].numerical = 0;
- sort_criteria[i].ord =
- zebraExplain_lookup_attr_str(zh->reg->zei,
- zinfo_index_category_sort,
- 0, sk->u.sortField);
- if (sks->which != Z_SortKeySpec_null
- && sort_criteria[i].ord == -1)
+ yaz_log(log_level_sort, "key %d is of type sortField", i+1);
+ for (ib = 0; ib < numbases; ib++)
{
- zebra_setError(zh,
- YAZ_BIB1_CANNOT_SORT_ACCORDING_TO_SEQUENCE, 0);
- return ZEBRA_FAIL;
+ zebraExplain_curDatabase(zh->reg->zei, zh->basenames[ib]);
+ sort_criteria[i].numerical[ib] = 0;
+ sort_criteria[i].ord[ib] =
+ zebraExplain_lookup_attr_str(zh->reg->zei,
+ zinfo_index_category_sort,
+ 0, sk->u.sortField);
+ if (sks->which != Z_SortKeySpec_null
+ && sort_criteria[i].ord[ib] == -1)
+ {
+ zebra_setError(zh,
+ YAZ_BIB1_CANNOT_SORT_ACCORDING_TO_SEQUENCE, 0);
+ return ZEBRA_FAIL;
+ }
}
break;
case Z_SortKey_elementSpec:
- yaz_log(log_level_sort, "key %d is of type elementSpec",
- i+1);
+ yaz_log(log_level_sort, "key %d is of type elementSpec", i+1);
zebra_setError(zh, YAZ_BIB1_CANNOT_SORT_ACCORDING_TO_SEQUENCE, 0);
return ZEBRA_FAIL;
case Z_SortKey_sortAttributes:
yaz_log(log_level_sort, "key %d is of type sortAttributes", i+1);
- res = zebra_sort_get_ord(zh, sk->u.sortAttributes,
-
- &sort_criteria[i].ord,
- &sort_criteria[i].numerical);
- if (sks->which != Z_SortKeySpec_null && res != ZEBRA_OK)
- return ZEBRA_FAIL;
+ /* for every database we searched, get the sort index file
+ id (ord) and its numerical indication and store them in
+ the sort_criteria */
+ for (ib = 0; ib < numbases; ib++)
+ {
+ zebraExplain_curDatabase(zh->reg->zei, zh->basenames[ib]);
+ if (zebra_sort_get_ord(zh, sk->u.sortAttributes,
+ &sort_criteria[i].ord[ib],
+ &sort_criteria[i].numerical[ib]) !=
+ ZEBRA_OK && sks->which != Z_SortKeySpec_null)
+ return ZEBRA_FAIL;
+ }
break;
}
- if (zebraExplain_lookup_ord(zh->reg->zei, sort_criteria[i].ord,
+ /* right now we look up the index type based on the first database
+ if the index_type's can differ between the indexes of different
+ databases (which i guess they can?) then we have to store the
+ index types for each database, just like the ord and numerical */
+ if (zebraExplain_lookup_ord(zh->reg->zei, sort_criteria[i].ord[0],
&sort_criteria[i].index_type,
0, 0))
{
}
}
/* allocate space for each cmpare buf + one extra for tmp comparison */
+ /* cmp_buf is an array of array, the first dimension is the criteria and the second dimension are
+ all other result entries to compare against. This is slowly filled when records are processed.
+ tmp_cmp_buf is an array with a value of the current record for each criteria
+ */
for (i = 0; i<num_criteria; i++)
{
cmp_buf[i] = xmalloc(sset->sort_info->max_entries
kno++;
if (this_sys != psysno)
{
+ int database_no = 0;
if ((sset->hits & 255) == 0 && zh->break_handler_func)
{
if (zh->break_handler_func(zh->break_handler_data))
}
(sset->hits)++;
psysno = this_sys;
- resultSetInsertSort(zh, sset,
+
+ /* determine database from the term, but only bother if more than
+ one database is in use*/
+ if (numbases > 1 && termid->ol)
+ {
+ const char *this_db = 0;
+ if (zebraExplain_lookup_ord(zh->reg->zei, termid->ol->ord, 0, &this_db, 0)
+ == 0 && this_db)
+ {
+ for (ib = 0; ib < numbases; ib++)
+ if (!strcmp(this_db, zh->basenames[ib]))
+ database_no = ib;
+ }
+ }
+#if 0
+ yaz_log(YLOG_LOG, "sysno=" ZINT_FORMAT " database_no=%d", this_sys,
+ database_no);
+ ord_list_print(termid->ol);
+#endif
+ resultSetInsertSort(zh, sset, database_no,
sort_criteria, num_criteria, psysno, cmp_buf,
tmp_cmp_buf);
}
}
rset_close(rfd);
+ /* free the compare buffers */
for (i = 0; i<num_criteria; i++)
{
xfree(cmp_buf[i]);
}
yaz_log(log_level_sort, ZINT_FORMAT " keys, " ZINT_FORMAT " sysnos, sort",
- kno, sset->hits);
+ kno, sset->hits);
for (i = 0; i < numTerms; i++)
yaz_log(log_level_sort, "term=\"%s\" type=%s count=" ZINT_FORMAT,
- terms[i]->name, terms[i]->flags, terms[i]->rset->hits_count);
+ terms[i]->name, terms[i]->flags, terms[i]->rset->hits_count);
*sort_status = Z_SortResponse_success;
return ZEBRA_OK;
}
kno++;
if (log_level_searchhits)
key_logdump_txt(log_level_searchhits, &key, termid->name);
- if (this_sys != psysno)
+ if (this_sys != psysno)
{ /* new record .. */
if (!(rfd->counted_items & 255) && zh->break_handler_func)
{
}
if (rfd->counted_items > rset->hits_limit)
stop_flag = 1;
+ if (stop_flag)
+ {
+ zebraSet->estimated_hit_count = 1;
+ break;
+ }
if (psysno)
{ /* only if we did have a previous record */
score = (*rc->calc)(handle, psysno, pstaticrank,
- &stop_flag);
+ &stop_flag);
/* insert the hit. A=Ascending */
resultSetInsertRank(zh, sort_info, psysno, score, 'A');
count++;
}
- if (stop_flag)
- {
- zebraSet->estimated_hit_count = 1;
- rset_set_hits_limit(rset, 0);
- break;
- }
psysno = this_sys;
if (zh->m_staticrank)
pstaticrank = key.mem[0];
TERMID *term_array = xmalloc(num_terms * sizeof(*term_array));
zint *hits_array = xmalloc(num_terms * sizeof(*hits_array));
int *approx_array = xmalloc(num_terms * sizeof(*approx_array));
-
+
trav_rset_for_termids(sset->rset, term_array,
hits_array, approx_array);
{
char *outbuf = termbuf;
size_t ret;
-
+
ret = yaz_iconv(zh->iconv_from_utf8, &inbuf, &inleft,
&outbuf, &outleft);
if (ret == (size_t)(-1))
*termlen = 0;
else
{
- yaz_iconv(zh->iconv_from_utf8, 0, 0,
+ yaz_iconv(zh->iconv_from_utf8, 0, 0,
&outbuf, &outleft);
*termlen = outbuf - termbuf;
}
NMEM nmem = nmem_create();
struct it_key key;
RSET rsets[2], rset_comb;
- RSET rset_temp = rset_create_temp(nmem, kc, kc->scope,
+ RSET rset_temp = rset_create_temp(nmem, kc, kc->scope,
res_get(zh->res, "setTmpDir"),0 );
-
+
TERMID termid;
RSFD rsfd = rset_open(rset_temp, RSETF_WRITE);
-
+
key.mem[0] = sysno;
key.mem[1] = 0;
key.mem[2] = 0;
rsets[0] = rset_temp;
rsets[1] = rset_dup(sset->rset);
-
+
rset_comb = rset_create_and(nmem, kc, kc->scope, 2, rsets);
rsfd = rset_open(rset_comb, RSETF_READ);
}
}
rset_close(rsfd);
-
+
rset_delete(rset_comb);
nmem_destroy(nmem);
kc->dec(kc);
return ZEBRA_OK;
}
-static ZEBRA_RES zebra_recid_to_sysno(ZebraHandle zh,
+static ZEBRA_RES zebra_recid_to_sysno(ZebraHandle zh,
const char **basenames, int num_bases,
zint recid,
zint *sysnos, int *no_sysnos)
ZEBRA_RES res = ZEBRA_OK;
int sysnos_offset = 0;
int i;
-
- if (zh->reg->isamb)
+
+ if (!zh->reg->isamb || !zh->m_segment_indexing)
+ {
+ if (sysnos_offset < *no_sysnos)
+ *sysnos = recid;
+ sysnos_offset++;
+ }
+ else
{
for (i = 0; res == ZEBRA_OK && i < num_bases; i++)
{
char ord_buf[32];
int ord_len = key_SU_encode(ord, ord_buf);
char *info;
-
+
ord_buf[ord_len] = '\0';
-
+
info = dict_lookup(zh->reg->dict, ord_buf);
if (info)
{
struct it_key key_until, key_found;
int i = 0;
int r;
-
+
memcpy(&isam_p, info+1, sizeof(ISAM_P));
-
+
pt = isamb_pp_open(zh->reg->isamb, isam_p, 2);
if (!pt)
res = ZEBRA_FAIL;
key_until.mem[i++] = 0; /* segment */
key_until.mem[i++] = 0;
key_until.len = i;
-
+
r = isamb_pp_forward(pt, &key_found, &key_until);
while (r && key_found.mem[0] == recid)
{
if (sysnos_offset < *no_sysnos)
- sysnos[sysnos_offset] =
+ sysnos[sysnos_offset++] =
key_found.mem[key_found.len-1];
-
- yaz_log(YLOG_LOG, "Found " ZINT_FORMAT,
- key_found.mem[key_found.len-1]);
r = isamb_pp_read(pt, &key_found);
- sysnos_offset++;
}
isamb_pp_close(pt);
}
return res;
}
-ZEBRA_RES zebra_result_recid_to_sysno(ZebraHandle zh,
+ZEBRA_RES zebra_result_recid_to_sysno(ZebraHandle zh,
const char *setname,
zint recid,
zint *sysnos, int *no_sysnos)
return zebra_recid_to_sysno(zh, basenames, num_bases,
recid, sysnos, no_sysnos);
}
-
+
+void zebra_count_set(ZebraHandle zh, RSET rset, zint *count,
+ zint approx_limit)
+{
+ zint psysno = 0;
+ struct it_key key;
+ RSFD rfd;
+
+ yaz_log(YLOG_DEBUG, "count_set");
+
+ rset->hits_limit = approx_limit;
+
+ *count = 0;
+ rfd = rset_open(rset, RSETF_READ);
+ while (rset_read(rfd, &key,0 /* never mind terms */))
+ {
+ if (key.mem[0] != psysno)
+ {
+ psysno = key.mem[0];
+ if (rfd->counted_items >= rset->hits_limit)
+ break;
+ }
+ }
+ rset_close(rfd);
+ *count = rset->hits_count;
+}
+
/*
* Local variables:
* c-basic-offset: 4
+ * c-file-style: "Stroustrup"
* indent-tabs-mode: nil
* End:
* vim: shiftwidth=4 tabstop=8 expandtab
projects / idzebra-moved-to-github.git / blobdiff