X-Git-Url: http://git.indexdata.com/?p=idzebra-moved-to-github.git;a=blobdiff_plain;f=index%2Fzsets.c;h=4a8c74c39c8c26f6638ee1fd0e89069a0b05138a;hp=5944d76b2d711d89698d7f4c62e485a266ba22e5;hb=d87f70faee1aeead5d32c388bfde8c41e859fb2f;hpb=b2084f54a96f78eede08dda6f98d5ee1f502e91b diff --git a/index/zsets.c b/index/zsets.c index 5944d76..4a8c74c 100644 --- a/index/zsets.c +++ b/index/zsets.c @@ -1,8 +1,5 @@ -/* $Id: zsets.c,v 1.129 2007-12-04 11:13:17 adam Exp $ - Copyright (C) 1995-2007 - Index Data ApS - -This file is part of the Zebra server. +/* This file is part of the Zebra server. + Copyright (C) 1994-2011 Index Data Zebra is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free @@ -109,13 +106,14 @@ static ZEBRA_RES resultSetSearch(ZebraHandle zh, NMEM nmem, NMEM rset_nmem, sort_sequence->num_specs = 10; /* FIXME - Hard-coded number */ sort_sequence->specs = (Z_SortKeySpec **) nmem_malloc(nmem, sort_sequence->num_specs * - sizeof(*sort_sequence->specs)); + sizeof(*sort_sequence->specs)); for (i = 0; inum_specs; i++) sort_sequence->specs[i] = 0; rpn_get_top_approx_limit(zh, rpn->RPNStructure, &sset->approx_limit); res = rpn_search_top(zh, rpn->RPNStructure, rpn->attributeSetId, + sset->approx_limit, nmem, rset_nmem, sort_sequence, sset->num_bases, sset->basenames, @@ -128,7 +126,8 @@ static ZEBRA_RES resultSetSearch(ZebraHandle zh, NMEM nmem, NMEM rset_nmem, for (i = 0; sort_sequence->specs[i]; i++) ; sort_sequence->num_specs = i; - rset->hits_limit = sset->approx_limit; + rset_set_hits_limit(rset, sset->approx_limit); + if (!i) { res = resultSetRank(zh, sset, rset, rset_nmem); @@ -136,7 +135,7 @@ static ZEBRA_RES resultSetSearch(ZebraHandle zh, NMEM nmem, NMEM rset_nmem, else { res = resultSetSortSingle(zh, nmem, sset, rset, - sort_sequence, &sort_status); + sort_sequence, &sort_status); } sset->rset = rset; return res; @@ -196,7 +195,7 @@ void resultSetAddTerm(ZebraHandle zh, ZebraSet s, int reg_type, s->term_entries_max = 1000; s->term_entries = nmem_malloc(s->nmem, s->term_entries_max * - sizeof(*s->term_entries)); + sizeof(*s->term_entries)); for (i = 0; i < s->term_entries_max; i++) s->term_entries[i].term = 0; } @@ -255,10 +254,10 @@ ZebraSet resultSetAdd(ZebraHandle zh, const char *name, int ov) s->sort_info->entries = (struct zset_sort_entry **) xmalloc(sizeof(*s->sort_info->entries) * - s->sort_info->max_entries); + s->sort_info->max_entries); s->sort_info->all_entries = (struct zset_sort_entry *) xmalloc(sizeof(*s->sort_info->all_entries) * - s->sort_info->max_entries); + s->sort_info->max_entries); for (i = 0; i < s->sort_info->max_entries; i++) s->sort_info->entries[i] = s->sort_info->all_entries + i; } @@ -296,7 +295,7 @@ ZebraSet resultSetGet(ZebraHandle zh, const char *name) int sort_status; yaz_log(log_level_resultsets, "resort %s", name); resultSetSortSingle(zh, nmem, s, s->rset, s->sortSpec, - &sort_status); + &sort_status); } nmem_destroy(nmem); } @@ -543,7 +542,7 @@ ZebraMetaRecord *zebra_meta_records_create(ZebraHandle zh, const char *name, } void zebra_meta_records_destroy(ZebraHandle zh, ZebraMetaRecord *records, - int num) + int num) { assert(zh); /* compiler shut up about unused arg */ xfree(records); @@ -551,12 +550,13 @@ void zebra_meta_records_destroy(ZebraHandle zh, ZebraMetaRecord *records, struct sortKeyInfo { int relation; - int ord; - int numerical; + int *ord; /* array of ord for each database searched */ + int *numerical; /* array of ord for each database searched */ const char *index_type; }; void resultSetInsertSort(ZebraHandle zh, ZebraSet sset, + int database_no, struct sortKeyInfo *criteria, int num_criteria, zint sysno, char *cmp_buf[], char *tmp_cmp_buf[]) @@ -564,18 +564,53 @@ void resultSetInsertSort(ZebraHandle zh, ZebraSet sset, struct zset_sort_entry *new_entry = NULL; struct zset_sort_info *sort_info = sset->sort_info; int i, j; + WRBUF w = wrbuf_alloc(); zebra_sort_sysno(zh->reg->sort_index, sysno); for (i = 0; ireg->sort_index, criteria[i].ord[database_no]); + wrbuf_rewind(w); + if (zebra_sort_read(zh->reg->sort_index, 0, w)) + { + /* consider each sort entry and take lowest/highest one + of the one as sorting key depending on whether sort is + ascending/descending */ + int off = 0; + while (off != wrbuf_len(w)) + { + size_t l = strlen(wrbuf_buf(w)+off); + assert(off < wrbuf_len(w)); + + if (l >= SORT_IDX_ENTRYSIZE) + l = SORT_IDX_ENTRYSIZE-1; + if ( (off == 0) + || (criteria[i].relation == 'A' + && strcmp(wrbuf_buf(w)+off, this_entry_buf) < 0) + || (criteria[i].relation == 'D' + && strcmp(wrbuf_buf(w)+off, this_entry_buf) > 0) + ) + { + memcpy(this_entry_buf, wrbuf_buf(w)+off, l); + this_entry_buf[l] = '\0'; + } + off += 1 + strlen(wrbuf_buf(w)+off); + } + } + } + else { - zebra_sort_type(zh->reg->sort_index, criteria[i].ord); - zebra_sort_read(zh->reg->sort_index, this_entry_buf); + yaz_log(log_level_sort, "criteria[i].ord is -1 so not reading from sort index"); } } + wrbuf_destroy(w); i = sort_info->num_entries; while (--i >= 0) { @@ -585,7 +620,7 @@ void resultSetInsertSort(ZebraHandle zh, ZebraSet sset, char *this_entry_buf = tmp_cmp_buf[j]; char *other_entry_buf = cmp_buf[j] + i * SORT_IDX_ENTRYSIZE; - if (criteria[j].numerical) + if (criteria[j].numerical[database_no]) { char this_entry_org[1024]; char other_entry_org[1024]; @@ -609,6 +644,8 @@ void resultSetInsertSort(ZebraHandle zh, ZebraSet sset, rel = memcmp(this_entry_buf, other_entry_buf, SORT_IDX_ENTRYSIZE); } + /* when the compare is equal, continue to next criteria, + else break out */ if (rel) break; } @@ -626,15 +663,19 @@ void resultSetInsertSort(ZebraHandle zh, ZebraSet sset, } } ++i; + yaz_log(log_level_sort, "ok, we want to insert record at position %d",i); j = sort_info->max_entries; - if (i == j) + if (i == j){ + yaz_log(log_level_sort, "sort_info->max_entries reached (%d) abort sort",j); return; + } if (sort_info->num_entries == j) --j; else j = (sort_info->num_entries)++; new_entry = sort_info->entries[j]; + /* move up all higher entries (to make room) */ while (j != i) { int k; @@ -647,8 +688,10 @@ void resultSetInsertSort(ZebraHandle zh, ZebraSet sset, sort_info->entries[j] = sort_info->entries[j-1]; --j; } + /* and insert the new entry at the correct place */ sort_info->entries[i] = new_entry; assert(new_entry); + /* and add this to the compare buffer */ for (i = 0; isortSpec = copy_SortKeySpecList(sort_sequence, sset->nmem); - return resultSetSortSingle (zh, nmem, sset, rset, sort_sequence, - sort_status); + return resultSetSortSingle(zh, nmem, sset, rset, sort_sequence, + sort_status); } ZEBRA_RES resultSetSortSingle(ZebraHandle zh, NMEM nmem, @@ -825,6 +868,7 @@ ZEBRA_RES resultSetSortSingle(ZebraHandle zh, NMEM nmem, int *sort_status) { int i; + int ib; int n = 0; zint kno = 0; zint psysno = 0; @@ -838,6 +882,9 @@ ZEBRA_RES resultSetSortSingle(ZebraHandle zh, NMEM nmem, TERMID *terms; int numTerms = 0; size_t sysno_mem_index = 0; + + int numbases = zh->num_basenames; + yaz_log(log_level_sort, "searching %d databases",numbases); if (zh->m_staticrank) sysno_mem_index = 1; @@ -853,14 +900,23 @@ ZEBRA_RES resultSetSortSingle(ZebraHandle zh, NMEM nmem, num_criteria = sort_sequence->num_specs; if (num_criteria > ZSET_SORT_MAX_LEVEL) num_criteria = ZSET_SORT_MAX_LEVEL; + /* set up the search criteria */ for (i = 0; i < num_criteria; i++) { Z_SortKeySpec *sks = sort_sequence->specs[i]; Z_SortKey *sk; - ZEBRA_RES res; - - sort_criteria[i].ord = -1; - sort_criteria[i].numerical = 0; + + sort_criteria[i].ord = (int *) + nmem_malloc(nmem, sizeof(int)*numbases); + sort_criteria[i].numerical = (int *) + nmem_malloc(nmem, sizeof(int)*numbases); + + /* initialize ord and numerical for each database */ + for (ib = 0; ib < numbases; ib++) + { + sort_criteria[i].ord[ib] = -1; + sort_criteria[i].numerical[ib] = 0; + } if (sks->which == Z_SortKeySpec_missingValueData) { @@ -890,37 +946,49 @@ ZEBRA_RES resultSetSortSingle(ZebraHandle zh, NMEM nmem, switch (sk->which) { case Z_SortKey_sortField: - yaz_log(log_level_sort, "key %d is of type sortField", - i+1); - sort_criteria[i].numerical = 0; - sort_criteria[i].ord = - zebraExplain_lookup_attr_str(zh->reg->zei, - zinfo_index_category_sort, - 0, sk->u.sortField); - if (sks->which != Z_SortKeySpec_null - && sort_criteria[i].ord == -1) + yaz_log(log_level_sort, "key %d is of type sortField", i+1); + for (ib = 0; ib < numbases; ib++) { - zebra_setError(zh, - YAZ_BIB1_CANNOT_SORT_ACCORDING_TO_SEQUENCE, 0); - return ZEBRA_FAIL; + zebraExplain_curDatabase(zh->reg->zei, zh->basenames[ib]); + sort_criteria[i].numerical[ib] = 0; + sort_criteria[i].ord[ib] = + zebraExplain_lookup_attr_str(zh->reg->zei, + zinfo_index_category_sort, + 0, sk->u.sortField); + if (sks->which != Z_SortKeySpec_null + && sort_criteria[i].ord[ib] == -1) + { + zebra_setError(zh, + YAZ_BIB1_CANNOT_SORT_ACCORDING_TO_SEQUENCE, 0); + return ZEBRA_FAIL; + } } break; case Z_SortKey_elementSpec: - yaz_log(log_level_sort, "key %d is of type elementSpec", - i+1); + yaz_log(log_level_sort, "key %d is of type elementSpec", i+1); zebra_setError(zh, YAZ_BIB1_CANNOT_SORT_ACCORDING_TO_SEQUENCE, 0); return ZEBRA_FAIL; case Z_SortKey_sortAttributes: yaz_log(log_level_sort, "key %d is of type sortAttributes", i+1); - res = zebra_sort_get_ord(zh, sk->u.sortAttributes, - - &sort_criteria[i].ord, - &sort_criteria[i].numerical); - if (sks->which != Z_SortKeySpec_null && res != ZEBRA_OK) - return ZEBRA_FAIL; + /* for every database we searched, get the sort index file + id (ord) and its numerical indication and store them in + the sort_criteria */ + for (ib = 0; ib < numbases; ib++) + { + zebraExplain_curDatabase(zh->reg->zei, zh->basenames[ib]); + if (zebra_sort_get_ord(zh, sk->u.sortAttributes, + &sort_criteria[i].ord[ib], + &sort_criteria[i].numerical[ib]) != + ZEBRA_OK && sks->which != Z_SortKeySpec_null) + return ZEBRA_FAIL; + } break; } - if (zebraExplain_lookup_ord(zh->reg->zei, sort_criteria[i].ord, + /* right now we look up the index type based on the first database + if the index_type's can differ between the indexes of different + databases (which i guess they can?) then we have to store the + index types for each database, just like the ord and numerical */ + if (zebraExplain_lookup_ord(zh->reg->zei, sort_criteria[i].ord[0], &sort_criteria[i].index_type, 0, 0)) { @@ -929,6 +997,10 @@ ZEBRA_RES resultSetSortSingle(ZebraHandle zh, NMEM nmem, } } /* allocate space for each cmpare buf + one extra for tmp comparison */ + /* cmp_buf is an array of array, the first dimension is the criteria and the second dimension are + all other result entries to compare against. This is slowly filled when records are processed. + tmp_cmp_buf is an array with a value of the current record for each criteria + */ for (i = 0; isort_info->max_entries @@ -944,6 +1016,7 @@ ZEBRA_RES resultSetSortSingle(ZebraHandle zh, NMEM nmem, kno++; if (this_sys != psysno) { + int database_no = 0; if ((sset->hits & 255) == 0 && zh->break_handler_func) { if (zh->break_handler_func(zh->break_handler_data)) @@ -954,13 +1027,33 @@ ZEBRA_RES resultSetSortSingle(ZebraHandle zh, NMEM nmem, } (sset->hits)++; psysno = this_sys; - resultSetInsertSort(zh, sset, + + /* determine database from the term, but only bother if more than + one database is in use*/ + if (numbases > 1 && termid->ol) + { + const char *this_db = 0; + if (zebraExplain_lookup_ord(zh->reg->zei, termid->ol->ord, 0, &this_db, 0) + == 0 && this_db) + { + for (ib = 0; ib < numbases; ib++) + if (!strcmp(this_db, zh->basenames[ib])) + database_no = ib; + } + } +#if 0 + yaz_log(YLOG_LOG, "sysno=" ZINT_FORMAT " database_no=%d", this_sys, + database_no); + ord_list_print(termid->ol); +#endif + resultSetInsertSort(zh, sset, database_no, sort_criteria, num_criteria, psysno, cmp_buf, tmp_cmp_buf); } } rset_close(rfd); + /* free the compare buffers */ for (i = 0; ihits); for (i = 0; i < numTerms; i++) yaz_log(log_level_sort, "term=\"%s\" type=%s count=" ZINT_FORMAT, - terms[i]->name, terms[i]->flags, terms[i]->rset->hits_count); + terms[i]->name, terms[i]->flags, terms[i]->rset->hits_count); *sort_status = Z_SortResponse_success; return ZEBRA_OK; } @@ -1051,20 +1144,19 @@ ZEBRA_RES resultSetRank(ZebraHandle zh, ZebraSet zebraSet, } if (rfd->counted_items > rset->hits_limit) stop_flag = 1; + if (stop_flag) + { + zebraSet->estimated_hit_count = 1; + break; + } if (psysno) { /* only if we did have a previous record */ score = (*rc->calc)(handle, psysno, pstaticrank, - &stop_flag); + &stop_flag); /* insert the hit. A=Ascending */ resultSetInsertRank(zh, sort_info, psysno, score, 'A'); count++; } - if (stop_flag) - { - zebraSet->estimated_hit_count = 1; - rset_set_hits_limit(rset, 0); - break; - } psysno = this_sys; if (zh->m_staticrank) pstaticrank = key.mem[0]; @@ -1307,7 +1399,7 @@ static ZEBRA_RES zebra_recid_to_sysno(ZebraHandle zh, int sysnos_offset = 0; int i; - if (!zh->reg->isamb) + if (!zh->reg->isamb || !zh->m_segment_indexing) { if (sysnos_offset < *no_sysnos) *sysnos = recid; @@ -1367,11 +1459,9 @@ static ZEBRA_RES zebra_recid_to_sysno(ZebraHandle zh, while (r && key_found.mem[0] == recid) { if (sysnos_offset < *no_sysnos) - sysnos[sysnos_offset] = + sysnos[sysnos_offset++] = key_found.mem[key_found.len-1]; - r = isamb_pp_read(pt, &key_found); - sysnos_offset++; } isamb_pp_close(pt); } @@ -1432,6 +1522,7 @@ void zebra_count_set(ZebraHandle zh, RSET rset, zint *count, /* * Local variables: * c-basic-offset: 4 + * c-file-style: "Stroustrup" * indent-tabs-mode: nil * End: * vim: shiftwidth=4 tabstop=8 expandtab