X-Git-Url: http://git.indexdata.com/?p=idzebra-moved-to-github.git;a=blobdiff_plain;f=index%2Fzsets.c;h=4a8c74c39c8c26f6638ee1fd0e89069a0b05138a;hp=ce615ffe31b76a74111c8ffc608a06cbff77b919;hb=d87f70faee1aeead5d32c388bfde8c41e859fb2f;hpb=d5170943ca365ace791c3254db15cc852df56d6b diff --git a/index/zsets.c b/index/zsets.c index ce615ff..4a8c74c 100644 --- a/index/zsets.c +++ b/index/zsets.c @@ -1,5 +1,5 @@ /* This file is part of the Zebra server. - Copyright (C) 1995-2008 Index Data + Copyright (C) 1994-2011 Index Data Zebra is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free @@ -106,13 +106,14 @@ static ZEBRA_RES resultSetSearch(ZebraHandle zh, NMEM nmem, NMEM rset_nmem, sort_sequence->num_specs = 10; /* FIXME - Hard-coded number */ sort_sequence->specs = (Z_SortKeySpec **) nmem_malloc(nmem, sort_sequence->num_specs * - sizeof(*sort_sequence->specs)); + sizeof(*sort_sequence->specs)); for (i = 0; inum_specs; i++) sort_sequence->specs[i] = 0; rpn_get_top_approx_limit(zh, rpn->RPNStructure, &sset->approx_limit); res = rpn_search_top(zh, rpn->RPNStructure, rpn->attributeSetId, + sset->approx_limit, nmem, rset_nmem, sort_sequence, sset->num_bases, sset->basenames, @@ -125,7 +126,8 @@ static ZEBRA_RES resultSetSearch(ZebraHandle zh, NMEM nmem, NMEM rset_nmem, for (i = 0; sort_sequence->specs[i]; i++) ; sort_sequence->num_specs = i; - rset->hits_limit = sset->approx_limit; + rset_set_hits_limit(rset, sset->approx_limit); + if (!i) { res = resultSetRank(zh, sset, rset, rset_nmem); @@ -133,7 +135,7 @@ static ZEBRA_RES resultSetSearch(ZebraHandle zh, NMEM nmem, NMEM rset_nmem, else { res = resultSetSortSingle(zh, nmem, sset, rset, - sort_sequence, &sort_status); + sort_sequence, &sort_status); } sset->rset = rset; return res; @@ -193,7 +195,7 @@ void resultSetAddTerm(ZebraHandle zh, ZebraSet s, int reg_type, s->term_entries_max = 1000; s->term_entries = nmem_malloc(s->nmem, s->term_entries_max * - sizeof(*s->term_entries)); + sizeof(*s->term_entries)); for (i = 0; i < s->term_entries_max; i++) s->term_entries[i].term = 0; } @@ -252,10 +254,10 @@ ZebraSet resultSetAdd(ZebraHandle zh, const char *name, int ov) s->sort_info->entries = (struct zset_sort_entry **) xmalloc(sizeof(*s->sort_info->entries) * - s->sort_info->max_entries); + s->sort_info->max_entries); s->sort_info->all_entries = (struct zset_sort_entry *) xmalloc(sizeof(*s->sort_info->all_entries) * - s->sort_info->max_entries); + s->sort_info->max_entries); for (i = 0; i < s->sort_info->max_entries; i++) s->sort_info->entries[i] = s->sort_info->all_entries + i; } @@ -293,7 +295,7 @@ ZebraSet resultSetGet(ZebraHandle zh, const char *name) int sort_status; yaz_log(log_level_resultsets, "resort %s", name); resultSetSortSingle(zh, nmem, s, s->rset, s->sortSpec, - &sort_status); + &sort_status); } nmem_destroy(nmem); } @@ -540,7 +542,7 @@ ZebraMetaRecord *zebra_meta_records_create(ZebraHandle zh, const char *name, } void zebra_meta_records_destroy(ZebraHandle zh, ZebraMetaRecord *records, - int num) + int num) { assert(zh); /* compiler shut up about unused arg */ xfree(records); @@ -548,21 +550,21 @@ void zebra_meta_records_destroy(ZebraHandle zh, ZebraMetaRecord *records, struct sortKeyInfo { int relation; - int *ord; // array of ord for each database searched - int *numerical;// array of ord for each database searched + int *ord; /* array of ord for each database searched */ + int *numerical; /* array of ord for each database searched */ const char *index_type; }; void resultSetInsertSort(ZebraHandle zh, ZebraSet sset, + int database_no, struct sortKeyInfo *criteria, int num_criteria, zint sysno, - char *cmp_buf[], char *tmp_cmp_buf[], int *cached_success_db) + char *cmp_buf[], char *tmp_cmp_buf[]) { struct zset_sort_entry *new_entry = NULL; struct zset_sort_info *sort_info = sset->sort_info; int i, j; - int scan_db,scan_count; - int numbases = zh->num_basenames; + WRBUF w = wrbuf_alloc(); zebra_sort_sysno(zh->reg->sort_index, sysno); for (i = 0; inumbases){ - // well...we scanned all databases and still nothing...give up - yaz_log(log_level_sort, "zebra_sort_read failed (record not found in indices)"); - break; - } - - // the criteria[i].ord is the file id of the sort index - yaz_log(log_level_sort, "pre zebra_sort_type ord is %d", criteria[i].ord[scan_db]); - zebra_sort_type(zh->reg->sort_index, criteria[i].ord[scan_db]); - if(zebra_sort_read(zh->reg->sort_index, this_entry_buf)){ - // allright, found it - // cache this db so we start trying from this db for next record - *cached_success_db=scan_db; - break; - }else{ - yaz_log(log_level_sort, "record not found in database, trying next one"); - scan_db++; - if(scan_db>=numbases){ - scan_db=0; + yaz_log(log_level_sort, "pre zebra_sort_type ord is %d", + criteria[i].ord[database_no]); + zebra_sort_type(zh->reg->sort_index, criteria[i].ord[database_no]); + wrbuf_rewind(w); + if (zebra_sort_read(zh->reg->sort_index, 0, w)) + { + /* consider each sort entry and take lowest/highest one + of the one as sorting key depending on whether sort is + ascending/descending */ + int off = 0; + while (off != wrbuf_len(w)) + { + size_t l = strlen(wrbuf_buf(w)+off); + assert(off < wrbuf_len(w)); + + if (l >= SORT_IDX_ENTRYSIZE) + l = SORT_IDX_ENTRYSIZE-1; + if ( (off == 0) + || (criteria[i].relation == 'A' + && strcmp(wrbuf_buf(w)+off, this_entry_buf) < 0) + || (criteria[i].relation == 'D' + && strcmp(wrbuf_buf(w)+off, this_entry_buf) > 0) + ) + { + memcpy(this_entry_buf, wrbuf_buf(w)+off, l); + this_entry_buf[l] = '\0'; } + off += 1 + strlen(wrbuf_buf(w)+off); } } - - }else{ + } + else + { yaz_log(log_level_sort, "criteria[i].ord is -1 so not reading from sort index"); } } + wrbuf_destroy(w); i = sort_info->num_entries; while (--i >= 0) { @@ -616,15 +620,11 @@ void resultSetInsertSort(ZebraHandle zh, ZebraSet sset, char *this_entry_buf = tmp_cmp_buf[j]; char *other_entry_buf = cmp_buf[j] + i * SORT_IDX_ENTRYSIZE; - if (criteria[j].numerical[*cached_success_db]) + if (criteria[j].numerical[database_no]) { char this_entry_org[1024]; char other_entry_org[1024]; double diff; - // when searching multiple databases, we use the index type of - // the first one. So if they differ between databases, we have a problem here - // we could store the index_type for each database, but if we didn't find the - // record in any sort index, then we still don't know to which database it belongs. const char *index_type = criteria[j].index_type; zebra_term_untrans(zh, index_type, this_entry_org, this_entry_buf); @@ -644,7 +644,8 @@ void resultSetInsertSort(ZebraHandle zh, ZebraSet sset, rel = memcmp(this_entry_buf, other_entry_buf, SORT_IDX_ENTRYSIZE); } - // when the compare is equal, continue to next criteria, else break out + /* when the compare is equal, continue to next criteria, + else break out */ if (rel) break; } @@ -674,7 +675,7 @@ void resultSetInsertSort(ZebraHandle zh, ZebraSet sset, else j = (sort_info->num_entries)++; new_entry = sort_info->entries[j]; - // move up all higher entries (to make room) + /* move up all higher entries (to make room) */ while (j != i) { int k; @@ -687,10 +688,10 @@ void resultSetInsertSort(ZebraHandle zh, ZebraSet sset, sort_info->entries[j] = sort_info->entries[j-1]; --j; } - // and insert the new entry at the correct place + /* and insert the new entry at the correct place */ sort_info->entries[i] = new_entry; assert(new_entry); - // and add this to the compare buffer + /* and add this to the compare buffer */ for (i = 0; isortSpec = copy_SortKeySpecList(sort_sequence, sset->nmem); - return resultSetSortSingle (zh, nmem, sset, rset, sort_sequence, - sort_status); + return resultSetSortSingle(zh, nmem, sset, rset, sort_sequence, + sort_status); } ZEBRA_RES resultSetSortSingle(ZebraHandle zh, NMEM nmem, @@ -868,7 +869,6 @@ ZEBRA_RES resultSetSortSingle(ZebraHandle zh, NMEM nmem, { int i; int ib; - int cached_success_db = 0; int n = 0; zint kno = 0; zint psysno = 0; @@ -900,23 +900,22 @@ ZEBRA_RES resultSetSortSingle(ZebraHandle zh, NMEM nmem, num_criteria = sort_sequence->num_specs; if (num_criteria > ZSET_SORT_MAX_LEVEL) num_criteria = ZSET_SORT_MAX_LEVEL; - // set up the search criteria + /* set up the search criteria */ for (i = 0; i < num_criteria; i++) { Z_SortKeySpec *sks = sort_sequence->specs[i]; Z_SortKey *sk; - ZEBRA_RES res; sort_criteria[i].ord = (int *) nmem_malloc(nmem, sizeof(int)*numbases); sort_criteria[i].numerical = (int *) nmem_malloc(nmem, sizeof(int)*numbases); - // initialize ord and numerical for each database + /* initialize ord and numerical for each database */ for (ib = 0; ib < numbases; ib++) { - sort_criteria[i].ord[ib] = -1; - sort_criteria[i].numerical[ib] = 0; + sort_criteria[i].ord[ib] = -1; + sort_criteria[i].numerical[ib] = 0; } if (sks->which == Z_SortKeySpec_missingValueData) @@ -947,8 +946,7 @@ ZEBRA_RES resultSetSortSingle(ZebraHandle zh, NMEM nmem, switch (sk->which) { case Z_SortKey_sortField: - yaz_log(log_level_sort, "key %d is of type sortField", - i+1); + yaz_log(log_level_sort, "key %d is of type sortField", i+1); for (ib = 0; ib < numbases; ib++) { zebraExplain_curDatabase(zh->reg->zei, zh->basenames[ib]); @@ -967,29 +965,29 @@ ZEBRA_RES resultSetSortSingle(ZebraHandle zh, NMEM nmem, } break; case Z_SortKey_elementSpec: - yaz_log(log_level_sort, "key %d is of type elementSpec", - i+1); + yaz_log(log_level_sort, "key %d is of type elementSpec", i+1); zebra_setError(zh, YAZ_BIB1_CANNOT_SORT_ACCORDING_TO_SEQUENCE, 0); return ZEBRA_FAIL; case Z_SortKey_sortAttributes: yaz_log(log_level_sort, "key %d is of type sortAttributes", i+1); - // for every database we searched, get the sort index file id (ord) - // and its numerical indication and store them in the sort_criteria + /* for every database we searched, get the sort index file + id (ord) and its numerical indication and store them in + the sort_criteria */ for (ib = 0; ib < numbases; ib++) { zebraExplain_curDatabase(zh->reg->zei, zh->basenames[ib]); - res = zebra_sort_get_ord(zh, sk->u.sortAttributes, + if (zebra_sort_get_ord(zh, sk->u.sortAttributes, &sort_criteria[i].ord[ib], - &sort_criteria[i].numerical[ib]); + &sort_criteria[i].numerical[ib]) != + ZEBRA_OK && sks->which != Z_SortKeySpec_null) + return ZEBRA_FAIL; } - - if (sks->which != Z_SortKeySpec_null && res != ZEBRA_OK) - return ZEBRA_FAIL; break; } - // right now we look up the index type based on the first database - // if the index_type's can differ between the indexes of different databases (which i guess they can?) - // then we have to store the index types for each database, just like the ord and numerical + /* right now we look up the index type based on the first database + if the index_type's can differ between the indexes of different + databases (which i guess they can?) then we have to store the + index types for each database, just like the ord and numerical */ if (zebraExplain_lookup_ord(zh->reg->zei, sort_criteria[i].ord[0], &sort_criteria[i].index_type, 0, 0)) @@ -1018,6 +1016,7 @@ ZEBRA_RES resultSetSortSingle(ZebraHandle zh, NMEM nmem, kno++; if (this_sys != psysno) { + int database_no = 0; if ((sset->hits & 255) == 0 && zh->break_handler_func) { if (zh->break_handler_func(zh->break_handler_data)) @@ -1028,14 +1027,33 @@ ZEBRA_RES resultSetSortSingle(ZebraHandle zh, NMEM nmem, } (sset->hits)++; psysno = this_sys; - resultSetInsertSort(zh, sset, + + /* determine database from the term, but only bother if more than + one database is in use*/ + if (numbases > 1 && termid->ol) + { + const char *this_db = 0; + if (zebraExplain_lookup_ord(zh->reg->zei, termid->ol->ord, 0, &this_db, 0) + == 0 && this_db) + { + for (ib = 0; ib < numbases; ib++) + if (!strcmp(this_db, zh->basenames[ib])) + database_no = ib; + } + } +#if 0 + yaz_log(YLOG_LOG, "sysno=" ZINT_FORMAT " database_no=%d", this_sys, + database_no); + ord_list_print(termid->ol); +#endif + resultSetInsertSort(zh, sset, database_no, sort_criteria, num_criteria, psysno, cmp_buf, - tmp_cmp_buf, &cached_success_db); + tmp_cmp_buf); } } rset_close(rfd); - // free the compare buffers + /* free the compare buffers */ for (i = 0; ihits); for (i = 0; i < numTerms; i++) yaz_log(log_level_sort, "term=\"%s\" type=%s count=" ZINT_FORMAT, - terms[i]->name, terms[i]->flags, terms[i]->rset->hits_count); + terms[i]->name, terms[i]->flags, terms[i]->rset->hits_count); *sort_status = Z_SortResponse_success; return ZEBRA_OK; } @@ -1126,20 +1144,19 @@ ZEBRA_RES resultSetRank(ZebraHandle zh, ZebraSet zebraSet, } if (rfd->counted_items > rset->hits_limit) stop_flag = 1; + if (stop_flag) + { + zebraSet->estimated_hit_count = 1; + break; + } if (psysno) { /* only if we did have a previous record */ score = (*rc->calc)(handle, psysno, pstaticrank, - &stop_flag); + &stop_flag); /* insert the hit. A=Ascending */ resultSetInsertRank(zh, sort_info, psysno, score, 'A'); count++; } - if (stop_flag) - { - zebraSet->estimated_hit_count = 1; - rset_set_hits_limit(rset, 0); - break; - } psysno = this_sys; if (zh->m_staticrank) pstaticrank = key.mem[0]; @@ -1505,6 +1522,7 @@ void zebra_count_set(ZebraHandle zh, RSET rset, zint *count, /* * Local variables: * c-basic-offset: 4 + * c-file-style: "Stroustrup" * indent-tabs-mode: nil * End: * vim: shiftwidth=4 tabstop=8 expandtab