From 593927cb1897c1e3163c284448eff7fee6ddad51 Mon Sep 17 00:00:00 2001 From: Adam Dickmeiss Date: Fri, 28 Oct 2005 07:25:30 +0000 Subject: [PATCH] Fix leak for rset_key_control --- index/Makefile.am | 4 +- index/extract.c | 535 +++++++++++++++++++++++++++++++++++++++++++++-------- index/index.h | 29 ++- index/kinput.c | 13 +- index/retrieve.c | 18 +- index/zebraapi.c | 11 +- index/zsets.c | 3 +- 7 files changed, 518 insertions(+), 95 deletions(-) diff --git a/index/Makefile.am b/index/Makefile.am index 3e1d7b6..5f78bd8 100644 --- a/index/Makefile.am +++ b/index/Makefile.am @@ -1,4 +1,4 @@ -## $Id: Makefile.am,v 1.33 2005-09-15 09:27:18 pop Exp $ +## $Id: Makefile.am,v 1.34 2005-10-28 07:25:30 adam Exp $ noinst_PROGRAMS = apitest kdump @@ -9,7 +9,7 @@ libidzebra_api_la_SOURCES = dir.c dirs.c trav.c kinput.c kcompare.c \ zebraapi.c api_swig.c \ zinfo.c invstat.c sortidx.c compact.c zsets.c zrpn.c \ rank1.c trunc.c retrieve.c extract.c rankstatic.c \ - index.h recindex.h recindxp.h \ + index.h recindex.h recindxp.h reckeys.c reckeys.h \ zinfo.h zserver.h zvrank.c limit.c kcontrol.c bin_PROGRAMS = zebraidx zebrasrv zebrash diff --git a/index/extract.c b/index/extract.c index dcd4206..836b6de 100644 --- a/index/extract.c +++ b/index/extract.c @@ -1,4 +1,4 @@ -/* $Id: extract.c,v 1.195 2005-09-16 09:58:38 adam Exp $ +/* $Id: extract.c,v 1.196 2005-10-28 07:25:30 adam Exp $ Copyright (C) 1995-2005 Index Data ApS @@ -43,8 +43,6 @@ Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA #define USE_SHELLSORT 0 -#define REC_MULTI_SKIP 0 - #if USE_SHELLSORT static void shellsort(void *ar, int r, size_t s, int (*cmp)(const void *a, const void *b)) @@ -101,14 +99,21 @@ static void extract_init (struct recExtractCtrl *p, RecWord *w) } static void searchRecordKey(ZebraHandle zh, +#if NEW_REC_KEYS + zebra_rec_keys_t reckeys, +#else const struct recKeys *reckeys, +#endif int attrSetS, int attrUseS, const char **ws, int ws_length) { +#if NEW_REC_KEYS +#else void *decode_handle = iscz1_start(); int off = 0; int startSeq = -1; int seqno = 0; +#endif int i; int ch; @@ -120,6 +125,33 @@ static void searchRecordKey(ZebraHandle zh, if (ch < 0) return ; +#if NEW_REC_KEYS + if (zebra_rec_keys_rewind(reckeys)) + { + int startSeq = -1; + const char *str; + size_t slen; + struct it_key key; + zint seqno; + while (zebra_rec_keys_read(reckeys, &str, &slen, &key)) + { + assert(key.len <= 4 && key.len > 2); + + seqno = key.mem[key.len-1]; + + if (key.mem[0] == ch) + { + int woff; + + if (startSeq == -1) + startSeq = seqno; + woff = seqno - startSeq; + if (woff >= 0 && woff < ws_length) + ws[woff] = str; + } + } + } +#else while (off < reckeys->buf_used) { const char *src = reckeys->buf + off; @@ -148,6 +180,7 @@ static void searchRecordKey(ZebraHandle zh, } iscz1_stop(decode_handle); assert (off == reckeys->buf_used); +#endif } struct file_read_info { @@ -216,7 +249,11 @@ static void file_end (void *handle, off_t offset) } static char *fileMatchStr (ZebraHandle zh, +#if NEW_REC_KEYS + zebra_rec_keys_t reckeys, +#else struct recKeys *reckeys, +#endif const char *fname, const char *spec) { static char dstBuf[2048]; /* static here ??? */ @@ -371,11 +408,14 @@ struct recordLogInfo { struct recordGroup *rGroup; }; +#if NEW_REC_KEYS +#else void create_rec_keys_codec(struct recKeys *keys) { keys->buf_used = 0; iscz1_reset(keys->codec_handle); } +#endif static void init_extractCtrl(ZebraHandle zh, struct recExtractCtrl *ctrl) { @@ -418,9 +458,18 @@ static int file_extract_record(ZebraHandle zh, if (fi->fd != -1) { /* we are going to read from a file, so prepare the extraction */ +#if NEW_REC_KEYS + zebra_rec_keys_reset(zh->reg->keys); +#else create_rec_keys_codec(&zh->reg->keys); +#endif + #if NATTR +#if NEW_REC_KEYS + zebra_rec_keys_reset(zh->reg->sortKeys); +#else create_rec_keys_codec(&zh->reg->sortKeys); +#endif #else zh->reg->sortKeys.buf_used = 0; #endif @@ -494,9 +543,13 @@ static int file_extract_record(ZebraHandle zh, if (matchStr == 0 && zh->m_record_id && *zh->m_record_id) { - +#if NEW_REC_KEYS + matchStr = fileMatchStr (zh, zh->reg->keys, fname, + zh->m_record_id); +#else matchStr = fileMatchStr (zh, &zh->reg->keys, fname, zh->m_record_id); +#endif if (!matchStr) { yaz_log(YLOG_WARN, "Bad match criteria"); @@ -513,7 +566,13 @@ static int file_extract_record(ZebraHandle zh, } } } - if (! *sysno && zh->reg->keys.buf_used == 0) + if (! *sysno +#if NEW_REC_KEYS + && zebra_rec_keys_empty(zh->reg->keys) +#else + && zh->reg->keys.buf_used == 0 +#endif + ) { /* the extraction process returned no information - the record is probably empty - unless flagShowRecords is in use */ @@ -551,17 +610,30 @@ static int file_extract_record(ZebraHandle zh, dict_insert (zh->reg->matchDict, matchStr, sizeof(*sysno), sysno); } extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys); +#if NEW_REC_KEYS + extract_flushRecordKeys (zh, *sysno, 1, zh->reg->keys, + recordAttr->staticrank); +#else extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys, recordAttr->staticrank); - +#endif zh->records_inserted++; } else { /* record already exists */ +#if NEW_REC_KEYS + zebra_rec_keys_t delkeys = zebra_rec_keys_open(); + +#else struct recKeys delkeys; +#endif #if NATTR +#if NEW_REC_KEYS + zebra_rec_keys_t sortKeys = zebra_rec_keys_open(); +#else struct recKeys sortKeys; +#endif #else struct sortKeys sortKeys; #endif @@ -571,44 +643,56 @@ static int file_extract_record(ZebraHandle zh, recordAttr = rec_init_attr (zh->reg->zei, rec); -#if REC_MULTI_SKIP - if (!force_update && recordAttr->runNumber == - zebraExplain_runNumberIncrement (zh->reg->zei, 0)) - { - yaz_log (YLOG_LOG, "run number = " ZINT_FORMAT, - recordAttr->runNumber); - yaz_log (YLOG_LOG, "skipped %s %s " PRINTF_OFF_T, - zh->m_record_type, fname, recordOffset); - extract_flushSortKeys (zh, *sysno, -1, &zh->reg->sortKeys); - rec_rm (&rec); - logRecord (zh); - return 1; - } -#endif +#if NEW_REC_KEYS + zebra_rec_keys_set_buf(delkeys, + rec->info[recInfo_delKeys], + rec->size[recInfo_delKeys], + 0); +#else /* flush old keys for sort&search etc. */ delkeys.buf_used = rec->size[recInfo_delKeys]; delkeys.buf = rec->info[recInfo_delKeys]; +#endif +#if NEW_REC_KEYS && NATTR + zebra_rec_keys_set_buf(sortKeys, + rec->info[recInfo_sortKeys], + rec->size[recInfo_sortKeys], + 0); + extract_flushSortKeys (zh, *sysno, 0, sortKeys); +#else sortKeys.buf_used = rec->size[recInfo_sortKeys]; sortKeys.buf = rec->info[recInfo_sortKeys]; - extract_flushSortKeys (zh, *sysno, 0, &sortKeys); +#endif + +#if NEW_REC_KEYS + extract_flushRecordKeys (zh, *sysno, 0, delkeys, + recordAttr->staticrank); /* old values */ +#else extract_flushRecordKeys (zh, *sysno, 0, &delkeys, recordAttr->staticrank); /* old values */ +#endif if (deleteFlag) { /* record going to be deleted */ - if (!delkeys.buf_used) + if ( +#if NEW_REC_KEYS + zebra_rec_keys_empty(delkeys) +#else + !delkeys.buf_used +#endif + ) { yaz_log (YLOG_LOG, "delete %s %s " PRINTF_OFF_T, - zh->m_record_type, fname, recordOffset); + zh->m_record_type, fname, recordOffset); yaz_log (YLOG_WARN, "cannot delete file above, storeKeys false"); } else { if (zh->records_processed < zh->m_file_verbose_limit) yaz_log (YLOG_LOG, "delete %s %s " PRINTF_OFF_T, - zh->m_record_type, fname, recordOffset); + zh->m_record_type, fname, recordOffset); zh->records_deleted++; if (matchStr) dict_delete (zh->reg->matchDict, matchStr); @@ -621,7 +705,13 @@ static int file_extract_record(ZebraHandle zh, else { /* record going to be updated */ - if (!delkeys.buf_used) + if ( +#if NEW_REC_KEYS + zebra_rec_keys_empty(delkeys) +#else + !delkeys.buf_used +#endif + ) { yaz_log (YLOG_LOG, "update %s %s " PRINTF_OFF_T, zh->m_record_type, fname, recordOffset); @@ -635,11 +725,22 @@ static int file_extract_record(ZebraHandle zh, zh->m_record_type, fname, recordOffset); recordAttr->staticrank = extractCtrl.staticrank; extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys); +#if NEW_REC_KEYS + extract_flushRecordKeys (zh, *sysno, 1, zh->reg->keys, + recordAttr->staticrank); +#else extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys, recordAttr->staticrank); +#endif zh->records_updated++; } } +#if NEW_REC_KEYS + zebra_rec_keys_close(delkeys); +#endif +#if NATTR && NEW_REC_KWYS + zebra_rec_keys_close(sortKeys); +#endif } /* update file type */ xfree (rec->info[recInfo_fileType]); @@ -653,12 +754,25 @@ static int file_extract_record(ZebraHandle zh, /* update delete keys */ xfree (rec->info[recInfo_delKeys]); - if (zh->reg->keys.buf_used > 0 && zh->m_store_keys == 1) + if ( +#if NEW_REC_KEYS + !zebra_rec_keys_empty(zh->reg->keys) +#else + zh->reg->keys.buf_used > 0 +#endif + && zh->m_store_keys == 1) { +#if NEW_REC_KEYS + zebra_rec_keys_get_buf(zh->reg->keys, + &rec->info[recInfo_delKeys], + &rec->size[recInfo_delKeys]); +#else + rec->size[recInfo_delKeys] = zh->reg->keys.buf_used; rec->info[recInfo_delKeys] = zh->reg->keys.buf; zh->reg->keys.buf = NULL; zh->reg->keys.buf_max = 0; +#endif } else { @@ -847,16 +961,16 @@ int fileExtract (ZebraHandle zh, SYSNO *sysno, const char *fname, If not, and a record is provided, then sysno is got from there */ -ZEBRA_RES buffer_extract_record (ZebraHandle zh, - const char *buf, size_t buf_size, - int delete_flag, - int test_mode, - const char *recordType, - SYSNO *sysno, - const char *match_criteria, - const char *fname, - int force_update, - int allow_update) +ZEBRA_RES buffer_extract_record(ZebraHandle zh, + const char *buf, size_t buf_size, + int delete_flag, + int test_mode, + const char *recordType, + SYSNO *sysno, + const char *match_criteria, + const char *fname, + int force_update, + int allow_update) { RecordAttr *recordAttr; struct recExtractCtrl extractCtrl; @@ -888,9 +1002,17 @@ ZEBRA_RES buffer_extract_record (ZebraHandle zh, extractCtrl.first_record = 1; extractCtrl.fh = &fc; +#if NEW_REC_KEYS + zebra_rec_keys_reset(zh->reg->keys); +#else create_rec_keys_codec(&zh->reg->keys); +#endif #if NATTR +#if NEW_REC_KEYS + zebra_rec_keys_reset(zh->reg->sortKeys); +#else create_rec_keys_codec(&zh->reg->sortKeys); +#endif #else zh->reg->sortKeys.buf_used = 0; #endif @@ -966,8 +1088,13 @@ ZEBRA_RES buffer_extract_record (ZebraHandle zh, matchStr = match_criteria; } else { if (zh->m_record_id && *zh->m_record_id) { +#if NEW_REC_KEYS + matchStr = fileMatchStr (zh, zh->reg->keys, pr_fname, + zh->m_record_id); +#else matchStr = fileMatchStr (zh, &zh->reg->keys, pr_fname, zh->m_record_id); +#endif if (!matchStr) { yaz_log (YLOG_WARN, "Bad match criteria (recordID)"); @@ -984,7 +1111,13 @@ ZEBRA_RES buffer_extract_record (ZebraHandle zh, } } } - if (zh->reg->keys.buf_used == 0) + if ( +#if NEW_REC_KEYS + zebra_rec_keys_empty(zh->reg->keys) +#else + zh->reg->keys.buf_used == 0 +#endif +) { /* the extraction process returned no information - the record is probably empty - unless flagShowRecords is in use */ @@ -1019,15 +1152,31 @@ ZEBRA_RES buffer_extract_record (ZebraHandle zh, sizeof(*sysno), sysno); } extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys); + +#if NEW_REC_KEYS +#if 0 + print_rec_keys(zh, zh->reg->keys); +#endif + extract_flushRecordKeys (zh, *sysno, 1, zh->reg->keys, + recordAttr->staticrank); +#else +#if 0 + print_rec_keys(zh, &zh->reg->keys); +#endif extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys, - recordAttr->staticrank); + recordAttr->staticrank); +#endif zh->records_inserted++; } else { /* record already exists */ +#if NEW_REC_KEYS + zebra_rec_keys_t delkeys = zebra_rec_keys_open(); +#else struct recKeys delkeys; +#endif #if NATTR struct recKeys sortKeys; #else @@ -1047,36 +1196,41 @@ ZEBRA_RES buffer_extract_record (ZebraHandle zh, assert (rec); recordAttr = rec_init_attr (zh->reg->zei, rec); - -#if REC_MULTI_SKIP - if (!force_update) { - if (recordAttr->runNumber == - zebraExplain_runNumberIncrement (zh->reg->zei, 0)) - { - if (show_progress) - yaz_log (YLOG_LOG, "skipped %s %s %ld", recordType, - pr_fname, (long) recordOffset); - extract_flushSortKeys (zh, *sysno, -1, &zh->reg->sortKeys); - rec_rm (&rec); - logRecord(zh); - return ZEBRA_FAIL; - } - } -#endif +#if NEW_REC_KEYS + zebra_rec_keys_set_buf(delkeys, + rec->info[recInfo_delKeys], + rec->size[recInfo_delKeys], + 0); +#else delkeys.buf_used = rec->size[recInfo_delKeys]; delkeys.buf = rec->info[recInfo_delKeys]; - +#endif sortKeys.buf_used = rec->size[recInfo_sortKeys]; sortKeys.buf = rec->info[recInfo_sortKeys]; +#if NEW_REC_KEYS && NATTR + extract_flushSortKeys (zh, *sysno, 0, sortKeys); +#else extract_flushSortKeys (zh, *sysno, 0, &sortKeys); +#endif +#if NEW_REC_KEYS + extract_flushRecordKeys (zh, *sysno, 0, delkeys, + recordAttr->staticrank); +#else extract_flushRecordKeys (zh, *sysno, 0, &delkeys, recordAttr->staticrank); +#endif if (delete_flag) { /* record going to be deleted */ - if (!delkeys.buf_used) + if ( +#if NEW_REC_KEYS + zebra_rec_keys_empty(delkeys) +#else + !delkeys.buf_used +#endif + ) { if (show_progress) { @@ -1103,7 +1257,13 @@ ZEBRA_RES buffer_extract_record (ZebraHandle zh, else { /* record going to be updated */ - if (!delkeys.buf_used) + if ( +#if NEW_REC_KEYS + zebra_rec_keys_empty(delkeys) +#else + !delkeys.buf_used +#endif + ) { if (show_progress) { @@ -1119,11 +1279,22 @@ ZEBRA_RES buffer_extract_record (ZebraHandle zh, pr_fname, (long) recordOffset); recordAttr->staticrank = extractCtrl.staticrank; extract_flushSortKeys (zh, *sysno, 1, &zh->reg->sortKeys); +#if NEW_REC_KEYS + extract_flushRecordKeys (zh, *sysno, 1, zh->reg->keys, + recordAttr->staticrank); +#else extract_flushRecordKeys (zh, *sysno, 1, &zh->reg->keys, recordAttr->staticrank); +#endif zh->records_updated++; } } +#if NEW_REC_KEYS + zebra_rec_keys_close(delkeys); +#endif +#if NEW_REC_KEYS && NATTR + zebra_rec_keys_close(sortKeys); +#endif } /* update file type */ xfree (rec->info[recInfo_fileType]); @@ -1137,19 +1308,31 @@ ZEBRA_RES buffer_extract_record (ZebraHandle zh, /* update delete keys */ xfree (rec->info[recInfo_delKeys]); - if (zh->reg->keys.buf_used > 0 && zh->m_store_keys == 1) + if ( +#if NEW_REC_KEYS + !zebra_rec_keys_empty(zh->reg->keys) +#else + zh->reg->keys.buf_used > 0 +#endif + && zh->m_store_keys == 1) { +#if NEW_REC_KEYS + zebra_rec_keys_get_buf(zh->reg->keys, + &rec->info[recInfo_delKeys], + &rec->size[recInfo_delKeys]); +#else + rec->size[recInfo_delKeys] = zh->reg->keys.buf_used; rec->info[recInfo_delKeys] = zh->reg->keys.buf; zh->reg->keys.buf = NULL; zh->reg->keys.buf_max = 0; +#endif } else { rec->info[recInfo_delKeys] = NULL; rec->size[recInfo_delKeys] = 0; } - /* update sort keys */ xfree (rec->info[recInfo_sortKeys]); @@ -1225,9 +1408,18 @@ int explain_extract (void *handle, Record rec, data1_node *n) abort (); } +#if NEW_REC_KEYS + zebra_rec_keys_reset(zh->reg->keys); +#else create_rec_keys_codec(&zh->reg->keys); +#endif + #if NATTR +#if NEW_REC_KEYS + zebra_rec_keys_reset(zh->reg->sortKeys); +#else create_rec_keys_codec(&zh->reg->sortKeys); +#endif #else zh->reg->sortKeys.buf_used = 0; #endif @@ -1251,30 +1443,68 @@ int explain_extract (void *handle, Record rec, data1_node *n) if (rec->size[recInfo_delKeys]) { +#if NEW_REC_KEYS + zebra_rec_keys_t delkeys = zebra_rec_keys_open(); +#else struct recKeys delkeys; +#endif + #if NATTR +#if NEW_REC_KEYS + zebra_rec_keys_t sortkeys = zzebra_rec_keys_open(); +#else struct recKeys sortkeys; +#endif #else struct sortKeys sortkeys; #endif +#if NEW_REC_KEYS + zebra_rec_keys_set_buf(delkeys, rec->info[recInfo_delKeys], + rec->size[recInfo_delKeys], + 0); + extract_flushRecordKeys (zh, rec->sysno, 0, delkeys, 0); + zebra_rec_keys_close(delkeys); +#else delkeys.buf_used = rec->size[recInfo_delKeys]; delkeys.buf = rec->info[recInfo_delKeys]; + extract_flushRecordKeys (zh, rec->sysno, 0, &delkeys, 0); +#endif +#if NATTR && NEW_REC_KEYS + zebra_rec_keys_set_buf(sortkeys, rec->info[recInfo_sortKeys], + rec->size[recInfo_sortKeys], + 0); + extract_flushSortKeys (zh, rec->sysno, 0, sortkeys); + zebra_rec_keys_close(sortkeys); +#else sortkeys.buf_used = rec->size[recInfo_sortKeys]; sortkeys.buf = rec->info[recInfo_sortKeys]; - extract_flushSortKeys (zh, rec->sysno, 0, &sortkeys); - extract_flushRecordKeys (zh, rec->sysno, 0, &delkeys, 0); +#endif } +#if NEW_REC_KEYS + extract_flushRecordKeys (zh, rec->sysno, 1, zh->reg->keys, 0); +#else extract_flushRecordKeys (zh, rec->sysno, 1, &zh->reg->keys, 0); +#endif +#if NATTR && NEW_REC_KEYS + extract_flushSortKeys (zh, rec->sysno, 1, zh->reg->sortKeys); +#else extract_flushSortKeys (zh, rec->sysno, 1, &zh->reg->sortKeys); +#endif xfree (rec->info[recInfo_delKeys]); +#if NEW_REC_KEYS + zebra_rec_keys_get_buf(zh->reg->keys, + &rec->info[recInfo_delKeys], + &rec->size[recInfo_delKeys]); +#else rec->size[recInfo_delKeys] = zh->reg->keys.buf_used; rec->info[recInfo_delKeys] = zh->reg->keys.buf; zh->reg->keys.buf = NULL; zh->reg->keys.buf_max = 0; +#endif xfree (rec->info[recInfo_sortKeys]); rec->size[recInfo_sortKeys] = zh->reg->sortKeys.buf_used; @@ -1286,12 +1516,20 @@ int explain_extract (void *handle, Record rec, data1_node *n) } void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno, - int cmd, struct recKeys *reckeys, + int cmd, +#if NEW_REC_KEYS + zebra_rec_keys_t reckeys, +#else + struct recKeys *reckeys, +#endif zint staticrank) { +#if NEW_REC_KEYS +#else void *decode_handle = iscz1_start(); int off = 0; int ch = 0; +#endif ZebraExplainInfo zei = zh->reg->zei; if (!zh->reg->key_buf) @@ -1312,6 +1550,65 @@ void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno, } zebraExplain_recordCountIncrement (zei, cmd ? 1 : -1); +#if NEW_REC_KEYS + if (zebra_rec_keys_rewind(reckeys)) + { + size_t slen; + const char *str; + struct it_key key_in; + while(zebra_rec_keys_read(reckeys, &str, &slen, &key_in)) + { + int ch = 0; + struct it_key key_out; + zint *keyp = key_out.mem; + + assert(key_in.len == 4); + + /* check for buffer overflow */ + if (zh->reg->key_buf_used + 1024 > + (zh->reg->ptr_top -zh->reg->ptr_i)*sizeof(char*)) + extract_flushWriteKeys (zh, 0); + + ++(zh->reg->ptr_i); + assert(zh->reg->ptr_i > 0); + (zh->reg->key_buf)[zh->reg->ptr_top - zh->reg->ptr_i] = + (char*)zh->reg->key_buf + zh->reg->key_buf_used; + + /* encode the ordinal value (field/use/attribute) .. */ + ch = (int) key_in.mem[0]; + zh->reg->key_buf_used += + key_SU_encode(ch, (char*)zh->reg->key_buf + + zh->reg->key_buf_used); + + /* copy the 0-terminated stuff from str to output */ + memcpy((char*)zh->reg->key_buf + zh->reg->key_buf_used, str, slen); + zh->reg->key_buf_used += slen; + ((char*)zh->reg->key_buf)[(zh->reg->key_buf_used)++] = '\0'; + + /* the delete/insert indicator */ + ((char*)zh->reg->key_buf)[(zh->reg->key_buf_used)++] = cmd; + + if (zh->m_staticrank) /* rank config enabled ? */ + { + *keyp++ = staticrank; + key_out.len = 4; + } + else + key_out.len = 3; + + if (key_in.mem[1]) /* filter specified record ID */ + *keyp++ = key_in.mem[1]; + else + *keyp++ = sysno; + *keyp++ = key_in.mem[2]; /* section_id */ + *keyp++ = key_in.mem[3]; /* sequence .. */ + + memcpy((char*)zh->reg->key_buf + zh->reg->key_buf_used, + &key_out, sizeof(key_out)); + (zh->reg->key_buf_used) += sizeof(key_out); + } + } +#else while (off < reckeys->buf_used) { const char *src = reckeys->buf + off; @@ -1325,7 +1622,7 @@ void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno, if (zh->reg->key_buf_used + 1024 > (zh->reg->ptr_top -zh->reg->ptr_i)*sizeof(char*)) - extract_flushWriteKeys (zh,0); + extract_flushWriteKeys(zh, 0); ++(zh->reg->ptr_i); assert(zh->reg->ptr_i > 0); (zh->reg->key_buf)[zh->reg->ptr_top - zh->reg->ptr_i] = @@ -1334,8 +1631,8 @@ void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno, ch = (int) key_in.mem[0]; /* ordinal for field/use/attribute */ zh->reg->key_buf_used += - key_SU_encode (ch,((char*)zh->reg->key_buf) + - zh->reg->key_buf_used); + key_SU_encode(ch, ((char*)zh->reg->key_buf) + + zh->reg->key_buf_used); while (*src) ((char*)zh->reg->key_buf) [(zh->reg->key_buf_used)++] = *src++; src++; @@ -1364,6 +1661,7 @@ void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno, } assert (off == reckeys->buf_used); iscz1_stop(decode_handle); +#endif } void extract_flushWriteKeys (ZebraHandle zh, int final) @@ -1500,6 +1798,15 @@ void extract_flushWriteKeys (ZebraHandle zh, int final) zh->reg->key_buf_used = 0; } +#if NEW_REC_KEYS +void extract_add_it_key (ZebraHandle zh, + zebra_rec_keys_t *keys, + int reg_type, + const char *str, int slen, struct it_key *key) +{ + zebra_rec_keys_write(*keys, reg_type, str, slen, key); +} +#else void extract_add_it_key (ZebraHandle zh, struct recKeys *keys, int reg_type, @@ -1528,15 +1835,47 @@ void extract_add_it_key (ZebraHandle zh, *dst++ = '\0'; keys->buf_used = dst - keys->buf; } +#endif -ZEBRA_RES zebra_snippets_rec_keys(ZebraHandle zh, struct recKeys *reckeys, +ZEBRA_RES zebra_snippets_rec_keys(ZebraHandle zh, +#if NEW_REC_KEYS + zebra_rec_keys_t reckeys, +#else + struct recKeys *reckeys, +#endif zebra_snippets *snippets) { - void *decode_handle = iscz1_start(); - int off = 0; - int seqno = 0; NMEM nmem = nmem_create(); +#if NEW_REC_KEYS + if (zebra_rec_keys_rewind(reckeys)) + { + const char *str; + size_t slen; + struct it_key key; + while (zebra_rec_keys_read(reckeys, &str, &slen, &key)) + { + char dst_buf[IT_MAX_WORD]; + char *dst_term = dst_buf; + int ord, seqno; + int index_type; + assert(key.len <= 4 && key.len > 2); + seqno = (int) key.mem[key.len-1]; + ord = key.mem[0]; + + zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type, + 0/* db */, 0/* set */, 0/* use */); + assert(index_type); + zebra_term_untrans_iconv(zh, nmem, index_type, + &dst_term, str); + zebra_snippets_append(snippets, seqno, ord, dst_term); + nmem_reset(nmem); + } + } +#else + int off = 0; + int seqno = 0; + void *decode_handle = iscz1_start(); assert(reckeys->buf); while (off < reckeys->buf_used) { @@ -1564,18 +1903,50 @@ ZEBRA_RES zebra_snippets_rec_keys(ZebraHandle zh, struct recKeys *reckeys, off = src - reckeys->buf; nmem_reset(nmem); } - nmem_destroy(nmem); iscz1_stop(decode_handle); +#endif + nmem_destroy(nmem); return ZEBRA_OK; } -void print_rec_keys(ZebraHandle zh, struct recKeys *reckeys) +void print_rec_keys(ZebraHandle zh, +#if NEW_REC_KEYS + zebra_rec_keys_t reckeys +#else + struct recKeys *reckeys +#endif +) { - void *decode_handle = iscz1_start(); +#if NEW_REC_KEYS + yaz_log(YLOG_LOG, "print_rec_keys"); + if (zebra_rec_keys_rewind(reckeys)) + { + const char *str; + size_t slen; + struct it_key key; + while (zebra_rec_keys_read(reckeys, &str, &slen, &key)) + { + char dst_buf[IT_MAX_WORD]; + int seqno; + int index_type; + const char *db = 0; + assert(key.len <= 4 && key.len > 2); + + zebraExplain_lookup_ord(zh->reg->zei, + key.mem[0], &index_type, &db, 0, 0); + + seqno = (int) key.mem[key.len-1]; + + zebra_term_untrans(zh, index_type, dst_buf, str); + + yaz_log(YLOG_LOG, "ord=" ZINT_FORMAT " seqno=%d term=%s", + key.mem[0], seqno, dst_buf); + } + } +#else int off = 0; int seqno = 0; - NMEM nmem = nmem_create(); - + void *decode_handle = iscz1_start(); yaz_log(YLOG_LOG, "print_rec_keys buf=%p sz=%d", reckeys->buf, reckeys->buf_used); assert(reckeys->buf); @@ -1584,28 +1955,28 @@ void print_rec_keys(ZebraHandle zh, struct recKeys *reckeys) const char *src = reckeys->buf + off; struct it_key key; char *dst = (char*) &key; - int attrSet, attrUse; char dst_buf[IT_MAX_WORD]; - char *dst_term = dst_buf; + int index_type; + const char *db = 0; iscz1_decode(decode_handle, &dst, &src); assert(key.len <= 4 && key.len > 2); - attrSet = (int) key.mem[0] >> 16; - attrUse = (int) key.mem[0] & 65535; seqno = (int) key.mem[key.len-1]; + + zebraExplain_lookup_ord(zh->reg->zei, + key.mem[0], &index_type, &db, 0, 0); - zebra_term_untrans_iconv(zh, nmem, src[0], &dst_term, src+1); + zebra_term_untrans(zh, index_type, dst_buf, src); yaz_log(YLOG_LOG, "ord=" ZINT_FORMAT " seqno=%d term=%s", - key.mem[0], seqno, dst_term); + key.mem[0], seqno, dst_buf); while (*src++) ; off = src - reckeys->buf; - nmem_reset(nmem); } - nmem_destroy(nmem); iscz1_stop(decode_handle); +#endif } void extract_add_index_string (RecWord *p, const char *str, int length) diff --git a/index/index.h b/index/index.h index fe7002c..5f15228 100644 --- a/index/index.h +++ b/index/index.h @@ -1,4 +1,4 @@ -/* $Id: index.h,v 1.152 2005-10-20 18:28:10 quinn Exp $ +/* $Id: index.h,v 1.153 2005-10-28 07:25:30 adam Exp $ Copyright (C) 1995-2005 Index Data ApS @@ -192,12 +192,18 @@ typedef struct zebra_rank_class { struct zebra_rank_class *next; } *ZebraRankClass; +#define NEW_REC_KEYS 1 + +#if NEW_REC_KEYS +#include "reckeys.h" +#else struct recKeys { int buf_used; int buf_max; char *buf; void *codec_handle; }; +#endif #if NATTR @@ -234,7 +240,12 @@ struct zebra_register { int stop_flag; int active; /* 0=shutdown, 1=enabled and inactive, 2=activated */ +#if NEW_REC_KEYS + zebra_rec_keys_t keys; +#else struct recKeys keys; +#endif + #if NATTR struct recKeys sortKeys; #else @@ -476,15 +487,31 @@ off_t zebra_record_int_tell (void *fh); int zebra_record_int_read (void *fh, char *buf, size_t count); void zebra_record_int_end (void *fh, off_t offset); +#if NEW_REC_KEYS +void print_rec_keys(ZebraHandle zh, zebra_rec_keys_t reckeys); +#else void print_rec_keys(ZebraHandle zh, struct recKeys *reckeys); +#endif + +#if NEW_REC_KEYS +ZEBRA_RES zebra_snippets_rec_keys(ZebraHandle zh, zebra_rec_keys_t reckeys, + zebra_snippets *snippets); +#else ZEBRA_RES zebra_snippets_rec_keys(ZebraHandle zh, struct recKeys *reckeys, zebra_snippets *snippets); +#endif ZEBRA_RES zebra_snippets_hit_vector(ZebraHandle zh, const char *setname, zint sysno, zebra_snippets *snippets); +#if NEW_REC_KEYS +void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno, + int cmd, zebra_rec_keys_t reckeys, + zint staticrank); +#else void extract_flushRecordKeys (ZebraHandle zh, SYSNO sysno, int cmd, struct recKeys *reckeys, zint staticrank); +#endif #if NATTR void extract_flushSortKeys (ZebraHandle zh, SYSNO sysno, int cmd, struct recKeys *skp); diff --git a/index/kinput.c b/index/kinput.c index 748ced9..79e9f04 100644 --- a/index/kinput.c +++ b/index/kinput.c @@ -1,4 +1,4 @@ -/* $Id: kinput.c,v 1.67 2005-09-16 09:58:39 adam Exp $ +/* $Id: kinput.c,v 1.68 2005-10-28 07:25:30 adam Exp $ Copyright (C) 1995-2005 Index Data ApS @@ -397,18 +397,20 @@ static int heap_read_one (struct heap_info *hi, char *name, char *key) #define PR_KEY_LOW 0 #define PR_KEY_TOP 0 +#if 0 static void pkey(const char *b, int mode) { key_logdump_txt(YLOG_LOG, b, mode ? "i" : "d"); } +#endif #if 1 /* for debugging only */ -static void print_dict_item (ZebraHandle zh, const char *s) +static void print_dict_item(ZebraHandle zh, const char *s) { char dst[IT_MAX_WORD+1]; int ord; - int len = key_SU_decode (&ord, (const unsigned char *) s); + int len = key_SU_decode(&ord, (const unsigned char *) s); int index_type; const char *db = 0; @@ -416,7 +418,8 @@ static void print_dict_item (ZebraHandle zh, const char *s) ord, &index_type, &db, 0, 0); zebra_term_untrans(zh, index_type, dst, s + len); - yaz_log (YLOG_LOG, "%s", dst); + + yaz_log(YLOG_LOG, "ord=%d term=%s", ord, dst); } #endif @@ -656,7 +659,7 @@ int heap_inpb(struct heap_cread_info *hci, struct heap_info *hi) hi->no_diffs++; #if 0 - print_dict_item (hi->zh, hci->cur_name); + print_dict_item(hi->zh, hci->cur_name); #endif if ((dict_info = dict_lookup (hi->reg->dict, hci->cur_name))) { diff --git a/index/retrieve.c b/index/retrieve.c index 08257c4..d60a989 100644 --- a/index/retrieve.c +++ b/index/retrieve.c @@ -1,4 +1,4 @@ -/* $Id: retrieve.c,v 1.35 2005-08-26 10:13:31 adam Exp $ +/* $Id: retrieve.c,v 1.36 2005-10-28 07:25:30 adam Exp $ Copyright (C) 1995-2005 Index Data ApS @@ -219,12 +219,24 @@ int zebra_record_fetch (ZebraHandle zh, SYSNO sysno, int score, if (1) { /* snippets code */ - struct recKeys reckeys; zebra_snippets *snippet; + +#if NEW_REC_KEYS + zebra_rec_keys_t reckeys = zebra_rec_keys_open(); + + zebra_rec_keys_set_buf(reckeys, + rec->info[recInfo_delKeys], + rec->size[recInfo_delKeys], + 0); + zebra_snippets_rec_keys(zh, reckeys, retrieveCtrl.doc_snippet); + zebra_rec_keys_close(reckeys); +#else + struct recKeys reckeys; reckeys.buf = rec->info[recInfo_delKeys]; reckeys.buf_used = rec->size[recInfo_delKeys]; - zebra_snippets_rec_keys(zh, &reckeys, retrieveCtrl.doc_snippet); +#endif + #if 0 /* for debugging purposes */ diff --git a/index/zebraapi.c b/index/zebraapi.c index 5782aeb..d419085 100644 --- a/index/zebraapi.c +++ b/index/zebraapi.c @@ -1,4 +1,4 @@ -/* $Id: zebraapi.c,v 1.191 2005-10-21 18:29:04 adam Exp $ +/* $Id: zebraapi.c,v 1.192 2005-10-28 07:25:30 adam Exp $ Copyright (C) 1995-2005 Index Data ApS @@ -326,9 +326,13 @@ struct zebra_register *zebra_register_open(ZebraService zs, const char *name, reg->key_buf = 0; +#if NEW_REC_KEYS + reg->keys = zebra_rec_keys_open(); +#else reg->keys.buf_max = 0; reg->keys.buf = 0; reg->keys.codec_handle = iscz1_start(); +#endif reg->sortKeys.buf = 0; reg->sortKeys.buf_max = 0; @@ -497,6 +501,9 @@ static void zebra_register_close (ZebraService zs, struct zebra_register *reg) data1_destroy (reg->dh); xfree(reg->sortKeys.buf); +#if NEW_REC_KEYS + zebra_rec_keys_close(reg->keys); +#else xfree(reg->keys.buf); if (reg->keys.codec_handle) iscz1_stop(reg->keys.codec_handle); @@ -504,6 +511,8 @@ static void zebra_register_close (ZebraService zs, struct zebra_register *reg) if (reg->sortKeys.codec_handle) iscz1_stop(reg->sortKeys.codec_handle); #endif +#endif + xfree(reg->key_buf); xfree(reg->name); xfree(reg); diff --git a/index/zsets.c b/index/zsets.c index 1b76097..949ace4 100644 --- a/index/zsets.c +++ b/index/zsets.c @@ -1,4 +1,4 @@ -/* $Id: zsets.c,v 1.95 2005-08-26 10:13:31 adam Exp $ +/* $Id: zsets.c,v 1.96 2005-10-28 07:25:30 adam Exp $ Copyright (C) 1995-2005 Index Data ApS @@ -1119,6 +1119,7 @@ ZEBRA_RES zebra_snippets_hit_vector(ZebraHandle zh, const char *setname, rset_delete(rset_comb); nmem_destroy(nmem); + kc->dec(kc); } return ZEBRA_OK; } -- 1.7.10.4