X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=index%2Fextract.c;h=d9ed681bbb83945145099c1275a50696f6888c24;hb=5623222cf968d493c2bc9506c42b8344de60c4e5;hp=f6fab5d8188988cae95fd5748731357cc2357c46;hpb=21f90a8618faec6bee8d125c12088b74db8eb8b9;p=idzebra-moved-to-github.git diff --git a/index/extract.c b/index/extract.c index f6fab5d..d9ed681 100644 --- a/index/extract.c +++ b/index/extract.c @@ -1,4 +1,4 @@ -/* $Id: extract.c,v 1.252 2007-03-14 11:48:32 adam Exp $ +/* $Id: extract.c,v 1.260 2007-08-21 13:27:04 adam Exp $ Copyright (C) 1995-2007 Index Data ApS @@ -40,6 +40,16 @@ static int log_level_extract = 0; static int log_level_details = 0; static int log_level_initialized = 0; +/* 1 if we use eliminitate identical delete/insert keys */ +/* eventually this the 0-case code will be removed */ +#define FLUSH2 1 + +void extract_flush_record_keys2(ZebraHandle zh, zint sysno, + zebra_rec_keys_t ins_keys, + zint ins_rank, + zebra_rec_keys_t del_keys, + zint del_rank); + static void zebra_init_log_level(void) { if (!log_level_initialized) @@ -81,6 +91,20 @@ static void logRecord (ZebraHandle zh) } } +static void init_extractCtrl(ZebraHandle zh, struct recExtractCtrl *ctrl) +{ + int i; + for (i = 0; i<256; i++) + { + if (zebra_maps_is_positioned(zh->reg->zebra_maps, i)) + ctrl->seqno[i] = 1; + else + ctrl->seqno[i] = 0; + } + ctrl->flagShowRecords = !zh->m_flag_rw; +} + + static void extract_add_index_string (RecWord *p, zinfo_index_category_t cat, const char *str, int length); @@ -98,6 +122,213 @@ static void extract_init(struct recExtractCtrl *p, RecWord *w) w->segment = 0; } +struct snip_rec_info { + ZebraHandle zh; + zebra_snippets *snippets; +}; + + +static void snippet_add_complete_field(RecWord *p, int ord) +{ + struct snip_rec_info *h = p->extractCtrl->handle; + ZebraHandle zh = h->zh; + + const char *b = p->term_buf; + char buf[IT_MAX_WORD+1]; + const char **map = 0; + int i = 0, remain = p->term_len; + const char *start = b; + const char *last = 0; + + if (remain > 0) + map = zebra_maps_input (zh->reg->zebra_maps, p->index_type, &b, remain, 1); + + while (remain > 0 && i < IT_MAX_WORD) + { + while (map && *map && **map == *CHR_SPACE) + { + remain = p->term_len - (b - p->term_buf); + + if (i == 0) + start = b; /* set to first non-ws area */ + if (remain > 0) + { + int first = i ? 0 : 1; /* first position */ + + map = zebra_maps_input(zh->reg->zebra_maps, p->index_type, + &b, remain, first); + } + else + map = 0; + } + if (!map) + break; + + if (i && i < IT_MAX_WORD) + buf[i++] = *CHR_SPACE; + while (map && *map && **map != *CHR_SPACE) + { + const char *cp = *map; + + if (**map == *CHR_CUT) + { + i = 0; + } + else + { + if (i >= IT_MAX_WORD) + break; + while (i < IT_MAX_WORD && *cp) + buf[i++] = *(cp++); + } + last = b; + remain = p->term_len - (b - p->term_buf); + if (remain > 0) + { + map = zebra_maps_input (zh->reg->zebra_maps, p->index_type, &b, + remain, 0); + } + else + map = 0; + } + } + if (!i) + return; + if (last && start != last) + zebra_snippets_appendn(h->snippets, p->seqno, 0, ord, + start, last - start); +} + +static void snippet_add_incomplete_field(RecWord *p, int ord) +{ + struct snip_rec_info *h = p->extractCtrl->handle; + ZebraHandle zh = h->zh; + const char *b = p->term_buf; + int remain = p->term_len; + int first = 1; + const char **map = 0; + const char *start = b; + + if (remain > 0) + map = zebra_maps_input(zh->reg->zebra_maps, p->index_type, &b, remain, 0); + + while (map) + { + char buf[IT_MAX_WORD+1]; + const char *last = b; + int i, remain; + + /* Skip spaces */ + while (map && *map && **map == *CHR_SPACE) + { + remain = p->term_len - (b - p->term_buf); + last = b; + if (remain > 0) + map = zebra_maps_input(zh->reg->zebra_maps, p->index_type, &b, + remain, 0); + else + map = 0; + } + if (!map) + break; + if (start != last) + { + zebra_snippets_appendn(h->snippets, p->seqno, 1, ord, + start, last - start); + + } + start = last; + + i = 0; + while (map && *map && **map != *CHR_SPACE) + { + const char *cp = *map; + + while (i < IT_MAX_WORD && *cp) + buf[i++] = *(cp++); + remain = p->term_len - (b - p->term_buf); + last = b; + if (remain > 0) + map = zebra_maps_input(zh->reg->zebra_maps, p->index_type, &b, remain, 0); + else + map = 0; + } + if (!i) + return; + + if (first) + { + first = 0; + if (zebra_maps_is_first_in_field(zh->reg->zebra_maps, p->index_type)) + { + /* first in field marker */ + p->seqno++; + } + } + if (start != last) + zebra_snippets_appendn(h->snippets, p->seqno, 0, ord, + start, last - start); + start = last; + p->seqno++; + } + +} + +static void snippet_token_add(RecWord *p) +{ + struct snip_rec_info *h = p->extractCtrl->handle; + ZebraHandle zh = h->zh; + + if (zebra_maps_is_index(zh->reg->zebra_maps, p->index_type)) + { + ZebraExplainInfo zei = zh->reg->zei; + int ch = zebraExplain_lookup_attr_str( + zei, zinfo_index_category_index, p->index_type, p->index_name); + + if (zebra_maps_is_complete (h->zh->reg->zebra_maps, p->index_type)) + snippet_add_complete_field (p, ch); + else + snippet_add_incomplete_field(p, ch); + } +} + +static void snippet_schema_add( + struct recExtractCtrl *p, Odr_oid *oid) +{ + +} + +void extract_snippet(ZebraHandle zh, zebra_snippets *sn, + struct ZebraRecStream *stream, + RecType rt, void *recTypeClientData) +{ + struct recExtractCtrl extractCtrl; + struct snip_rec_info info; + int r; + + extractCtrl.stream = stream; + extractCtrl.first_record = 1; + extractCtrl.init = extract_init; + extractCtrl.tokenAdd = snippet_token_add; + extractCtrl.schemaAdd = snippet_schema_add; + assert(zh->reg); + assert(zh->reg->dh); + + extractCtrl.dh = zh->reg->dh; + + info.zh = zh; + info.snippets = sn; + extractCtrl.handle = &info; + extractCtrl.match_criteria[0] = '\0'; + extractCtrl.staticrank = 0; + extractCtrl.action = action_insert; + + init_extractCtrl(zh, &extractCtrl); + + r = (*rt->extract)(recTypeClientData, &extractCtrl); + +} + static void searchRecordKey(ZebraHandle zh, zebra_rec_keys_t reckeys, const char *index_name, @@ -295,19 +526,6 @@ struct recordLogInfo { struct recordGroup *rGroup; }; -static void init_extractCtrl(ZebraHandle zh, struct recExtractCtrl *ctrl) -{ - int i; - for (i = 0; i<256; i++) - { - if (zebra_maps_is_positioned(zh->reg->zebra_maps, i)) - ctrl->seqno[i] = 1; - else - ctrl->seqno[i] = 0; - } - ctrl->flagShowRecords = !zh->m_flag_rw; -} - static void all_matches_add(struct recExtractCtrl *ctrl) { RecWord word; @@ -543,6 +761,20 @@ ZEBRA_RES zebra_extract_records_stream(ZebraHandle zh, } +static WRBUF wrbuf_hex_str(const char *cstr) +{ + size_t i; + WRBUF w = wrbuf_alloc(); + for (i = 0; cstr[i]; i++) + { + if (cstr[i] < ' ' || cstr[i] > 126) + wrbuf_printf(w, "\\%02X", cstr[i] & 0xff); + else + wrbuf_putc(w, cstr[i]); + } + return w; +} + ZEBRA_RES zebra_extract_record_stream(ZebraHandle zh, struct ZebraRecStream *stream, enum zebra_recctrl_action_t action, @@ -600,13 +832,19 @@ ZEBRA_RES zebra_extract_record_stream(ZebraHandle zh, extractCtrl.handle = zh; extractCtrl.match_criteria[0] = '\0'; extractCtrl.staticrank = 0; + extractCtrl.action = action; init_extractCtrl(zh, &extractCtrl); - + extract_set_store_data_prepare(&extractCtrl); r = (*recType->extract)(recTypeClientData, &extractCtrl); + if (action == action_update) + { + action = extractCtrl.action; + } + switch (r) { case RECCTRL_EXTRACT_EOF: @@ -672,12 +910,20 @@ ZEBRA_RES zebra_extract_record_stream(ZebraHandle zh, int db_ord = zebraExplain_get_database_ord(zh->reg->zei); char *rinfo = dict_lookup_ord(zh->reg->matchDict, db_ord, matchStr); + + + if (log_level_extract) + { + WRBUF w = wrbuf_hex_str(matchStr); + yaz_log(log_level_extract, "matchStr: %s", wrbuf_cstr(w)); + wrbuf_destroy(w); + } if (rinfo) { assert(*rinfo == sizeof(*sysno)); memcpy (sysno, rinfo+1, sizeof(*sysno)); } - } + } } if (zebra_rec_keys_empty(zh->reg->keys)) { @@ -717,7 +963,6 @@ ZEBRA_RES zebra_extract_record_stream(ZebraHandle zh, yaz_log(YLOG_WARN, "Negative staticrank for record. Set to 0"); extractCtrl.staticrank = 0; } - recordAttr->staticrank = extractCtrl.staticrank; if (matchStr) { @@ -727,8 +972,15 @@ ZEBRA_RES zebra_extract_record_stream(ZebraHandle zh, } extract_flush_sort_keys(zh, *sysno, 1, zh->reg->sortKeys); +#if FLUSH2 + extract_flush_record_keys2(zh, *sysno, + zh->reg->keys, extractCtrl.staticrank, + 0, recordAttr->staticrank); +#else extract_flush_record_keys(zh, *sysno, 1, zh->reg->keys, - recordAttr->staticrank); + extractCtrl.staticrank); +#endif + recordAttr->staticrank = extractCtrl.staticrank; zh->records_inserted++; } else @@ -763,11 +1015,17 @@ ZEBRA_RES zebra_extract_record_stream(ZebraHandle zh, 0); extract_flush_sort_keys(zh, *sysno, 0, sortKeys); +#if !FLUSH2 extract_flush_record_keys(zh, *sysno, 0, delkeys, recordAttr->staticrank); +#endif if (action == action_delete) { /* record going to be deleted */ +#if FLUSH2 + extract_flush_record_keys2(zh, *sysno, 0, recordAttr->staticrank, + delkeys, recordAttr->staticrank); +#endif if (zebra_rec_keys_empty(delkeys)) { yaz_log(YLOG_LOG, "delete %s %s " ZINT_FORMAT, recordType, @@ -788,6 +1046,8 @@ ZEBRA_RES zebra_extract_record_stream(ZebraHandle zh, } rec_del (zh->reg->records, &rec); } + zebra_rec_keys_close(delkeys); + zebra_rec_keys_close(sortKeys); rec_free(&rec); logRecord(zh); return ZEBRA_OK; @@ -797,10 +1057,17 @@ ZEBRA_RES zebra_extract_record_stream(ZebraHandle zh, if (show_progress) yaz_log(YLOG_LOG, "update %s %s " ZINT_FORMAT, recordType, pr_fname, (zint) start_offset); - recordAttr->staticrank = extractCtrl.staticrank; extract_flush_sort_keys(zh, *sysno, 1, zh->reg->sortKeys); - extract_flush_record_keys(zh, *sysno, 1, zh->reg->keys, - recordAttr->staticrank); + +#if FLUSH2 + extract_flush_record_keys2(zh, *sysno, + zh->reg->keys, extractCtrl.staticrank, + delkeys, recordAttr->staticrank); +#else + extract_flush_record_keys(zh, *sysno, 1, + zh->reg->keys, extractCtrl.staticrank); +#endif + recordAttr->staticrank = extractCtrl.staticrank; zh->records_updated++; } zebra_rec_keys_close(delkeys); @@ -916,6 +1183,8 @@ ZEBRA_RES zebra_extract_explain(void *handle, Record rec, data1_node *n) extractCtrl.flagShowRecords = 0; extractCtrl.match_criteria[0] = '\0'; extractCtrl.staticrank = 0; + extractCtrl.action = action_update; + extractCtrl.handle = handle; extractCtrl.first_record = 1; @@ -933,7 +1202,13 @@ ZEBRA_RES zebra_extract_explain(void *handle, Record rec, data1_node *n) zebra_rec_keys_set_buf(delkeys, rec->info[recInfo_delKeys], rec->size[recInfo_delKeys], 0); +#if FLUSH2 + extract_flush_record_keys2(zh, rec->sysno, + zh->reg->keys, 0, delkeys, 0); +#else extract_flush_record_keys(zh, rec->sysno, 0, delkeys, 0); + extract_flush_record_keys(zh, rec->sysno, 1, zh->reg->keys, 0); +#endif zebra_rec_keys_close(delkeys); zebra_rec_keys_set_buf(sortkeys, rec->info[recInfo_sortKeys], @@ -943,7 +1218,14 @@ ZEBRA_RES zebra_extract_explain(void *handle, Record rec, data1_node *n) extract_flush_sort_keys(zh, rec->sysno, 0, sortkeys); zebra_rec_keys_close(sortkeys); } - extract_flush_record_keys(zh, rec->sysno, 1, zh->reg->keys, 0); + else + { +#if FLUSH2 + extract_flush_record_keys2(zh, rec->sysno, zh->reg->keys, 0, 0, 0); +#else + extract_flush_record_keys(zh, rec->sysno, 1, zh->reg->keys, 0); +#endif + } extract_flush_sort_keys(zh, rec->sysno, 1, zh->reg->sortKeys); xfree (rec->info[recInfo_delKeys]); @@ -1074,6 +1356,79 @@ void extract_rec_keys_adjust(ZebraHandle zh, int is_insert, } } +void extract_flush_record_keys2(ZebraHandle zh, zint sysno, + zebra_rec_keys_t ins_keys, zint ins_rank, + zebra_rec_keys_t del_keys, zint del_rank) +{ + ZebraExplainInfo zei = zh->reg->zei; + int normal = 0; + int optimized = 0; + + if (!zh->reg->key_block) + { + int mem = 1024*1024 * atoi( res_get_def( zh->res, "memmax", "8")); + const char *key_tmp_dir = res_get_def (zh->res, "keyTmpDir", "."); + int use_threads = atoi(res_get_def (zh->res, "threads", "1")); + zh->reg->key_block = key_block_create(mem, key_tmp_dir, use_threads); + } + + if (ins_keys) + { + extract_rec_keys_adjust(zh, 1, ins_keys); + if (!del_keys) + zebraExplain_recordCountIncrement (zei, 1); + zebra_rec_keys_rewind(ins_keys); + } + if (del_keys) + { + extract_rec_keys_adjust(zh, 0, del_keys); + if (!ins_keys) + zebraExplain_recordCountIncrement (zei, -1); + zebra_rec_keys_rewind(del_keys); + } + + while (1) + { + size_t del_slen; + const char *del_str; + struct it_key del_key_in; + int del = 0; + + size_t ins_slen; + const char *ins_str; + struct it_key ins_key_in; + int ins = 0; + + if (del_keys) + del = zebra_rec_keys_read(del_keys, &del_str, &del_slen, + &del_key_in); + if (ins_keys) + ins = zebra_rec_keys_read(ins_keys, &ins_str, &ins_slen, + &ins_key_in); + + if (del && ins && ins_rank == del_rank + && !key_compare(&del_key_in, &ins_key_in) + && ins_slen == del_slen && !memcmp(del_str, ins_str, del_slen)) + { + optimized++; + continue; + } + if (!del && !ins) + break; + + normal++; + if (del) + key_block_write(zh->reg->key_block, sysno, + &del_key_in, 0, del_str, del_slen, + del_rank, zh->m_staticrank); + if (ins) + key_block_write(zh->reg->key_block, sysno, + &ins_key_in, 1, ins_str, ins_slen, + ins_rank, zh->m_staticrank); + } + yaz_log(log_level_extract, "normal=%d optimized=%d", normal, optimized); +} + void extract_flush_record_keys(ZebraHandle zh, zint sysno, int cmd, zebra_rec_keys_t reckeys, zint staticrank) @@ -1098,6 +1453,10 @@ void extract_flush_record_keys(ZebraHandle zh, zint sysno, int cmd, } zebraExplain_recordCountIncrement (zei, cmd ? 1 : -1); +#if 0 + yaz_log(YLOG_LOG, "sysno=" ZINT_FORMAT " cmd=%d", sysno, cmd); + print_rec_keys(zh, reckeys); +#endif if (zebra_rec_keys_rewind(reckeys)) { size_t slen; @@ -1112,7 +1471,6 @@ void extract_flush_record_keys(ZebraHandle zh, zint sysno, int cmd, } } - ZEBRA_RES zebra_rec_keys_to_snippets(ZebraHandle zh, zebra_rec_keys_t reckeys, zebra_snippets *snippets) @@ -1139,7 +1497,7 @@ ZEBRA_RES zebra_rec_keys_to_snippets(ZebraHandle zh, assert(index_type); zebra_term_untrans_iconv(zh, nmem, index_type, &dst_term, str); - zebra_snippets_append(snippets, seqno, ord, dst_term); + zebra_snippets_append(snippets, seqno, 0, ord, dst_term); nmem_reset(nmem); } } @@ -1389,9 +1747,9 @@ static void extract_token_add(RecWord *p) ZebraHandle zh = p->extractCtrl->handle; WRBUF wrbuf; - if (log_level_extract) + if (log_level_details) { - yaz_log(log_level_extract, "extract_token_add " + yaz_log(log_level_details, "extract_token_add " "type=%c index=%s seqno=" ZINT_FORMAT " s=%.*s", p->index_type, p->index_name, p->seqno, p->term_len, p->term_buf);