X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=index%2Fextract.c;h=a93373ce8f259656e1c6d37299c33ae9bb483729;hb=b3f079a571e404665620e8eb3bb1dd7db1ff6669;hp=23f0b6ed2102c96a8a6cb1fa326b83d5d15fe681;hpb=a030c87bc444608639905eca95e29f84a4f1d991;p=idzebra-moved-to-github.git diff --git a/index/extract.c b/index/extract.c index 23f0b6e..a93373c 100644 --- a/index/extract.c +++ b/index/extract.c @@ -21,6 +21,9 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA \brief indexes records and extract tokens for indexing and sorting */ +#if HAVE_CONFIG_H +#include +#endif #include #include #include @@ -47,11 +50,18 @@ static int log_level_initialized = 0; /* eventually this the 0-case code will be removed */ #define FLUSH2 1 -void extract_flush_record_keys2(ZebraHandle zh, zint sysno, - zebra_rec_keys_t ins_keys, - zint ins_rank, - zebra_rec_keys_t del_keys, - zint del_rank); +#if FLUSH2 +static void extract_flush_record_keys2(ZebraHandle zh, zint sysno, + zebra_rec_keys_t ins_keys, + zint ins_rank, + zebra_rec_keys_t del_keys, + zint del_rank); +#else +static void extract_flush_record_keys(ZebraHandle zh, zint sysno, + int cmd, + zebra_rec_keys_t reckeys, + zint staticrank); +#endif static void zebra_init_log_level(void) { @@ -139,69 +149,10 @@ static void snippet_add_complete_field(RecWord *p, int ord, zebra_map_t zm) { struct snip_rec_info *h = p->extractCtrl->handle; - - const char *b = p->term_buf; - char buf[IT_MAX_WORD+1]; - const char **map = 0; - int i = 0, remain = p->term_len; - const char *start = b; - const char *last = 0; - - if (remain > 0) - map = zebra_maps_input(zm, &b, remain, 1); - - while (remain > 0 && i < IT_MAX_WORD) - { - while (map && *map && **map == *CHR_SPACE) - { - remain = p->term_len - (b - p->term_buf); - - if (i == 0) - start = b; /* set to first non-ws area */ - if (remain > 0) - { - int first = i ? 0 : 1; /* first position */ - - map = zebra_maps_input(zm, &b, remain, first); - } - else - map = 0; - } - if (!map) - break; - - if (i && i < IT_MAX_WORD) - buf[i++] = *CHR_SPACE; - while (map && *map && **map != *CHR_SPACE) - { - const char *cp = *map; - - if (**map == *CHR_CUT) - { - i = 0; - } - else - { - if (i >= IT_MAX_WORD) - break; - while (i < IT_MAX_WORD && *cp) - buf[i++] = *(cp++); - } - last = b; - remain = p->term_len - (b - p->term_buf); - if (remain > 0) - { - map = zebra_maps_input(zm, &b, remain, 0); - } - else - map = 0; - } - } - if (!i) - return; - if (last && start != last && zebra_maps_is_index(zm)) + if (p->term_len && p->term_buf && zebra_maps_is_index(zm)) zebra_snippets_appendn(h->snippets, p->seqno, 0, ord, - start, last - start); + p->term_buf, p->term_len); + p->seqno++; } static void snippet_add_incomplete_field(RecWord *p, int ord, zebra_map_t zm) @@ -219,8 +170,7 @@ static void snippet_add_incomplete_field(RecWord *p, int ord, zebra_map_t zm) while (map) { - char buf[IT_MAX_WORD+1]; - int i, remain; + int remain; /* Skip spaces */ while (map && *map && **map == *CHR_SPACE) @@ -238,17 +188,10 @@ static void snippet_add_incomplete_field(RecWord *p, int ord, zebra_map_t zm) { zebra_snippets_appendn(h->snippets, p->seqno, 1, ord, start, last - start); - } start = last; - - i = 0; while (map && *map && **map != *CHR_SPACE) { - const char *cp = *map; - - while (i < IT_MAX_WORD && *cp) - buf[i++] = *(cp++); remain = p->term_len - (b - p->term_buf); last = b; if (remain > 0) @@ -256,8 +199,8 @@ static void snippet_add_incomplete_field(RecWord *p, int ord, zebra_map_t zm) else map = 0; } - if (!i) - return; + if (start == last) + return ; if (first) { @@ -334,7 +277,6 @@ void extract_snippet(ZebraHandle zh, zebra_snippets *sn, { struct recExtractCtrl extractCtrl; struct snip_rec_info info; - int r; extractCtrl.stream = stream; extractCtrl.first_record = 1; @@ -357,8 +299,7 @@ void extract_snippet(ZebraHandle zh, zebra_snippets *sn, extractCtrl.setStoreData = 0; - r = (*rt->extract)(recTypeClientData, &extractCtrl); - + (*rt->extract)(recTypeClientData, &extractCtrl); } static void searchRecordKey(ZebraHandle zh, @@ -624,7 +565,6 @@ ZEBRA_RES zebra_extract_file(ZebraHandle zh, zint *sysno, const char *fname, char gprefix[128]; char ext[128]; char ext_res[128]; - struct file_read_info *fi = 0; const char *original_record_type = 0; RecType recType; void *recTypeClientData; @@ -690,7 +630,6 @@ ZEBRA_RES zebra_extract_file(ZebraHandle zh, zint *sysno, const char *fname, if (sysno && (action == action_delete || action == action_a_delete)) { streamp = 0; - fi = 0; } else { @@ -1455,9 +1394,11 @@ void extract_rec_keys_adjust(ZebraHandle zh, int is_insert, } } -void extract_flush_record_keys2(ZebraHandle zh, zint sysno, - zebra_rec_keys_t ins_keys, zint ins_rank, - zebra_rec_keys_t del_keys, zint del_rank) +#if FLUSH2 +static void extract_flush_record_keys2( + ZebraHandle zh, zint sysno, + zebra_rec_keys_t ins_keys, zint ins_rank, + zebra_rec_keys_t del_keys, zint del_rank) { ZebraExplainInfo zei = zh->reg->zei; int normal = 0; @@ -1527,7 +1468,50 @@ void extract_flush_record_keys2(ZebraHandle zh, zint sysno, } yaz_log(log_level_extract, "normal=%d optimized=%d", normal, optimized); } +#else +static void extract_flush_record_keys( + ZebraHandle zh, zint sysno, int cmd, + zebra_rec_keys_t reckeys, + zint staticrank) +{ + ZebraExplainInfo zei = zh->reg->zei; + extract_rec_keys_adjust(zh, cmd, reckeys); + + if (log_level_details) + { + yaz_log(log_level_details, "Keys for record " ZINT_FORMAT " %s", + sysno, cmd ? "insert" : "delete"); + extract_rec_keys_log(zh, cmd, reckeys, log_level_details); + } + + if (!zh->reg->key_block) + { + int mem = 1024*1024 * atoi( res_get_def( zh->res, "memmax", "8")); + const char *key_tmp_dir = res_get_def(zh->res, "keyTmpDir", "."); + int use_threads = atoi(res_get_def(zh->res, "threads", "1")); + zh->reg->key_block = key_block_create(mem, key_tmp_dir, use_threads); + } + zebraExplain_recordCountIncrement(zei, cmd ? 1 : -1); + +#if 0 + yaz_log(YLOG_LOG, "sysno=" ZINT_FORMAT " cmd=%d", sysno, cmd); + print_rec_keys(zh, reckeys); +#endif + if (zebra_rec_keys_rewind(reckeys)) + { + size_t slen; + const char *str; + struct it_key key_in; + while(zebra_rec_keys_read(reckeys, &str, &slen, &key_in)) + { + key_block_write(zh->reg->key_block, sysno, + &key_in, cmd, str, slen, + staticrank, zh->m_staticrank); + } + } +} +#endif ZEBRA_RES zebra_rec_keys_to_snippets1(ZebraHandle zh, zebra_rec_keys_t reckeys, @@ -1803,6 +1787,7 @@ static void extract_add_complete_field(RecWord *p, zebra_map_t zm) if (!i) return; extract_add_string(p, zm, buf, i); + p->seqno++; } static void extract_add_icu(RecWord *p, zebra_map_t zm) @@ -1813,6 +1798,11 @@ static void extract_add_icu(RecWord *p, zebra_map_t zm) zebra_map_tokenize_start(zm, p->term_buf, p->term_len); while (zebra_map_tokenize_next(zm, &res_buf, &res_len, 0, 0)) { + if (res_len > IT_MAX_WORD) + { + yaz_log(YLOG_LOG, "Truncating long term %ld", (long) res_len); + res_len = IT_MAX_WORD; + } extract_add_string(p, zm, res_buf, res_len); p->seqno++; } @@ -1838,7 +1828,6 @@ static void extract_token_add(RecWord *p) { ZebraHandle zh = p->extractCtrl->handle; zebra_map_t zm = zebra_map_get_or_add(zh->reg->zebra_maps, p->index_type); - WRBUF wrbuf; if (log_level_details) { @@ -1847,11 +1836,6 @@ static void extract_token_add(RecWord *p) p->index_type, p->index_name, p->seqno, p->term_len, p->term_buf); } - if ((wrbuf = zebra_replace(zm, 0, p->term_buf, p->term_len))) - { - p->term_buf = wrbuf_buf(wrbuf); - p->term_len = wrbuf_len(wrbuf); - } if (zebra_maps_is_icu(zm)) { extract_add_icu(p, zm);