X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;ds=sidebyside;f=index%2Fextract.c;h=a4688e52933986da810a8d85c9735a3c4bb8a35c;hb=3bb2fd863b1664f1ff71dadb540313f301ca234f;hp=e61483c9114363dd532f1c00374066ced26398e4;hpb=1d471f499b0be3a3990ad252ff9c19da0462f9aa;p=idzebra-moved-to-github.git diff --git a/index/extract.c b/index/extract.c index e61483c..a4688e5 100644 --- a/index/extract.c +++ b/index/extract.c @@ -1,5 +1,5 @@ /* This file is part of the Zebra server. - Copyright (C) 1994-2009 Index Data + Copyright (C) 1994-2011 Index Data Zebra is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free @@ -21,6 +21,9 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA \brief indexes records and extract tokens for indexing and sorting */ +#if HAVE_CONFIG_H +#include +#endif #include #include #include @@ -47,11 +50,18 @@ static int log_level_initialized = 0; /* eventually this the 0-case code will be removed */ #define FLUSH2 1 -void extract_flush_record_keys2(ZebraHandle zh, zint sysno, - zebra_rec_keys_t ins_keys, - zint ins_rank, - zebra_rec_keys_t del_keys, - zint del_rank); +#if FLUSH2 +static void extract_flush_record_keys2(ZebraHandle zh, zint sysno, + zebra_rec_keys_t ins_keys, + zint ins_rank, + zebra_rec_keys_t del_keys, + zint del_rank); +#else +static void extract_flush_record_keys(ZebraHandle zh, zint sysno, + int cmd, + zebra_rec_keys_t reckeys, + zint staticrank); +#endif static void zebra_init_log_level(void) { @@ -139,69 +149,10 @@ static void snippet_add_complete_field(RecWord *p, int ord, zebra_map_t zm) { struct snip_rec_info *h = p->extractCtrl->handle; - - const char *b = p->term_buf; - char buf[IT_MAX_WORD+1]; - const char **map = 0; - int i = 0, remain = p->term_len; - const char *start = b; - const char *last = 0; - - if (remain > 0) - map = zebra_maps_input(zm, &b, remain, 1); - - while (remain > 0 && i < IT_MAX_WORD) - { - while (map && *map && **map == *CHR_SPACE) - { - remain = p->term_len - (b - p->term_buf); - - if (i == 0) - start = b; /* set to first non-ws area */ - if (remain > 0) - { - int first = i ? 0 : 1; /* first position */ - - map = zebra_maps_input(zm, &b, remain, first); - } - else - map = 0; - } - if (!map) - break; - - if (i && i < IT_MAX_WORD) - buf[i++] = *CHR_SPACE; - while (map && *map && **map != *CHR_SPACE) - { - const char *cp = *map; - - if (**map == *CHR_CUT) - { - i = 0; - } - else - { - if (i >= IT_MAX_WORD) - break; - while (i < IT_MAX_WORD && *cp) - buf[i++] = *(cp++); - } - last = b; - remain = p->term_len - (b - p->term_buf); - if (remain > 0) - { - map = zebra_maps_input(zm, &b, remain, 0); - } - else - map = 0; - } - } - if (!i) - return; - if (last && start != last && zebra_maps_is_index(zm)) + if (p->term_len && p->term_buf && zebra_maps_is_index(zm)) zebra_snippets_appendn(h->snippets, p->seqno, 0, ord, - start, last - start); + p->term_buf, p->term_len); + p->seqno++; } static void snippet_add_incomplete_field(RecWord *p, int ord, zebra_map_t zm) @@ -219,8 +170,7 @@ static void snippet_add_incomplete_field(RecWord *p, int ord, zebra_map_t zm) while (map) { - char buf[IT_MAX_WORD+1]; - int i, remain; + int remain; /* Skip spaces */ while (map && *map && **map == *CHR_SPACE) @@ -238,17 +188,10 @@ static void snippet_add_incomplete_field(RecWord *p, int ord, zebra_map_t zm) { zebra_snippets_appendn(h->snippets, p->seqno, 1, ord, start, last - start); - } start = last; - - i = 0; while (map && *map && **map != *CHR_SPACE) { - const char *cp = *map; - - while (i < IT_MAX_WORD && *cp) - buf[i++] = *(cp++); remain = p->term_len - (b - p->term_buf); last = b; if (remain > 0) @@ -256,8 +199,8 @@ static void snippet_add_incomplete_field(RecWord *p, int ord, zebra_map_t zm) else map = 0; } - if (!i) - return; + if (start == last) + return ; if (first) { @@ -334,7 +277,6 @@ void extract_snippet(ZebraHandle zh, zebra_snippets *sn, { struct recExtractCtrl extractCtrl; struct snip_rec_info info; - int r; extractCtrl.stream = stream; extractCtrl.first_record = 1; @@ -357,8 +299,7 @@ void extract_snippet(ZebraHandle zh, zebra_snippets *sn, extractCtrl.setStoreData = 0; - r = (*rt->extract)(recTypeClientData, &extractCtrl); - + (*rt->extract)(recTypeClientData, &extractCtrl); } static void searchRecordKey(ZebraHandle zh, @@ -624,7 +565,6 @@ ZEBRA_RES zebra_extract_file(ZebraHandle zh, zint *sysno, const char *fname, char gprefix[128]; char ext[128]; char ext_res[128]; - struct file_read_info *fi = 0; const char *original_record_type = 0; RecType recType; void *recTypeClientData; @@ -690,7 +630,6 @@ ZEBRA_RES zebra_extract_file(ZebraHandle zh, zint *sysno, const char *fname, if (sysno && (action == action_delete || action == action_a_delete)) { streamp = 0; - fi = 0; } else { @@ -1333,6 +1272,56 @@ ZEBRA_RES zebra_extract_explain(void *handle, Record rec, data1_node *n) return ZEBRA_OK; } +void zebra_it_key_str_dump(ZebraHandle zh, struct it_key *key, + const char *str, size_t slen, NMEM nmem, int level) +{ + char keystr[200]; /* room for zints to print */ + char *dst_term = 0; + int ord = CAST_ZINT_TO_INT(key->mem[0]); + const char *index_type; + int i; + const char *string_index; + + zebraExplain_lookup_ord(zh->reg->zei, ord, &index_type, + 0/* db */, &string_index); + assert(index_type); + zebra_term_untrans_iconv(zh, nmem, index_type, + &dst_term, str); + *keystr = '\0'; + for (i = 0; i < key->len; i++) + { + sprintf(keystr + strlen(keystr), ZINT_FORMAT " ", key->mem[i]); + } + + if (*str < CHR_BASE_CHAR) + { + int i; + char dst_buf[200]; /* room for special chars */ + + strcpy(dst_buf , "?"); + + if (!strcmp(str, "")) + strcpy(dst_buf, "alwaysmatches"); + if (!strcmp(str, FIRST_IN_FIELD_STR)) + strcpy(dst_buf, "firstinfield"); + else if (!strcmp(str, CHR_UNKNOWN)) + strcpy(dst_buf, "unknown"); + else if (!strcmp(str, CHR_SPACE)) + strcpy(dst_buf, "space"); + + for (i = 0; ireg->zei, ord, &index_type, - 0/* db */, &string_index); - assert(index_type); - zebra_term_untrans_iconv(zh, nmem, index_type, - &dst_term, str); - *keystr = '\0'; - for (i = 0; ireg->zei; int normal = 0; @@ -1522,7 +1468,50 @@ void extract_flush_record_keys2(ZebraHandle zh, zint sysno, } yaz_log(log_level_extract, "normal=%d optimized=%d", normal, optimized); } +#else +static void extract_flush_record_keys( + ZebraHandle zh, zint sysno, int cmd, + zebra_rec_keys_t reckeys, + zint staticrank) +{ + ZebraExplainInfo zei = zh->reg->zei; + + extract_rec_keys_adjust(zh, cmd, reckeys); + + if (log_level_details) + { + yaz_log(log_level_details, "Keys for record " ZINT_FORMAT " %s", + sysno, cmd ? "insert" : "delete"); + extract_rec_keys_log(zh, cmd, reckeys, log_level_details); + } + + if (!zh->reg->key_block) + { + int mem = 1024*1024 * atoi( res_get_def( zh->res, "memmax", "8")); + const char *key_tmp_dir = res_get_def(zh->res, "keyTmpDir", "."); + int use_threads = atoi(res_get_def(zh->res, "threads", "1")); + zh->reg->key_block = key_block_create(mem, key_tmp_dir, use_threads); + } + zebraExplain_recordCountIncrement(zei, cmd ? 1 : -1); +#if 0 + yaz_log(YLOG_LOG, "sysno=" ZINT_FORMAT " cmd=%d", sysno, cmd); + print_rec_keys(zh, reckeys); +#endif + if (zebra_rec_keys_rewind(reckeys)) + { + size_t slen; + const char *str; + struct it_key key_in; + while(zebra_rec_keys_read(reckeys, &str, &slen, &key_in)) + { + key_block_write(zh->reg->key_block, sysno, + &key_in, cmd, str, slen, + staticrank, zh->m_staticrank); + } + } +} +#endif ZEBRA_RES zebra_rec_keys_to_snippets1(ZebraHandle zh, zebra_rec_keys_t reckeys, @@ -1798,6 +1787,7 @@ static void extract_add_complete_field(RecWord *p, zebra_map_t zm) if (!i) return; extract_add_string(p, zm, buf, i); + p->seqno++; } static void extract_add_icu(RecWord *p, zebra_map_t zm)