From 34b6f9669f88fef08ad911000f2b66fb2c3429b9 Mon Sep 17 00:00:00 2001 From: Adam Dickmeiss Date: Mon, 10 Dec 2007 17:06:07 +0000 Subject: [PATCH] Handle display buf for searches (ICU). Extend zebra_map_tokenize_next to return display terms. For RPN-query evaluation (rpnsearch.c) store display term in WRBUF rather than buffer. --- include/zebramap.h | 5 +- index/extract.c | 4 +- index/rpnsearch.c | 189 ++++++++++++++++++++++++++-------------------------- util/zebramap.c | 11 ++- 4 files changed, 108 insertions(+), 101 deletions(-) diff --git a/include/zebramap.h b/include/zebramap.h index db4343c..58feb56 100644 --- a/include/zebramap.h +++ b/include/zebramap.h @@ -1,4 +1,4 @@ -/* $Id: zebramap.h,v 1.28 2007-12-07 14:09:09 adam Exp $ +/* $Id: zebramap.h,v 1.29 2007-12-10 17:06:07 adam Exp $ Copyright (C) 1995-2007 Index Data ApS @@ -100,7 +100,8 @@ int zebra_map_tokenize_start(zebra_map_t zm, YAZ_EXPORT int zebra_map_tokenize_next(zebra_map_t zm, - const char **result_buf, size_t *result_len); + const char **result_buf, size_t *result_len, + const char **display_buf, size_t *display_len); YAZ_END_CDECL diff --git a/index/extract.c b/index/extract.c index 48dd978..cc54d67 100644 --- a/index/extract.c +++ b/index/extract.c @@ -1,4 +1,4 @@ -/* $Id: extract.c,v 1.271 2007-12-07 14:09:09 adam Exp $ +/* $Id: extract.c,v 1.272 2007-12-10 17:06:08 adam Exp $ Copyright (C) 1995-2007 Index Data ApS @@ -1737,7 +1737,7 @@ static void extract_add_icu(RecWord *p, zebra_map_t zm) ch = zebraExplain_add_attr_str(zh->reg->zei, cat, p->index_type, p->index_name); zebra_map_tokenize_start(zm, p->term_buf, p->term_len); - while (zebra_map_tokenize_next(zm, &res_buf, &res_len)) + while (zebra_map_tokenize_next(zm, &res_buf, &res_len, 0, 0)) { int i = 0; key.mem[i++] = ch; diff --git a/index/rpnsearch.c b/index/rpnsearch.c index d1b9b88..18bef0b 100644 --- a/index/rpnsearch.c +++ b/index/rpnsearch.c @@ -1,4 +1,4 @@ -/* $Id: rpnsearch.c,v 1.27 2007-12-07 14:09:09 adam Exp $ +/* $Id: rpnsearch.c,v 1.28 2007-12-10 17:06:08 adam Exp $ Copyright (C) 1995-2007 Index Data ApS @@ -210,12 +210,12 @@ static void esc_str(char *out_buf, size_t out_size, static void add_non_space(const char *start, const char *end, WRBUF term_dict, - char *dst_term, int *dst_ptr, + WRBUF display_term, const char **map, int q_map_match) { size_t sz = end - start; - memcpy(dst_term + *dst_ptr, start, sz); - (*dst_ptr) += sz; + + wrbuf_write(display_term, start, sz); if (!q_map_match) { while (start < end) @@ -238,17 +238,20 @@ static void add_non_space(const char *start, const char *end, static int term_100_icu(zebra_map_t zm, const char **src, WRBUF term_dict, int space_split, - char *dst_term) + WRBUF display_term) { int i; const char *res_buf = 0; size_t res_len = 0; - if (!zebra_map_tokenize_next(zm, &res_buf, &res_len)) + const char *display_buf; + size_t display_len; + if (!zebra_map_tokenize_next(zm, &res_buf, &res_len, + &display_buf, &display_len)) { *src += strlen(*src); return 0; } - strcat(dst_term, *src); + wrbuf_write(display_term, display_buf, display_len); for (i = 0; i < res_len; i++) { if (strchr(REGEX_CHARS, res_buf[i])) @@ -263,18 +266,17 @@ static int term_100_icu(zebra_map_t zm, /* term_100: handle term, where trunc = none(no operators at all) */ static int term_100(zebra_map_t zm, const char **src, WRBUF term_dict, int space_split, - char *dst_term) + WRBUF display_term) { const char *s0; const char **map; int i = 0; - int j = 0; const char *space_start = 0; const char *space_end = 0; if (zebra_maps_is_icu(zm)) - return term_100_icu(zm, src, term_dict, space_split, dst_term); + return term_100_icu(zm, src, term_dict, space_split, display_term); if (!term_pre(zm, src, NULL, NULL, !space_split)) return 0; @@ -303,7 +305,7 @@ static int term_100(zebra_map_t zm, { if (strchr(REGEX_CHARS, *space_start)) wrbuf_putc(term_dict, '\\'); - dst_term[j++] = *space_start; + wrbuf_putc(display_term, *space_start); wrbuf_putc(term_dict, *space_start); space_start++; @@ -314,10 +316,8 @@ static int term_100(zebra_map_t zm, } i++; - add_non_space(s1, s0, term_dict, dst_term, &j, - map, q_map_match); + add_non_space(s1, s0, term_dict, display_term, map, q_map_match); } - dst_term[j] = '\0'; *src = s0; return i; } @@ -325,12 +325,11 @@ static int term_100(zebra_map_t zm, /* term_101: handle term, where trunc = Process # */ static int term_101(zebra_map_t zm, const char **src, WRBUF term_dict, int space_split, - char *dst_term) + WRBUF display_term) { const char *s0; const char **map; int i = 0; - int j = 0; if (!term_pre(zm, src, "#", "#", !space_split)) return 0; @@ -341,7 +340,8 @@ static int term_101(zebra_map_t zm, { i++; wrbuf_puts(term_dict, ".*"); - dst_term[j++] = *s0++; + wrbuf_putc(display_term, *s0); + s0++; } else { @@ -352,11 +352,9 @@ static int term_101(zebra_map_t zm, break; i++; - add_non_space(s1, s0, term_dict, dst_term, &j, - map, q_map_match); + add_non_space(s1, s0, term_dict, display_term, map, q_map_match); } } - dst_term[j++] = '\0'; *src = s0; return i; } @@ -364,10 +362,9 @@ static int term_101(zebra_map_t zm, /* term_103: handle term, where trunc = re-2 (regular expressions) */ static int term_103(zebra_map_t zm, const char **src, WRBUF term_dict, int *errors, int space_split, - char *dst_term) + WRBUF display_term) { int i = 0; - int j = 0; const char *s0; const char **map; @@ -386,7 +383,7 @@ static int term_103(zebra_map_t zm, const char **src, { if (strchr("^\\()[].*+?|-", *s0)) { - dst_term[j++] = *s0; + wrbuf_putc(display_term, *s0); wrbuf_putc(term_dict, *s0); s0++; i++; @@ -400,11 +397,9 @@ static int term_103(zebra_map_t zm, const char **src, break; i++; - add_non_space(s1, s0, term_dict, dst_term, &j, - map, q_map_match); + add_non_space(s1, s0, term_dict, display_term, map, q_map_match); } } - dst_term[j] = '\0'; *src = s0; return i; @@ -412,20 +407,19 @@ static int term_103(zebra_map_t zm, const char **src, /* term_103: handle term, where trunc = re-1 (regular expressions) */ static int term_102(zebra_map_t zm, const char **src, - WRBUF term_dict, int space_split, char *dst_term) + WRBUF term_dict, int space_split, WRBUF display_term) { - return term_103(zm, src, term_dict, NULL, space_split, dst_term); + return term_103(zm, src, term_dict, NULL, space_split, display_term); } /* term_104: handle term, process # and ! */ static int term_104(zebra_map_t zm, const char **src, - WRBUF term_dict, int space_split, char *dst_term) + WRBUF term_dict, int space_split, WRBUF display_term) { const char *s0; const char **map; int i = 0; - int j = 0; if (!term_pre(zm, src, "?*#", "?*#", !space_split)) return 0; @@ -435,14 +429,16 @@ static int term_104(zebra_map_t zm, const char **src, if (*s0 == '?') { i++; - dst_term[j++] = *s0++; + wrbuf_putc(display_term, *s0); + s0++; if (*s0 >= '0' && *s0 <= '9') { int limit = 0; while (*s0 >= '0' && *s0 <= '9') { limit = limit * 10 + (*s0 - '0'); - dst_term[j++] = *s0++; + wrbuf_putc(display_term, *s0); + s0++; } if (limit > 20) limit = 20; @@ -460,13 +456,15 @@ static int term_104(zebra_map_t zm, const char **src, { i++; wrbuf_puts(term_dict, ".*"); - dst_term[j++] = *s0++; + wrbuf_putc(display_term, *s0); + s0++; } else if (*s0 == '#') { i++; wrbuf_puts(term_dict, "."); - dst_term[j++] = *s0++; + wrbuf_putc(display_term, *s0); + s0++; } else { @@ -477,11 +475,9 @@ static int term_104(zebra_map_t zm, const char **src, break; i++; - add_non_space(s1, s0, term_dict, dst_term, &j, - map, q_map_match); + add_non_space(s1, s0, term_dict, display_term, map, q_map_match); } } - dst_term[j++] = '\0'; *src = s0; return i; } @@ -489,12 +485,11 @@ static int term_104(zebra_map_t zm, const char **src, /* term_105/106: handle term, where trunc = Process * and ! and right trunc */ static int term_105(zebra_map_t zm, const char **src, WRBUF term_dict, int space_split, - char *dst_term, int right_truncate) + WRBUF display_term, int right_truncate) { const char *s0; const char **map; int i = 0; - int j = 0; if (!term_pre(zm, src, "*!", "*!", !space_split)) return 0; @@ -505,13 +500,15 @@ static int term_105(zebra_map_t zm, const char **src, { i++; wrbuf_puts(term_dict, ".*"); - dst_term[j++] = *s0++; + wrbuf_putc(display_term, *s0); + s0++; } else if (*s0 == '!') { i++; wrbuf_putc(term_dict, '.'); - dst_term[j++] = *s0++; + wrbuf_putc(display_term, *s0); + s0++; } else { @@ -522,13 +519,11 @@ static int term_105(zebra_map_t zm, const char **src, break; i++; - add_non_space(s1, s0, term_dict, dst_term, &j, - map, q_map_match); + add_non_space(s1, s0, term_dict, display_term, map, q_map_match); } } if (right_truncate) wrbuf_puts(term_dict, ".*"); - dst_term[j++] = '\0'; *src = s0; return i; } @@ -673,7 +668,8 @@ void string_rel_add_char(WRBUF term_p, WRBUF wsrc, int *indx) static int string_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt, const char **term_sub, WRBUF term_dict, const Odr_oid *attributeSet, - zebra_map_t zm, int space_split, char *term_dst, + zebra_map_t zm, int space_split, + WRBUF display_term, int *error_code) { AttrType relation; @@ -689,7 +685,7 @@ static int string_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt, switch (relation_value) { case 1: - if (!term_100(zm, term_sub, term_component, space_split, term_dst)) + if (!term_100(zm, term_sub, term_component, space_split, display_term)) { wrbuf_destroy(term_component); return 0; @@ -723,7 +719,7 @@ static int string_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt, wrbuf_putc(term_dict, ')'); break; case 2: - if (!term_100(zm, term_sub, term_component, space_split, term_dst)) + if (!term_100(zm, term_sub, term_component, space_split, display_term)) { wrbuf_destroy(term_component); return 0; @@ -758,7 +754,7 @@ static int string_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt, wrbuf_putc(term_dict, ')'); break; case 5: - if (!term_100(zm, term_sub, term_component, space_split, term_dst)) + if (!term_100(zm, term_sub, term_component, space_split, display_term)) { wrbuf_destroy(term_component); return 0; @@ -791,7 +787,7 @@ static int string_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt, wrbuf_putc(term_dict, ')'); break; case 4: - if (!term_100(zm, term_sub, term_component, space_split, term_dst)) + if (!term_100(zm, term_sub, term_component, space_split, display_term)) { wrbuf_destroy(term_component); return 0; @@ -832,7 +828,7 @@ static int string_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt, if (!**term_sub) return 1; yaz_log(log_level_rpn, "Relation ="); - if (!term_100(zm, term_sub, term_component, space_split, term_dst)) + if (!term_100(zm, term_sub, term_component, space_split, display_term)) { wrbuf_destroy(term_component); return 0; @@ -862,7 +858,7 @@ static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt, const Odr_oid *attributeSet, NMEM stream, struct grep_info *grep_info, const char *index_type, int complete_flag, - char *term_dst, + WRBUF display_term, const char *xpath_use, struct ord_list **ol, zebra_map_t zm); @@ -922,7 +918,6 @@ static ZEBRA_RES search_term(ZebraHandle zh, const Odr_oid *attributeSet, NMEM stream, struct grep_info *grep_info, const char *index_type, int complete_flag, - char *term_dst, const char *rank_type, const char *xpath_use, NMEM rset_nmem, @@ -935,6 +930,7 @@ static ZEBRA_RES search_term(ZebraHandle zh, zint hits_limit_value; const char *term_ref_id_str = 0; WRBUF term_dict = wrbuf_alloc(); + WRBUF display_term = wrbuf_alloc(); *rset = 0; zebra_term_limits_APT(zh, zapt, &hits_limit_value, &term_ref_id_str, stream); @@ -942,22 +938,23 @@ static ZEBRA_RES search_term(ZebraHandle zh, res = string_term(zh, zapt, term_sub, term_dict, attributeSet, stream, grep_info, index_type, complete_flag, - term_dst, xpath_use, &ol, zm); + display_term, xpath_use, &ol, zm); wrbuf_destroy(term_dict); - if (res != ZEBRA_OK) - return res; - if (!*term_sub) /* no more terms ? */ - return res; - yaz_log(log_level_rpn, "term: %s", term_dst); - *rset = rset_trunc(zh, grep_info->isam_p_buf, - grep_info->isam_p_indx, term_dst, - strlen(term_dst), rank_type, 1 /* preserve pos */, - zapt->term->which, rset_nmem, - kc, kc->scope, ol, index_type, hits_limit_value, - term_ref_id_str); - if (!*rset) - return ZEBRA_FAIL; - return ZEBRA_OK; + if (res == ZEBRA_OK && *term_sub) + { + yaz_log(log_level_rpn, "term: %s", wrbuf_cstr(display_term)); + *rset = rset_trunc(zh, grep_info->isam_p_buf, + grep_info->isam_p_indx, wrbuf_buf(display_term), + wrbuf_len(display_term), rank_type, + 1 /* preserve pos */, + zapt->term->which, rset_nmem, + kc, kc->scope, ol, index_type, hits_limit_value, + term_ref_id_str); + if (!*rset) + res = ZEBRA_FAIL; + } + wrbuf_destroy(display_term); + return res; } static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt, @@ -966,7 +963,7 @@ static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt, const Odr_oid *attributeSet, NMEM stream, struct grep_info *grep_info, const char *index_type, int complete_flag, - char *term_dst, + WRBUF display_term, const char *xpath_use, struct ord_list **ol, zebra_map_t zm) @@ -1023,7 +1020,7 @@ static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt, case 100: /* do not truncate */ if (!string_relation(zh, zapt, &termp, term_dict, attributeSet, - zm, space_split, term_dst, + zm, space_split, display_term, &relation_error)) { if (relation_error) @@ -1037,7 +1034,7 @@ static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt, break; case 1: /* right truncation */ wrbuf_putc(term_dict, '('); - if (!term_100(zm, &termp, term_dict, space_split, term_dst)) + if (!term_100(zm, &termp, term_dict, space_split, display_term)) { *term_sub = 0; return ZEBRA_OK; @@ -1046,7 +1043,7 @@ static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt, break; case 2: /* keft truncation */ wrbuf_puts(term_dict, "(.*"); - if (!term_100(zm, &termp, term_dict, space_split, term_dst)) + if (!term_100(zm, &termp, term_dict, space_split, display_term)) { *term_sub = 0; return ZEBRA_OK; @@ -1055,7 +1052,7 @@ static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt, break; case 3: /* left&right truncation */ wrbuf_puts(term_dict, "(.*"); - if (!term_100(zm, &termp, term_dict, space_split, term_dst)) + if (!term_100(zm, &termp, term_dict, space_split, display_term)) { *term_sub = 0; return ZEBRA_OK; @@ -1064,7 +1061,7 @@ static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt, break; case 101: /* process # in term */ wrbuf_putc(term_dict, '('); - if (!term_101(zm, &termp, term_dict, space_split, term_dst)) + if (!term_101(zm, &termp, term_dict, space_split, display_term)) { *term_sub = 0; return ZEBRA_OK; @@ -1073,7 +1070,7 @@ static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt, break; case 102: /* Regexp-1 */ wrbuf_putc(term_dict, '('); - if (!term_102(zm, &termp, term_dict, space_split, term_dst)) + if (!term_102(zm, &termp, term_dict, space_split, display_term)) { *term_sub = 0; return ZEBRA_OK; @@ -1084,7 +1081,7 @@ static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt, regex_range = 1; wrbuf_putc(term_dict, '('); if (!term_103(zm, &termp, term_dict, ®ex_range, - space_split, term_dst)) + space_split, display_term)) { *term_sub = 0; return ZEBRA_OK; @@ -1093,7 +1090,7 @@ static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt, break; case 104: /* process # and ! in term */ wrbuf_putc(term_dict, '('); - if (!term_104(zm, &termp, term_dict, space_split, term_dst)) + if (!term_104(zm, &termp, term_dict, space_split, display_term)) { *term_sub = 0; return ZEBRA_OK; @@ -1102,7 +1099,7 @@ static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt, break; case 105: /* process * and ! in term */ wrbuf_putc(term_dict, '('); - if (!term_105(zm, &termp, term_dict, space_split, term_dst, 1)) + if (!term_105(zm, &termp, term_dict, space_split, display_term, 1)) { *term_sub = 0; return ZEBRA_OK; @@ -1111,7 +1108,7 @@ static ZEBRA_RES string_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt, break; case 106: /* process * and ! in term */ wrbuf_putc(term_dict, '('); - if (!term_105(zm, &termp, term_dict, space_split, term_dst, 0)) + if (!term_105(zm, &termp, term_dict, space_split, display_term, 0)) { *term_sub = 0; return ZEBRA_OK; @@ -1235,16 +1232,14 @@ static ZEBRA_RES search_terms_chrmap(ZebraHandle zh, struct rset_key_control *kc, zebra_map_t zm) { - char term_dst[IT_MAX_WORD+1]; struct grep_info grep_info; const char *termp = termz; int alloc_sets = 0; *num_result_sets = 0; - *term_dst = 0; if (grep_info_prepare(zh, zapt, &grep_info, index_type) == ZEBRA_FAIL) return ZEBRA_FAIL; - while(1) + while (1) { ZEBRA_RES res; @@ -1261,7 +1256,7 @@ static ZEBRA_RES search_terms_chrmap(ZebraHandle zh, res = search_term(zh, zapt, &termp, attributeSet, stream, &grep_info, index_type, complete_flag, - term_dst, rank_type, + rank_type, xpath_use, rset_nmem, &(*result_sets)[*num_result_sets], kc, zm); @@ -1595,7 +1590,7 @@ static int numeric_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt, struct grep_info *grep_info, int *max_pos, zebra_map_t zm, - char *term_dst, + WRBUF display_term, int *error_code) { AttrType relation; @@ -1614,7 +1609,7 @@ static int numeric_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt, { case 1: yaz_log(log_level_rpn, "Relation <"); - if (!term_100(zm, term_sub, term_num, 1, term_dst)) + if (!term_100(zm, term_sub, term_num, 1, display_term)) { wrbuf_destroy(term_num); return 0; @@ -1624,7 +1619,7 @@ static int numeric_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt, break; case 2: yaz_log(log_level_rpn, "Relation <="); - if (!term_100(zm, term_sub, term_num, 1, term_dst)) + if (!term_100(zm, term_sub, term_num, 1, display_term)) { wrbuf_destroy(term_num); return 0; @@ -1634,7 +1629,7 @@ static int numeric_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt, break; case 4: yaz_log(log_level_rpn, "Relation >="); - if (!term_100(zm, term_sub, term_num, 1, term_dst)) + if (!term_100(zm, term_sub, term_num, 1, display_term)) { wrbuf_destroy(term_num); return 0; @@ -1644,7 +1639,7 @@ static int numeric_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt, break; case 5: yaz_log(log_level_rpn, "Relation >"); - if (!term_100(zm, term_sub, term_num, 1, term_dst)) + if (!term_100(zm, term_sub, term_num, 1, display_term)) { wrbuf_destroy(term_num); return 0; @@ -1655,7 +1650,7 @@ static int numeric_relation(ZebraHandle zh, Z_AttributesPlusTerm *zapt, case -1: case 3: yaz_log(log_level_rpn, "Relation ="); - if (!term_100(zm, term_sub, term_num, 1, term_dst)) + if (!term_100(zm, term_sub, term_num, 1, display_term)) { wrbuf_destroy(term_num); return 0; @@ -1691,7 +1686,7 @@ static ZEBRA_RES numeric_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt, const Odr_oid *attributeSet, NMEM stream, struct grep_info *grep_info, const char *index_type, int complete_flag, - char *term_dst, + WRBUF display_term, const char *xpath_use, struct ord_list **ol) { @@ -1731,7 +1726,7 @@ static ZEBRA_RES numeric_term(ZebraHandle zh, Z_AttributesPlusTerm *zapt, if (!numeric_relation(zh, zapt, &termp, term_dict, attributeSet, grep_info, &max_pos, zm, - term_dst, &relation_error)) + display_term, &relation_error)) { if (relation_error) { @@ -1760,7 +1755,6 @@ static ZEBRA_RES rpn_search_APT_numeric(ZebraHandle zh, RSET *rset, struct rset_key_control *kc) { - char term_dst[IT_MAX_WORD+1]; const char *termp = termz; RSET *result_sets = 0; int num_result_sets = 0; @@ -1780,6 +1774,7 @@ static ZEBRA_RES rpn_search_APT_numeric(ZebraHandle zh, { struct ord_list *ol; WRBUF term_dict = wrbuf_alloc(); + WRBUF display_term = wrbuf_alloc(); if (alloc_sets == num_result_sets) { int add = 10; @@ -1795,20 +1790,24 @@ static ZEBRA_RES rpn_search_APT_numeric(ZebraHandle zh, res = numeric_term(zh, zapt, &termp, term_dict, attributeSet, stream, &grep_info, index_type, complete_flag, - term_dst, xpath_use, &ol); + display_term, xpath_use, &ol); wrbuf_destroy(term_dict); if (res == ZEBRA_FAIL || termp == 0) + { + wrbuf_destroy(display_term); break; - yaz_log(YLOG_DEBUG, "term: %s", term_dst); + } + yaz_log(YLOG_DEBUG, "term: %s", wrbuf_cstr(display_term)); result_sets[num_result_sets] = rset_trunc(zh, grep_info.isam_p_buf, - grep_info.isam_p_indx, term_dst, - strlen(term_dst), rank_type, + grep_info.isam_p_indx, wrbuf_buf(display_term), + wrbuf_len(display_term), rank_type, 0 /* preserve position */, zapt->term->which, rset_nmem, kc, kc->scope, ol, index_type, hits_limit_value, term_ref_id_str); + wrbuf_destroy(display_term); if (!result_sets[num_result_sets]) break; num_result_sets++; diff --git a/util/zebramap.c b/util/zebramap.c index d2cd37f..f92a271 100644 --- a/util/zebramap.c +++ b/util/zebramap.c @@ -1,4 +1,4 @@ -/* $Id: zebramap.c,v 1.73 2007-12-07 14:09:10 adam Exp $ +/* $Id: zebramap.c,v 1.74 2007-12-10 17:06:08 adam Exp $ Copyright (C) 1995-2007 Index Data ApS @@ -641,7 +641,8 @@ static int tokenize_simple(zebra_map_t zm, int zebra_map_tokenize_next(zebra_map_t zm, - const char **result_buf, size_t *result_len) + const char **result_buf, size_t *result_len, + const char **display_buf, size_t *display_len) { assert(zm->use_chain); @@ -659,6 +660,12 @@ int zebra_map_tokenize_next(zebra_map_t zm, *result_len = strlen(*result_buf); + if (display_buf) + { + *display_buf = icu_chain_token_display(zm->icu_chain); + if (display_len) + *display_len = strlen(*display_buf); + } if (zm->debug) { wrbuf_rewind(zm->print_str); -- 1.7.10.4