X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=src%2Ficu_tokenizer.c;h=954ebc66fd957ee26adec0177db772624b75a03f;hb=03dccce903176e61f889b1558834aa5a09f9f8c0;hp=54cfbf3ab98b0bc011331b04560ceb0ece1cbba1;hpb=43a9d38d20c1b1bcd1a03b2445a501d27526bd35;p=yaz-moved-to-github.git diff --git a/src/icu_tokenizer.c b/src/icu_tokenizer.c index 54cfbf3..954ebc6 100644 --- a/src/icu_tokenizer.c +++ b/src/icu_tokenizer.c @@ -1,5 +1,5 @@ /* This file is part of the YAZ toolkit. - * Copyright (C) 1995-2011 Index Data + * Copyright (C) 1995-2012 Index Data * See the file LICENSE for details. */ @@ -195,14 +195,16 @@ int32_t icu_tokenizer_next_token(struct icu_tokenizer * tokenizer, { tokenizer->token_count++; tokenizer->token_id++; - } else { - tokenizer->token_id = 0; } + else + tokenizer->token_id = 0; + tokenizer->token_start = tkn_start; tokenizer->token_end = tkn_end; /* copying into token buffer if it exists */ - if (tkn16){ + if (tkn16) + { if (tkn16->utf16_cap < tkn_len) icu_buf_utf16_resize(tkn16, (size_t) tkn_len * 2);