X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=src%2Ficu_tokenizer.c;h=954ebc66fd957ee26adec0177db772624b75a03f;hb=03dccce903176e61f889b1558834aa5a09f9f8c0;hp=e8d3123270f3ee2be2777f3066f5aa18f3d43001;hpb=527a008f2f3b1a19a968a7a1275132a04aa4d945;p=yaz-moved-to-github.git

diff --git a/src/icu_tokenizer.c b/src/icu_tokenizer.c
index e8d3123..954ebc6 100644
--- a/src/icu_tokenizer.c
+++ b/src/icu_tokenizer.c
@@ -1,5 +1,5 @@
 /* This file is part of the YAZ toolkit.
- * Copyright (C) 1995-2010 Index Data
+ * Copyright (C) 1995-2012 Index Data
  * See the file LICENSE for details.
  */
 
@@ -61,8 +61,8 @@ static void icu_tokenizer_reset(struct icu_tokenizer *tokenizer,
 
 struct icu_tokenizer *icu_tokenizer_clone(struct icu_tokenizer *old)
 {
-    uint32_t bufferSize = 10000;
-    UErrorCode status = 0;
+    int32_t bufferSize = U_BRK_SAFECLONE_BUFFERSIZE;
+    UErrorCode status = U_ZERO_ERROR;
     struct icu_tokenizer * tokenizer
         = (struct icu_tokenizer *) xmalloc(sizeof(struct icu_tokenizer));
 
@@ -195,14 +195,16 @@ int32_t icu_tokenizer_next_token(struct icu_tokenizer * tokenizer,
     {
         tokenizer->token_count++;
         tokenizer->token_id++;
-    } else {
-        tokenizer->token_id = 0;    
     }
+    else
+        tokenizer->token_id = 0;    
+
     tokenizer->token_start = tkn_start;
     tokenizer->token_end = tkn_end;    
 
     /* copying into token buffer if it exists */
-    if (tkn16){
+    if (tkn16)
+    {
         if (tkn16->utf16_cap < tkn_len)
             icu_buf_utf16_resize(tkn16, (size_t) tkn_len * 2);