ICU test refactor. Hide icu_tokenizer struct.

[yaz-moved-to-github.git] / src / icu_tokenizer.c

diff --git a/src/icu_tokenizer.c b/src/icu_tokenizer.c
index c7bba7f..f9b4926 100644 (file)
--- a/src/icu_tokenizer.c
+++ b/src/icu_tokenizer.c
@@ -1,5 +1,5 @@
 /* This file is part of the YAZ toolkit.
- * Copyright (C) 1995-2009 Index Data
+ * Copyright (C) 1995-2010 Index Data
  * See the file LICENSE for details.
  */
 
@@ -26,6 +26,25 @@
 #include <unicode/ustring.h>  /* some more string fcns*/
 #include <unicode/uchar.h>    /* char names           */
 
+struct icu_tokenizer
+{
+    char action;
+    UBreakIterator* bi;
+    struct icu_buf_utf16 * buf16;
+    int32_t token_count;
+    int32_t token_id;
+    int32_t token_start;
+    int32_t token_end;
+/*
+  keep always invariant
+  0 <= token_start 
+  <= token_end 
+  <= buf16->utf16_len
+  and invariant
+  0 <= token_id <= token_count
+*/
+};
+
 struct icu_tokenizer *icu_tokenizer_create(const char *locale, char action,
                                            UErrorCode *status)
 {
@@ -34,7 +53,7 @@ struct icu_tokenizer *icu_tokenizer_create(const char *locale, char action,
 
     tokenizer->action = action;
     tokenizer->bi = 0;
-    tokenizer->buf16 = 0;
+    tokenizer->buf16 = icu_buf_utf16_create(0);
     tokenizer->token_count = 0;
     tokenizer->token_id = 0;
     tokenizer->token_start = 0;
@@ -81,6 +100,7 @@ void icu_tokenizer_destroy(struct icu_tokenizer * tokenizer)
 {
     if (tokenizer)
     {
+        icu_buf_utf16_destroy(tokenizer->buf16);
         if (tokenizer->bi)
             ubrk_close(tokenizer->bi);
         xfree(tokenizer);
@@ -94,13 +114,15 @@ int icu_tokenizer_attach(struct icu_tokenizer * tokenizer,
     if (!tokenizer || !tokenizer->bi || !src16)
         return 0;
 
-    tokenizer->buf16 = src16;
+    icu_buf_utf16_copy(tokenizer->buf16, src16);
+
     tokenizer->token_count = 0;
     tokenizer->token_id = 0;
     tokenizer->token_start = 0;
     tokenizer->token_end = 0;
 
-    ubrk_setText(tokenizer->bi, src16->utf16, src16->utf16_len, status);
+    ubrk_setText(tokenizer->bi,
+                 tokenizer->buf16->utf16, tokenizer->buf16->utf16_len, status);
      
     if (U_FAILURE(*status))
         return 0;