X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=src%2Ficu_tokenizer.c;h=d6ec7fee7f6aa710a4cb6b7fa3131a7dd10c7264;hb=b56b6242e4e9c0011b9a2dae6d67c3ef0789251f;hp=f9b4926c8ae410aacac32ab99fa519b3887bdd10;hpb=ded9dadedab82a379e0e52d78a61fe9632ac870c;p=yaz-moved-to-github.git

diff --git a/src/icu_tokenizer.c b/src/icu_tokenizer.c
index f9b4926..d6ec7fe 100644
--- a/src/icu_tokenizer.c
+++ b/src/icu_tokenizer.c
@@ -19,6 +19,7 @@
 
 #include <yaz/log.h>
 
+#include <assert.h>
 #include <string.h>
 #include <stdlib.h>
 #include <stdio.h>
@@ -45,12 +46,9 @@ struct icu_tokenizer
 */
 };
 
-struct icu_tokenizer *icu_tokenizer_create(const char *locale, char action,
-                                           UErrorCode *status)
+static void icu_tokenizer_reset(struct icu_tokenizer *tokenizer,
+                                char action)
 {
-    struct icu_tokenizer * tokenizer
-        = (struct icu_tokenizer *) xmalloc(sizeof(struct icu_tokenizer));
-
     tokenizer->action = action;
     tokenizer->bi = 0;
     tokenizer->buf16 = icu_buf_utf16_create(0);
@@ -58,7 +56,32 @@ struct icu_tokenizer *icu_tokenizer_create(const char *locale, char action,
     tokenizer->token_id = 0;
     tokenizer->token_start = 0;
     tokenizer->token_end = 0;
+    tokenizer->bi = 0;
+}
+
+struct icu_tokenizer *icu_tokenizer_clone(struct icu_tokenizer *old)
+{
+    int32_t bufferSize = U_BRK_SAFECLONE_BUFFERSIZE;
+    UErrorCode status = U_ZERO_ERROR;
+    struct icu_tokenizer * tokenizer
+        = (struct icu_tokenizer *) xmalloc(sizeof(struct icu_tokenizer));
+
+    assert(old);
+    icu_tokenizer_reset(tokenizer, old->action);
+    assert(old->bi);
+    tokenizer->bi = ubrk_safeClone(old->bi, NULL, &bufferSize, &status);
+    if (U_SUCCESS(status))
+        return tokenizer;
+    return tokenizer;
+}
+
+struct icu_tokenizer *icu_tokenizer_create(const char *locale, char action,
+                                           UErrorCode *status)
+{
+    struct icu_tokenizer * tokenizer
+        = (struct icu_tokenizer *) xmalloc(sizeof(struct icu_tokenizer));
 
+    icu_tokenizer_reset(tokenizer, action);
     switch (tokenizer->action)
     {    
     case 'l':