Fix icu_iter_get_norm returns invalid size for some input YAZ-834

The icu_iter_get_org_info returns information based on string
as it appears on tokenize step.

diff --git a/src/icu_chain.c b/src/icu_chain.c
index b672975..d0a2857 100644 (file)
--- a/src/icu_chain.c
+++ b/src/icu_chain.c
@@ -426,6 +426,8 @@ struct icu_buf_utf16 *icu_iter_invoke(yaz_icu_iter_t iter,
                 struct icu_buf_utf16 *src = dst;
 
                 icu_tokenizer_attach(step->u.tokenizer, src, &iter->status);
+                iter->utf8_base = iter->utf16_base = 0;
+                icu_buf_utf16_copy(iter->org, src);
                 icu_buf_utf16_destroy(src);
             }
             dst = icu_buf_utf16_create(0);