X-Git-Url: http://git.indexdata.com/?p=yaz-moved-to-github.git;a=blobdiff_plain;f=src%2Ficu_chain.c;h=bcb27b435fa61f1c0200950350e85b4f6fe406e1;hp=df313eb70c7a3a5e746e713736248fbf8cadb468;hb=be73a431cd4e151c77d3024b7ee09197e4a4910a;hpb=e662834ae7643141ab51c887afdbc40038494b1b

diff --git a/src/icu_chain.c b/src/icu_chain.c
index df313eb..bcb27b4 100644
--- a/src/icu_chain.c
+++ b/src/icu_chain.c
@@ -1,5 +1,5 @@
 /* This file is part of the YAZ toolkit.
- * Copyright (C) 1995-2009 Index Data
+ * Copyright (C) 1995-2013 Index Data
  * See the file LICENSE for details.
  */
 
@@ -17,8 +17,11 @@
 
 #include <yaz/icu_I18N.h>
 
-#include <yaz/log.h>
+#include <yaz/stemmer.h>
 
+#include <yaz/log.h>
+#include <yaz/nmem.h>
+#include <yaz/nmem_xml.h>
 #include <string.h>
 #include <stdlib.h>
 #include <stdio.h>
@@ -29,67 +32,39 @@
 
 enum icu_chain_step_type {
     ICU_chain_step_type_none,
-    ICU_chain_step_type_display,   /* convert to utf8 display format */
-    ICU_chain_step_type_casemap,   /* apply utf16 charmap */
-    ICU_chain_step_type_transform, /* apply utf16 transform */
-    ICU_chain_step_type_tokenize,  /* apply utf16 tokenization */
-    ICU_chain_step_type_transliterate  /* apply utf16 tokenization */
+    ICU_chain_step_type_display,        /* convert to utf8 display format */
+    ICU_chain_step_type_casemap,        /* apply utf16 charmap */
+    ICU_chain_step_type_transform,      /* apply utf16 transform */
+    ICU_chain_step_type_tokenize,       /* apply utf16 tokenization */
+    ICU_chain_step_type_transliterate,  /* apply utf16 tokenization */
+    YAZ_chain_step_type_stemming,       /* apply utf16 stemming (YAZ) */
+    ICU_chain_step_type_join
 };
 
-#define USE_ITER 1
-
 struct icu_chain_step
 {
     /* type and action object */
     enum icu_chain_step_type type;
     union {
-	struct icu_casemap * casemap;
-	struct icu_transform * transform;
-	struct icu_tokenizer * tokenizer;  
+	struct icu_casemap   *casemap;
+	struct icu_transform *transform;
+	struct icu_tokenizer *tokenizer;
+        yaz_stemmer_p         stemmer;
+        struct icu_buf_utf16 *join;
     } u;
-    struct icu_chain_step * previous;
-#if USE_ITER
-#else
-    /* temprary post-action utf16 buffer */
-    struct icu_buf_utf16 * buf16;  
-    int more_tokens;
-    int need_new_token;
-#endif
+    struct icu_chain_step *previous;
 };
 
-
 struct icu_chain
 {
-#if USE_ITER
-    struct icu_iter *iter;
-#endif
-
+    yaz_icu_iter_t iter;
     char *locale;
     int sort;
 
-    UCollator * coll;
-    
-#if USE_ITER
-#else
-    const char * src8cstr;
+    UCollator *coll;
 
-    /* number of tokens returned so far */
-    int32_t token_count;
-#endif
-    
-    /* utf8 output buffers */
-    struct icu_buf_utf8 * norm8;
-#if USE_ITER
-#else
-    struct icu_buf_utf8 * display8;
-    struct icu_buf_utf8 * sort8;
-    
-    /* utf16 source buffer */
-    struct icu_buf_utf16 * src16;
-#endif
-    
     /* linked list of chain steps */
-    struct icu_chain_step * steps;
+    struct icu_chain_step *csteps;
 };
 
 int icu_check_status(UErrorCode status)
@@ -97,62 +72,65 @@ int icu_check_status(UErrorCode status)
     if (U_FAILURE(status))
     {
         yaz_log(YLOG_WARN, "ICU: %d %s\n", status, u_errorName(status));
-        return 0;   
+        return 0;
     }
     return 1;
 }
 
-static struct icu_chain_step *icu_chain_step_create(
-    struct icu_chain * chain,  enum icu_chain_step_type type,
-    const uint8_t * rule, 
-#if USE_ITER
-#else
-    struct icu_buf_utf16 * buf16,
-#endif
-    UErrorCode *status)
+static struct icu_chain_step *icu_chain_insert_step(
+    struct icu_chain *chain, enum icu_chain_step_type type,
+    const char *rule, UErrorCode *status)
 {
-    struct icu_chain_step * step = 0;
-    
-    if(!chain || !type || !rule)
-        return 0;
+    struct icu_chain_step *step = 0;
 
-    step = (struct icu_chain_step *) xmalloc(sizeof(struct icu_chain_step));
+    assert(chain);
+    assert(type);
 
+    step = (struct icu_chain_step *) xmalloc(sizeof(*step));
     step->type = type;
-#if USE_ITER
-#else
-    step->buf16 = buf16;
-#endif
-    /* create auxilary objects */
+
     switch (step->type)
     {
     case ICU_chain_step_type_display:
         break;
     case ICU_chain_step_type_casemap:
+        assert(rule);
         step->u.casemap = icu_casemap_create(rule[0], status);
         break;
     case ICU_chain_step_type_transform:
+        assert(rule);
         /* rule omitted. Only ID used */
-        step->u.transform = icu_transform_create((const char *) rule, 'f',
-                                                 0, status);
+        step->u.transform = icu_transform_create(rule, 'f', 0, status);
         break;
     case ICU_chain_step_type_tokenize:
-        step->u.tokenizer = icu_tokenizer_create((char *) chain->locale, 
-                                                 (char) rule[0], status);
+        assert(rule);
+        step->u.tokenizer = icu_tokenizer_create(chain->locale, rule[0], status);
         break;
     case ICU_chain_step_type_transliterate:
+        assert(rule);
         /* we pass a dummy ID to utrans_openU.. */
-        step->u.transform = icu_transform_create("custom", 'f',
-                                                 (const char *) rule, status);
+        step->u.transform = icu_transform_create("custom", 'f', rule, status);
+        break;
+    case YAZ_chain_step_type_stemming:
+        assert(rule);
+        step->u.stemmer = yaz_stemmer_create(chain->locale, rule, status);
+        break;
+    case ICU_chain_step_type_join:
+        assert(rule);
+        step->u.join = icu_buf_utf16_create(0);
+        icu_utf16_from_utf8_cstr(step->u.join, rule, status);
         break;
     default:
         break;
     }
+    step->previous = chain->csteps;
+    chain->csteps = step;
+
     return step;
 }
 
 
-static void icu_chain_step_destroy(struct icu_chain_step * step)
+static void icu_chain_step_destroy(struct icu_chain_step *step)
 {
     if (!step)
         return;
@@ -165,25 +143,19 @@ static void icu_chain_step_destroy(struct icu_chain_step * step)
         break;
     case ICU_chain_step_type_casemap:
         icu_casemap_destroy(step->u.casemap);
-#if USE_ITER
-#else
-        icu_buf_utf16_destroy(step->buf16);
-#endif
         break;
     case ICU_chain_step_type_transform:
     case ICU_chain_step_type_transliterate:
         icu_transform_destroy(step->u.transform);
-#if USE_ITER
-#else
-        icu_buf_utf16_destroy(step->buf16);
-#endif
         break;
     case ICU_chain_step_type_tokenize:
         icu_tokenizer_destroy(step->u.tokenizer);
-#if USE_ITER
-#else
-        icu_buf_utf16_destroy(step->buf16);
-#endif
+        break;
+    case YAZ_chain_step_type_stemming:
+        yaz_stemmer_destroy(step->u.stemmer);
+        break;
+    case ICU_chain_step_type_join:
+        icu_buf_utf16_destroy(step->u.join);
         break;
     default:
         break;
@@ -191,128 +163,169 @@ static void icu_chain_step_destroy(struct icu_chain_step * step)
     xfree(step);
 }
 
-struct icu_chain *icu_chain_create(const char *locale, int sort,
-                                   UErrorCode * status)
+struct icu_chain_step *icu_chain_step_clone(struct icu_chain_step *old)
 {
-    struct icu_chain * chain 
-        = (struct icu_chain *) xmalloc(sizeof(struct icu_chain));
-
-    *status = U_ZERO_ERROR;
-
-#if USE_ITER
-    chain->iter = 0;
-#endif
-    chain->locale = xstrdup(locale);
+    struct icu_chain_step *step = 0;
+    struct icu_chain_step **sp = &step;
+    while (old)
+    {
+        *sp = (struct icu_chain_step *) xmalloc(sizeof(**sp));
+        (*sp)->type = old->type;
 
-    chain->sort = sort;
+        switch ((*sp)->type)
+        {
+        case ICU_chain_step_type_display:
+            break;
+        case ICU_chain_step_type_casemap:
+            (*sp)->u.casemap = icu_casemap_clone(old->u.casemap);
+            break;
+        case ICU_chain_step_type_transform:
+        case ICU_chain_step_type_transliterate:
+            (*sp)->u.transform = icu_transform_clone(old->u.transform);
+            break;
+        case ICU_chain_step_type_tokenize:
+            (*sp)->u.tokenizer = icu_tokenizer_clone(old->u.tokenizer);
+            break;
+        case YAZ_chain_step_type_stemming:
+            (*sp)->u.stemmer = yaz_stemmer_clone(old->u.stemmer);
+            break;
+        case ICU_chain_step_type_none:
+            break;
+        case ICU_chain_step_type_join:
+            (*sp)->u.join = icu_buf_utf16_create(0);
+            (*sp)->u.join = icu_buf_utf16_copy((*sp)->u.join, old->u.join);
+            break;
+        }
+        old = old->previous;
+        sp = &(*sp)->previous;
+    }
+    *sp = 0;
+    return step;
+}
 
-    chain->coll = ucol_open((const char *) chain->locale, status);
+struct icu_chain *icu_chain_create(const char *locale, int sort,
+                                   UErrorCode *status)
+{
+    struct icu_chain *chain;
+    UCollator *coll = ucol_open(locale, status);
 
     if (U_FAILURE(*status))
         return 0;
-#if USE_ITER
-#else
-    chain->token_count = 0;
-    chain->src8cstr = 0;
-#endif
 
-    chain->norm8 = icu_buf_utf8_create(0);
-#if USE_ITER
-#else
-    chain->display8 = icu_buf_utf8_create(0);
-    chain->sort8 = icu_buf_utf8_create(0);
-    chain->src16 = icu_buf_utf16_create(0);
-#endif
-
-    chain->steps = 0;
+    chain = (struct icu_chain *) xmalloc(sizeof(*chain));
+    chain->iter = 0;
+    chain->locale = xstrdup(locale);
+    chain->sort = sort;
+    chain->coll = coll;
+    chain->csteps = 0;
 
     return chain;
 }
 
-void icu_chain_destroy(struct icu_chain * chain)
+void icu_chain_destroy(struct icu_chain *chain)
 {
     if (chain)
     {
         if (chain->coll)
             ucol_close(chain->coll);
 
-        icu_buf_utf8_destroy(chain->norm8);
-#if USE_ITER
         if (chain->iter)
             icu_iter_destroy(chain->iter);
-#else
-        icu_buf_utf8_destroy(chain->display8);
-        icu_buf_utf8_destroy(chain->sort8);
-        icu_buf_utf16_destroy(chain->src16);
-#endif        
-    
-        icu_chain_step_destroy(chain->steps);
+        icu_chain_step_destroy(chain->csteps);
         xfree(chain->locale);
         xfree(chain);
     }
 }
 
-static struct icu_chain_step *icu_chain_insert_step(
-    struct icu_chain * chain, enum icu_chain_step_type type,
-    const uint8_t * rule, UErrorCode *status);
-
-struct icu_chain * icu_chain_xml_config(const xmlNode *xml_node, 
-                                        int sort,
-                                        UErrorCode * status)
+struct icu_chain *icu_chain_xml_config(const xmlNode *xml_node,
+                                       int sort,
+                                       UErrorCode *status)
 {
     xmlNode *node = 0;
-    struct icu_chain * chain = 0;
-   
+    int no_errors = 0;
+    struct icu_chain *chain = 0;
+    NMEM nmem = 0;
+
     *status = U_ZERO_ERROR;
 
-    if (!xml_node ||xml_node->type != XML_ELEMENT_NODE)
-        return 0;
-    
+    if (xml_node && xml_node->type == XML_ELEMENT_NODE)
     {
-        xmlChar * xml_locale = xmlGetProp((xmlNode *) xml_node, 
-                                          (xmlChar *) "locale");
-        
+        xmlChar *xml_locale = xmlGetProp((xmlNode *) xml_node,
+                                         (xmlChar *) "locale");
         if (xml_locale)
         {
             chain = icu_chain_create((const char *) xml_locale, sort, status);
             xmlFree(xml_locale);
         }
-        
     }
+
     if (!chain)
         return 0;
 
+    nmem = nmem_create();
     for (node = xml_node->children; node; node = node->next)
     {
-        xmlChar *xml_rule;
-        struct icu_chain_step * step = 0;
+        char *rule = 0;
+        struct icu_chain_step *step = 0;
+        struct _xmlAttr *attr;
 
+        nmem_reset(nmem);
         if (node->type != XML_ELEMENT_NODE)
             continue;
 
-        xml_rule = xmlGetProp(node, (xmlChar *) "rule");
+        for (attr = node->properties; attr; attr = attr->next)
+        {
+            if (!strcmp((const char *) attr->name, "rule"))
+            {
+                rule = nmem_text_node_cdata(attr->children, nmem);
+            }
+            else
+            {
+                yaz_log(YLOG_WARN, "Unsupported attribute '%s' for "
+                        "element '%s'", attr->name, node->name);
+                no_errors++;
+            }
+        }
+        if (!rule && node->children)
+            rule = nmem_text_node_cdata(node->children, nmem);
 
+        if (!rule && strcmp((const char *) node->name, "display"))
+        {
+            yaz_log(YLOG_WARN, "Missing attribute rule for element %s",
+                    (const char *) node->name);
+            no_errors++;
+            continue;
+        }
         if (!strcmp((const char *) node->name, "casemap"))
-            step = icu_chain_insert_step(chain, ICU_chain_step_type_casemap, 
-                                         (const uint8_t *) xml_rule, status);
+            step = icu_chain_insert_step(chain,
+                                         ICU_chain_step_type_casemap,
+                                         rule, status);
         else if (!strcmp((const char *) node->name, "transform"))
-            step = icu_chain_insert_step(chain, ICU_chain_step_type_transform, 
-                                         (const uint8_t *) xml_rule, status);
+            step = icu_chain_insert_step(chain,
+                                         ICU_chain_step_type_transform,
+                                         rule, status);
         else if (!strcmp((const char *) node->name, "transliterate"))
-            step = icu_chain_insert_step(chain, ICU_chain_step_type_transliterate, 
-                                         (const uint8_t *) xml_rule, status);
+            step = icu_chain_insert_step(chain,
+                                         ICU_chain_step_type_transliterate,
+                                         rule, status);
         else if (!strcmp((const char *) node->name, "tokenize"))
-            step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize, 
-                                         (const uint8_t *) xml_rule, status);
+            step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize,
+                                         rule, status);
         else if (!strcmp((const char *) node->name, "display"))
-            step = icu_chain_insert_step(chain, ICU_chain_step_type_display, 
-                                         (const uint8_t *) "", status);
+            step = icu_chain_insert_step(chain, ICU_chain_step_type_display,
+                                         rule, status);
+        else if (!strcmp((const char *) node->name, "stemming"))
+            step = icu_chain_insert_step(chain, YAZ_chain_step_type_stemming,
+                                         rule, status);
+        else if (!strcmp((const char *) node->name, "join"))
+            step = icu_chain_insert_step(chain, ICU_chain_step_type_join,
+                                         rule, status);
         else if (!strcmp((const char *) node->name, "normalize"))
         {
             yaz_log(YLOG_WARN, "Element %s is deprecated. "
                     "Use transform instead", node->name);
-            step = icu_chain_insert_step(chain, ICU_chain_step_type_transform, 
-                                         (const uint8_t *) xml_rule, status);
+            step = icu_chain_insert_step(chain, ICU_chain_step_type_transform,
+                                         rule, status);
         }
         else if (!strcmp((const char *) node->name, "index")
                  || !strcmp((const char *) node->name, "sortkey"))
@@ -323,185 +336,41 @@ struct icu_chain * icu_chain_xml_config(const xmlNode *xml_node,
         else
         {
             yaz_log(YLOG_WARN, "Unknown element %s", node->name);
-            icu_chain_destroy(chain);
-            return 0;
+            no_errors++;
+            continue;
+        }
+        if (!step)
+        {
+            yaz_log(YLOG_WARN, "Step not created for %s", node->name);
+            no_errors++;
         }
-        xmlFree(xml_rule);
         if (step && U_FAILURE(*status))
         {
-            icu_chain_destroy(chain);
-            return 0;
+            no_errors++;
+            break;
         }
     }
-    return chain;
-}
-
-static struct icu_chain_step *icu_chain_insert_step(
-    struct icu_chain * chain, enum icu_chain_step_type type,
-    const uint8_t * rule, UErrorCode *status)
-{    
-    struct icu_chain_step * step = 0;
-#if USE_ITER
-#else
-    struct icu_buf_utf16 * src16 = 0;
-    struct icu_buf_utf16 * buf16 = 0;
-#endif
-    if (!chain || !type || !rule)
-        return 0;
-
-#if USE_ITER
-#else
-    /* assign utf16 src buffers as needed */
-    if (chain->steps && chain->steps->buf16)
-        src16 = chain->steps->buf16;
-    else if (chain->src16)
-        src16 = chain->src16;
-    else
-        return 0;
-
-    /* create utf16 destination buffers as needed, or */
-    switch (type)
-    {
-    case ICU_chain_step_type_display:
-        buf16 = src16;
-        break;
-    case ICU_chain_step_type_casemap:
-        buf16 = icu_buf_utf16_create(0);
-        break;
-    case ICU_chain_step_type_transform:
-    case ICU_chain_step_type_transliterate:
-        buf16 = icu_buf_utf16_create(0);
-        break;
-    case ICU_chain_step_type_tokenize:
-        buf16 = icu_buf_utf16_create(0);
-        break;
-        break;
-    default:
-        break;
-    }
-#endif
-    /* create actual chain step with this buffer */
-    step = icu_chain_step_create(chain, type, rule,
-#if USE_ITER
-#else
-                                 buf16,
-#endif
-                                 status);
-
-    step->previous = chain->steps;
-    chain->steps = step;
-
-    return step;
-}
-
-#if USE_ITER
-#else
-static int icu_chain_step_next_token(struct icu_chain * chain,
-                                     struct icu_chain_step * step,
-                                     UErrorCode *status)
-{
-    struct icu_buf_utf16 * src16 = 0;
-    int got_new_token = 0;
-
-    if (!chain || !chain->src16 || !step || !step->more_tokens)
-        return 0;
-
-    /* assign utf16 src buffers as needed, advance in previous steps
-       tokens until non-zero token met, and setting stop condition */
-
-    if (step->previous)
-    {
-        src16 = step->previous->buf16;
-        /* tokens might be killed in previous steps, therefore looping */
-
-        while (step->need_new_token 
-               && step->previous->more_tokens
-               && !got_new_token)
-            got_new_token
-                = icu_chain_step_next_token(chain, step->previous, status);
-    }
-    else 
-    { /* first step can only work once on chain->src16 input buffer */
-        src16 = chain->src16;
-        step->more_tokens = 0;
-        got_new_token = 1;
-    }
-
-    if (!src16)
-        return 0;
-
-    /* stop if nothing to process */
-    if (step->need_new_token && !got_new_token)
-    {
-        step->more_tokens = 0;
-        return 0;
-    }
-
-    /* either an old token not finished yet, or a new token, thus
-       perform the work, eventually put this steps output in 
-       step->buf16 or the chains UTF8 output buffers  */
-
-    switch (step->type)
+    nmem_destroy(nmem);
+    if (no_errors)
     {
-    case ICU_chain_step_type_display:
-        icu_utf16_to_utf8(chain->display8, src16, status);
-        break;
-    case ICU_chain_step_type_casemap:
-        icu_casemap_casemap(step->u.casemap,
-                            step->buf16, src16, status,
-                            chain->locale);
-        break;
-    case ICU_chain_step_type_transform:
-    case ICU_chain_step_type_transliterate:
-        icu_transform_trans(step->u.transform,
-                            step->buf16, src16, status);
-        break;
-    case ICU_chain_step_type_tokenize:
-        /* attach to new src16 token only first time during splitting */
-        if (step->need_new_token)
-        {
-            icu_tokenizer_attach(step->u.tokenizer, src16, status);
-            step->need_new_token = 0;
-        }
-
-        /* splitting one src16 token into multiple buf16 tokens */
-        step->more_tokens
-            = icu_tokenizer_next_token(step->u.tokenizer,
-                                       step->buf16, status);
-
-        /* make sure to get new previous token if this one had been used up
-           by recursive call to _same_ step */
-
-        if (!step->more_tokens)
-        {
-            step->more_tokens = icu_chain_step_next_token(chain, step, status);
-            return step->more_tokens;  /* avoid one token count too much! */
-        }
-        break;
-    default:
+        icu_chain_destroy(chain);
         return 0;
-        break;
     }
-
-    if (U_FAILURE(*status))
-        return 0;
-
-    /* if token disappered into thin air, tell caller */
-    /* if (!step->buf16->utf16_len && !step->more_tokens) */ 
-    /*    return 0; */ 
-
-    return 1;
+    return chain;
 }
-#endif
 
 struct icu_iter {
     struct icu_chain *chain;
     struct icu_buf_utf16 *last;
+    struct icu_buf_utf16 *org;
     UErrorCode status;
     struct icu_buf_utf8 *display;
     struct icu_buf_utf8 *sort8;
-    struct icu_buf_utf16 *input;
+    struct icu_buf_utf8 *result;
     int token_count;
+    size_t org_start;
+    size_t org_len;
+    struct icu_chain_step *steps;
 };
 
 void icu_utf16_print(struct icu_buf_utf16 *src16)
@@ -511,7 +380,6 @@ void icu_utf16_print(struct icu_buf_utf16 *src16)
     struct icu_buf_utf8 *dst8 = icu_buf_utf8_create(0);
     icu_utf16_to_utf8(dst8, src16, &status);
 
-    assert(status != 1234);
     if (U_FAILURE(status))
     {
         printf("failure");
@@ -524,7 +392,7 @@ void icu_utf16_print(struct icu_buf_utf16 *src16)
     icu_buf_utf8_destroy(dst8);
 }
 
-struct icu_buf_utf16 *icu_iter_invoke(struct icu_iter *iter,
+struct icu_buf_utf16 *icu_iter_invoke(yaz_icu_iter_t iter,
                                       struct icu_chain_step *step,
                                       struct icu_buf_utf16 *src)
 {
@@ -533,7 +401,7 @@ struct icu_buf_utf16 *icu_iter_invoke(struct icu_iter *iter,
     else
     {
         struct icu_buf_utf16 *dst = icu_iter_invoke(iter, step->previous, src);
-        
+
         switch (step->type)
         {
         case ICU_chain_step_type_casemap:
@@ -557,7 +425,8 @@ struct icu_buf_utf16 *icu_iter_invoke(struct icu_iter *iter,
             }
             dst = icu_buf_utf16_create(0);
             iter->status = U_ZERO_ERROR;
-            if (!icu_tokenizer_next_token(step->u.tokenizer, dst, &iter->status))
+            if (!icu_tokenizer_next_token(step->u.tokenizer, dst, &iter->status,
+                                          &iter->org_start, &iter->org_len))
             {
                 icu_buf_utf16_destroy(dst);
                 dst = 0;
@@ -577,6 +446,31 @@ struct icu_buf_utf16 *icu_iter_invoke(struct icu_iter *iter,
             if (dst)
                 icu_utf16_to_utf8(iter->display, dst, &iter->status);
             break;
+        case YAZ_chain_step_type_stemming:
+            if (dst)
+            {
+                struct icu_buf_utf16 *src = dst;
+                dst = icu_buf_utf16_create(0);
+                yaz_stemmer_stem(step->u.stemmer, dst, src, &iter->status);
+                icu_buf_utf16_destroy(src);
+            }
+            break;
+        case ICU_chain_step_type_join:
+            if (dst)
+            {
+                while (1)
+                {
+                    struct icu_buf_utf16 *dst1 =
+                        icu_iter_invoke(iter, step->previous, 0);
+
+                    if (!dst1)
+                        break; 
+                    dst = icu_buf_utf16_append(dst, step->u.join);
+                    dst = icu_buf_utf16_append(dst, dst1);
+                    icu_buf_utf16_destroy(dst1);
+                }
+            }
+            break;
         default:
             assert(0);
         }
@@ -584,226 +478,163 @@ struct icu_buf_utf16 *icu_iter_invoke(struct icu_iter *iter,
     }
 }
 
-struct icu_iter *icu_iter_create(struct icu_chain *chain,
-                                 const char *src8cstr)
+yaz_icu_iter_t icu_iter_create(struct icu_chain *chain)
 {
-    if (!src8cstr)
-        return 0;
-    else
-    {
-        struct icu_iter *iter = xmalloc(sizeof(*iter));
-        iter->chain = chain;
-        iter->status = U_ZERO_ERROR;
-        iter->display = icu_buf_utf8_create(0);
-        iter->sort8 = icu_buf_utf8_create(0);
-        iter->token_count = 0;
-        iter->last = 0; /* no last returned string (yet) */
-
-        /* fill and assign input string.. It will be 0 after
-           first iteration */
-        iter->input =  icu_buf_utf16_create(0);
-        icu_utf16_from_utf8_cstr(iter->input, src8cstr, &iter->status);
-        return iter;
+    yaz_icu_iter_t iter = xmalloc(sizeof(*iter));
+    iter->chain = chain;
+    iter->status = U_ZERO_ERROR;
+    iter->display = icu_buf_utf8_create(0);
+    iter->sort8 = icu_buf_utf8_create(0);
+    iter->result = icu_buf_utf8_create(0);
+    iter->org = icu_buf_utf16_create(0);
+    iter->last = 0; /* no last returned string (yet) */
+    iter->steps = icu_chain_step_clone(chain->csteps);
+    iter->token_count = 0;
+
+    return iter;
+}
 
-    }
+void icu_iter_first(yaz_icu_iter_t iter, const char *src8cstr)
+{
+    struct icu_buf_utf16 *src = icu_buf_utf16_create(0);
+    icu_utf16_from_utf8_cstr(src, src8cstr, &iter->status);
+    icu_buf_utf16_copy(iter->org, src);
+    iter->token_count = 0;
+    iter->org_start = 0;
+    iter->org_len = src->utf16_len;
+    iter->last = icu_iter_invoke(iter, iter->steps, src);
 }
 
-void icu_iter_destroy(struct icu_iter *iter)
+void icu_iter_destroy(yaz_icu_iter_t iter)
 {
     if (iter)
     {
         icu_buf_utf8_destroy(iter->display);
         icu_buf_utf8_destroy(iter->sort8);
-        if (iter->input)
-            icu_buf_utf16_destroy(iter->input);
+        icu_buf_utf8_destroy(iter->result);
+        icu_buf_utf16_destroy(iter->org);
+        icu_chain_step_destroy(iter->steps);
         xfree(iter);
     }
 }
 
-int icu_iter_next(struct icu_iter *iter, struct icu_buf_utf8 *result)
+int icu_iter_next(yaz_icu_iter_t iter)
 {
-    if (!iter->input && iter->last == 0)
+    if (iter->token_count && iter->last)
+        iter->last = icu_iter_invoke(iter, iter->steps, 0);
+    if (!iter->last)
         return 0;
     else
     {
-        /* on first call, iter->input is the input string. Thereafter: 0. */
-        iter->last = icu_iter_invoke(iter, iter->chain->steps, iter->input);
-        iter->input = 0;
-        
-        if (!iter->last)
-            return 0;
-
         iter->token_count++;
-
         if (iter->chain->sort)
-        {        
+        {
             icu_sortkey8_from_utf16(iter->chain->coll,
                                     iter->sort8, iter->last,
                                     &iter->status);
         }
-        icu_utf16_to_utf8(result, iter->last, &iter->status);
+        icu_utf16_to_utf8(iter->result, iter->last, &iter->status);
         icu_buf_utf16_destroy(iter->last);
 
         return 1;
     }
 }
 
-const char *icu_iter_get_sortkey(struct icu_iter *iter)
+const char *icu_iter_get_norm(yaz_icu_iter_t iter)
+{
+    return icu_buf_utf8_to_cstr(iter->result);
+}
+
+const char *icu_iter_get_sortkey(yaz_icu_iter_t iter)
 {
     return icu_buf_utf8_to_cstr(iter->sort8);
 }
 
-const char *icu_iter_get_display(struct icu_iter *iter)
-{ 
-    return icu_buf_utf8_to_cstr(iter->display);   
+const char *icu_iter_get_display(yaz_icu_iter_t iter)
+{
+    return icu_buf_utf8_to_cstr(iter->display);
 }
 
-int icu_chain_assign_cstr(struct icu_chain * chain, const char * src8cstr, 
-                          UErrorCode *status)
+int icu_iter_get_token_number(yaz_icu_iter_t iter)
 {
-#if USE_ITER
-    if (chain->iter)
-        icu_iter_destroy(chain->iter);
-    chain->iter = icu_iter_create(chain, src8cstr);
-    return 1;
-#else
-    struct icu_chain_step * stp = 0; 
+    return iter->token_count;
+}
 
-    if (!chain || !src8cstr)
-        return 0;
 
-    chain->src8cstr = src8cstr;
+void icu_iter_get_org_info(yaz_icu_iter_t iter, size_t *start, size_t *len)
+{
+    /* save full length of org since we're gonna cut it */
+    int32_t save_len = iter->org->utf16_len;
 
-    stp = chain->steps;
-    
-    /* clear token count */
-    chain->token_count = 0;
+    struct icu_buf_utf8 *tmp = icu_buf_utf8_create(0);
+    UErrorCode status = U_ZERO_ERROR;
 
-    /* clear all steps stop states */
-    while (stp)
-    {
-        stp->more_tokens = 1;
-        stp->need_new_token = 1;
-        stp = stp->previous;
-    }
-    
-    /* finally convert UTF8 to UTF16 string if needed */
-    if (chain->steps || chain->sort)
-        icu_utf16_from_utf8_cstr(chain->src16, chain->src8cstr, status);
-            
-    if (U_FAILURE(*status))
-        return 0;
+    iter->org->utf16_len = iter->org_start;
+    icu_utf16_to_utf8(tmp, iter->org, &status);
+    if (U_SUCCESS(status))
+        *start = tmp->utf8_len;
+    else
+        *start = 0;
+    iter->org->utf16_len = iter->org_start + iter->org_len;
+    icu_utf16_to_utf8(tmp, iter->org, &status);
+    if (U_SUCCESS(status))
+        *len = tmp->utf8_len - *start;
+    else
+        *len = 0;
+    iter->org->utf16_len = save_len;
+}
 
+int icu_chain_assign_cstr(struct icu_chain *chain, const char *src8cstr,
+                          UErrorCode *status)
+{
+    if (chain->iter)
+        icu_iter_destroy(chain->iter);
+    chain->iter = icu_iter_create(chain);
+    icu_iter_first(chain->iter, src8cstr);
     return 1;
-#endif
 }
 
-int icu_chain_next_token(struct icu_chain * chain, UErrorCode *status)
+int icu_chain_next_token(struct icu_chain *chain, UErrorCode *status)
 {
-#if USE_ITER
-    *status = U_ZERO_ERROR;
-    return icu_iter_next(chain->iter, chain->norm8);
-#else
-    int got_token = 0;
-    
     *status = U_ZERO_ERROR;
-
-    if (!chain)
-        return 0;
-
-    /* special case with no steps - same as index type binary */
-    if (!chain->steps)
-    {
-        if (chain->token_count)
-            return 0;
-        else
-        {
-            chain->token_count++;
-            
-            if (chain->sort)
-                icu_sortkey8_from_utf16(chain->coll,
-                                        chain->sort8, chain->steps->buf16,
-                                        status);
-            return chain->token_count;
-        }
-    }
-    /* usual case, one or more icu chain steps existing */
-    else 
-    {
-        while (!got_token && chain->steps && chain->steps->more_tokens)
-            got_token = icu_chain_step_next_token(chain, chain->steps, status);
-
-        if (got_token)
-        {
-            chain->token_count++;
-
-            icu_utf16_to_utf8(chain->norm8, chain->steps->buf16, status);
-            
-            if (chain->sort)
-                icu_sortkey8_from_utf16(chain->coll,
-                                        chain->sort8, chain->steps->buf16,
-                                        status);
-            return chain->token_count;
-        }
-    }
-        
-    return 0;
-#endif
+    return icu_iter_next(chain->iter);
 }
 
-int icu_chain_token_number(struct icu_chain * chain)
+int icu_chain_token_number(struct icu_chain *chain)
 {
-#if USE_ITER
     if (chain && chain->iter)
         return chain->iter->token_count;
     return 0;
-#else
-    if (!chain)
-        return 0;
-    
-    return chain->token_count;
-#endif
 }
 
-const char * icu_chain_token_display(struct icu_chain * chain)
+const char *icu_chain_token_display(struct icu_chain *chain)
 {
-#if USE_ITER
     if (chain->iter)
         return icu_iter_get_display(chain->iter);
-#else
-    if (chain->display8)
-        return icu_buf_utf8_to_cstr(chain->display8);
-#endif
     return 0;
 }
 
-const char * icu_chain_token_norm(struct icu_chain * chain)
+const char *icu_chain_token_norm(struct icu_chain *chain)
 {
-#if USE_ITER
-    if (chain->norm8)
-        return icu_buf_utf8_to_cstr(chain->norm8);
-#else
-    if (!chain->steps)
-        return chain->src8cstr;
-
-    if (chain->norm8)
-        return icu_buf_utf8_to_cstr(chain->norm8);
-#endif
+    if (chain->iter)
+        return icu_iter_get_norm(chain->iter);
     return 0;
 }
 
-const char * icu_chain_token_sortkey(struct icu_chain * chain)
+const char *icu_chain_token_sortkey(struct icu_chain *chain)
 {
-#if USE_ITER
     if (chain->iter)
         return icu_iter_get_sortkey(chain->iter);
-#else
-    if (chain->sort8)
-        return icu_buf_utf8_to_cstr(chain->sort8);
-#endif
     return 0;
 }
 
+void icu_chain_get_org_info(struct icu_chain *chain, size_t *start, size_t *len)
+{
+    if (chain->iter)
+        icu_iter_get_org_info(chain->iter, start, len);
+}
+
+
 #endif /* YAZ_HAVE_ICU */
 
 /*