X-Git-Url: http://git.indexdata.com/?p=yaz-moved-to-github.git;a=blobdiff_plain;f=src%2Ficu_chain.c;h=bcb27b435fa61f1c0200950350e85b4f6fe406e1;hp=e6044d464b1ee4160d63d99f3bdc453b56bcc9d2;hb=be73a431cd4e151c77d3024b7ee09197e4a4910a;hpb=af3b1f509a27cb38ca5c9cb7a22cd384f5ca7c70

diff --git a/src/icu_chain.c b/src/icu_chain.c
index e6044d4..bcb27b4 100644
--- a/src/icu_chain.c
+++ b/src/icu_chain.c
@@ -1,5 +1,5 @@
 /* This file is part of the YAZ toolkit.
- * Copyright (C) 1995-2010 Index Data
+ * Copyright (C) 1995-2013 Index Data
  * See the file LICENSE for details.
  */
 
@@ -17,6 +17,8 @@
 
 #include <yaz/icu_I18N.h>
 
+#include <yaz/stemmer.h>
+
 #include <yaz/log.h>
 #include <yaz/nmem.h>
 #include <yaz/nmem_xml.h>
@@ -30,11 +32,13 @@
 
 enum icu_chain_step_type {
     ICU_chain_step_type_none,
-    ICU_chain_step_type_display,   /* convert to utf8 display format */
-    ICU_chain_step_type_casemap,   /* apply utf16 charmap */
-    ICU_chain_step_type_transform, /* apply utf16 transform */
-    ICU_chain_step_type_tokenize,  /* apply utf16 tokenization */
-    ICU_chain_step_type_transliterate  /* apply utf16 tokenization */
+    ICU_chain_step_type_display,        /* convert to utf8 display format */
+    ICU_chain_step_type_casemap,        /* apply utf16 charmap */
+    ICU_chain_step_type_transform,      /* apply utf16 transform */
+    ICU_chain_step_type_tokenize,       /* apply utf16 tokenization */
+    ICU_chain_step_type_transliterate,  /* apply utf16 tokenization */
+    YAZ_chain_step_type_stemming,       /* apply utf16 stemming (YAZ) */
+    ICU_chain_step_type_join
 };
 
 struct icu_chain_step
@@ -42,9 +46,11 @@ struct icu_chain_step
     /* type and action object */
     enum icu_chain_step_type type;
     union {
-	struct icu_casemap *casemap;
+	struct icu_casemap   *casemap;
 	struct icu_transform *transform;
-	struct icu_tokenizer *tokenizer;  
+	struct icu_tokenizer *tokenizer;
+        yaz_stemmer_p         stemmer;
+        struct icu_buf_utf16 *join;
     } u;
     struct icu_chain_step *previous;
 };
@@ -56,7 +62,7 @@ struct icu_chain
     int sort;
 
     UCollator *coll;
-    
+
     /* linked list of chain steps */
     struct icu_chain_step *csteps;
 };
@@ -66,48 +72,60 @@ int icu_check_status(UErrorCode status)
     if (U_FAILURE(status))
     {
         yaz_log(YLOG_WARN, "ICU: %d %s\n", status, u_errorName(status));
-        return 0;   
+        return 0;
     }
     return 1;
 }
 
-static struct icu_chain_step *icu_chain_step_create(
-    struct icu_chain *chain,  enum icu_chain_step_type type,
-    const uint8_t *rule, UErrorCode *status)
+static struct icu_chain_step *icu_chain_insert_step(
+    struct icu_chain *chain, enum icu_chain_step_type type,
+    const char *rule, UErrorCode *status)
 {
     struct icu_chain_step *step = 0;
-    
-    if (!chain || !type || !rule)
-        return 0;
 
-    step = (struct icu_chain_step *) xmalloc(sizeof(*step));
+    assert(chain);
+    assert(type);
 
+    step = (struct icu_chain_step *) xmalloc(sizeof(*step));
     step->type = type;
-    /* create auxilary objects */
+
     switch (step->type)
     {
     case ICU_chain_step_type_display:
         break;
     case ICU_chain_step_type_casemap:
+        assert(rule);
         step->u.casemap = icu_casemap_create(rule[0], status);
         break;
     case ICU_chain_step_type_transform:
+        assert(rule);
         /* rule omitted. Only ID used */
-        step->u.transform = icu_transform_create((const char *) rule, 'f',
-                                                 0, status);
+        step->u.transform = icu_transform_create(rule, 'f', 0, status);
         break;
     case ICU_chain_step_type_tokenize:
-        step->u.tokenizer = icu_tokenizer_create((char *) chain->locale, 
-                                                 (char) rule[0], status);
+        assert(rule);
+        step->u.tokenizer = icu_tokenizer_create(chain->locale, rule[0], status);
         break;
     case ICU_chain_step_type_transliterate:
+        assert(rule);
         /* we pass a dummy ID to utrans_openU.. */
-        step->u.transform = icu_transform_create("custom", 'f',
-                                                 (const char *) rule, status);
+        step->u.transform = icu_transform_create("custom", 'f', rule, status);
+        break;
+    case YAZ_chain_step_type_stemming:
+        assert(rule);
+        step->u.stemmer = yaz_stemmer_create(chain->locale, rule, status);
+        break;
+    case ICU_chain_step_type_join:
+        assert(rule);
+        step->u.join = icu_buf_utf16_create(0);
+        icu_utf16_from_utf8_cstr(step->u.join, rule, status);
         break;
     default:
         break;
     }
+    step->previous = chain->csteps;
+    chain->csteps = step;
+
     return step;
 }
 
@@ -133,6 +151,12 @@ static void icu_chain_step_destroy(struct icu_chain_step *step)
     case ICU_chain_step_type_tokenize:
         icu_tokenizer_destroy(step->u.tokenizer);
         break;
+    case YAZ_chain_step_type_stemming:
+        yaz_stemmer_destroy(step->u.stemmer);
+        break;
+    case ICU_chain_step_type_join:
+        icu_buf_utf16_destroy(step->u.join);
+        break;
     default:
         break;
     }
@@ -147,7 +171,7 @@ struct icu_chain_step *icu_chain_step_clone(struct icu_chain_step *old)
     {
         *sp = (struct icu_chain_step *) xmalloc(sizeof(**sp));
         (*sp)->type = old->type;
-        
+
         switch ((*sp)->type)
         {
         case ICU_chain_step_type_display:
@@ -162,8 +186,15 @@ struct icu_chain_step *icu_chain_step_clone(struct icu_chain_step *old)
         case ICU_chain_step_type_tokenize:
             (*sp)->u.tokenizer = icu_tokenizer_clone(old->u.tokenizer);
             break;
+        case YAZ_chain_step_type_stemming:
+            (*sp)->u.stemmer = yaz_stemmer_clone(old->u.stemmer);
+            break;
         case ICU_chain_step_type_none:
             break;
+        case ICU_chain_step_type_join:
+            (*sp)->u.join = icu_buf_utf16_create(0);
+            (*sp)->u.join = icu_buf_utf16_copy((*sp)->u.join, old->u.join);
+            break;
         }
         old = old->previous;
         sp = &(*sp)->previous;
@@ -175,21 +206,17 @@ struct icu_chain_step *icu_chain_step_clone(struct icu_chain_step *old)
 struct icu_chain *icu_chain_create(const char *locale, int sort,
                                    UErrorCode *status)
 {
-    struct icu_chain *chain 
-        = (struct icu_chain *) xmalloc(sizeof(*chain));
+    struct icu_chain *chain;
+    UCollator *coll = ucol_open(locale, status);
 
-    *status = U_ZERO_ERROR;
+    if (U_FAILURE(*status))
+        return 0;
 
+    chain = (struct icu_chain *) xmalloc(sizeof(*chain));
     chain->iter = 0;
     chain->locale = xstrdup(locale);
-
     chain->sort = sort;
-
-    chain->coll = ucol_open((const char *) chain->locale, status);
-
-    if (U_FAILURE(*status))
-        return 0;
-
+    chain->coll = coll;
     chain->csteps = 0;
 
     return chain;
@@ -210,11 +237,7 @@ void icu_chain_destroy(struct icu_chain *chain)
     }
 }
 
-static struct icu_chain_step *icu_chain_insert_step(
-    struct icu_chain *chain, enum icu_chain_step_type type,
-    const uint8_t *rule, UErrorCode *status);
-
-struct icu_chain *icu_chain_xml_config(const xmlNode *xml_node, 
+struct icu_chain *icu_chain_xml_config(const xmlNode *xml_node,
                                        int sort,
                                        UErrorCode *status)
 {
@@ -222,23 +245,20 @@ struct icu_chain *icu_chain_xml_config(const xmlNode *xml_node,
     int no_errors = 0;
     struct icu_chain *chain = 0;
     NMEM nmem = 0;
-   
+
     *status = U_ZERO_ERROR;
 
-    if (!xml_node ||xml_node->type != XML_ELEMENT_NODE)
-        return 0;
-    
+    if (xml_node && xml_node->type == XML_ELEMENT_NODE)
     {
-        xmlChar *xml_locale = xmlGetProp((xmlNode *) xml_node, 
+        xmlChar *xml_locale = xmlGetProp((xmlNode *) xml_node,
                                          (xmlChar *) "locale");
-        
         if (xml_locale)
         {
             chain = icu_chain_create((const char *) xml_locale, sort, status);
             xmlFree(xml_locale);
         }
-        
     }
+
     if (!chain)
         return 0;
 
@@ -264,33 +284,48 @@ struct icu_chain *icu_chain_xml_config(const xmlNode *xml_node,
                 yaz_log(YLOG_WARN, "Unsupported attribute '%s' for "
                         "element '%s'", attr->name, node->name);
                 no_errors++;
-                continue;
             }
         }
         if (!rule && node->children)
             rule = nmem_text_node_cdata(node->children, nmem);
-        
+
+        if (!rule && strcmp((const char *) node->name, "display"))
+        {
+            yaz_log(YLOG_WARN, "Missing attribute rule for element %s",
+                    (const char *) node->name);
+            no_errors++;
+            continue;
+        }
         if (!strcmp((const char *) node->name, "casemap"))
-            step = icu_chain_insert_step(chain, ICU_chain_step_type_casemap, 
-                                         (const uint8_t *) rule, status);
+            step = icu_chain_insert_step(chain,
+                                         ICU_chain_step_type_casemap,
+                                         rule, status);
         else if (!strcmp((const char *) node->name, "transform"))
-            step = icu_chain_insert_step(chain, ICU_chain_step_type_transform, 
-                                         (const uint8_t *) rule, status);
+            step = icu_chain_insert_step(chain,
+                                         ICU_chain_step_type_transform,
+                                         rule, status);
         else if (!strcmp((const char *) node->name, "transliterate"))
-            step = icu_chain_insert_step(chain, ICU_chain_step_type_transliterate, 
-                                         (const uint8_t *) rule, status);
+            step = icu_chain_insert_step(chain,
+                                         ICU_chain_step_type_transliterate,
+                                         rule, status);
         else if (!strcmp((const char *) node->name, "tokenize"))
-            step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize, 
-                                         (const uint8_t *) rule, status);
+            step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize,
+                                         rule, status);
         else if (!strcmp((const char *) node->name, "display"))
-            step = icu_chain_insert_step(chain, ICU_chain_step_type_display, 
-                                         (const uint8_t *) "", status);
+            step = icu_chain_insert_step(chain, ICU_chain_step_type_display,
+                                         rule, status);
+        else if (!strcmp((const char *) node->name, "stemming"))
+            step = icu_chain_insert_step(chain, YAZ_chain_step_type_stemming,
+                                         rule, status);
+        else if (!strcmp((const char *) node->name, "join"))
+            step = icu_chain_insert_step(chain, ICU_chain_step_type_join,
+                                         rule, status);
         else if (!strcmp((const char *) node->name, "normalize"))
         {
             yaz_log(YLOG_WARN, "Element %s is deprecated. "
                     "Use transform instead", node->name);
-            step = icu_chain_insert_step(chain, ICU_chain_step_type_transform, 
-                                         (const uint8_t *) rule, status);
+            step = icu_chain_insert_step(chain, ICU_chain_step_type_transform,
+                                         rule, status);
         }
         else if (!strcmp((const char *) node->name, "index")
                  || !strcmp((const char *) node->name, "sortkey"))
@@ -304,6 +339,11 @@ struct icu_chain *icu_chain_xml_config(const xmlNode *xml_node,
             no_errors++;
             continue;
         }
+        if (!step)
+        {
+            yaz_log(YLOG_WARN, "Step not created for %s", node->name);
+            no_errors++;
+        }
         if (step && U_FAILURE(*status))
         {
             no_errors++;
@@ -319,34 +359,17 @@ struct icu_chain *icu_chain_xml_config(const xmlNode *xml_node,
     return chain;
 }
 
-
-static struct icu_chain_step *icu_chain_insert_step(
-    struct icu_chain *chain, enum icu_chain_step_type type,
-    const uint8_t *rule, UErrorCode *status)
-{    
-    struct icu_chain_step *step = 0;
-    if (!chain || !type || !rule)
-        return 0;
-
-    /* create actual chain step with this buffer */
-    step = icu_chain_step_create(chain, type, rule,
-                                 status);
-
-    step->previous = chain->csteps;
-    chain->csteps = step;
-
-    return step;
-}
-
 struct icu_iter {
     struct icu_chain *chain;
     struct icu_buf_utf16 *last;
+    struct icu_buf_utf16 *org;
     UErrorCode status;
     struct icu_buf_utf8 *display;
     struct icu_buf_utf8 *sort8;
     struct icu_buf_utf8 *result;
-    struct icu_buf_utf16 *input;
     int token_count;
+    size_t org_start;
+    size_t org_len;
     struct icu_chain_step *steps;
 };
 
@@ -357,7 +380,6 @@ void icu_utf16_print(struct icu_buf_utf16 *src16)
     struct icu_buf_utf8 *dst8 = icu_buf_utf8_create(0);
     icu_utf16_to_utf8(dst8, src16, &status);
 
-    assert(status != 1234);
     if (U_FAILURE(status))
     {
         printf("failure");
@@ -379,7 +401,7 @@ struct icu_buf_utf16 *icu_iter_invoke(yaz_icu_iter_t iter,
     else
     {
         struct icu_buf_utf16 *dst = icu_iter_invoke(iter, step->previous, src);
-        
+
         switch (step->type)
         {
         case ICU_chain_step_type_casemap:
@@ -403,7 +425,8 @@ struct icu_buf_utf16 *icu_iter_invoke(yaz_icu_iter_t iter,
             }
             dst = icu_buf_utf16_create(0);
             iter->status = U_ZERO_ERROR;
-            if (!icu_tokenizer_next_token(step->u.tokenizer, dst, &iter->status))
+            if (!icu_tokenizer_next_token(step->u.tokenizer, dst, &iter->status,
+                                          &iter->org_start, &iter->org_len))
             {
                 icu_buf_utf16_destroy(dst);
                 dst = 0;
@@ -423,6 +446,31 @@ struct icu_buf_utf16 *icu_iter_invoke(yaz_icu_iter_t iter,
             if (dst)
                 icu_utf16_to_utf8(iter->display, dst, &iter->status);
             break;
+        case YAZ_chain_step_type_stemming:
+            if (dst)
+            {
+                struct icu_buf_utf16 *src = dst;
+                dst = icu_buf_utf16_create(0);
+                yaz_stemmer_stem(step->u.stemmer, dst, src, &iter->status);
+                icu_buf_utf16_destroy(src);
+            }
+            break;
+        case ICU_chain_step_type_join:
+            if (dst)
+            {
+                while (1)
+                {
+                    struct icu_buf_utf16 *dst1 =
+                        icu_iter_invoke(iter, step->previous, 0);
+
+                    if (!dst1)
+                        break; 
+                    dst = icu_buf_utf16_append(dst, step->u.join);
+                    dst = icu_buf_utf16_append(dst, dst1);
+                    icu_buf_utf16_destroy(dst1);
+                }
+            }
+            break;
         default:
             assert(0);
         }
@@ -438,22 +486,23 @@ yaz_icu_iter_t icu_iter_create(struct icu_chain *chain)
     iter->display = icu_buf_utf8_create(0);
     iter->sort8 = icu_buf_utf8_create(0);
     iter->result = icu_buf_utf8_create(0);
+    iter->org = icu_buf_utf16_create(0);
     iter->last = 0; /* no last returned string (yet) */
     iter->steps = icu_chain_step_clone(chain->csteps);
-    iter->input = 0;
+    iter->token_count = 0;
 
     return iter;
 }
 
 void icu_iter_first(yaz_icu_iter_t iter, const char *src8cstr)
 {
-    if (iter->input)
-        icu_buf_utf16_destroy(iter->input);
-    iter->input = icu_buf_utf16_create(0);
+    struct icu_buf_utf16 *src = icu_buf_utf16_create(0);
+    icu_utf16_from_utf8_cstr(src, src8cstr, &iter->status);
+    icu_buf_utf16_copy(iter->org, src);
     iter->token_count = 0;
-    /* fill and assign input string.. It will be 0 after
-       first iteration */
-    icu_utf16_from_utf8_cstr(iter->input, src8cstr, &iter->status);
+    iter->org_start = 0;
+    iter->org_len = src->utf16_len;
+    iter->last = icu_iter_invoke(iter, iter->steps, src);
 }
 
 void icu_iter_destroy(yaz_icu_iter_t iter)
@@ -463,8 +512,7 @@ void icu_iter_destroy(yaz_icu_iter_t iter)
         icu_buf_utf8_destroy(iter->display);
         icu_buf_utf8_destroy(iter->sort8);
         icu_buf_utf8_destroy(iter->result);
-        if (iter->input)
-            icu_buf_utf16_destroy(iter->input);
+        icu_buf_utf16_destroy(iter->org);
         icu_chain_step_destroy(iter->steps);
         xfree(iter);
     }
@@ -472,23 +520,15 @@ void icu_iter_destroy(yaz_icu_iter_t iter)
 
 int icu_iter_next(yaz_icu_iter_t iter)
 {
-    if (!iter->input && iter->last == 0)
+    if (iter->token_count && iter->last)
+        iter->last = icu_iter_invoke(iter, iter->steps, 0);
+    if (!iter->last)
         return 0;
     else
     {
-        /* on first call, iter->input is the input string. Thereafter: 0. */
-        iter->last = icu_iter_invoke(iter, iter->steps ?
-                                     iter->steps : iter->chain->csteps,
-                                     iter->input);
-        iter->input = 0;
-        
-        if (!iter->last)
-            return 0;
-
         iter->token_count++;
-
         if (iter->chain->sort)
-        {        
+        {
             icu_sortkey8_from_utf16(iter->chain->coll,
                                     iter->sort8, iter->last,
                                     &iter->status);
@@ -511,16 +551,40 @@ const char *icu_iter_get_sortkey(yaz_icu_iter_t iter)
 }
 
 const char *icu_iter_get_display(yaz_icu_iter_t iter)
-{ 
-    return icu_buf_utf8_to_cstr(iter->display);   
+{
+    return icu_buf_utf8_to_cstr(iter->display);
 }
 
 int icu_iter_get_token_number(yaz_icu_iter_t iter)
-{ 
+{
     return iter->token_count;
 }
 
-int icu_chain_assign_cstr(struct icu_chain *chain, const char *src8cstr, 
+
+void icu_iter_get_org_info(yaz_icu_iter_t iter, size_t *start, size_t *len)
+{
+    /* save full length of org since we're gonna cut it */
+    int32_t save_len = iter->org->utf16_len;
+
+    struct icu_buf_utf8 *tmp = icu_buf_utf8_create(0);
+    UErrorCode status = U_ZERO_ERROR;
+
+    iter->org->utf16_len = iter->org_start;
+    icu_utf16_to_utf8(tmp, iter->org, &status);
+    if (U_SUCCESS(status))
+        *start = tmp->utf8_len;
+    else
+        *start = 0;
+    iter->org->utf16_len = iter->org_start + iter->org_len;
+    icu_utf16_to_utf8(tmp, iter->org, &status);
+    if (U_SUCCESS(status))
+        *len = tmp->utf8_len - *start;
+    else
+        *len = 0;
+    iter->org->utf16_len = save_len;
+}
+
+int icu_chain_assign_cstr(struct icu_chain *chain, const char *src8cstr,
                           UErrorCode *status)
 {
     if (chain->iter)
@@ -564,6 +628,13 @@ const char *icu_chain_token_sortkey(struct icu_chain *chain)
     return 0;
 }
 
+void icu_chain_get_org_info(struct icu_chain *chain, size_t *start, size_t *len)
+{
+    if (chain->iter)
+        icu_iter_get_org_info(chain->iter, start, len);
+}
+
+
 #endif /* YAZ_HAVE_ICU */
 
 /*