X-Git-Url: http://git.indexdata.com/?p=yaz-moved-to-github.git;a=blobdiff_plain;f=src%2Ficu_chain.c;h=de2e627c6ac5e980c87b807242f98b574422e6a7;hp=5354e89fc9855163ba6502b4c4e65e8ae20b072f;hb=94b1547e5951e1e01bf5180159e74095cd0527f4;hpb=0e948cf6b84b6e601f3c7c12c83921e2753cdd1b

diff --git a/src/icu_chain.c b/src/icu_chain.c
index 5354e89..de2e627 100644
--- a/src/icu_chain.c
+++ b/src/icu_chain.c
@@ -1,5 +1,5 @@
 /* This file is part of the YAZ toolkit.
- * Copyright (C) 1995-2010 Index Data
+ * Copyright (C) 1995-2013 Index Data
  * See the file LICENSE for details.
  */
 
@@ -17,8 +17,11 @@
 
 #include <yaz/icu_I18N.h>
 
-#include <yaz/log.h>
+#include <yaz/stemmer.h>
 
+#include <yaz/log.h>
+#include <yaz/nmem.h>
+#include <yaz/nmem_xml.h>
 #include <string.h>
 #include <stdlib.h>
 #include <stdio.h>
@@ -29,11 +32,13 @@
 
 enum icu_chain_step_type {
     ICU_chain_step_type_none,
-    ICU_chain_step_type_display,   /* convert to utf8 display format */
-    ICU_chain_step_type_casemap,   /* apply utf16 charmap */
-    ICU_chain_step_type_transform, /* apply utf16 transform */
-    ICU_chain_step_type_tokenize,  /* apply utf16 tokenization */
-    ICU_chain_step_type_transliterate  /* apply utf16 tokenization */
+    ICU_chain_step_type_display,        /* convert to utf8 display format */
+    ICU_chain_step_type_casemap,        /* apply utf16 charmap */
+    ICU_chain_step_type_transform,      /* apply utf16 transform */
+    ICU_chain_step_type_tokenize,       /* apply utf16 tokenization */
+    ICU_chain_step_type_transliterate,  /* apply utf16 tokenization */
+    YAZ_chain_step_type_stemming,       /* apply utf16 stemming (YAZ) */
+    ICU_chain_step_type_join
 };
 
 struct icu_chain_step
@@ -41,9 +46,11 @@ struct icu_chain_step
     /* type and action object */
     enum icu_chain_step_type type;
     union {
-	struct icu_casemap *casemap;
+	struct icu_casemap   *casemap;
 	struct icu_transform *transform;
-	struct icu_tokenizer *tokenizer;  
+	struct icu_tokenizer *tokenizer;
+        yaz_stemmer_p         stemmer;
+        struct icu_buf_utf16 *join;
     } u;
     struct icu_chain_step *previous;
 };
@@ -55,7 +62,7 @@ struct icu_chain
     int sort;
 
     UCollator *coll;
-    
+
     /* linked list of chain steps */
     struct icu_chain_step *csteps;
 };
@@ -65,48 +72,60 @@ int icu_check_status(UErrorCode status)
     if (U_FAILURE(status))
     {
         yaz_log(YLOG_WARN, "ICU: %d %s\n", status, u_errorName(status));
-        return 0;   
+        return 0;
     }
     return 1;
 }
 
-static struct icu_chain_step *icu_chain_step_create(
-    struct icu_chain *chain,  enum icu_chain_step_type type,
-    const uint8_t *rule, UErrorCode *status)
+static struct icu_chain_step *icu_chain_insert_step(
+    struct icu_chain *chain, enum icu_chain_step_type type,
+    const char *rule, UErrorCode *status)
 {
     struct icu_chain_step *step = 0;
-    
-    if (!chain || !type || !rule)
-        return 0;
 
-    step = (struct icu_chain_step *) xmalloc(sizeof(*step));
+    assert(chain);
+    assert(type);
 
+    step = (struct icu_chain_step *) xmalloc(sizeof(*step));
     step->type = type;
-    /* create auxilary objects */
+
     switch (step->type)
     {
     case ICU_chain_step_type_display:
         break;
     case ICU_chain_step_type_casemap:
+        assert(rule);
         step->u.casemap = icu_casemap_create(rule[0], status);
         break;
     case ICU_chain_step_type_transform:
+        assert(rule);
         /* rule omitted. Only ID used */
-        step->u.transform = icu_transform_create((const char *) rule, 'f',
-                                                 0, status);
+        step->u.transform = icu_transform_create(rule, 'f', 0, status);
         break;
     case ICU_chain_step_type_tokenize:
-        step->u.tokenizer = icu_tokenizer_create((char *) chain->locale, 
-                                                 (char) rule[0], status);
+        assert(rule);
+        step->u.tokenizer = icu_tokenizer_create(chain->locale, rule[0], status);
         break;
     case ICU_chain_step_type_transliterate:
+        assert(rule);
         /* we pass a dummy ID to utrans_openU.. */
-        step->u.transform = icu_transform_create("custom", 'f',
-                                                 (const char *) rule, status);
+        step->u.transform = icu_transform_create("custom", 'f', rule, status);
+        break;
+    case YAZ_chain_step_type_stemming:
+        assert(rule);
+        step->u.stemmer = yaz_stemmer_create(chain->locale, rule, status);
+        break;
+    case ICU_chain_step_type_join:
+        assert(rule);
+        step->u.join = icu_buf_utf16_create(0);
+        icu_utf16_from_utf8_cstr(step->u.join, rule, status);
         break;
     default:
         break;
     }
+    step->previous = chain->csteps;
+    chain->csteps = step;
+
     return step;
 }
 
@@ -132,6 +151,12 @@ static void icu_chain_step_destroy(struct icu_chain_step *step)
     case ICU_chain_step_type_tokenize:
         icu_tokenizer_destroy(step->u.tokenizer);
         break;
+    case YAZ_chain_step_type_stemming:
+        yaz_stemmer_destroy(step->u.stemmer);
+        break;
+    case ICU_chain_step_type_join:
+        icu_buf_utf16_destroy(step->u.join);
+        break;
     default:
         break;
     }
@@ -146,7 +171,7 @@ struct icu_chain_step *icu_chain_step_clone(struct icu_chain_step *old)
     {
         *sp = (struct icu_chain_step *) xmalloc(sizeof(**sp));
         (*sp)->type = old->type;
-        
+
         switch ((*sp)->type)
         {
         case ICU_chain_step_type_display:
@@ -161,8 +186,15 @@ struct icu_chain_step *icu_chain_step_clone(struct icu_chain_step *old)
         case ICU_chain_step_type_tokenize:
             (*sp)->u.tokenizer = icu_tokenizer_clone(old->u.tokenizer);
             break;
+        case YAZ_chain_step_type_stemming:
+            (*sp)->u.stemmer = yaz_stemmer_clone(old->u.stemmer);
+            break;
         case ICU_chain_step_type_none:
             break;
+        case ICU_chain_step_type_join:
+            (*sp)->u.join = icu_buf_utf16_create(0);
+            (*sp)->u.join = icu_buf_utf16_copy((*sp)->u.join, old->u.join);
+            break;
         }
         old = old->previous;
         sp = &(*sp)->previous;
@@ -174,21 +206,17 @@ struct icu_chain_step *icu_chain_step_clone(struct icu_chain_step *old)
 struct icu_chain *icu_chain_create(const char *locale, int sort,
                                    UErrorCode *status)
 {
-    struct icu_chain *chain 
-        = (struct icu_chain *) xmalloc(sizeof(*chain));
+    struct icu_chain *chain;
+    UCollator *coll = ucol_open(locale, status);
 
-    *status = U_ZERO_ERROR;
+    if (U_FAILURE(*status))
+        return 0;
 
+    chain = (struct icu_chain *) xmalloc(sizeof(*chain));
     chain->iter = 0;
     chain->locale = xstrdup(locale);
-
     chain->sort = sort;
-
-    chain->coll = ucol_open((const char *) chain->locale, status);
-
-    if (U_FAILURE(*status))
-        return 0;
-
+    chain->coll = coll;
     chain->csteps = 0;
 
     return chain;
@@ -209,69 +237,95 @@ void icu_chain_destroy(struct icu_chain *chain)
     }
 }
 
-static struct icu_chain_step *icu_chain_insert_step(
-    struct icu_chain *chain, enum icu_chain_step_type type,
-    const uint8_t *rule, UErrorCode *status);
-
-struct icu_chain *icu_chain_xml_config(const xmlNode *xml_node, 
+struct icu_chain *icu_chain_xml_config(const xmlNode *xml_node,
                                        int sort,
                                        UErrorCode *status)
 {
     xmlNode *node = 0;
+    int no_errors = 0;
     struct icu_chain *chain = 0;
-   
+    NMEM nmem = 0;
+
     *status = U_ZERO_ERROR;
 
-    if (!xml_node ||xml_node->type != XML_ELEMENT_NODE)
-        return 0;
-    
+    if (xml_node && xml_node->type == XML_ELEMENT_NODE)
     {
-        xmlChar *xml_locale = xmlGetProp((xmlNode *) xml_node, 
+        xmlChar *xml_locale = xmlGetProp((xmlNode *) xml_node,
                                          (xmlChar *) "locale");
-        
         if (xml_locale)
         {
             chain = icu_chain_create((const char *) xml_locale, sort, status);
             xmlFree(xml_locale);
         }
-        
     }
+
     if (!chain)
         return 0;
 
+    nmem = nmem_create();
     for (node = xml_node->children; node; node = node->next)
     {
-        xmlChar *xml_rule;
+        char *rule = 0;
         struct icu_chain_step *step = 0;
+        struct _xmlAttr *attr;
 
+        nmem_reset(nmem);
         if (node->type != XML_ELEMENT_NODE)
             continue;
 
-        xml_rule = xmlGetProp(node, (xmlChar *) "rule");
-
-        yaz_log(YLOG_LOG, "rule=%s", xml_rule);
+        for (attr = node->properties; attr; attr = attr->next)
+        {
+            if (!strcmp((const char *) attr->name, "rule"))
+            {
+                rule = nmem_text_node_cdata(attr->children, nmem);
+            }
+            else
+            {
+                yaz_log(YLOG_WARN, "Unsupported attribute '%s' for "
+                        "element '%s'", attr->name, node->name);
+                no_errors++;
+            }
+        }
+        if (!rule && node->children)
+            rule = nmem_text_node_cdata(node->children, nmem);
 
+        if (!rule && strcmp((const char *) node->name, "display"))
+        {
+            yaz_log(YLOG_WARN, "Missing attribute rule for element %s",
+                    (const char *) node->name);
+            no_errors++;
+            continue;
+        }
         if (!strcmp((const char *) node->name, "casemap"))
-            step = icu_chain_insert_step(chain, ICU_chain_step_type_casemap, 
-                                         (const uint8_t *) xml_rule, status);
+            step = icu_chain_insert_step(chain,
+                                         ICU_chain_step_type_casemap,
+                                         rule, status);
         else if (!strcmp((const char *) node->name, "transform"))
-            step = icu_chain_insert_step(chain, ICU_chain_step_type_transform, 
-                                         (const uint8_t *) xml_rule, status);
+            step = icu_chain_insert_step(chain,
+                                         ICU_chain_step_type_transform,
+                                         rule, status);
         else if (!strcmp((const char *) node->name, "transliterate"))
-            step = icu_chain_insert_step(chain, ICU_chain_step_type_transliterate, 
-                                         (const uint8_t *) xml_rule, status);
+            step = icu_chain_insert_step(chain,
+                                         ICU_chain_step_type_transliterate,
+                                         rule, status);
         else if (!strcmp((const char *) node->name, "tokenize"))
-            step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize, 
-                                         (const uint8_t *) xml_rule, status);
+            step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize,
+                                         rule, status);
         else if (!strcmp((const char *) node->name, "display"))
-            step = icu_chain_insert_step(chain, ICU_chain_step_type_display, 
-                                         (const uint8_t *) "", status);
+            step = icu_chain_insert_step(chain, ICU_chain_step_type_display,
+                                         rule, status);
+        else if (!strcmp((const char *) node->name, "stemming"))
+            step = icu_chain_insert_step(chain, YAZ_chain_step_type_stemming,
+                                         rule, status);
+        else if (!strcmp((const char *) node->name, "join"))
+            step = icu_chain_insert_step(chain, ICU_chain_step_type_join,
+                                         rule, status);
         else if (!strcmp((const char *) node->name, "normalize"))
         {
             yaz_log(YLOG_WARN, "Element %s is deprecated. "
                     "Use transform instead", node->name);
-            step = icu_chain_insert_step(chain, ICU_chain_step_type_transform, 
-                                         (const uint8_t *) xml_rule, status);
+            step = icu_chain_insert_step(chain, ICU_chain_step_type_transform,
+                                         rule, status);
         }
         else if (!strcmp((const char *) node->name, "index")
                  || !strcmp((const char *) node->name, "sortkey"))
@@ -282,36 +336,27 @@ struct icu_chain *icu_chain_xml_config(const xmlNode *xml_node,
         else
         {
             yaz_log(YLOG_WARN, "Unknown element %s", node->name);
-            icu_chain_destroy(chain);
-            return 0;
+            no_errors++;
+            continue;
+        }
+        if (!step)
+        {
+            yaz_log(YLOG_WARN, "Step not created for %s", node->name);
+            no_errors++;
         }
-        xmlFree(xml_rule);
         if (step && U_FAILURE(*status))
         {
-            icu_chain_destroy(chain);
-            return 0;
+            no_errors++;
+            break;
         }
     }
-    return chain;
-}
-
-
-static struct icu_chain_step *icu_chain_insert_step(
-    struct icu_chain *chain, enum icu_chain_step_type type,
-    const uint8_t *rule, UErrorCode *status)
-{    
-    struct icu_chain_step *step = 0;
-    if (!chain || !type || !rule)
+    nmem_destroy(nmem);
+    if (no_errors)
+    {
+        icu_chain_destroy(chain);
         return 0;
-
-    /* create actual chain step with this buffer */
-    step = icu_chain_step_create(chain, type, rule,
-                                 status);
-
-    step->previous = chain->csteps;
-    chain->csteps = step;
-
-    return step;
+    }
+    return chain;
 }
 
 struct icu_iter {
@@ -323,6 +368,8 @@ struct icu_iter {
     struct icu_buf_utf8 *result;
     struct icu_buf_utf16 *input;
     int token_count;
+    size_t org_start;
+    size_t org_len;
     struct icu_chain_step *steps;
 };
 
@@ -333,7 +380,6 @@ void icu_utf16_print(struct icu_buf_utf16 *src16)
     struct icu_buf_utf8 *dst8 = icu_buf_utf8_create(0);
     icu_utf16_to_utf8(dst8, src16, &status);
 
-    assert(status != 1234);
     if (U_FAILURE(status))
     {
         printf("failure");
@@ -355,7 +401,7 @@ struct icu_buf_utf16 *icu_iter_invoke(yaz_icu_iter_t iter,
     else
     {
         struct icu_buf_utf16 *dst = icu_iter_invoke(iter, step->previous, src);
-        
+
         switch (step->type)
         {
         case ICU_chain_step_type_casemap:
@@ -379,7 +425,8 @@ struct icu_buf_utf16 *icu_iter_invoke(yaz_icu_iter_t iter,
             }
             dst = icu_buf_utf16_create(0);
             iter->status = U_ZERO_ERROR;
-            if (!icu_tokenizer_next_token(step->u.tokenizer, dst, &iter->status))
+            if (!icu_tokenizer_next_token(step->u.tokenizer, dst, &iter->status,
+                                          &iter->org_start, &iter->org_len))
             {
                 icu_buf_utf16_destroy(dst);
                 dst = 0;
@@ -399,6 +446,31 @@ struct icu_buf_utf16 *icu_iter_invoke(yaz_icu_iter_t iter,
             if (dst)
                 icu_utf16_to_utf8(iter->display, dst, &iter->status);
             break;
+        case YAZ_chain_step_type_stemming:
+            if (dst)
+            {
+                struct icu_buf_utf16 *src = dst;
+                dst = icu_buf_utf16_create(0);
+                yaz_stemmer_stem(step->u.stemmer, dst, src, &iter->status);
+                icu_buf_utf16_destroy(src);
+            }
+            break;
+        case ICU_chain_step_type_join:
+            if (dst)
+            {
+                while (1)
+                {
+                    struct icu_buf_utf16 *dst1 =
+                        icu_iter_invoke(iter, step->previous, 0);
+
+                    if (!dst1)
+                        break; 
+                    dst = icu_buf_utf16_append(dst, step->u.join);
+                    dst = icu_buf_utf16_append(dst, dst1);
+                    icu_buf_utf16_destroy(dst1);
+                }
+            }
+            break;
         default:
             assert(0);
         }
@@ -430,6 +502,8 @@ void icu_iter_first(yaz_icu_iter_t iter, const char *src8cstr)
     /* fill and assign input string.. It will be 0 after
        first iteration */
     icu_utf16_from_utf8_cstr(iter->input, src8cstr, &iter->status);
+    iter->org_start = 0;
+    iter->org_len = iter->input->utf16_len;
 }
 
 void icu_iter_destroy(yaz_icu_iter_t iter)
@@ -453,18 +527,17 @@ int icu_iter_next(yaz_icu_iter_t iter)
     else
     {
         /* on first call, iter->input is the input string. Thereafter: 0. */
-        iter->last = icu_iter_invoke(iter, iter->steps ?
-                                     iter->steps : iter->chain->csteps,
-                                     iter->input);
+        assert(iter->steps || !iter->chain->csteps);
+        iter->last = icu_iter_invoke(iter, iter->steps, iter->input);
         iter->input = 0;
-        
+
         if (!iter->last)
             return 0;
 
         iter->token_count++;
 
         if (iter->chain->sort)
-        {        
+        {
             icu_sortkey8_from_utf16(iter->chain->coll,
                                     iter->sort8, iter->last,
                                     &iter->status);
@@ -487,16 +560,23 @@ const char *icu_iter_get_sortkey(yaz_icu_iter_t iter)
 }
 
 const char *icu_iter_get_display(yaz_icu_iter_t iter)
-{ 
-    return icu_buf_utf8_to_cstr(iter->display);   
+{
+    return icu_buf_utf8_to_cstr(iter->display);
 }
 
 int icu_iter_get_token_number(yaz_icu_iter_t iter)
-{ 
+{
     return iter->token_count;
 }
 
-int icu_chain_assign_cstr(struct icu_chain *chain, const char *src8cstr, 
+
+void icu_iter_get_org_info(yaz_icu_iter_t iter, size_t *start, size_t *len)
+{
+    *start = iter->org_start;
+    *len = iter->org_len;
+}
+
+int icu_chain_assign_cstr(struct icu_chain *chain, const char *src8cstr,
                           UErrorCode *status)
 {
     if (chain->iter)
@@ -540,6 +620,13 @@ const char *icu_chain_token_sortkey(struct icu_chain *chain)
     return 0;
 }
 
+void icu_chain_get_org_info(struct icu_chain *chain, size_t *start, size_t *len)
+{
+    if (chain->iter)
+        icu_iter_get_org_info(chain->iter, start, len);
+}
+
+
 #endif /* YAZ_HAVE_ICU */
 
 /*