X-Git-Url: http://git.indexdata.com/?p=yaz-moved-to-github.git;a=blobdiff_plain;f=src%2Ficu_chain.c;h=8cc9a6457fc05b5cc89bb510c933b48b8548b61c;hp=c6f21ef642dd1ea9a6831930751a07dbcf1cb059;hb=9e04bbdbec82ea6f5795dc2d16bd9876ecc7762f;hpb=ed329d8fb558a860215b29aea90ff688488bcf45

diff --git a/src/icu_chain.c b/src/icu_chain.c
index c6f21ef..8cc9a64 100644
--- a/src/icu_chain.c
+++ b/src/icu_chain.c
@@ -17,6 +17,8 @@
 
 #include <yaz/icu_I18N.h>
 
+#include <yaz/stemmer.h>
+
 #include <yaz/log.h>
 #include <yaz/nmem.h>
 #include <yaz/nmem_xml.h>
@@ -30,11 +32,12 @@
 
 enum icu_chain_step_type {
     ICU_chain_step_type_none,
-    ICU_chain_step_type_display,   /* convert to utf8 display format */
-    ICU_chain_step_type_casemap,   /* apply utf16 charmap */
-    ICU_chain_step_type_transform, /* apply utf16 transform */
-    ICU_chain_step_type_tokenize,  /* apply utf16 tokenization */
-    ICU_chain_step_type_transliterate  /* apply utf16 tokenization */
+    ICU_chain_step_type_display,        /* convert to utf8 display format */
+    ICU_chain_step_type_casemap,        /* apply utf16 charmap */
+    ICU_chain_step_type_transform,      /* apply utf16 transform */
+    ICU_chain_step_type_tokenize,       /* apply utf16 tokenization */
+    ICU_chain_step_type_transliterate,  /* apply utf16 tokenization */
+    YAZ_chain_step_type_stemming        /* apply utf16 stemming (YAZ) */
 };
 
 struct icu_chain_step
@@ -42,9 +45,10 @@ struct icu_chain_step
     /* type and action object */
     enum icu_chain_step_type type;
     union {
-	struct icu_casemap *casemap;
+	struct icu_casemap   *casemap;
 	struct icu_transform *transform;
 	struct icu_tokenizer *tokenizer;  
+        yaz_stemmer_p         stemmer;
     } u;
     struct icu_chain_step *previous;
 };
@@ -71,8 +75,8 @@ int icu_check_status(UErrorCode status)
     return 1;
 }
 
-static struct icu_chain_step *icu_chain_step_create(
-    struct icu_chain *chain,  enum icu_chain_step_type type,
+static struct icu_chain_step *icu_chain_insert_step(
+    struct icu_chain *chain, enum icu_chain_step_type type,
     const uint8_t *rule, UErrorCode *status)
 {
     struct icu_chain_step *step = 0;
@@ -105,9 +109,15 @@ static struct icu_chain_step *icu_chain_step_create(
         step->u.transform = icu_transform_create("custom", 'f',
                                                  (const char *) rule, status);
         break;
+    case YAZ_chain_step_type_stemming:
+        step->u.stemmer = yaz_stemmer_create((char *) chain->locale, (const char *) rule, status);
+        break;
     default:
         break;
     }
+    step->previous = chain->csteps;
+    chain->csteps = step;
+
     return step;
 }
 
@@ -133,6 +143,9 @@ static void icu_chain_step_destroy(struct icu_chain_step *step)
     case ICU_chain_step_type_tokenize:
         icu_tokenizer_destroy(step->u.tokenizer);
         break;
+    case YAZ_chain_step_type_stemming:
+        yaz_stemmer_destroy(step->u.stemmer);
+        break;
     default:
         break;
     }
@@ -162,6 +175,9 @@ struct icu_chain_step *icu_chain_step_clone(struct icu_chain_step *old)
         case ICU_chain_step_type_tokenize:
             (*sp)->u.tokenizer = icu_tokenizer_clone(old->u.tokenizer);
             break;
+        case YAZ_chain_step_type_stemming:
+            yaz_stemmer_clone(step->u.stemmer);
+            break;
         case ICU_chain_step_type_none:
             break;
         }
@@ -270,13 +286,6 @@ struct icu_chain *icu_chain_xml_config(const xmlNode *xml_node,
         if (!rule && node->children)
             rule = nmem_text_node_cdata(node->children, nmem);
         
-        if (!rule || !*rule)
-        {
-            yaz_log(YLOG_WARN, "Missing rule for element '%s'", node->name);
-            no_errors++;
-            continue;
-        }
-            
         if (!strcmp((const char *) node->name, "casemap"))
             step = icu_chain_insert_step(chain, ICU_chain_step_type_casemap, 
                                          (const uint8_t *) rule, status);
@@ -292,6 +301,9 @@ struct icu_chain *icu_chain_xml_config(const xmlNode *xml_node,
         else if (!strcmp((const char *) node->name, "display"))
             step = icu_chain_insert_step(chain, ICU_chain_step_type_display, 
                                          (const uint8_t *) "", status);
+        else if (!strcmp((const char *) node->name, "stemming"))
+            step = icu_chain_insert_step(chain, YAZ_chain_step_type_stemming,
+                                         (const uint8_t *) rule, status);
         else if (!strcmp((const char *) node->name, "normalize"))
         {
             yaz_log(YLOG_WARN, "Element %s is deprecated. "
@@ -326,25 +338,6 @@ struct icu_chain *icu_chain_xml_config(const xmlNode *xml_node,
     return chain;
 }
 
-
-static struct icu_chain_step *icu_chain_insert_step(
-    struct icu_chain *chain, enum icu_chain_step_type type,
-    const uint8_t *rule, UErrorCode *status)
-{    
-    struct icu_chain_step *step = 0;
-    if (!chain || !type || !rule)
-        return 0;
-
-    /* create actual chain step with this buffer */
-    step = icu_chain_step_create(chain, type, rule,
-                                 status);
-
-    step->previous = chain->csteps;
-    chain->csteps = step;
-
-    return step;
-}
-
 struct icu_iter {
     struct icu_chain *chain;
     struct icu_buf_utf16 *last;
@@ -430,6 +423,15 @@ struct icu_buf_utf16 *icu_iter_invoke(yaz_icu_iter_t iter,
             if (dst)
                 icu_utf16_to_utf8(iter->display, dst, &iter->status);
             break;
+        case YAZ_chain_step_type_stemming:
+            if (dst)
+            {
+                struct icu_buf_utf16 *src = dst;
+                dst = icu_buf_utf16_create(0);
+                yaz_stemmer_stem(step->u.stemmer, dst, src, &iter->status);
+                icu_buf_utf16_destroy(src);
+            }
+            break;
         default:
             assert(0);
         }