New ICU chain rule, join, to join tokens

diff --git a/include/yaz/icu_I18N.h b/include/yaz/icu_I18N.h
index 5bcbb3f..b26cb60 100644 (file)
--- a/include/yaz/icu_I18N.h
+++ b/include/yaz/icu_I18N.h
@@ -66,6 +66,11 @@ struct icu_buf_utf16 * icu_buf_utf16_resize(struct icu_buf_utf16 * buf16,
 struct icu_buf_utf16 *icu_buf_utf16_copy(struct icu_buf_utf16 * dest16,
                                          const struct icu_buf_utf16 * src16);
 
 struct icu_buf_utf16 *icu_buf_utf16_copy(struct icu_buf_utf16 * dest16,
                                          const struct icu_buf_utf16 * src16);
 

+struct icu_buf_utf16 *icu_buf_utf16_append(struct icu_buf_utf16 *dest16,
+                                           const struct icu_buf_utf16 * src16);
+
+void icu_buf_utf16_log(const char *lead, struct icu_buf_utf16 *src16);
+

 void icu_buf_utf16_destroy(struct icu_buf_utf16 * buf16);
 
 struct icu_buf_utf8;
 void icu_buf_utf16_destroy(struct icu_buf_utf16 * buf16);
 
 struct icu_buf_utf8;

diff --git a/src/icu_chain.c b/src/icu_chain.c
index f62e9bb..e7b9b4e 100644 (file)
--- a/src/icu_chain.c
+++ b/src/icu_chain.c
@@ -37,7 +37,8 @@ enum icu_chain_step_type {
     ICU_chain_step_type_transform,      /* apply utf16 transform */
     ICU_chain_step_type_tokenize,       /* apply utf16 tokenization */
     ICU_chain_step_type_transliterate,  /* apply utf16 tokenization */
     ICU_chain_step_type_transform,      /* apply utf16 transform */
     ICU_chain_step_type_tokenize,       /* apply utf16 tokenization */
     ICU_chain_step_type_transliterate,  /* apply utf16 tokenization */

-    YAZ_chain_step_type_stemming        /* apply utf16 stemming (YAZ) */
+    YAZ_chain_step_type_stemming,       /* apply utf16 stemming (YAZ) */
+    ICU_chain_step_type_join

 };
 
 struct icu_chain_step
 };
 
 struct icu_chain_step

@@ -49,6 +50,7 @@ struct icu_chain_step
        struct icu_transform *transform;
        struct icu_tokenizer *tokenizer;
         yaz_stemmer_p         stemmer;
        struct icu_transform *transform;
        struct icu_tokenizer *tokenizer;
         yaz_stemmer_p         stemmer;

+        struct icu_buf_utf16 *join;

     } u;
     struct icu_chain_step *previous;
 };
     } u;
     struct icu_chain_step *previous;
 };

@@ -77,7 +79,7 @@ int icu_check_status(UErrorCode status)
 
 static struct icu_chain_step *icu_chain_insert_step(
     struct icu_chain *chain, enum icu_chain_step_type type,
 
 static struct icu_chain_step *icu_chain_insert_step(
     struct icu_chain *chain, enum icu_chain_step_type type,

-    const uint8_t *rule, UErrorCode *status)
+    const char *rule, UErrorCode *status)

 {
     struct icu_chain_step *step = 0;
 
 {
     struct icu_chain_step *step = 0;
 

@@ -97,21 +99,21 @@ static struct icu_chain_step *icu_chain_insert_step(
         break;
     case ICU_chain_step_type_transform:
         /* rule omitted. Only ID used */
         break;
     case ICU_chain_step_type_transform:
         /* rule omitted. Only ID used */

-        step->u.transform = icu_transform_create((const char *) rule, 'f',
-                                                 0, status);
+        step->u.transform = icu_transform_create(rule, 'f', 0, status);

         break;
     case ICU_chain_step_type_tokenize:
         break;
     case ICU_chain_step_type_tokenize:

-        step->u.tokenizer = icu_tokenizer_create(chain->locale,
-                                                 (char) rule[0], status);
+        step->u.tokenizer = icu_tokenizer_create(chain->locale, rule[0], status);

         break;
     case ICU_chain_step_type_transliterate:
         /* we pass a dummy ID to utrans_openU.. */
         break;
     case ICU_chain_step_type_transliterate:
         /* we pass a dummy ID to utrans_openU.. */

-        step->u.transform = icu_transform_create("custom", 'f',
-                                                 (const char *) rule, status);
+        step->u.transform = icu_transform_create("custom", 'f', rule, status);

         break;
     case YAZ_chain_step_type_stemming:
         break;
     case YAZ_chain_step_type_stemming:

-        step->u.stemmer = yaz_stemmer_create(chain->locale,
-                                             (const char *) rule, status);
+        step->u.stemmer = yaz_stemmer_create(chain->locale, rule, status);
+        break;
+    case ICU_chain_step_type_join:
+        step->u.join = icu_buf_utf16_create(0);
+        icu_utf16_from_utf8_cstr(step->u.join, rule, status);

         break;
     default:
         break;
         break;
     default:
         break;

@@ -147,6 +149,9 @@ static void icu_chain_step_destroy(struct icu_chain_step *step)
     case YAZ_chain_step_type_stemming:
         yaz_stemmer_destroy(step->u.stemmer);
         break;
     case YAZ_chain_step_type_stemming:
         yaz_stemmer_destroy(step->u.stemmer);
         break;

+    case ICU_chain_step_type_join:
+        icu_buf_utf16_destroy(step->u.join);
+        break;

     default:
         break;
     }
     default:
         break;
     }

@@ -181,6 +186,10 @@ struct icu_chain_step *icu_chain_step_clone(struct icu_chain_step *old)
             break;
         case ICU_chain_step_type_none:
             break;
             break;
         case ICU_chain_step_type_none:
             break;

+        case ICU_chain_step_type_join:
+            (*sp)->u.join = icu_buf_utf16_create(0);
+            (*sp)->u.join = icu_buf_utf16_copy((*sp)->u.join, old->u.join);
+            break;

         }
         old = old->previous;
         sp = &(*sp)->previous;
         }
         old = old->previous;
         sp = &(*sp)->previous;

@@ -277,29 +286,37 @@ struct icu_chain *icu_chain_xml_config(const xmlNode *xml_node,
             rule = nmem_text_node_cdata(node->children, nmem);
 
         if (!strcmp((const char *) node->name, "casemap"))
             rule = nmem_text_node_cdata(node->children, nmem);
 
         if (!strcmp((const char *) node->name, "casemap"))

-            step = icu_chain_insert_step(chain, ICU_chain_step_type_casemap,
-                                         (const uint8_t *) rule, status);
+            step = icu_chain_insert_step(chain,
+                                         ICU_chain_step_type_casemap,
+                                         rule, status);

         else if (!strcmp((const char *) node->name, "transform"))
         else if (!strcmp((const char *) node->name, "transform"))

-            step = icu_chain_insert_step(chain, ICU_chain_step_type_transform,
-                                         (const uint8_t *) rule, status);
+            step = icu_chain_insert_step(chain,
+                                         ICU_chain_step_type_transform,
+                                         rule, status);

         else if (!strcmp((const char *) node->name, "transliterate"))
         else if (!strcmp((const char *) node->name, "transliterate"))

-            step = icu_chain_insert_step(chain, ICU_chain_step_type_transliterate,
-                                         (const uint8_t *) rule, status);
+            step = icu_chain_insert_step(chain,
+                                         ICU_chain_step_type_transliterate,
+                                         rule, status);

         else if (!strcmp((const char *) node->name, "tokenize"))
             step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize,
         else if (!strcmp((const char *) node->name, "tokenize"))
             step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize,

-                                         (const uint8_t *) rule, status);
+                                         rule, status);

         else if (!strcmp((const char *) node->name, "display"))
             step = icu_chain_insert_step(chain, ICU_chain_step_type_display,
         else if (!strcmp((const char *) node->name, "display"))
             step = icu_chain_insert_step(chain, ICU_chain_step_type_display,

-                                         (const uint8_t *) "", status);
+                                         "", status);

         else if (!strcmp((const char *) node->name, "stemming"))
             step = icu_chain_insert_step(chain, YAZ_chain_step_type_stemming,
         else if (!strcmp((const char *) node->name, "stemming"))
             step = icu_chain_insert_step(chain, YAZ_chain_step_type_stemming,

-                                         (const uint8_t *) rule, status);
+                                         rule, status);
+        else if (!strcmp((const char *) node->name, "join"))
+        {
+            step = icu_chain_insert_step(chain, ICU_chain_step_type_join,
+                                         rule, status);
+        }

         else if (!strcmp((const char *) node->name, "normalize"))
         {
             yaz_log(YLOG_WARN, "Element %s is deprecated. "
                     "Use transform instead", node->name);
             step = icu_chain_insert_step(chain, ICU_chain_step_type_transform,
         else if (!strcmp((const char *) node->name, "normalize"))
         {
             yaz_log(YLOG_WARN, "Element %s is deprecated. "
                     "Use transform instead", node->name);
             step = icu_chain_insert_step(chain, ICU_chain_step_type_transform,

-                                         (const uint8_t *) rule, status);
+                                         rule, status);

         }
         else if (!strcmp((const char *) node->name, "index")
                  || !strcmp((const char *) node->name, "sortkey"))
         }
         else if (!strcmp((const char *) node->name, "index")
                  || !strcmp((const char *) node->name, "sortkey"))

@@ -313,6 +330,11 @@ struct icu_chain *icu_chain_xml_config(const xmlNode *xml_node,
             no_errors++;
             continue;
         }
             no_errors++;
             continue;
         }

+        if (!step)
+        {
+            yaz_log(YLOG_WARN, "Step not created for %s", node->name);
+            no_errors++;
+        }

         if (step && U_FAILURE(*status))
         {
             no_errors++;
         if (step && U_FAILURE(*status))
         {
             no_errors++;

@@ -421,6 +443,22 @@ struct icu_buf_utf16 *icu_iter_invoke(yaz_icu_iter_t iter,
                 icu_buf_utf16_destroy(src);
             }
             break;
                 icu_buf_utf16_destroy(src);
             }
             break;

+        case ICU_chain_step_type_join:
+            if (dst)
+            {
+                while (1)
+                {
+                    struct icu_buf_utf16 *dst1 =
+                        icu_iter_invoke(iter, step->previous, 0);
+
+                    if (!dst1)
+                        break; 
+                    dst = icu_buf_utf16_append(dst, step->u.join);
+                    dst = icu_buf_utf16_append(dst, dst1);
+                    icu_buf_utf16_destroy(dst1);
+                }
+            }
+            break;

         default:
             assert(0);
         }
         default:
             assert(0);
         }

diff --git a/src/icu_utf16.c b/src/icu_utf16.c
index 65298c2..466f7af 100644 (file)
--- a/src/icu_utf16.c
+++ b/src/icu_utf16.c
@@ -22,6 +22,7 @@
 #include <string.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
 #include <stdio.h>

+#include <assert.h>

 
 #include <unicode/ustring.h>  /* some more string fcns*/
 #include <unicode/uchar.h>    /* char names           */
 
 #include <unicode/ustring.h>  /* some more string fcns*/
 #include <unicode/uchar.h>    /* char names           */

@@ -68,17 +69,14 @@ struct icu_buf_utf16 * icu_buf_utf16_resize(struct icu_buf_utf16 * buf16,
         else
             buf16->utf16
                 = (UChar *) xrealloc(buf16->utf16, sizeof(UChar) * capacity);
         else
             buf16->utf16
                 = (UChar *) xrealloc(buf16->utf16, sizeof(UChar) * capacity);

-
-        icu_buf_utf16_clear(buf16);
-        buf16->utf16_cap = capacity;

     }
     else
     {
         xfree(buf16->utf16);
         buf16->utf16 = 0;
         buf16->utf16_len = 0;
     }
     else
     {
         xfree(buf16->utf16);
         buf16->utf16 = 0;
         buf16->utf16_len = 0;

-        buf16->utf16_cap = 0;

     }
     }

+    buf16->utf16_cap = capacity;

     return buf16;
 }
 
     return buf16;
 }
 

@@ -98,6 +96,27 @@ struct icu_buf_utf16 * icu_buf_utf16_copy(struct icu_buf_utf16 * dest16,
     return dest16;
 }
 
     return dest16;
 }
 

+
+struct icu_buf_utf16 *icu_buf_utf16_append(struct icu_buf_utf16 *dest16,
+                                           const struct icu_buf_utf16 * src16)
+{
+    assert(dest16);
+    if (!src16)
+        return dest16;
+    if (dest16 == src16)
+        return 0;
+
+    if (dest16->utf16_cap <= src16->utf16_len + dest16->utf16_len)
+        icu_buf_utf16_resize(dest16, dest16->utf16_len + src16->utf16_len * 2);
+
+    u_strncpy(dest16->utf16 + dest16->utf16_len,
+              src16->utf16, src16->utf16_len);
+    dest16->utf16_len += src16->utf16_len;
+
+    return dest16;
+}
+
+

 void icu_buf_utf16_destroy(struct icu_buf_utf16 * buf16)
 {
     if (buf16)
 void icu_buf_utf16_destroy(struct icu_buf_utf16 * buf16)
 {
     if (buf16)

@@ -105,6 +124,21 @@ void icu_buf_utf16_destroy(struct icu_buf_utf16 * buf16)
     xfree(buf16);
 }
 
     xfree(buf16);
 }
 

+void icu_buf_utf16_log(const char *lead, struct icu_buf_utf16 *src16)
+{
+    if (src16)
+    {
+        struct icu_buf_utf8 *dst8 = icu_buf_utf8_create(0);
+        UErrorCode status = U_ZERO_ERROR;
+        icu_utf16_to_utf8(dst8, src16, &status);
+        yaz_log(YLOG_LOG, "%s=%s", lead, dst8->utf8);
+        icu_buf_utf8_destroy(dst8);
+    }
+    else
+    {
+        yaz_log(YLOG_LOG, "%s=NULL", lead);
+    }
+}

 
 #endif /* YAZ_HAVE_ICU */
 
 
 #endif /* YAZ_HAVE_ICU */