ICU chain passes directives display, norm, sort, and normalize. Directives tokenize...

diff --git a/src/icu_I18N.c b/src/icu_I18N.c
index 8d67c76..d791aba 100644 (file)
--- a/src/icu_I18N.c
+++ b/src/icu_I18N.c
@@ -1,4 +1,4 @@
-/* $Id: icu_I18N.c,v 1.15 2007-05-16 19:12:00 marc Exp $
+/* $Id: icu_I18N.c,v 1.16 2007-05-16 19:50:01 marc Exp $
    Copyright (c) 2006-2007, Index Data.
 
    This file is part of Pazpar2.
@@ -938,9 +938,10 @@ int icu_chain_step_next_token(struct icu_chain * chain,
                                  step->buf16, src16, status);
         break;
     case ICU_chain_step_type_tokenize:
-        // step->more_tokens
-        //       = icu_tokenizer_next_token(step->u.tokenizer,
-        //                               step->buf16, status);
+        icu_tokenizer_attach(step->u.tokenizer, src16, status);
+        step->more_tokens
+            = icu_tokenizer_next_token(step->u.tokenizer,
+                                       step->buf16, status);
         break;
     default:
         return 0;

diff --git a/src/test_icu_I18N.c b/src/test_icu_I18N.c
index e0453c0..00671f3 100644 (file)
--- a/src/test_icu_I18N.c
+++ b/src/test_icu_I18N.c
@@ -1,4 +1,4 @@
-/* $Id: test_icu_I18N.c,v 1.20 2007-05-16 19:12:00 marc Exp $
+/* $Id: test_icu_I18N.c,v 1.21 2007-05-16 19:50:01 marc Exp $
    Copyright (c) 2006-2007, Index Data.
 
    This file is part of Pazpar2.
@@ -507,9 +507,9 @@ void test_icu_I18N_chain(int argc, char **argv)
 /*     step = icu_chain_insert_step(chain, ICU_chain_step_type_normalize, */
 /*                                  (const uint8_t *) "[:Control:] Any-Remove", */
 /*                                  &status); */
-/*     step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize, */
-/*                                  (const uint8_t *) "w", */
-/*                                  &status); */
+    step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize,
+                                 (const uint8_t *) "l",
+                                 &status);
 /*     step = icu_chain_insert_step(chain, ICU_chain_step_type_normalize, */
 /*                                  (const uint8_t *) */
 /*                                  "[[:WhiteSpace:][:Punctuation:]] Any-Remove", */
@@ -517,12 +517,12 @@ void test_icu_I18N_chain(int argc, char **argv)
     step = icu_chain_insert_step(chain, ICU_chain_step_type_display,
                                  (const uint8_t *)"",
                                  &status);
-    step = icu_chain_insert_step(chain, ICU_chain_step_type_normalize,
-                                 (const uint8_t *) "Lower",
-                                 &status);
-    step = icu_chain_insert_step(chain, ICU_chain_step_type_norm,
-                                 (const uint8_t *)"",
-                                 &status);
+/*     step = icu_chain_insert_step(chain, ICU_chain_step_type_normalize, */
+/*                                  (const uint8_t *) "Lower", */
+/*                                  &status); */
+/*     step = icu_chain_insert_step(chain, ICU_chain_step_type_norm, */
+/*                                  (const uint8_t *)"", */
+/*                                  &status); */
 /*     step = icu_chain_insert_step(chain, ICU_chain_step_type_sort, */
 /*                                  (const uint8_t *)"", */
 /*                                  &status); */