ICU test refactor. Hide icu_tokenizer struct.

diff --git a/include/yaz/icu_I18N.h b/include/yaz/icu_I18N.h
index 3ad54e2..b330446 100644 (file)
--- a/include/yaz/icu_I18N.h
+++ b/include/yaz/icu_I18N.h
@@ -120,25 +120,7 @@ void icu_sortkey8_from_utf16(UCollator *coll,
                              struct icu_buf_utf16 * src16,
                              UErrorCode * status);
 
                              struct icu_buf_utf16 * src16,
                              UErrorCode * status);
 

-struct icu_tokenizer
-{
-    char action;
-    UBreakIterator* bi;
-    struct icu_buf_utf16 * buf16;
-    int32_t token_count;
-    int32_t token_id;
-    int32_t token_start;
-    int32_t token_end;
-/*
-  keep always invariant
-  0 <= token_start 
-  <= token_end 
-  <= buf16->utf16_len
-  and invariant
-  0 <= token_id <= token_count
-*/
-};
-
+struct icu_tokenizer;

 struct icu_tokenizer * icu_tokenizer_create(const char *locale, char action,
                                             UErrorCode *status);
 
 struct icu_tokenizer * icu_tokenizer_create(const char *locale, char action,
                                             UErrorCode *status);
 

diff --git a/src/icu_tokenizer.c b/src/icu_tokenizer.c
index 55409ac..f9b4926 100644 (file)
--- a/src/icu_tokenizer.c
+++ b/src/icu_tokenizer.c
@@ -26,6 +26,25 @@
 #include <unicode/ustring.h>  /* some more string fcns*/
 #include <unicode/uchar.h>    /* char names           */
 
 #include <unicode/ustring.h>  /* some more string fcns*/
 #include <unicode/uchar.h>    /* char names           */
 

+struct icu_tokenizer
+{
+    char action;
+    UBreakIterator* bi;
+    struct icu_buf_utf16 * buf16;
+    int32_t token_count;
+    int32_t token_id;
+    int32_t token_start;
+    int32_t token_end;
+/*
+  keep always invariant
+  0 <= token_start 
+  <= token_end 
+  <= buf16->utf16_len
+  and invariant
+  0 <= token_id <= token_count
+*/
+};
+

 struct icu_tokenizer *icu_tokenizer_create(const char *locale, char action,
                                            UErrorCode *status)
 {
 struct icu_tokenizer *icu_tokenizer_create(const char *locale, char action,
                                            UErrorCode *status)
 {

diff --git a/test/Makefile.am b/test/Makefile.am
index 35f819c..08c0eda 100644 (file)
--- a/test/Makefile.am
+++ b/test/Makefile.am
@@ -51,6 +51,7 @@ tstodrcodec.c tstodrcodec.h: tstodr.asn $(YAZCOMP)
 LDADD = ../src/libyaz.la
 tst_icu_I18N_LDADD = ../src/libyaz_icu.la $(ICU_LIBS)
 
 LDADD = ../src/libyaz.la
 tst_icu_I18N_LDADD = ../src/libyaz_icu.la $(ICU_LIBS)
 

+

 CONFIG_CLEAN_FILES=*.log
 
 tstxmalloc_SOURCES = tstxmalloc.c
 CONFIG_CLEAN_FILES=*.log
 
 tstxmalloc_SOURCES = tstxmalloc.c

@@ -75,6 +76,7 @@ tst_retrieval_SOURCES = tst_retrieval.c
 tst_tpath_SOURCES = tst_tpath.c
 tst_timing_SOURCES = tst_timing.c
 tst_query_charset_SOURCES = tst_query_charset.c
 tst_tpath_SOURCES = tst_tpath.c
 tst_timing_SOURCES = tst_timing.c
 tst_query_charset_SOURCES = tst_query_charset.c

+tst_icu_I18N_CFLAGS = $(PTHREAD_CFLAGS)

 tst_icu_I18N_SOURCES = tst_icu_I18N.c
 tst_match_glob_SOURCES = tst_match_glob.c
 tst_rpn2cql_SOURCES = tst_rpn2cql.c
 tst_icu_I18N_SOURCES = tst_icu_I18N.c
 tst_match_glob_SOURCES = tst_match_glob.c
 tst_rpn2cql_SOURCES = tst_rpn2cql.c

diff --git a/test/tst_icu_I18N.c b/test/tst_icu_I18N.c
index a97b065..484e9cc 100644 (file)
--- a/test/tst_icu_I18N.c
+++ b/test/tst_icu_I18N.c
@@ -17,10 +17,15 @@
 
 #include <yaz/test.h>
 #include <yaz/log.h>
 
 #include <yaz/test.h>
 #include <yaz/log.h>

+#include <yaz/wrbuf.h>

 
 #if YAZ_HAVE_ICU
 #include <yaz/icu_I18N.h>
 
 
 #if YAZ_HAVE_ICU
 #include <yaz/icu_I18N.h>
 

+#if YAZ_POSIX_THREADS
+#include <pthread.h>
+#endif
+

 #include <string.h>
 #include <stdlib.h>
 
 #include <string.h>
 #include <stdlib.h>
 

@@ -32,7 +37,7 @@ struct icu_termmap
 };
 
 
 };
 
 

-int icu_termmap_cmp(const void *vp1, const void *vp2)
+static int icu_termmap_cmp(const void *vp1, const void *vp2)

 {
     struct icu_termmap *itmp1 = *(struct icu_termmap **) vp1;
     struct icu_termmap *itmp2 = *(struct icu_termmap **) vp2;
 {
     struct icu_termmap *itmp1 = *(struct icu_termmap **) vp1;
     struct icu_termmap *itmp2 = *(struct icu_termmap **) vp2;

@@ -45,8 +50,8 @@ int icu_termmap_cmp(const void *vp1, const void *vp2)
 }
 
 
 }
 
 

-int test_icu_casemap(const char * locale, char action,
-                     const char * src8cstr, const char * chk8cstr)
+static int test_icu_casemap(const char * locale, char action,
+                            const char * src8cstr, const char * chk8cstr)

 {
     int success = 0;
     UErrorCode status = U_ZERO_ERROR;
 {
     int success = 0;
     UErrorCode status = U_ZERO_ERROR;

@@ -98,7 +103,7 @@ int test_icu_casemap(const char * locale, char action,
     return success;
 }
 
     return success;
 }
 

-void test_icu_I18N_casemap(int argc, char **argv)
+static void check_icu_casemap(void)

 {
     /* Locale 'en' */
 
 {
     /* Locale 'en' */
 

@@ -159,8 +164,8 @@ void test_icu_I18N_casemap(int argc, char **argv)
 
 }
 
 
 }
 

-int test_icu_sortmap(const char * locale, int src_list_len,
-                     const char ** src_list, const char ** chk_list)
+static int test_icu_sortmap(const char * locale, int src_list_len,
+                            const char ** src_list, const char ** chk_list)

 {
     int success = 1;
 
 {
     int success = 1;
 

@@ -176,7 +181,7 @@ int test_icu_sortmap(const char * locale, int src_list_len,
     UCollator *coll = ucol_open(locale, &status); 
     icu_check_status(status);
 
     UCollator *coll = ucol_open(locale, &status); 
     icu_check_status(status);
 

-    if(U_FAILURE(status))
+    if (U_FAILURE(status))

         return 0;
 
     /* assigning display terms and sort keys using buf 8 and buf16 */
         return 0;
 
     /* assigning display terms and sort keys using buf 8 and buf16 */

@@ -239,7 +244,7 @@ int test_icu_sortmap(const char * locale, int src_list_len,
     return success;  
 }
 
     return success;  
 }
 

-void test_icu_I18N_sortmap(int argc, char **argv)
+static void check_icu_sortmap(void)

 {
     /* successful tests */
     size_t en_1_len = 6;
 {
     /* successful tests */
     size_t en_1_len = 6;

@@ -268,12 +273,11 @@ void test_icu_I18N_sortmap(int argc, char **argv)
         YAZ_CHECK(test_icu_sortmap("de_AT", de_1_len, de_1_src, de_1_cck));
         YAZ_CHECK(test_icu_sortmap("de_DE", de_1_len, de_1_src, de_1_cck));
     }
         YAZ_CHECK(test_icu_sortmap("de_AT", de_1_len, de_1_src, de_1_cck));
         YAZ_CHECK(test_icu_sortmap("de_DE", de_1_len, de_1_src, de_1_cck));
     }

-    

 }
 
 }
 

-int test_icu_normalizer(const char * rules8cstr,
-                        const char * src8cstr,
-                        const char * chk8cstr)
+static int test_icu_normalizer(const char * rules8cstr,
+                               const char * src8cstr,
+                               const char * chk8cstr)

 {
     int success = 0;
     
 {
     int success = 0;
     

@@ -296,7 +300,7 @@ int test_icu_normalizer(const char * rules8cstr,
     icu_check_status(status);
 
 
     icu_check_status(status);
 
 

-    if(!strcmp((const char *) dest8->utf8, 
+    if (!strcmp((const char *) dest8->utf8, 

                (const char *) chk8cstr))
         success = 1;
     else
                (const char *) chk8cstr))
         success = 1;
     else

@@ -317,7 +321,7 @@ int test_icu_normalizer(const char * rules8cstr,
     return success;
 }
 
     return success;
 }
 

-void test_icu_I18N_normalizer(int argc, char **argv)
+static void check_icu_normalizer(void)

 {
     YAZ_CHECK(test_icu_normalizer("[:Punctuation:] Any-Remove",
                                   "Don't shoot!",
 {
     YAZ_CHECK(test_icu_normalizer("[:Punctuation:] Any-Remove",
                                   "Don't shoot!",

@@ -349,8 +353,8 @@ void test_icu_I18N_normalizer(int argc, char **argv)
                                   "a cote de l'alcove ovoide"));
 }
 
                                   "a cote de l'alcove ovoide"));
 }
 

-int test_icu_tokenizer(const char * locale, char action,
-                       const char * src8cstr, int count)
+static int test_icu_tokenizer(const char * locale, char action,
+                              const char * src8cstr, int count)

 {
     int success = 1;
 
 {
     int success = 1;
 

@@ -372,17 +376,18 @@ int test_icu_tokenizer(const char * locale, char action,
     /* attach text buffer to tokenizer */
     icu_tokenizer_attach(tokenizer, src16, &status);    
     icu_check_status(status);
     /* attach text buffer to tokenizer */
     icu_tokenizer_attach(tokenizer, src16, &status);    
     icu_check_status(status);

-    YAZ_CHECK(tokenizer->bi);

 
     /* perform work on tokens */
 
     /* perform work on tokens */

-    while(icu_tokenizer_next_token(tokenizer, tkn16, &status)){
+    while (icu_tokenizer_next_token(tokenizer, tkn16, &status))
+    {

         icu_check_status(status);
 
         /* converting to UTF8 */
         icu_utf16_to_utf8(tkn8, tkn16, &status);
     }
 
         icu_check_status(status);
 
         /* converting to UTF8 */
         icu_utf16_to_utf8(tkn8, tkn16, &status);
     }
 

-    if (count != icu_tokenizer_token_count(tokenizer)){
+    if (count != icu_tokenizer_token_count(tokenizer))
+    {

         success = 0;
         yaz_log(YLOG_LOG, "Tokenizer '%s:%c' Error:", locale, action);
         yaz_log(YLOG_LOG, " Input:  '%s'", src8cstr);
         success = 0;
         yaz_log(YLOG_LOG, "Tokenizer '%s:%c' Error:", locale, action);
         yaz_log(YLOG_LOG, " Input:  '%s'", src8cstr);

@@ -398,7 +403,7 @@ int test_icu_tokenizer(const char * locale, char action,
     return success;
 }
 
     return success;
 }
 

-void test_icu_I18N_tokenizer(int argc, char **argv)
+static void check_icu_tokenizer(void)

 {
     const char * en_str 
         = "O Romeo, Romeo! wherefore art thou Romeo?";
 {
     const char * en_str 
         = "O Romeo, Romeo! wherefore art thou Romeo?";

@@ -420,7 +425,7 @@ void test_icu_I18N_tokenizer(int argc, char **argv)
     }
 }
 
     }
 }
 

-void test_icu_I18N_chain(int argc, char **argv)
+static void check_icu_chain(void)

 {
     const char * en_str 
         = "O Romeo, Romeo! wherefore art thou\t Romeo?";
 {
     const char * en_str 
         = "O Romeo, Romeo! wherefore art thou\t Romeo?";

@@ -478,7 +483,7 @@ void test_icu_I18N_chain(int argc, char **argv)
 }
 
 
 }
 
 

-void test_bug_1140(void)
+static void check_bug_1140(void)

 {
     UErrorCode status = U_ZERO_ERROR;
     struct icu_chain * chain = 0;
 {
     UErrorCode status = U_ZERO_ERROR;
     struct icu_chain * chain = 0;

@@ -539,7 +544,7 @@ void test_bug_1140(void)
 }
 
 
 }
 
 

-void test_chain_empty_token(void)
+static void check_chain_empty_token(void)

 {
     UErrorCode status = U_ZERO_ERROR;
     struct icu_chain * chain = 0;
 {
     UErrorCode status = U_ZERO_ERROR;
     struct icu_chain * chain = 0;

@@ -576,7 +581,7 @@ void test_chain_empty_token(void)
     icu_chain_destroy(chain);
 }
 
     icu_chain_destroy(chain);
 }
 

-void test_chain_empty_chain(void)
+static void check_chain_empty_chain(void)

 {
     UErrorCode status = U_ZERO_ERROR;
     struct icu_chain * chain = 0;
 {
     UErrorCode status = U_ZERO_ERROR;
     struct icu_chain * chain = 0;

@@ -617,7 +622,7 @@ void test_chain_empty_chain(void)
     icu_chain_destroy(chain);
 }
 
     icu_chain_destroy(chain);
 }
 

-void test_icu_iter1(void)
+static void check_icu_iter1(void)

 {
     UErrorCode status = U_ZERO_ERROR;
     struct icu_chain * chain = 0;
 {
     UErrorCode status = U_ZERO_ERROR;
     struct icu_chain * chain = 0;

@@ -659,14 +664,75 @@ void test_icu_iter1(void)
     icu_chain_destroy(chain);
 }
 
     icu_chain_destroy(chain);
 }
 

+static int test_iter(struct icu_chain *chain, const char *input,
+                     const char *expected)
+{
+    struct icu_iter *iter = icu_iter_create(chain, input);
+    WRBUF result;
+    struct icu_buf_utf8 *token;
+
+    if (!iter)
+    {
+        yaz_log(YLOG_WARN, "test_iter: input=%s !iter", input);
+        return 0;
+    }
+
+    token = icu_buf_utf8_create(0);
+    result = wrbuf_alloc();
+    while (icu_iter_next(iter, token))
+    {
+        wrbuf_puts(result, "[");
+        wrbuf_write(result, (const char *) token->utf8, (int) token->utf8_len);
+        wrbuf_puts(result, "]");
+    }
+    icu_buf_utf8_destroy(token);
+
+    icu_iter_destroy(iter);
+
+    if (strcmp(expected, wrbuf_cstr(result)))
+    {
+        yaz_log(YLOG_WARN, "test_iter: input=%s expected=%s got=%s",
+                input, expected, wrbuf_cstr(result));
+        wrbuf_destroy(result);
+        return 0;
+    }
+    wrbuf_destroy(result);
+    return 1;
+}
+
+static void *iter_thread(void *p)
+{
+    struct icu_chain *chain = (struct icu_chain *) p;
+    int i;
+    
+    for (i = 0; i < 10000; i++)
+    {
+        YAZ_CHECK(test_iter(chain, "Adobe Acrobat Reader, 1991-1999.",
+                            "[adobe][acrobat][reader][1991][][1999][]"));
+    }
+    return 0;
+}

 
 

-void test_icu_iter2(void)
+static void check_iter_threads(struct icu_chain *chain)
+{
+#if YAZ_POSIX_THREADS
+#define NO_THREADS 1
+
+    pthread_t t[NO_THREADS];
+    int i;
+
+    for (i = 0; i < NO_THREADS; i++)
+        pthread_create(t + i, 0, iter_thread, chain);
+
+    for (i = 0; i < NO_THREADS; i++)
+        pthread_join(t[i], 0);
+#endif
+}
+static void check_icu_iter2(void)

 {
     UErrorCode status = U_ZERO_ERROR;
     struct icu_chain * chain = 0;
     xmlNode *xml_node;
 {
     UErrorCode status = U_ZERO_ERROR;
     struct icu_chain * chain = 0;
     xmlNode *xml_node;

-    struct icu_iter *iter;
-    struct icu_buf_utf8 *token;

 
     const char * xml_str = "<icu locale=\"en\">"
         "<transform rule=\"[:Control:] Any-Remove\"/>"
 
     const char * xml_str = "<icu locale=\"en\">"
         "<transform rule=\"[:Control:] Any-Remove\"/>"

@@ -691,18 +757,11 @@ void test_icu_iter2(void)
     xmlFreeDoc(doc);
     YAZ_CHECK(chain);
     
     xmlFreeDoc(doc);
     YAZ_CHECK(chain);
     

-    iter = icu_iter_create(chain, "Adobe Acrobat Reader, 1991-1999.");
-    YAZ_CHECK(iter);
-    if (!iter)
-        return;
-    token = icu_buf_utf8_create(0);
-    while (icu_iter_next(iter, token))
-    {
-        yaz_log(YLOG_LOG, "[%.*s]", (int) token->utf8_len, token->utf8);
-    }
-    icu_buf_utf8_destroy(token);
+    YAZ_CHECK(test_iter(chain, "Adobe Acrobat Reader, 1991-1999.",
+                        "[adobe][acrobat][reader][1991][][1999][]"));
+
+    check_iter_threads(chain);

 
 

-    icu_iter_destroy(iter);

     icu_chain_destroy(chain);
 }
 
     icu_chain_destroy(chain);
 }
 

@@ -715,17 +774,17 @@ int main(int argc, char **argv)
 
 #if YAZ_HAVE_ICU
 
 
 #if YAZ_HAVE_ICU
 

-    test_icu_I18N_casemap(argc, argv);
-    test_icu_I18N_sortmap(argc, argv);
-    test_icu_I18N_normalizer(argc, argv); 
-    test_icu_I18N_tokenizer(argc, argv);
-    test_icu_I18N_chain(argc, argv);
-    test_chain_empty_token();
-    test_chain_empty_chain();
-    test_icu_iter1();
-    test_icu_iter2();
+    check_icu_casemap();
+    check_icu_sortmap();
+    check_icu_normalizer();
+    check_icu_tokenizer();
+    check_icu_chain();
+    check_chain_empty_token();
+    check_chain_empty_chain();
+    check_icu_iter1();
+    check_icu_iter2();

     
     

-    test_bug_1140();
+    check_bug_1140();

 
 #else /* YAZ_HAVE_ICU */
 
 
 #else /* YAZ_HAVE_ICU */