From ded9dadedab82a379e0e52d78a61fe9632ac870c Mon Sep 17 00:00:00 2001 From: Adam Dickmeiss Date: Mon, 1 Feb 2010 15:05:44 +0100 Subject: [PATCH] ICU test refactor. Hide icu_tokenizer struct. --- include/yaz/icu_I18N.h | 20 +----- src/icu_tokenizer.c | 19 ++++++ test/Makefile.am | 2 + test/tst_icu_I18N.c | 157 +++++++++++++++++++++++++++++++++--------------- 4 files changed, 130 insertions(+), 68 deletions(-) diff --git a/include/yaz/icu_I18N.h b/include/yaz/icu_I18N.h index 3ad54e2..b330446 100644 --- a/include/yaz/icu_I18N.h +++ b/include/yaz/icu_I18N.h @@ -120,25 +120,7 @@ void icu_sortkey8_from_utf16(UCollator *coll, struct icu_buf_utf16 * src16, UErrorCode * status); -struct icu_tokenizer -{ - char action; - UBreakIterator* bi; - struct icu_buf_utf16 * buf16; - int32_t token_count; - int32_t token_id; - int32_t token_start; - int32_t token_end; -/* - keep always invariant - 0 <= token_start - <= token_end - <= buf16->utf16_len - and invariant - 0 <= token_id <= token_count -*/ -}; - +struct icu_tokenizer; struct icu_tokenizer * icu_tokenizer_create(const char *locale, char action, UErrorCode *status); diff --git a/src/icu_tokenizer.c b/src/icu_tokenizer.c index 55409ac..f9b4926 100644 --- a/src/icu_tokenizer.c +++ b/src/icu_tokenizer.c @@ -26,6 +26,25 @@ #include /* some more string fcns*/ #include /* char names */ +struct icu_tokenizer +{ + char action; + UBreakIterator* bi; + struct icu_buf_utf16 * buf16; + int32_t token_count; + int32_t token_id; + int32_t token_start; + int32_t token_end; +/* + keep always invariant + 0 <= token_start + <= token_end + <= buf16->utf16_len + and invariant + 0 <= token_id <= token_count +*/ +}; + struct icu_tokenizer *icu_tokenizer_create(const char *locale, char action, UErrorCode *status) { diff --git a/test/Makefile.am b/test/Makefile.am index 35f819c..08c0eda 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -51,6 +51,7 @@ tstodrcodec.c tstodrcodec.h: tstodr.asn $(YAZCOMP) LDADD = ../src/libyaz.la tst_icu_I18N_LDADD = ../src/libyaz_icu.la $(ICU_LIBS) + CONFIG_CLEAN_FILES=*.log tstxmalloc_SOURCES = tstxmalloc.c @@ -75,6 +76,7 @@ tst_retrieval_SOURCES = tst_retrieval.c tst_tpath_SOURCES = tst_tpath.c tst_timing_SOURCES = tst_timing.c tst_query_charset_SOURCES = tst_query_charset.c +tst_icu_I18N_CFLAGS = $(PTHREAD_CFLAGS) tst_icu_I18N_SOURCES = tst_icu_I18N.c tst_match_glob_SOURCES = tst_match_glob.c tst_rpn2cql_SOURCES = tst_rpn2cql.c diff --git a/test/tst_icu_I18N.c b/test/tst_icu_I18N.c index a97b065..484e9cc 100644 --- a/test/tst_icu_I18N.c +++ b/test/tst_icu_I18N.c @@ -17,10 +17,15 @@ #include #include +#include #if YAZ_HAVE_ICU #include +#if YAZ_POSIX_THREADS +#include +#endif + #include #include @@ -32,7 +37,7 @@ struct icu_termmap }; -int icu_termmap_cmp(const void *vp1, const void *vp2) +static int icu_termmap_cmp(const void *vp1, const void *vp2) { struct icu_termmap *itmp1 = *(struct icu_termmap **) vp1; struct icu_termmap *itmp2 = *(struct icu_termmap **) vp2; @@ -45,8 +50,8 @@ int icu_termmap_cmp(const void *vp1, const void *vp2) } -int test_icu_casemap(const char * locale, char action, - const char * src8cstr, const char * chk8cstr) +static int test_icu_casemap(const char * locale, char action, + const char * src8cstr, const char * chk8cstr) { int success = 0; UErrorCode status = U_ZERO_ERROR; @@ -98,7 +103,7 @@ int test_icu_casemap(const char * locale, char action, return success; } -void test_icu_I18N_casemap(int argc, char **argv) +static void check_icu_casemap(void) { /* Locale 'en' */ @@ -159,8 +164,8 @@ void test_icu_I18N_casemap(int argc, char **argv) } -int test_icu_sortmap(const char * locale, int src_list_len, - const char ** src_list, const char ** chk_list) +static int test_icu_sortmap(const char * locale, int src_list_len, + const char ** src_list, const char ** chk_list) { int success = 1; @@ -176,7 +181,7 @@ int test_icu_sortmap(const char * locale, int src_list_len, UCollator *coll = ucol_open(locale, &status); icu_check_status(status); - if(U_FAILURE(status)) + if (U_FAILURE(status)) return 0; /* assigning display terms and sort keys using buf 8 and buf16 */ @@ -239,7 +244,7 @@ int test_icu_sortmap(const char * locale, int src_list_len, return success; } -void test_icu_I18N_sortmap(int argc, char **argv) +static void check_icu_sortmap(void) { /* successful tests */ size_t en_1_len = 6; @@ -268,12 +273,11 @@ void test_icu_I18N_sortmap(int argc, char **argv) YAZ_CHECK(test_icu_sortmap("de_AT", de_1_len, de_1_src, de_1_cck)); YAZ_CHECK(test_icu_sortmap("de_DE", de_1_len, de_1_src, de_1_cck)); } - } -int test_icu_normalizer(const char * rules8cstr, - const char * src8cstr, - const char * chk8cstr) +static int test_icu_normalizer(const char * rules8cstr, + const char * src8cstr, + const char * chk8cstr) { int success = 0; @@ -296,7 +300,7 @@ int test_icu_normalizer(const char * rules8cstr, icu_check_status(status); - if(!strcmp((const char *) dest8->utf8, + if (!strcmp((const char *) dest8->utf8, (const char *) chk8cstr)) success = 1; else @@ -317,7 +321,7 @@ int test_icu_normalizer(const char * rules8cstr, return success; } -void test_icu_I18N_normalizer(int argc, char **argv) +static void check_icu_normalizer(void) { YAZ_CHECK(test_icu_normalizer("[:Punctuation:] Any-Remove", "Don't shoot!", @@ -349,8 +353,8 @@ void test_icu_I18N_normalizer(int argc, char **argv) "a cote de l'alcove ovoide")); } -int test_icu_tokenizer(const char * locale, char action, - const char * src8cstr, int count) +static int test_icu_tokenizer(const char * locale, char action, + const char * src8cstr, int count) { int success = 1; @@ -372,17 +376,18 @@ int test_icu_tokenizer(const char * locale, char action, /* attach text buffer to tokenizer */ icu_tokenizer_attach(tokenizer, src16, &status); icu_check_status(status); - YAZ_CHECK(tokenizer->bi); /* perform work on tokens */ - while(icu_tokenizer_next_token(tokenizer, tkn16, &status)){ + while (icu_tokenizer_next_token(tokenizer, tkn16, &status)) + { icu_check_status(status); /* converting to UTF8 */ icu_utf16_to_utf8(tkn8, tkn16, &status); } - if (count != icu_tokenizer_token_count(tokenizer)){ + if (count != icu_tokenizer_token_count(tokenizer)) + { success = 0; yaz_log(YLOG_LOG, "Tokenizer '%s:%c' Error:", locale, action); yaz_log(YLOG_LOG, " Input: '%s'", src8cstr); @@ -398,7 +403,7 @@ int test_icu_tokenizer(const char * locale, char action, return success; } -void test_icu_I18N_tokenizer(int argc, char **argv) +static void check_icu_tokenizer(void) { const char * en_str = "O Romeo, Romeo! wherefore art thou Romeo?"; @@ -420,7 +425,7 @@ void test_icu_I18N_tokenizer(int argc, char **argv) } } -void test_icu_I18N_chain(int argc, char **argv) +static void check_icu_chain(void) { const char * en_str = "O Romeo, Romeo! wherefore art thou\t Romeo?"; @@ -478,7 +483,7 @@ void test_icu_I18N_chain(int argc, char **argv) } -void test_bug_1140(void) +static void check_bug_1140(void) { UErrorCode status = U_ZERO_ERROR; struct icu_chain * chain = 0; @@ -539,7 +544,7 @@ void test_bug_1140(void) } -void test_chain_empty_token(void) +static void check_chain_empty_token(void) { UErrorCode status = U_ZERO_ERROR; struct icu_chain * chain = 0; @@ -576,7 +581,7 @@ void test_chain_empty_token(void) icu_chain_destroy(chain); } -void test_chain_empty_chain(void) +static void check_chain_empty_chain(void) { UErrorCode status = U_ZERO_ERROR; struct icu_chain * chain = 0; @@ -617,7 +622,7 @@ void test_chain_empty_chain(void) icu_chain_destroy(chain); } -void test_icu_iter1(void) +static void check_icu_iter1(void) { UErrorCode status = U_ZERO_ERROR; struct icu_chain * chain = 0; @@ -659,14 +664,75 @@ void test_icu_iter1(void) icu_chain_destroy(chain); } +static int test_iter(struct icu_chain *chain, const char *input, + const char *expected) +{ + struct icu_iter *iter = icu_iter_create(chain, input); + WRBUF result; + struct icu_buf_utf8 *token; + + if (!iter) + { + yaz_log(YLOG_WARN, "test_iter: input=%s !iter", input); + return 0; + } + + token = icu_buf_utf8_create(0); + result = wrbuf_alloc(); + while (icu_iter_next(iter, token)) + { + wrbuf_puts(result, "["); + wrbuf_write(result, (const char *) token->utf8, (int) token->utf8_len); + wrbuf_puts(result, "]"); + } + icu_buf_utf8_destroy(token); + + icu_iter_destroy(iter); + + if (strcmp(expected, wrbuf_cstr(result))) + { + yaz_log(YLOG_WARN, "test_iter: input=%s expected=%s got=%s", + input, expected, wrbuf_cstr(result)); + wrbuf_destroy(result); + return 0; + } + wrbuf_destroy(result); + return 1; +} + +static void *iter_thread(void *p) +{ + struct icu_chain *chain = (struct icu_chain *) p; + int i; + + for (i = 0; i < 10000; i++) + { + YAZ_CHECK(test_iter(chain, "Adobe Acrobat Reader, 1991-1999.", + "[adobe][acrobat][reader][1991][][1999][]")); + } + return 0; +} -void test_icu_iter2(void) +static void check_iter_threads(struct icu_chain *chain) +{ +#if YAZ_POSIX_THREADS +#define NO_THREADS 1 + + pthread_t t[NO_THREADS]; + int i; + + for (i = 0; i < NO_THREADS; i++) + pthread_create(t + i, 0, iter_thread, chain); + + for (i = 0; i < NO_THREADS; i++) + pthread_join(t[i], 0); +#endif +} +static void check_icu_iter2(void) { UErrorCode status = U_ZERO_ERROR; struct icu_chain * chain = 0; xmlNode *xml_node; - struct icu_iter *iter; - struct icu_buf_utf8 *token; const char * xml_str = "" "" @@ -691,18 +757,11 @@ void test_icu_iter2(void) xmlFreeDoc(doc); YAZ_CHECK(chain); - iter = icu_iter_create(chain, "Adobe Acrobat Reader, 1991-1999."); - YAZ_CHECK(iter); - if (!iter) - return; - token = icu_buf_utf8_create(0); - while (icu_iter_next(iter, token)) - { - yaz_log(YLOG_LOG, "[%.*s]", (int) token->utf8_len, token->utf8); - } - icu_buf_utf8_destroy(token); + YAZ_CHECK(test_iter(chain, "Adobe Acrobat Reader, 1991-1999.", + "[adobe][acrobat][reader][1991][][1999][]")); + + check_iter_threads(chain); - icu_iter_destroy(iter); icu_chain_destroy(chain); } @@ -715,17 +774,17 @@ int main(int argc, char **argv) #if YAZ_HAVE_ICU - test_icu_I18N_casemap(argc, argv); - test_icu_I18N_sortmap(argc, argv); - test_icu_I18N_normalizer(argc, argv); - test_icu_I18N_tokenizer(argc, argv); - test_icu_I18N_chain(argc, argv); - test_chain_empty_token(); - test_chain_empty_chain(); - test_icu_iter1(); - test_icu_iter2(); + check_icu_casemap(); + check_icu_sortmap(); + check_icu_normalizer(); + check_icu_tokenizer(); + check_icu_chain(); + check_chain_empty_token(); + check_chain_empty_chain(); + check_icu_iter1(); + check_icu_iter2(); - test_bug_1140(); + check_bug_1140(); #else /* YAZ_HAVE_ICU */ -- 1.7.10.4