X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=test%2Ftst_icu_I18N.c;h=2ad005ca0fe2a4b2b1b06645d426008f12f511e6;hb=055f23ee3692349a03f681ef1a0d5fd70c0cc770;hp=fcb438d1d1567792fe7fdc6ee21291035c0b32d8;hpb=3a98cd4b508df847d4aa2b7ed7c1a7213766bad3;p=yaz-moved-to-github.git diff --git a/test/tst_icu_I18N.c b/test/tst_icu_I18N.c index fcb438d..2ad005c 100644 --- a/test/tst_icu_I18N.c +++ b/test/tst_icu_I18N.c @@ -1,5 +1,5 @@ /* This file is part of the YAZ toolkit. - * Copyright (C) 1995-2009 Index Data + * Copyright (C) 1995-2010 Index Data * See the file LICENSE for details. */ @@ -16,10 +16,16 @@ #endif #include +#include +#include #if YAZ_HAVE_ICU #include +#if YAZ_POSIX_THREADS +#include +#endif + #include #include @@ -31,7 +37,7 @@ struct icu_termmap }; -int icu_termmap_cmp(const void *vp1, const void *vp2) +static int icu_termmap_cmp(const void *vp1, const void *vp2) { struct icu_termmap *itmp1 = *(struct icu_termmap **) vp1; struct icu_termmap *itmp2 = *(struct icu_termmap **) vp2; @@ -44,8 +50,8 @@ int icu_termmap_cmp(const void *vp1, const void *vp2) } -int test_icu_casemap(const char * locale, char action, - const char * src8cstr, const char * chk8cstr) +static int test_icu_casemap(const char * locale, char action, + const char * src8cstr, const char * chk8cstr) { int success = 0; UErrorCode status = U_ZERO_ERROR; @@ -77,12 +83,15 @@ int test_icu_casemap(const char * locale, char action, success = 0; /* report failures */ - if (!success){ - printf("\nERROR\n"); - printf("original string: '%s' (%d)\n", src8cstr, src8cstr_len); - printf("icu_casemap '%s:%c' '%s' (%d)\n", - locale, action, dest8->utf8, dest8->utf8_len); - printf("expected string: '%s' (%d)\n", chk8cstr, chk8cstr_len); + if (!success) + { + yaz_log(YLOG_WARN, "test_icu_casemap failed"); + yaz_log(YLOG_LOG, "Original string: '%s' (%d)", + src8cstr, src8cstr_len); + yaz_log(YLOG_LOG, "icu_casemap '%s:%c' '%s' (%d)", + locale, action, dest8->utf8, dest8->utf8_len); + yaz_log(YLOG_LOG, "expected string: '%s' (%d)", + chk8cstr, chk8cstr_len); } /* clean the buffers */ @@ -94,7 +103,7 @@ int test_icu_casemap(const char * locale, char action, return success; } -void test_icu_I18N_casemap(int argc, char **argv) +static void check_icu_casemap(void) { /* Locale 'en' */ @@ -155,8 +164,8 @@ void test_icu_I18N_casemap(int argc, char **argv) } -int test_icu_sortmap(const char * locale, int src_list_len, - const char ** src_list, const char ** chk_list) +static int test_icu_sortmap(const char * locale, int src_list_len, + const char ** src_list, const char ** chk_list) { int success = 1; @@ -172,7 +181,7 @@ int test_icu_sortmap(const char * locale, int src_list_len, UCollator *coll = ucol_open(locale, &status); icu_check_status(status); - if(U_FAILURE(status)) + if (U_FAILURE(status)) return 0; /* assigning display terms and sort keys using buf 8 and buf16 */ @@ -209,22 +218,19 @@ int test_icu_sortmap(const char * locale, int src_list_len, if (!success) { - printf("\nERROR\n"); - printf("Input str: '%s' : ", locale); + yaz_log(YLOG_LOG, "ERROR"); + yaz_log(YLOG_LOG, "Input str:'%s':", locale); for (i = 0; i < src_list_len; i++) { - printf(" '%s'", list[i]->disp_term); + yaz_log(YLOG_LOG, " '%s'", list[i]->disp_term); } - printf("\n"); - printf("ICU sort: '%s' : ", locale); + yaz_log(YLOG_LOG, "ICU sort: '%s':", locale); for (i = 0; i < src_list_len; i++) { - printf(" '%s'", list[i]->disp_term); + yaz_log(YLOG_LOG, " '%s'", list[i]->disp_term); } - printf("\n"); - printf("Expected: '%s' : ", locale); + yaz_log(YLOG_LOG, "Expected: '%s':", locale); for (i = 0; i < src_list_len; i++) { - printf(" '%s'", chk_list[i]); + yaz_log(YLOG_LOG, " '%s'", chk_list[i]); } - printf("\n"); } for (i = 0; i < src_list_len; i++) @@ -238,7 +244,7 @@ int test_icu_sortmap(const char * locale, int src_list_len, return success; } -void test_icu_I18N_sortmap(int argc, char **argv) +static void check_icu_sortmap(void) { /* successful tests */ size_t en_1_len = 6; @@ -267,12 +273,11 @@ void test_icu_I18N_sortmap(int argc, char **argv) YAZ_CHECK(test_icu_sortmap("de_AT", de_1_len, de_1_src, de_1_cck)); YAZ_CHECK(test_icu_sortmap("de_DE", de_1_len, de_1_src, de_1_cck)); } - } -int test_icu_normalizer(const char * rules8cstr, - const char * src8cstr, - const char * chk8cstr) +static int test_icu_normalizer(const char * rules8cstr, + const char * src8cstr, + const char * chk8cstr) { int success = 0; @@ -295,17 +300,17 @@ int test_icu_normalizer(const char * rules8cstr, icu_check_status(status); - if(!strcmp((const char *) dest8->utf8, + if (!strcmp((const char *) dest8->utf8, (const char *) chk8cstr)) success = 1; else { success = 0; - printf("Normalization\n"); - printf("Rules: '%s'\n", rules8cstr); - printf("Input: '%s'\n", src8cstr); - printf("Normalized: '%s'\n", dest8->utf8); - printf("Expected: '%s'\n", chk8cstr); + yaz_log(YLOG_LOG, "Normalization"); + yaz_log(YLOG_LOG, " Rules: '%s'", rules8cstr); + yaz_log(YLOG_LOG, " Input: '%s'", src8cstr); + yaz_log(YLOG_LOG, " Normalized: '%s'", dest8->utf8); + yaz_log(YLOG_LOG, " Expected: '%s'", chk8cstr); } icu_transform_destroy(transform); @@ -316,7 +321,7 @@ int test_icu_normalizer(const char * rules8cstr, return success; } -void test_icu_I18N_normalizer(int argc, char **argv) +static void check_icu_normalizer(void) { YAZ_CHECK(test_icu_normalizer("[:Punctuation:] Any-Remove", "Don't shoot!", @@ -348,8 +353,8 @@ void test_icu_I18N_normalizer(int argc, char **argv) "a cote de l'alcove ovoide")); } -int test_icu_tokenizer(const char * locale, char action, - const char * src8cstr, int count) +static int test_icu_tokenizer(const char * locale, char action, + const char * src8cstr, int count) { int success = 1; @@ -371,22 +376,23 @@ int test_icu_tokenizer(const char * locale, char action, /* attach text buffer to tokenizer */ icu_tokenizer_attach(tokenizer, src16, &status); icu_check_status(status); - YAZ_CHECK(tokenizer->bi); /* perform work on tokens */ - while(icu_tokenizer_next_token(tokenizer, tkn16, &status)){ + while (icu_tokenizer_next_token(tokenizer, tkn16, &status)) + { icu_check_status(status); /* converting to UTF8 */ icu_utf16_to_utf8(tkn8, tkn16, &status); } - if (count != icu_tokenizer_token_count(tokenizer)){ + if (count != icu_tokenizer_token_count(tokenizer)) + { success = 0; - printf("\nTokenizer '%s:%c' Error: \n", locale, action); - printf("Input: '%s'\n", src8cstr); - printf("Tokens: %d", icu_tokenizer_token_count(tokenizer)); - printf(", expected: %d\n", count); + yaz_log(YLOG_LOG, "Tokenizer '%s:%c' Error:", locale, action); + yaz_log(YLOG_LOG, " Input: '%s'", src8cstr); + yaz_log(YLOG_LOG, " Tokens: %d", icu_tokenizer_token_count(tokenizer)); + yaz_log(YLOG_LOG, " Expected: %d", count); } icu_tokenizer_destroy(tokenizer); @@ -397,7 +403,7 @@ int test_icu_tokenizer(const char * locale, char action, return success; } -void test_icu_I18N_tokenizer(int argc, char **argv) +static void check_icu_tokenizer(void) { const char * en_str = "O Romeo, Romeo! wherefore art thou Romeo?"; @@ -419,7 +425,7 @@ void test_icu_I18N_tokenizer(int argc, char **argv) } } -void test_icu_I18N_chain(int argc, char **argv) +static void check_icu_chain(void) { const char * en_str = "O Romeo, Romeo! wherefore art thou\t Romeo?"; @@ -449,11 +455,10 @@ void test_icu_I18N_chain(int argc, char **argv) while (icu_chain_next_token(chain, &status)) { - ; - /* printf("%d '%s' '%s'\n", - icu_chain_token_number(chain), - icu_chain_token_norm(chain), - icu_chain_token_display(chain)); */ + yaz_log(YLOG_LOG, "%d '%s' '%s'", + icu_chain_token_number(chain), + icu_chain_token_norm(chain), + icu_chain_token_display(chain)); } YAZ_CHECK_EQ(icu_chain_token_number(chain), 7); @@ -463,11 +468,10 @@ void test_icu_I18N_chain(int argc, char **argv) while (icu_chain_next_token(chain, &status)) { - ; - /* printf("%d '%s' '%s'\n", - icu_chain_token_number(chain), - icu_chain_token_norm(chain), - icu_chain_token_display(chain)); */ + yaz_log(YLOG_LOG, "%d '%s' '%s'", + icu_chain_token_number(chain), + icu_chain_token_norm(chain), + icu_chain_token_display(chain)); } @@ -477,7 +481,7 @@ void test_icu_I18N_chain(int argc, char **argv) } -void test_bug_1140(void) +static void check_bug_1140(void) { UErrorCode status = U_ZERO_ERROR; struct icu_chain * chain = 0; @@ -538,7 +542,7 @@ void test_bug_1140(void) } -void test_chain_empty_token(void) +static void check_chain_empty_token(void) { UErrorCode status = U_ZERO_ERROR; struct icu_chain * chain = 0; @@ -575,7 +579,7 @@ void test_chain_empty_token(void) icu_chain_destroy(chain); } -void test_chain_empty_chain(void) +static void check_chain_empty_chain(void) { UErrorCode status = U_ZERO_ERROR; struct icu_chain * chain = 0; @@ -616,13 +620,12 @@ void test_chain_empty_chain(void) icu_chain_destroy(chain); } -void test_icu_iter1(void) +static void check_icu_iter1(void) { UErrorCode status = U_ZERO_ERROR; struct icu_chain * chain = 0; xmlNode *xml_node; - struct icu_iter *iter; - struct icu_buf_utf8 *token; + yaz_icu_iter_t iter; const char * xml_str = "" "" @@ -643,31 +646,110 @@ void test_icu_iter1(void) xmlFreeDoc(doc); YAZ_CHECK(chain); - iter = icu_iter_create(chain, "a string with 15 tokens and 8 displays"); + iter = icu_iter_create(chain); + icu_iter_first(iter, "a string with 15 tokens and 8 displays"); YAZ_CHECK(iter); if (!iter) return; -#if 1 - token = icu_buf_utf8_create(0); - while (icu_iter_next(iter, token)) + while (icu_iter_next(iter)) { - printf("[%.*s]", (int) token->utf8_len, token->utf8); + yaz_log(YLOG_LOG, "[%s]", icu_iter_get_norm(iter)); } - icu_buf_utf8_destroy(token); -#endif - icu_iter_destroy(iter); icu_chain_destroy(chain); } +static int test_iter(struct icu_chain *chain, const char *input, + const char *expected) +{ + yaz_icu_iter_t iter = icu_iter_create(chain); + WRBUF result, second; + int success = 1; + + if (!iter) + { + yaz_log(YLOG_WARN, "test_iter: input=%s !iter", input); + return 0; + } + + if (icu_iter_next(iter)) + { + yaz_log(YLOG_WARN, "test_iter: expecting 0 before icu_iter_first"); + return 0; + } + + result = wrbuf_alloc(); + icu_iter_first(iter, input); + while (icu_iter_next(iter)) + { + wrbuf_puts(result, "["); + wrbuf_puts(result, icu_iter_get_norm(iter)); + wrbuf_puts(result, "]"); + } + + second = wrbuf_alloc(); + icu_iter_first(iter, input); + while (icu_iter_next(iter)) + { + wrbuf_puts(second, "["); + wrbuf_puts(second, icu_iter_get_norm(iter)); + wrbuf_puts(second, "]"); + } + + icu_iter_destroy(iter); + + if (strcmp(expected, wrbuf_cstr(result))) + { + yaz_log(YLOG_WARN, "test_iter: input=%s expected=%s got=%s", + input, expected, wrbuf_cstr(result)); + success = 0; + } + + if (strcmp(expected, wrbuf_cstr(second))) + { + yaz_log(YLOG_WARN, "test_iter: input=%s expected=%s got=%s (2nd)", + input, expected, wrbuf_cstr(second)); + success = 0; + } + + wrbuf_destroy(result); + wrbuf_destroy(second); + return success; +} + +static void *iter_thread(void *p) +{ + struct icu_chain *chain = (struct icu_chain *) p; + int i; + + for (i = 0; i < 10000; i++) + { + YAZ_CHECK(test_iter(chain, "Adobe Acrobat Reader, 1991-1999.", + "[adobe][acrobat][reader][1991][][1999][]")); + } + return 0; +} -void test_icu_iter2(void) +static void check_iter_threads(struct icu_chain *chain) +{ +#if YAZ_POSIX_THREADS +#define NO_THREADS 1 + + pthread_t t[NO_THREADS]; + int i; + + for (i = 0; i < NO_THREADS; i++) + pthread_create(t + i, 0, iter_thread, chain); + + for (i = 0; i < NO_THREADS; i++) + pthread_join(t[i], 0); +#endif +} +static void check_icu_iter2(void) { UErrorCode status = U_ZERO_ERROR; struct icu_chain * chain = 0; xmlNode *xml_node; - struct icu_iter *iter; - struct icu_buf_utf8 *token; const char * xml_str = "" "" @@ -692,18 +774,11 @@ void test_icu_iter2(void) xmlFreeDoc(doc); YAZ_CHECK(chain); - iter = icu_iter_create(chain, "Adobe Acrobat Reader, 1991-1999."); - YAZ_CHECK(iter); - if (!iter) - return; - token = icu_buf_utf8_create(0); - while (icu_iter_next(iter, token)) - { - printf("[%.*s]", (int) token->utf8_len, token->utf8); - } - icu_buf_utf8_destroy(token); + YAZ_CHECK(test_iter(chain, "Adobe Acrobat Reader, 1991-1999.", + "[adobe][acrobat][reader][1991][][1999][]")); + + check_iter_threads(chain); - icu_iter_destroy(iter); icu_chain_destroy(chain); } @@ -716,22 +791,21 @@ int main(int argc, char **argv) #if YAZ_HAVE_ICU - test_icu_I18N_casemap(argc, argv); - test_icu_I18N_sortmap(argc, argv); - test_icu_I18N_normalizer(argc, argv); - test_icu_I18N_tokenizer(argc, argv); - test_icu_I18N_chain(argc, argv); - test_chain_empty_token(); - test_chain_empty_chain(); - test_icu_iter1(); - test_icu_iter2(); + check_icu_casemap(); + check_icu_sortmap(); + check_icu_normalizer(); + check_icu_tokenizer(); + check_icu_chain(); + check_chain_empty_token(); + check_chain_empty_chain(); + check_icu_iter1(); + check_icu_iter2(); - test_bug_1140(); + check_bug_1140(); #else /* YAZ_HAVE_ICU */ - printf("ICU unit tests omitted.\n" - "Please install libicu36-dev and icu-doc or similar\n"); + yaz_log(YLOG_LOG, "ICU unit tests omitted"); YAZ_CHECK(0 == 0); #endif /* YAZ_HAVE_ICU */