X-Git-Url: http://git.indexdata.com/?p=yaz-moved-to-github.git;a=blobdiff_plain;f=test%2Ftst_icu_I18N.c;h=2ad005ca0fe2a4b2b1b06645d426008f12f511e6;hp=35b4c56b3139c783a7035404104459d33b7b0654;hb=dc0ea3eaeb38d04f10118bfe62cd8e6c80033ce6;hpb=e005fd551460aab9ccefb561e42c746cc58fd302 diff --git a/test/tst_icu_I18N.c b/test/tst_icu_I18N.c index 35b4c56..2ad005c 100644 --- a/test/tst_icu_I18N.c +++ b/test/tst_icu_I18N.c @@ -1,23 +1,7 @@ -/* $Id: tst_icu_I18N.c,v 1.6 2007-10-24 14:48:17 marc Exp $ - Copyright (c) 2006-2007, Index Data. - - This file is part of Pazpar2. - - Pazpar2 is free software; you can redistribute it and/or modify it under - the terms of the GNU General Public License as published by the Free - Software Foundation; either version 2, or (at your option) any later - version. - - Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY - WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - for more details. - - You should have received a copy of the GNU General Public License - along with Pazpar2; see the file LICENSE. If not, write to the - Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA - 02111-1307, USA. -*/ +/* This file is part of the YAZ toolkit. + * Copyright (C) 1995-2010 Index Data + * See the file LICENSE for details. + */ /* DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 */ @@ -32,16 +16,19 @@ #endif #include +#include +#include -#if HAVE_ICU +#if YAZ_HAVE_ICU #include +#if YAZ_POSIX_THREADS +#include +#endif + #include #include -/* DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 */ - - #define MAX_KEY_SIZE 256 struct icu_termmap { @@ -50,8 +37,7 @@ struct icu_termmap }; - -int icu_termmap_cmp(const void *vp1, const void *vp2) +static int icu_termmap_cmp(const void *vp1, const void *vp2) { struct icu_termmap *itmp1 = *(struct icu_termmap **) vp1; struct icu_termmap *itmp2 = *(struct icu_termmap **) vp2; @@ -61,13 +47,11 @@ int icu_termmap_cmp(const void *vp1, const void *vp2) cmp = strcmp((const char *)itmp1->sort_key, (const char *)itmp2->sort_key); return cmp; -}; - - +} -int test_icu_casemap(const char * locale, char action, - const char * src8cstr, const char * chk8cstr) +static int test_icu_casemap(const char * locale, char action, + const char * src8cstr, const char * chk8cstr) { int success = 0; UErrorCode status = U_ZERO_ERROR; @@ -89,9 +73,7 @@ int test_icu_casemap(const char * locale, char action, /* converting to UTF8 */ icu_utf16_to_utf8(dest8, dest16, &status); - - /* determine success */ if (dest8->utf8 && (dest8->utf8_len == strlen(chk8cstr)) @@ -101,12 +83,15 @@ int test_icu_casemap(const char * locale, char action, success = 0; /* report failures */ - if (!success){ - printf("\nERROR\n"); - printf("original string: '%s' (%d)\n", src8cstr, src8cstr_len); - printf("icu_casemap '%s:%c' '%s' (%d)\n", - locale, action, dest8->utf8, dest8->utf8_len); - printf("expected string: '%s' (%d)\n", chk8cstr, chk8cstr_len); + if (!success) + { + yaz_log(YLOG_WARN, "test_icu_casemap failed"); + yaz_log(YLOG_LOG, "Original string: '%s' (%d)", + src8cstr, src8cstr_len); + yaz_log(YLOG_LOG, "icu_casemap '%s:%c' '%s' (%d)", + locale, action, dest8->utf8, dest8->utf8_len); + yaz_log(YLOG_LOG, "expected string: '%s' (%d)", + chk8cstr, chk8cstr_len); } /* clean the buffers */ @@ -114,18 +99,12 @@ int test_icu_casemap(const char * locale, char action, icu_buf_utf8_destroy(dest8); icu_buf_utf16_destroy(src16); icu_buf_utf16_destroy(dest16); - - + return success; } - - -/* DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 */ - -void test_icu_I18N_casemap(int argc, char **argv) +static void check_icu_casemap(void) { - /* Locale 'en' */ /* successful tests */ @@ -145,7 +124,6 @@ void test_icu_I18N_casemap(int argc, char **argv) "A ReD fOx hunTS sQUirriLs", "A Red Fox Hunts Squirrils")); - /* Locale 'da' */ /* success expected */ @@ -186,11 +164,8 @@ void test_icu_I18N_casemap(int argc, char **argv) } - -/* DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 */ - -int test_icu_sortmap(const char * locale, int src_list_len, - const char ** src_list, const char ** chk_list) +static int test_icu_sortmap(const char * locale, int src_list_len, + const char ** src_list, const char ** chk_list) { int success = 1; @@ -206,67 +181,61 @@ int test_icu_sortmap(const char * locale, int src_list_len, UCollator *coll = ucol_open(locale, &status); icu_check_status(status); - if(U_FAILURE(status)) + if (U_FAILURE(status)) return 0; /* assigning display terms and sort keys using buf 8 and buf16 */ - for( i = 0; i < src_list_len; i++) - { + for (i = 0; i < src_list_len; i++) + { - list[i] = (struct icu_termmap *) malloc(sizeof(struct icu_termmap)); + list[i] = (struct icu_termmap *) malloc(sizeof(struct icu_termmap)); - /* copy display term */ - strcpy(list[i]->disp_term, src_list[i]); + /* copy display term */ + strcpy(list[i]->disp_term, src_list[i]); - /* transforming to UTF16 */ - icu_utf16_from_utf8_cstr(buf16, list[i]->disp_term, &status); - icu_check_status(status); + /* transforming to UTF16 */ + icu_utf16_from_utf8_cstr(buf16, list[i]->disp_term, &status); + icu_check_status(status); - /* computing sortkeys */ - icu_sortkey8_from_utf16(coll, buf8, buf16, &status); - icu_check_status(status); + /* computing sortkeys */ + icu_sortkey8_from_utf16(coll, buf8, buf16, &status); + icu_check_status(status); - /* assigning sortkeys */ - memcpy(list[i]->sort_key, buf8->utf8, buf8->utf8_len); - } - + /* assigning sortkeys */ + memcpy(list[i]->sort_key, buf8->utf8, buf8->utf8_len); + } /* do the sorting */ - qsort(list, src_list_len, - sizeof(struct icu_termmap *), icu_termmap_cmp); + qsort(list, src_list_len, sizeof(struct icu_termmap *), icu_termmap_cmp); /* checking correct sorting */ - for (i = 0; i < src_list_len; i++){ + for (i = 0; i < src_list_len; i++) + { if (0 != strcmp(list[i]->disp_term, chk_list[i])){ success = 0; } } - if(!success){ - printf("\nERROR\n"); - printf("Input str: '%s' : ", locale); + if (!success) + { + yaz_log(YLOG_LOG, "ERROR"); + yaz_log(YLOG_LOG, "Input str:'%s':", locale); for (i = 0; i < src_list_len; i++) { - printf(" '%s'", list[i]->disp_term); + yaz_log(YLOG_LOG, " '%s'", list[i]->disp_term); } - printf("\n"); - printf("ICU sort: '%s' : ", locale); + yaz_log(YLOG_LOG, "ICU sort: '%s':", locale); for (i = 0; i < src_list_len; i++) { - printf(" '%s'", list[i]->disp_term); + yaz_log(YLOG_LOG, " '%s'", list[i]->disp_term); } - printf("\n"); - printf("Expected: '%s' : ", locale); + yaz_log(YLOG_LOG, "Expected: '%s':", locale); for (i = 0; i < src_list_len; i++) { - printf(" '%s'", chk_list[i]); + yaz_log(YLOG_LOG, " '%s'", chk_list[i]); } - printf("\n"); } - - - for( i = 0; i < src_list_len; i++) + for (i = 0; i < src_list_len; i++) free(list[i]); - - + ucol_close(coll); icu_buf_utf8_destroy(buf8); @@ -275,12 +244,8 @@ int test_icu_sortmap(const char * locale, int src_list_len, return success; } - -/* DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 */ - -void test_icu_I18N_sortmap(int argc, char **argv) +static void check_icu_sortmap(void) { - /* successful tests */ size_t en_1_len = 6; const char * en_1_src[6] = {"z", "K", "a", "A", "Z", "k"}; @@ -308,18 +273,11 @@ void test_icu_I18N_sortmap(int argc, char **argv) YAZ_CHECK(test_icu_sortmap("de_AT", de_1_len, de_1_src, de_1_cck)); YAZ_CHECK(test_icu_sortmap("de_DE", de_1_len, de_1_src, de_1_cck)); } - } - -/* DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 */ - - - - -int test_icu_normalizer(const char * rules8cstr, - const char * src8cstr, - const char * chk8cstr) +static int test_icu_normalizer(const char * rules8cstr, + const char * src8cstr, + const char * chk8cstr) { int success = 0; @@ -328,47 +286,43 @@ int test_icu_normalizer(const char * rules8cstr, struct icu_buf_utf16 * src16 = icu_buf_utf16_create(0); struct icu_buf_utf16 * dest16 = icu_buf_utf16_create(0); struct icu_buf_utf8 * dest8 = icu_buf_utf8_create(0); - struct icu_normalizer * normalizer - = icu_normalizer_create(rules8cstr, 'f', &status); + struct icu_transform * transform + = icu_transform_create(rules8cstr, 'f', 0, &status); icu_check_status(status); icu_utf16_from_utf8_cstr(src16, src8cstr, &status); icu_check_status(status); - icu_normalizer_normalize(normalizer, dest16, src16, &status); + icu_transform_trans(transform, dest16, src16, &status); icu_check_status(status); icu_utf16_to_utf8(dest8, dest16, &status); icu_check_status(status); - if(!strcmp((const char *) dest8->utf8, + if (!strcmp((const char *) dest8->utf8, (const char *) chk8cstr)) success = 1; - else { + else + { success = 0; - printf("Normalization\n"); - printf("Rules: '%s'\n", rules8cstr); - printf("Input: '%s'\n", src8cstr); - printf("Normalized: '%s'\n", dest8->utf8); - printf("Expected: '%s'\n", chk8cstr); + yaz_log(YLOG_LOG, "Normalization"); + yaz_log(YLOG_LOG, " Rules: '%s'", rules8cstr); + yaz_log(YLOG_LOG, " Input: '%s'", src8cstr); + yaz_log(YLOG_LOG, " Normalized: '%s'", dest8->utf8); + yaz_log(YLOG_LOG, " Expected: '%s'", chk8cstr); } - - icu_normalizer_destroy(normalizer); + icu_transform_destroy(transform); icu_buf_utf16_destroy(src16); icu_buf_utf16_destroy(dest16); icu_buf_utf8_destroy(dest8); return success; -}; - - -/* DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 */ +} -void test_icu_I18N_normalizer(int argc, char **argv) +static void check_icu_normalizer(void) { - YAZ_CHECK(test_icu_normalizer("[:Punctuation:] Any-Remove", "Don't shoot!", "Dont shoot")); @@ -394,18 +348,13 @@ void test_icu_I18N_normalizer(int argc, char **argv) " word4you? ", "word4you")); - YAZ_CHECK(test_icu_normalizer("NFD; [:Nonspacing Mark:] Remove; NFC", "à côté de l'alcôve ovoïde", "a cote de l'alcove ovoide")); - } - -/* DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 */ - -int test_icu_tokenizer(const char * locale, char action, - const char * src8cstr, int count) +static int test_icu_tokenizer(const char * locale, char action, + const char * src8cstr, int count) { int success = 1; @@ -427,22 +376,23 @@ int test_icu_tokenizer(const char * locale, char action, /* attach text buffer to tokenizer */ icu_tokenizer_attach(tokenizer, src16, &status); icu_check_status(status); - YAZ_CHECK(tokenizer->bi); /* perform work on tokens */ - while(icu_tokenizer_next_token(tokenizer, tkn16, &status)){ + while (icu_tokenizer_next_token(tokenizer, tkn16, &status)) + { icu_check_status(status); /* converting to UTF8 */ icu_utf16_to_utf8(tkn8, tkn16, &status); } - if (count != icu_tokenizer_token_count(tokenizer)){ + if (count != icu_tokenizer_token_count(tokenizer)) + { success = 0; - printf("\nTokenizer '%s:%c' Error: \n", locale, action); - printf("Input: '%s'\n", src8cstr); - printf("Tokens: %d", icu_tokenizer_token_count(tokenizer)); - printf(", expected: %d\n", count); + yaz_log(YLOG_LOG, "Tokenizer '%s:%c' Error:", locale, action); + yaz_log(YLOG_LOG, " Input: '%s'", src8cstr); + yaz_log(YLOG_LOG, " Tokens: %d", icu_tokenizer_token_count(tokenizer)); + yaz_log(YLOG_LOG, " Expected: %d", count); } icu_tokenizer_destroy(tokenizer); @@ -453,13 +403,8 @@ int test_icu_tokenizer(const char * locale, char action, return success; } - -/* DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 */ - -void test_icu_I18N_tokenizer(int argc, char **argv) +static void check_icu_tokenizer(void) { - - const char * en_str = "O Romeo, Romeo! wherefore art thou Romeo?"; @@ -468,8 +413,6 @@ void test_icu_I18N_tokenizer(int argc, char **argv) YAZ_CHECK(test_icu_tokenizer("en", 'w', en_str, 16)); YAZ_CHECK(test_icu_tokenizer("en", 'c', en_str, 41)); - - { const char * da_str = "Blåbærtærte. Denne kage stammer fra Finland. " @@ -480,11 +423,9 @@ void test_icu_I18N_tokenizer(int argc, char **argv) YAZ_CHECK(test_icu_tokenizer("da", 'w', da_str, 37)); YAZ_CHECK(test_icu_tokenizer("da", 'c', da_str, 110)); } - } - -void test_icu_I18N_chain(int argc, char **argv) +static void check_icu_chain(void) { const char * en_str = "O Romeo, Romeo! wherefore art thou\t Romeo?"; @@ -492,118 +433,77 @@ void test_icu_I18N_chain(int argc, char **argv) UErrorCode status = U_ZERO_ERROR; struct icu_chain * chain = 0; - - const char * xml_str = "" - "" + const char * xml_str = "" + "" "" - "" + "" "" "" - "" - "" - ""; + ""; xmlDoc *doc = xmlParseMemory(xml_str, strlen(xml_str)); xmlNode *xml_node = xmlDocGetRootElement(doc); YAZ_CHECK(xml_node); - // printf("ICU chain:\ninput: '%s'\n", en_str); - - - chain = icu_chain_xml_config(xml_node, (uint8_t *) "en", &status); - -#if 0 - chain = icu_chain_create((uint8_t *) "en:word", (uint8_t *) "en"); - step = icu_chain_insert_step(chain, ICU_chain_step_type_normalize, - (const uint8_t *) "[:Control:] Any-Remove", - &status); - step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize, - (const uint8_t *) "s", - &status); - step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize, - (const uint8_t *) "l", - &status); - step = icu_chain_insert_step(chain, ICU_chain_step_type_normalize, - (const uint8_t *) - "[[:WhiteSpace:][:Punctuation:]] Any-Remove", - &status); - step = icu_chain_insert_step(chain, ICU_chain_step_type_display, - (const uint8_t *)"", - &status); -/* step = icu_chain_insert_step(chain, ICU_chain_step_type_normalize, */ -/* (const uint8_t *) "Lower", */ -/* &status); */ - step = icu_chain_insert_step(chain, ICU_chain_step_type_casemap, - (const uint8_t *) "l", - &status); - step = icu_chain_insert_step(chain, ICU_chain_step_type_index, - (const uint8_t *)"", - &status); -/* step = icu_chain_insert_step(chain, ICU_chain_step_type_sortkey, */ -/* (const uint8_t *)"", */ -/* &status); */ - -#endif + chain = icu_chain_xml_config(xml_node, 0, &status); xmlFreeDoc(doc); YAZ_CHECK(chain); YAZ_CHECK(icu_chain_assign_cstr(chain, en_str, &status)); - while (icu_chain_next_token(chain, &status)){ - ; - // printf("%d '%s' '%s'\n", - // icu_chain_get_token_count(chain), - // icu_chain_get_norm(chain), - // icu_chain_get_display(chain)); + while (icu_chain_next_token(chain, &status)) + { + yaz_log(YLOG_LOG, "%d '%s' '%s'", + icu_chain_token_number(chain), + icu_chain_token_norm(chain), + icu_chain_token_display(chain)); } - YAZ_CHECK_EQ(icu_chain_get_token_count(chain), 7); + YAZ_CHECK_EQ(icu_chain_token_number(chain), 7); YAZ_CHECK(icu_chain_assign_cstr(chain, "what is this?", &status)); - while (icu_chain_next_token(chain, &status)){ - ; - //printf("%d '%s' '%s'\n", - // icu_chain_get_token_count(chain), - // icu_chain_get_norm(chain), - // icu_chain_get_display(chain)); + while (icu_chain_next_token(chain, &status)) + { + yaz_log(YLOG_LOG, "%d '%s' '%s'", + icu_chain_token_number(chain), + icu_chain_token_norm(chain), + icu_chain_token_display(chain)); } - YAZ_CHECK_EQ(icu_chain_get_token_count(chain), 3); + YAZ_CHECK_EQ(icu_chain_token_number(chain), 3); icu_chain_destroy(chain); } -void test_bug_1140(void) +static void check_bug_1140(void) { UErrorCode status = U_ZERO_ERROR; struct icu_chain * chain = 0; - const char * xml_str = "" + const char * xml_str = "" /* if the first rule is normalize instead. Then it works */ #if 0 - "" + "" #endif "" - "" + "" "" "" - "" - "" - ""; + ""; xmlDoc *doc = xmlParseMemory(xml_str, strlen(xml_str)); xmlNode *xml_node = xmlDocGetRootElement(doc); YAZ_CHECK(xml_node); - chain = icu_chain_xml_config(xml_node, (uint8_t *) "en", &status); + chain = icu_chain_xml_config(xml_node, 0, &status); xmlFreeDoc(doc); YAZ_CHECK(chain); @@ -612,113 +512,313 @@ void test_bug_1140(void) chain, "O Romeo, Romeo! wherefore art thou\t Romeo?", &status)); - while (icu_chain_next_token(chain, &status)){ + while (icu_chain_next_token(chain, &status)) + { ; - //printf("%d '%s' '%s'\n", - // icu_chain_get_token_count(chain), - // icu_chain_get_norm(chain), - // icu_chain_get_display(chain)); - + /* printf("%d '%s' '%s'\n", + icu_chain_token_number(chain), + icu_chain_token_norm(chain), + icu_chain_token_display(chain)); */ } - YAZ_CHECK_EQ(icu_chain_get_token_count(chain), 7); + YAZ_CHECK_EQ(icu_chain_token_number(chain), 7); YAZ_CHECK(icu_chain_assign_cstr(chain, "what is this?", &status)); - while (icu_chain_next_token(chain, &status)){ - ; - //printf("%d '%s' '%s'\n", - // icu_chain_get_token_count(chain), - // icu_chain_get_norm(chain), - // icu_chain_get_display(chain)); + while (icu_chain_next_token(chain, &status)) + { + ; + /* printf("%d '%s' '%s'\n", + icu_chain_token_number(chain), + icu_chain_token_norm(chain), + icu_chain_token_display(chain)); */ } /* we expect 'what' 'is' 'this', i.e. 3 tokens */ - YAZ_CHECK_EQ(icu_chain_get_token_count(chain), 3); + YAZ_CHECK_EQ(icu_chain_token_number(chain), 3); icu_chain_destroy(chain); } - -void test_chain_empty_token(void) +static void check_chain_empty_token(void) { UErrorCode status = U_ZERO_ERROR; struct icu_chain * chain = 0; - const char * xml_str = "" + const char * xml_str = "" "" - "" - "" - ""; + "" + ""; xmlDoc *doc = xmlParseMemory(xml_str, strlen(xml_str)); xmlNode *xml_node = xmlDocGetRootElement(doc); YAZ_CHECK(xml_node); - chain = icu_chain_xml_config(xml_node, (uint8_t *) "en", &status); + chain = icu_chain_xml_config(xml_node, 0, &status); xmlFreeDoc(doc); YAZ_CHECK(chain); YAZ_CHECK(icu_chain_assign_cstr( - chain, "a string with 15 wordbreaks and 8 tokens", + chain, "a string with 15 tokenss and 8 displays", &status)); - while (icu_chain_next_token(chain, &status)){ + while (icu_chain_next_token(chain, &status)) + { ; - //printf("%d '%s' '%s'\n", - // icu_chain_get_token_count(chain), - // icu_chain_get_norm(chain), - // icu_chain_get_display(chain)); + /* printf("%d '%s' '%s'\n", + icu_chain_token_number(chain), + icu_chain_token_norm(chain), + icu_chain_token_display(chain)); */ } - YAZ_CHECK_EQ(icu_chain_get_token_count(chain), 8); + YAZ_CHECK_EQ(icu_chain_token_number(chain), 15); icu_chain_destroy(chain); } -#endif // HAVE_ICU +static void check_chain_empty_chain(void) +{ + UErrorCode status = U_ZERO_ERROR; + struct icu_chain * chain = 0; -/* DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 */ + const char * xml_str = "" + ""; + + const char * src8 = "some 5487 weired !¤%&(/& sTuFf"; + char * dest8 = 0; -int main(int argc, char **argv) + xmlDoc *doc = xmlParseMemory(xml_str, strlen(xml_str)); + xmlNode *xml_node = xmlDocGetRootElement(doc); + YAZ_CHECK(xml_node); + + chain = icu_chain_xml_config(xml_node, 0, &status); + + xmlFreeDoc(doc); + YAZ_CHECK(chain); + + YAZ_CHECK(icu_chain_assign_cstr( + chain, src8, + &status)); + + while (icu_chain_next_token(chain, &status)) + { + ; + /* printf("%d '%s' '%s'\n", + icu_chain_token_number(chain), + icu_chain_token_norm(chain), + icu_chain_token_display(chain)); */ + } + + YAZ_CHECK_EQ(icu_chain_token_number(chain), 1); + + dest8 = (char *) icu_chain_token_norm(chain); + YAZ_CHECK_EQ(strcmp(src8, dest8), 0); + + icu_chain_destroy(chain); +} + +static void check_icu_iter1(void) { + UErrorCode status = U_ZERO_ERROR; + struct icu_chain * chain = 0; + xmlNode *xml_node; + yaz_icu_iter_t iter; - YAZ_CHECK_INIT(argc, argv); - YAZ_CHECK_LOG(); + const char * xml_str = "" + "" + "" + ""; + + xmlDoc *doc = xmlParseMemory(xml_str, strlen(xml_str)); + YAZ_CHECK(doc); + if (!doc) + return; + xml_node = xmlDocGetRootElement(doc); + YAZ_CHECK(xml_node); + if (!xml_node) + return ; -#if HAVE_ICU + chain = icu_chain_xml_config(xml_node, 0, &status); - test_icu_I18N_casemap(argc, argv); - test_icu_I18N_sortmap(argc, argv); - test_icu_I18N_normalizer(argc, argv); - test_icu_I18N_tokenizer(argc, argv); - test_icu_I18N_chain(argc, argv); - test_chain_empty_token(); - test_bug_1140(); + xmlFreeDoc(doc); + YAZ_CHECK(chain); + + iter = icu_iter_create(chain); + icu_iter_first(iter, "a string with 15 tokens and 8 displays"); + YAZ_CHECK(iter); + if (!iter) + return; + while (icu_iter_next(iter)) + { + yaz_log(YLOG_LOG, "[%s]", icu_iter_get_norm(iter)); + } + icu_iter_destroy(iter); + icu_chain_destroy(chain); +} -#else /* HAVE_ICU */ +static int test_iter(struct icu_chain *chain, const char *input, + const char *expected) +{ + yaz_icu_iter_t iter = icu_iter_create(chain); + WRBUF result, second; + int success = 1; - printf("ICU unit tests omitted.\n" - "Please install libicu36-dev and icu-doc or similar\n"); - YAZ_CHECK(0 == 0); + if (!iter) + { + yaz_log(YLOG_WARN, "test_iter: input=%s !iter", input); + return 0; + } -#endif /* HAVE_ICU */ - - YAZ_CHECK_TERM; + if (icu_iter_next(iter)) + { + yaz_log(YLOG_WARN, "test_iter: expecting 0 before icu_iter_first"); + return 0; + } + + result = wrbuf_alloc(); + icu_iter_first(iter, input); + while (icu_iter_next(iter)) + { + wrbuf_puts(result, "["); + wrbuf_puts(result, icu_iter_get_norm(iter)); + wrbuf_puts(result, "]"); + } + + second = wrbuf_alloc(); + icu_iter_first(iter, input); + while (icu_iter_next(iter)) + { + wrbuf_puts(second, "["); + wrbuf_puts(second, icu_iter_get_norm(iter)); + wrbuf_puts(second, "]"); + } + + icu_iter_destroy(iter); + + if (strcmp(expected, wrbuf_cstr(result))) + { + yaz_log(YLOG_WARN, "test_iter: input=%s expected=%s got=%s", + input, expected, wrbuf_cstr(result)); + success = 0; + } + + if (strcmp(expected, wrbuf_cstr(second))) + { + yaz_log(YLOG_WARN, "test_iter: input=%s expected=%s got=%s (2nd)", + input, expected, wrbuf_cstr(second)); + success = 0; + } + + wrbuf_destroy(result); + wrbuf_destroy(second); + return success; } +static void *iter_thread(void *p) +{ + struct icu_chain *chain = (struct icu_chain *) p; + int i; + + for (i = 0; i < 10000; i++) + { + YAZ_CHECK(test_iter(chain, "Adobe Acrobat Reader, 1991-1999.", + "[adobe][acrobat][reader][1991][][1999][]")); + } + return 0; +} -/* DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 */ +static void check_iter_threads(struct icu_chain *chain) +{ +#if YAZ_POSIX_THREADS +#define NO_THREADS 1 + + pthread_t t[NO_THREADS]; + int i; + + for (i = 0; i < NO_THREADS; i++) + pthread_create(t + i, 0, iter_thread, chain); + + for (i = 0; i < NO_THREADS; i++) + pthread_join(t[i], 0); +#endif +} +static void check_icu_iter2(void) +{ + UErrorCode status = U_ZERO_ERROR; + struct icu_chain * chain = 0; + xmlNode *xml_node; + + const char * xml_str = "" + "" + "" + "" + "" + "" + "" + ""; + xmlDoc *doc = xmlParseMemory(xml_str, strlen(xml_str)); + YAZ_CHECK(doc); + if (!doc) + return; + xml_node = xmlDocGetRootElement(doc); + YAZ_CHECK(xml_node); + if (!xml_node) + return ; + + chain = icu_chain_xml_config(xml_node, 0, &status); + + xmlFreeDoc(doc); + YAZ_CHECK(chain); + + YAZ_CHECK(test_iter(chain, "Adobe Acrobat Reader, 1991-1999.", + "[adobe][acrobat][reader][1991][][1999][]")); + + check_iter_threads(chain); + + icu_chain_destroy(chain); +} + +#endif /* YAZ_HAVE_ICU */ + +int main(int argc, char **argv) +{ + YAZ_CHECK_INIT(argc, argv); + YAZ_CHECK_LOG(); +#if YAZ_HAVE_ICU + + check_icu_casemap(); + check_icu_sortmap(); + check_icu_normalizer(); + check_icu_tokenizer(); + check_icu_chain(); + check_chain_empty_token(); + check_chain_empty_chain(); + check_icu_iter1(); + check_icu_iter2(); + + check_bug_1140(); + +#else /* YAZ_HAVE_ICU */ + + yaz_log(YLOG_LOG, "ICU unit tests omitted"); + YAZ_CHECK(0 == 0); + +#endif /* YAZ_HAVE_ICU */ + + YAZ_CHECK_TERM; +} /* * Local variables: * c-basic-offset: 4 + * c-file-style: "Stroustrup" * indent-tabs-mode: nil * End: * vim: shiftwidth=4 tabstop=8 expandtab */ +