X-Git-Url: http://git.indexdata.com/?p=yaz-moved-to-github.git;a=blobdiff_plain;f=test%2Ftst_icu_I18N.c;h=4bef6653126acfd62993a6cdfbde7058300968b8;hp=768e387cdff97809d28b94f7f5f49c0dbf4ca773;hb=d72929b7de6221054feb6a220dc72289b0f3e03d;hpb=0ffb25b0bf78f53399267664b71206c88980c4da diff --git a/test/tst_icu_I18N.c b/test/tst_icu_I18N.c index 768e387..4bef665 100644 --- a/test/tst_icu_I18N.c +++ b/test/tst_icu_I18N.c @@ -1,29 +1,16 @@ -/* $Id: tst_icu_I18N.c,v 1.1 2007-10-22 12:21:39 adam Exp $ - Copyright (c) 2006-2007, Index Data. - - This file is part of Pazpar2. - - Pazpar2 is free software; you can redistribute it and/or modify it under - the terms of the GNU General Public License as published by the Free - Software Foundation; either version 2, or (at your option) any later - version. - - Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY - WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - for more details. - - You should have received a copy of the GNU General Public License - along with Pazpar2; see the file LICENSE. If not, write to the - Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA - 02111-1307, USA. -*/ +/* + * Copyright (C) 2005-2007, Index Data ApS + * See the file LICENSE for details. + * + * $Id: tst_icu_I18N.c,v 1.15 2007-11-12 11:11:16 adam Exp $ + * + */ -// DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 +/* DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 */ #if HAVE_CONFIG_H -#include "cconfig.h" +#include "config.h" #endif #define USE_TIMING 0 @@ -33,23 +20,20 @@ #include - - -#ifdef HAVE_ICU +#if YAZ_HAVE_ICU #include #include #include -//#include -// DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 +/* DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 */ #define MAX_KEY_SIZE 256 struct icu_termmap { - uint8_t sort_key[MAX_KEY_SIZE]; // standard C string '\0' terminated - char disp_term[MAX_KEY_SIZE]; // standard C utf-8 string + uint8_t sort_key[MAX_KEY_SIZE]; /* standard C string '\0' terminated */ + char disp_term[MAX_KEY_SIZE]; /* standard C utf-8 string */ }; @@ -84,18 +68,18 @@ int test_icu_casemap(const char * locale, char action, int src8cstr_len = strlen(src8cstr); int chk8cstr_len = strlen(chk8cstr); - // converting to UTF16 + /* converting to UTF16 */ icu_utf16_from_utf8_cstr(src16, src8cstr, &status); - // perform case mapping + /* perform case mapping */ icu_utf16_casemap(dest16, src16, locale, action, &status); - // converting to UTF8 + /* converting to UTF8 */ icu_utf16_to_utf8(dest8, dest16, &status); - // determine success + /* determine success */ if (dest8->utf8 && (dest8->utf8_len == strlen(chk8cstr)) && !strcmp(chk8cstr, (const char *) dest8->utf8)) @@ -103,7 +87,7 @@ int test_icu_casemap(const char * locale, char action, else success = 0; - // report failures + /* report failures */ if (!success){ printf("\nERROR\n"); printf("original string: '%s' (%d)\n", src8cstr, src8cstr_len); @@ -112,7 +96,7 @@ int test_icu_casemap(const char * locale, char action, printf("expected string: '%s' (%d)\n", chk8cstr, chk8cstr_len); } - // clean the buffers + /* clean the buffers */ icu_buf_utf8_destroy(src8); icu_buf_utf8_destroy(dest8); icu_buf_utf16_destroy(src16); @@ -124,14 +108,14 @@ int test_icu_casemap(const char * locale, char action, -// DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 +/* DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 */ void test_icu_I18N_casemap(int argc, char **argv) { - // Locale 'en' + /* Locale 'en' */ - // sucessful tests + /* successful tests */ YAZ_CHECK(test_icu_casemap("en", 'l', "A ReD fOx hunTS sQUirriLs", "a red fox hunts squirrils")); @@ -149,9 +133,9 @@ void test_icu_I18N_casemap(int argc, char **argv) "A Red Fox Hunts Squirrils")); - // Locale 'da' + /* Locale 'da' */ - // sucess expected + /* success expected */ YAZ_CHECK(test_icu_casemap("da", 'l', "åh ÆbLE, øs fLØde i Åen efter bLåBærGRødeN", "åh æble, øs fløde i åen efter blåbærgrøden")); @@ -168,9 +152,9 @@ void test_icu_I18N_casemap(int argc, char **argv) "åh ÆbLE, øs fLØde i Åen efter bLåBærGRødeN", "Åh Æble, Øs Fløde I Åen Efter Blåbærgrøden")); - // Locale 'de' + /* Locale 'de' */ - // sucess expected + /* success expected */ YAZ_CHECK(test_icu_casemap("de", 'l', "zWÖlf ärgerliche Würste rollen ÜBer die StRAße", "zwölf ärgerliche würste rollen über die straße")); @@ -190,7 +174,7 @@ void test_icu_I18N_casemap(int argc, char **argv) } -// DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 +/* DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 */ int test_icu_sortmap(const char * locale, int src_list_len, const char ** src_list, const char ** chk_list) @@ -212,35 +196,33 @@ int test_icu_sortmap(const char * locale, int src_list_len, if(U_FAILURE(status)) return 0; - // assigning display terms and sort keys using buf 8 and buf16 + /* assigning display terms and sort keys using buf 8 and buf16 */ for( i = 0; i < src_list_len; i++) { list[i] = (struct icu_termmap *) malloc(sizeof(struct icu_termmap)); - // copy display term + /* copy display term */ strcpy(list[i]->disp_term, src_list[i]); - // transforming to UTF16 + /* transforming to UTF16 */ icu_utf16_from_utf8_cstr(buf16, list[i]->disp_term, &status); icu_check_status(status); - // computing sortkeys + /* computing sortkeys */ icu_sortkey8_from_utf16(coll, buf8, buf16, &status); icu_check_status(status); - // assigning sortkeys + /* assigning sortkeys */ memcpy(list[i]->sort_key, buf8->utf8, buf8->utf8_len); - //strncpy(list[i]->sort_key, buf8->utf8, buf8->utf8_len); - //strcpy((char *) list[i]->sort_key, (const char *) buf8->utf8); } - // do the sorting + /* do the sorting */ qsort(list, src_list_len, sizeof(struct icu_termmap *), icu_termmap_cmp); - // checking correct sorting + /* checking correct sorting */ for (i = 0; i < src_list_len; i++){ if (0 != strcmp(list[i]->disp_term, chk_list[i])){ success = 0; @@ -257,7 +239,6 @@ int test_icu_sortmap(const char * locale, int src_list_len, printf("ICU sort: '%s' : ", locale); for (i = 0; i < src_list_len; i++) { printf(" '%s'", list[i]->disp_term); - //printf("(%d|%d)", list[i]->sort_key[0],list[i]->sort_key[1]); } printf("\n"); printf("Expected: '%s' : ", locale); @@ -282,12 +263,12 @@ int test_icu_sortmap(const char * locale, int src_list_len, } -// DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 +/* DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 */ void test_icu_I18N_sortmap(int argc, char **argv) { - // sucessful tests + /* successful tests */ size_t en_1_len = 6; const char * en_1_src[6] = {"z", "K", "a", "A", "Z", "k"}; const char * en_1_cck[6] = {"a", "A", "k", "K", "z", "Z"}; @@ -297,25 +278,28 @@ void test_icu_I18N_sortmap(int argc, char **argv) YAZ_CHECK(test_icu_sortmap("en_GB", en_1_len, en_1_src, en_1_cck)); YAZ_CHECK(test_icu_sortmap("en_US", en_1_len, en_1_src, en_1_cck)); - // sucessful tests - size_t da_1_len = 6; - const char * da_1_src[6] = {"z", "å", "o", "æ", "a", "ø"}; - const char * da_1_cck[6] = {"a", "o", "z", "æ", "ø", "å"}; - YAZ_CHECK(test_icu_sortmap("da", da_1_len, da_1_src, da_1_cck)); - YAZ_CHECK(test_icu_sortmap("da_DK", da_1_len, da_1_src, da_1_cck)); - - // sucessful tests - size_t de_1_len = 9; - const char * de_1_src[9] = {"u", "ä", "o", "t", "s", "ß", "ü", "ö", "a"}; - const char * de_1_cck[9] = {"a","ä", "o", "ö", "s", "ß", "t", "u", "ü"}; - YAZ_CHECK(test_icu_sortmap("de", de_1_len, de_1_src, de_1_cck)); - YAZ_CHECK(test_icu_sortmap("de_AT", de_1_len, de_1_src, de_1_cck)); - YAZ_CHECK(test_icu_sortmap("de_DE", de_1_len, de_1_src, de_1_cck)); + /* successful tests */ + { + size_t da_1_len = 6; + const char * da_1_src[6] = {"z", "å", "o", "æ", "a", "ø"}; + const char * da_1_cck[6] = {"a", "o", "z", "æ", "ø", "å"}; + YAZ_CHECK(test_icu_sortmap("da", da_1_len, da_1_src, da_1_cck)); + YAZ_CHECK(test_icu_sortmap("da_DK", da_1_len, da_1_src, da_1_cck)); + } + /* successful tests */ + { + size_t de_1_len = 9; + const char * de_1_src[9] = {"u", "ä", "o", "t", "s", "ß", "ü", "ö", "a"}; + const char * de_1_cck[9] = {"a","ä", "o", "ö", "s", "ß", "t", "u", "ü"}; + YAZ_CHECK(test_icu_sortmap("de", de_1_len, de_1_src, de_1_cck)); + YAZ_CHECK(test_icu_sortmap("de_AT", de_1_len, de_1_src, de_1_cck)); + YAZ_CHECK(test_icu_sortmap("de_DE", de_1_len, de_1_src, de_1_cck)); + } } -// DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 +/* DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 */ @@ -367,7 +351,7 @@ int test_icu_normalizer(const char * rules8cstr, }; -// DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 +/* DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 */ void test_icu_I18N_normalizer(int argc, char **argv) { @@ -405,7 +389,7 @@ void test_icu_I18N_normalizer(int argc, char **argv) } -// DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 +/* DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 */ int test_icu_tokenizer(const char * locale, char action, const char * src8cstr, int count) @@ -416,38 +400,28 @@ int test_icu_tokenizer(const char * locale, char action, struct icu_buf_utf16 * src16 = icu_buf_utf16_create(0); struct icu_buf_utf16 * tkn16 = icu_buf_utf16_create(0); struct icu_buf_utf8 * tkn8 = icu_buf_utf8_create(0); + struct icu_tokenizer * tokenizer = 0; - //printf("Input: '%s'\n", src8cstr); - - // transforming to UTF16 + /* transforming to UTF16 */ icu_utf16_from_utf8_cstr(src16, src8cstr, &status); icu_check_status(status); - // set up tokenizer - struct icu_tokenizer * tokenizer - = icu_tokenizer_create(locale, action, &status); + /* set up tokenizer */ + tokenizer = icu_tokenizer_create(locale, action, &status); icu_check_status(status); YAZ_CHECK(tokenizer); - // attach text buffer to tokenizer + /* attach text buffer to tokenizer */ icu_tokenizer_attach(tokenizer, src16, &status); icu_check_status(status); YAZ_CHECK(tokenizer->bi); - // perform work on tokens - //printf("Tokens: "); + /* perform work on tokens */ while(icu_tokenizer_next_token(tokenizer, tkn16, &status)){ icu_check_status(status); - // converting to UTF8 + /* converting to UTF8 */ icu_utf16_to_utf8(tkn8, tkn16, &status); - - //printf("token %d %d %d %d '%s'\n", - // - // icu_tokenizer_token_start(tokenizer), - // icu_tokenizer_token_end(tokenizer), - // icu_tokenizer_token_length(tokenizer), - // tkn8->utf8); } if (count != icu_tokenizer_token_count(tokenizer)){ @@ -467,7 +441,7 @@ int test_icu_tokenizer(const char * locale, char action, } -// DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 +/* DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 */ void test_icu_I18N_tokenizer(int argc, char **argv) { @@ -483,14 +457,16 @@ void test_icu_I18N_tokenizer(int argc, char **argv) - const char * da_str - = "Blåbærtærte. Denne kage stammer fra Finland. " - "Den er med blåbær, men alle sommerens forskellige bær kan bruges."; - - YAZ_CHECK(test_icu_tokenizer("da", 's', da_str, 3)); - YAZ_CHECK(test_icu_tokenizer("dar", 'l', da_str, 17)); - YAZ_CHECK(test_icu_tokenizer("da", 'w', da_str, 37)); - YAZ_CHECK(test_icu_tokenizer("da", 'c', da_str, 110)); + { + const char * da_str + = "Blåbærtærte. Denne kage stammer fra Finland. " + "Den er med blåbær, men alle sommerens forskellige bær kan bruges."; + + YAZ_CHECK(test_icu_tokenizer("da", 's', da_str, 3)); + YAZ_CHECK(test_icu_tokenizer("dar", 'l', da_str, 17)); + YAZ_CHECK(test_icu_tokenizer("da", 'w', da_str, 37)); + YAZ_CHECK(test_icu_tokenizer("da", 'c', da_str, 110)); + } } @@ -500,63 +476,24 @@ void test_icu_I18N_chain(int argc, char **argv) const char * en_str = "O Romeo, Romeo! wherefore art thou\t Romeo?"; - printf("ICU chain:\ninput: '%s'\n", en_str); - UErrorCode status = U_ZERO_ERROR; - //struct icu_chain_step * step = 0; struct icu_chain * chain = 0; - const char * xml_str = "" - "" + const char * xml_str = "" + "" "" - "" + "" "" "" - "" - "" - ""; + ""; xmlDoc *doc = xmlParseMemory(xml_str, strlen(xml_str)); xmlNode *xml_node = xmlDocGetRootElement(doc); YAZ_CHECK(xml_node); - - chain = icu_chain_xml_config(xml_node, &status); - -#if 0 - chain = icu_chain_create((uint8_t *) "en:word", (uint8_t *) "en"); - step = icu_chain_insert_step(chain, ICU_chain_step_type_normalize, - (const uint8_t *) "[:Control:] Any-Remove", - &status); - step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize, - (const uint8_t *) "s", - &status); - step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize, - (const uint8_t *) "l", - &status); - step = icu_chain_insert_step(chain, ICU_chain_step_type_normalize, - (const uint8_t *) - "[[:WhiteSpace:][:Punctuation:]] Any-Remove", - &status); - step = icu_chain_insert_step(chain, ICU_chain_step_type_display, - (const uint8_t *)"", - &status); -/* step = icu_chain_insert_step(chain, ICU_chain_step_type_normalize, */ -/* (const uint8_t *) "Lower", */ -/* &status); */ - step = icu_chain_insert_step(chain, ICU_chain_step_type_casemap, - (const uint8_t *) "l", - &status); - step = icu_chain_insert_step(chain, ICU_chain_step_type_index, - (const uint8_t *)"", - &status); -/* step = icu_chain_insert_step(chain, ICU_chain_step_type_sortkey, */ -/* (const uint8_t *)"", */ -/* &status); */ - -#endif + chain = icu_chain_xml_config(xml_node, 0, &status); xmlFreeDoc(doc); YAZ_CHECK(chain); @@ -564,26 +501,28 @@ void test_icu_I18N_chain(int argc, char **argv) YAZ_CHECK(icu_chain_assign_cstr(chain, en_str, &status)); while (icu_chain_next_token(chain, &status)){ - printf("%d '%s' '%s'\n", - icu_chain_get_token_count(chain), - icu_chain_get_norm(chain), - icu_chain_get_display(chain)); + ; + /* printf("%d '%s' '%s'\n", + icu_chain_token_number(chain), + icu_chain_token_norm(chain), + icu_chain_token_display(chain)); */ } - YAZ_CHECK_EQ(icu_chain_get_token_count(chain), 7); + YAZ_CHECK_EQ(icu_chain_token_number(chain), 7); YAZ_CHECK(icu_chain_assign_cstr(chain, "what is this?", &status)); while (icu_chain_next_token(chain, &status)){ - printf("%d '%s' '%s'\n", - icu_chain_get_token_count(chain), - icu_chain_get_norm(chain), - icu_chain_get_display(chain)); + ; + /* printf("%d '%s' '%s'\n", + icu_chain_token_number(chain), + icu_chain_token_norm(chain), + icu_chain_token_display(chain)); */ } - YAZ_CHECK_EQ(icu_chain_get_token_count(chain), 3); + YAZ_CHECK_EQ(icu_chain_token_number(chain), 3); icu_chain_destroy(chain); } @@ -591,35 +530,27 @@ void test_icu_I18N_chain(int argc, char **argv) void test_bug_1140(void) { - const char * en_str - = "O Romeo, Romeo! wherefore art thou\t Romeo?"; - - printf("ICU chain:\ninput: '%s'\n", en_str); - UErrorCode status = U_ZERO_ERROR; - //struct icu_chain_step * step = 0; struct icu_chain * chain = 0; - const char * xml_str = "" + const char * xml_str = "" /* if the first rule is normalize instead. Then it works */ #if 0 - "" + "" #endif "" - "" + "" "" "" - "" - "" - ""; + ""; xmlDoc *doc = xmlParseMemory(xml_str, strlen(xml_str)); xmlNode *xml_node = xmlDocGetRootElement(doc); YAZ_CHECK(xml_node); - chain = icu_chain_xml_config(xml_node, &status); + chain = icu_chain_xml_config(xml_node, 0, &status); xmlFreeDoc(doc); YAZ_CHECK(chain); @@ -628,29 +559,116 @@ void test_bug_1140(void) chain, "O Romeo, Romeo! wherefore art thou\t Romeo?", &status)); - while (icu_chain_next_token(chain, &status)) + while (icu_chain_next_token(chain, &status)){ ; + /* printf("%d '%s' '%s'\n", + icu_chain_token_number(chain), + icu_chain_token_norm(chain), + icu_chain_token_display(chain)); */ - YAZ_CHECK_EQ(icu_chain_get_token_count(chain), 7); + } + + + YAZ_CHECK_EQ(icu_chain_token_number(chain), 7); YAZ_CHECK(icu_chain_assign_cstr(chain, "what is this?", &status)); while (icu_chain_next_token(chain, &status)){ - printf("%d '%s' '%s'\n", - icu_chain_get_token_count(chain), - icu_chain_get_norm(chain), - icu_chain_get_display(chain)); + ; + /* printf("%d '%s' '%s'\n", + icu_chain_token_number(chain), + icu_chain_token_norm(chain), + icu_chain_token_display(chain)); */ } /* we expect 'what' 'is' 'this', i.e. 3 tokens */ - YAZ_CHECK_EQ(icu_chain_get_token_count(chain), 3); + YAZ_CHECK_EQ(icu_chain_token_number(chain), 3); + + icu_chain_destroy(chain); +} + + + +void test_chain_empty_token(void) +{ + UErrorCode status = U_ZERO_ERROR; + struct icu_chain * chain = 0; + + const char * xml_str = "" + "" + "" + ""; + + xmlDoc *doc = xmlParseMemory(xml_str, strlen(xml_str)); + xmlNode *xml_node = xmlDocGetRootElement(doc); + YAZ_CHECK(xml_node); + + chain = icu_chain_xml_config(xml_node, 0, &status); + + xmlFreeDoc(doc); + YAZ_CHECK(chain); + + YAZ_CHECK(icu_chain_assign_cstr( + chain, "a string with 15 tokenss and 8 displays", + &status)); + + while (icu_chain_next_token(chain, &status)){ + ; + /* printf("%d '%s' '%s'\n", + icu_chain_token_number(chain), + icu_chain_token_norm(chain), + icu_chain_token_display(chain)); */ + } + + YAZ_CHECK_EQ(icu_chain_token_number(chain), 15); + + icu_chain_destroy(chain); +} + +void test_chain_empty_chain(void) +{ + UErrorCode status = U_ZERO_ERROR; + struct icu_chain * chain = 0; + + const char * xml_str = "" + ""; + + const char * src8 = "some 5487 weired !¤%&(/& sTuFf"; + char * dest8 = 0; + + xmlDoc *doc = xmlParseMemory(xml_str, strlen(xml_str)); + xmlNode *xml_node = xmlDocGetRootElement(doc); + YAZ_CHECK(xml_node); + + chain = icu_chain_xml_config(xml_node, 0, &status); + + xmlFreeDoc(doc); + YAZ_CHECK(chain); + + YAZ_CHECK(icu_chain_assign_cstr( + chain, src8, + &status)); + + while (icu_chain_next_token(chain, &status)){ + ; + /* printf("%d '%s' '%s'\n", + icu_chain_token_number(chain), + icu_chain_token_norm(chain), + icu_chain_token_display(chain)); */ + } + + YAZ_CHECK_EQ(icu_chain_token_number(chain), 1); + + dest8 = (char *) icu_chain_token_norm(chain); + YAZ_CHECK_EQ(strcmp(src8, dest8), 0); + icu_chain_destroy(chain); } -#endif // HAVE_ICU +#endif /* YAZ_HAVE_ICU */ -// DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 +/* DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 */ int main(int argc, char **argv) { @@ -658,29 +676,30 @@ int main(int argc, char **argv) YAZ_CHECK_INIT(argc, argv); YAZ_CHECK_LOG(); -#ifdef HAVE_ICU +#if YAZ_HAVE_ICU - //test_icu_I18N_casemap_failures(argc, argv); test_icu_I18N_casemap(argc, argv); test_icu_I18N_sortmap(argc, argv); - test_icu_I18N_normalizer(argc, argv); + test_icu_I18N_normalizer(argc, argv); test_icu_I18N_tokenizer(argc, argv); test_icu_I18N_chain(argc, argv); + test_chain_empty_token(); + test_chain_empty_chain(); test_bug_1140(); -#else // HAVE_ICU +#else /* YAZ_HAVE_ICU */ printf("ICU unit tests omitted.\n" "Please install libicu36-dev and icu-doc or similar\n"); YAZ_CHECK(0 == 0); -#endif // HAVE_ICU +#endif /* YAZ_HAVE_ICU */ YAZ_CHECK_TERM; } -// DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 +/* DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 */