X-Git-Url: http://git.indexdata.com/?p=yaz-moved-to-github.git;a=blobdiff_plain;f=test%2Ftst_icu_I18N.c;h=ef3cdc53ad047685bc19d8717e12c7c49ef2f794;hp=717e760ee3ee467411f1df45bb62a1d58b364e97;hb=69c2dee55aa204cab368989b8385eda8b83bf2e2;hpb=42e6fdb480e4de99b856018fd70475bd7fbdb928 diff --git a/test/tst_icu_I18N.c b/test/tst_icu_I18N.c index 717e760..ef3cdc5 100644 --- a/test/tst_icu_I18N.c +++ b/test/tst_icu_I18N.c @@ -1,23 +1,7 @@ -/* $Id: tst_icu_I18N.c,v 1.5 2007-10-24 13:23:34 marc Exp $ - Copyright (c) 2006-2007, Index Data. - - This file is part of Pazpar2. - - Pazpar2 is free software; you can redistribute it and/or modify it under - the terms of the GNU General Public License as published by the Free - Software Foundation; either version 2, or (at your option) any later - version. - - Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY - WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - for more details. - - You should have received a copy of the GNU General Public License - along with Pazpar2; see the file LICENSE. If not, write to the - Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA - 02111-1307, USA. -*/ +/* This file is part of the YAZ toolkit. + * Copyright (C) 1995-2009 Index Data + * See the file LICENSE for details. + */ /* DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 */ @@ -33,7 +17,7 @@ #include -#if HAVE_ICU +#if YAZ_HAVE_ICU #include #include @@ -328,14 +312,14 @@ int test_icu_normalizer(const char * rules8cstr, struct icu_buf_utf16 * src16 = icu_buf_utf16_create(0); struct icu_buf_utf16 * dest16 = icu_buf_utf16_create(0); struct icu_buf_utf8 * dest8 = icu_buf_utf8_create(0); - struct icu_normalizer * normalizer - = icu_normalizer_create(rules8cstr, 'f', &status); + struct icu_transform * transform + = icu_transform_create(rules8cstr, 'f', 0, &status); icu_check_status(status); icu_utf16_from_utf8_cstr(src16, src8cstr, &status); icu_check_status(status); - icu_normalizer_normalize(normalizer, dest16, src16, &status); + icu_transform_trans(transform, dest16, src16, &status); icu_check_status(status); icu_utf16_to_utf8(dest8, dest16, &status); @@ -355,7 +339,7 @@ int test_icu_normalizer(const char * rules8cstr, } - icu_normalizer_destroy(normalizer); + icu_transform_destroy(transform); icu_buf_utf16_destroy(src16); icu_buf_utf16_destroy(dest16); icu_buf_utf8_destroy(dest8); @@ -493,58 +477,20 @@ void test_icu_I18N_chain(int argc, char **argv) struct icu_chain * chain = 0; - const char * xml_str = "" - "" + const char * xml_str = "" + "" "" - "" + "" "" "" - "" - "" - ""; + ""; xmlDoc *doc = xmlParseMemory(xml_str, strlen(xml_str)); xmlNode *xml_node = xmlDocGetRootElement(doc); YAZ_CHECK(xml_node); - // printf("ICU chain:\ninput: '%s'\n", en_str); - - - chain = icu_chain_xml_config(xml_node, &status); - -#if 0 - chain = icu_chain_create((uint8_t *) "en:word", (uint8_t *) "en"); - step = icu_chain_insert_step(chain, ICU_chain_step_type_normalize, - (const uint8_t *) "[:Control:] Any-Remove", - &status); - step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize, - (const uint8_t *) "s", - &status); - step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize, - (const uint8_t *) "l", - &status); - step = icu_chain_insert_step(chain, ICU_chain_step_type_normalize, - (const uint8_t *) - "[[:WhiteSpace:][:Punctuation:]] Any-Remove", - &status); - step = icu_chain_insert_step(chain, ICU_chain_step_type_display, - (const uint8_t *)"", - &status); -/* step = icu_chain_insert_step(chain, ICU_chain_step_type_normalize, */ -/* (const uint8_t *) "Lower", */ -/* &status); */ - step = icu_chain_insert_step(chain, ICU_chain_step_type_casemap, - (const uint8_t *) "l", - &status); - step = icu_chain_insert_step(chain, ICU_chain_step_type_index, - (const uint8_t *)"", - &status); -/* step = icu_chain_insert_step(chain, ICU_chain_step_type_sortkey, */ -/* (const uint8_t *)"", */ -/* &status); */ - -#endif + chain = icu_chain_xml_config(xml_node, 0, &status); xmlFreeDoc(doc); YAZ_CHECK(chain); @@ -553,27 +499,27 @@ void test_icu_I18N_chain(int argc, char **argv) while (icu_chain_next_token(chain, &status)){ ; - // printf("%d '%s' '%s'\n", - // icu_chain_get_token_count(chain), - // icu_chain_get_norm(chain), - // icu_chain_get_display(chain)); + /* printf("%d '%s' '%s'\n", + icu_chain_token_number(chain), + icu_chain_token_norm(chain), + icu_chain_token_display(chain)); */ } - YAZ_CHECK_EQ(icu_chain_get_token_count(chain), 7); + YAZ_CHECK_EQ(icu_chain_token_number(chain), 7); YAZ_CHECK(icu_chain_assign_cstr(chain, "what is this?", &status)); while (icu_chain_next_token(chain, &status)){ ; - //printf("%d '%s' '%s'\n", - // icu_chain_get_token_count(chain), - // icu_chain_get_norm(chain), - // icu_chain_get_display(chain)); + /* printf("%d '%s' '%s'\n", + icu_chain_token_number(chain), + icu_chain_token_norm(chain), + icu_chain_token_display(chain)); */ } - YAZ_CHECK_EQ(icu_chain_get_token_count(chain), 3); + YAZ_CHECK_EQ(icu_chain_token_number(chain), 3); icu_chain_destroy(chain); } @@ -584,26 +530,24 @@ void test_bug_1140(void) UErrorCode status = U_ZERO_ERROR; struct icu_chain * chain = 0; - const char * xml_str = "" + const char * xml_str = "" /* if the first rule is normalize instead. Then it works */ #if 0 - "" + "" #endif "" - "" + "" "" "" - "" - "" - ""; + ""; xmlDoc *doc = xmlParseMemory(xml_str, strlen(xml_str)); xmlNode *xml_node = xmlDocGetRootElement(doc); YAZ_CHECK(xml_node); - chain = icu_chain_xml_config(xml_node, &status); + chain = icu_chain_xml_config(xml_node, 0, &status); xmlFreeDoc(doc); YAZ_CHECK(chain); @@ -614,28 +558,28 @@ void test_bug_1140(void) while (icu_chain_next_token(chain, &status)){ ; - //printf("%d '%s' '%s'\n", - // icu_chain_get_token_count(chain), - // icu_chain_get_norm(chain), - // icu_chain_get_display(chain)); + /* printf("%d '%s' '%s'\n", + icu_chain_token_number(chain), + icu_chain_token_norm(chain), + icu_chain_token_display(chain)); */ } - YAZ_CHECK_EQ(icu_chain_get_token_count(chain), 7); + YAZ_CHECK_EQ(icu_chain_token_number(chain), 7); YAZ_CHECK(icu_chain_assign_cstr(chain, "what is this?", &status)); while (icu_chain_next_token(chain, &status)){ ; - //printf("%d '%s' '%s'\n", - // icu_chain_get_token_count(chain), - // icu_chain_get_norm(chain), - // icu_chain_get_display(chain)); + /* printf("%d '%s' '%s'\n", + icu_chain_token_number(chain), + icu_chain_token_norm(chain), + icu_chain_token_display(chain)); */ } /* we expect 'what' 'is' 'this', i.e. 3 tokens */ - YAZ_CHECK_EQ(icu_chain_get_token_count(chain), 3); + YAZ_CHECK_EQ(icu_chain_token_number(chain), 3); icu_chain_destroy(chain); } @@ -647,39 +591,79 @@ void test_chain_empty_token(void) UErrorCode status = U_ZERO_ERROR; struct icu_chain * chain = 0; - const char * xml_str = "" + const char * xml_str = "" "" - "" - "" - ""; + "" + ""; xmlDoc *doc = xmlParseMemory(xml_str, strlen(xml_str)); xmlNode *xml_node = xmlDocGetRootElement(doc); YAZ_CHECK(xml_node); - chain = icu_chain_xml_config(xml_node, &status); + chain = icu_chain_xml_config(xml_node, 0, &status); xmlFreeDoc(doc); YAZ_CHECK(chain); YAZ_CHECK(icu_chain_assign_cstr( - chain, "a string with 15 wordbreaks and 8 tokens", + chain, "a string with 15 tokenss and 8 displays", &status)); while (icu_chain_next_token(chain, &status)){ ; - //printf("%d '%s' '%s'\n", - // icu_chain_get_token_count(chain), - // icu_chain_get_norm(chain), - // icu_chain_get_display(chain)); + /* printf("%d '%s' '%s'\n", + icu_chain_token_number(chain), + icu_chain_token_norm(chain), + icu_chain_token_display(chain)); */ } - YAZ_CHECK_EQ(icu_chain_get_token_count(chain), 8); + YAZ_CHECK_EQ(icu_chain_token_number(chain), 15); icu_chain_destroy(chain); } -#endif // HAVE_ICU +void test_chain_empty_chain(void) +{ + UErrorCode status = U_ZERO_ERROR; + struct icu_chain * chain = 0; + + const char * xml_str = "" + ""; + + const char * src8 = "some 5487 weired !¤%&(/& sTuFf"; + char * dest8 = 0; + + xmlDoc *doc = xmlParseMemory(xml_str, strlen(xml_str)); + xmlNode *xml_node = xmlDocGetRootElement(doc); + YAZ_CHECK(xml_node); + + chain = icu_chain_xml_config(xml_node, 0, &status); + + xmlFreeDoc(doc); + YAZ_CHECK(chain); + + YAZ_CHECK(icu_chain_assign_cstr( + chain, src8, + &status)); + + while (icu_chain_next_token(chain, &status)){ + ; + /* printf("%d '%s' '%s'\n", + icu_chain_token_number(chain), + icu_chain_token_norm(chain), + icu_chain_token_display(chain)); */ + } + + YAZ_CHECK_EQ(icu_chain_token_number(chain), 1); + + dest8 = (char *) icu_chain_token_norm(chain); + YAZ_CHECK_EQ(strcmp(src8, dest8), 0); + + + icu_chain_destroy(chain); +} + +#endif /* YAZ_HAVE_ICU */ /* DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 */ @@ -689,7 +673,7 @@ int main(int argc, char **argv) YAZ_CHECK_INIT(argc, argv); YAZ_CHECK_LOG(); -#if HAVE_ICU +#if YAZ_HAVE_ICU test_icu_I18N_casemap(argc, argv); test_icu_I18N_sortmap(argc, argv); @@ -697,15 +681,16 @@ int main(int argc, char **argv) test_icu_I18N_tokenizer(argc, argv); test_icu_I18N_chain(argc, argv); test_chain_empty_token(); + test_chain_empty_chain(); test_bug_1140(); -#else /* HAVE_ICU */ +#else /* YAZ_HAVE_ICU */ printf("ICU unit tests omitted.\n" "Please install libicu36-dev and icu-doc or similar\n"); YAZ_CHECK(0 == 0); -#endif /* HAVE_ICU */ +#endif /* YAZ_HAVE_ICU */ YAZ_CHECK_TERM; } @@ -718,7 +703,9 @@ int main(int argc, char **argv) /* * Local variables: * c-basic-offset: 4 + * c-file-style: "Stroustrup" * indent-tabs-mode: nil * End: * vim: shiftwidth=4 tabstop=8 expandtab */ +