X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=src%2Ftest_icu_I18N.c;h=bc55d7d38a6b439542fcd0eec3a659f86f509464;hb=8bef04f359fd6485b6efb0af122f53502a6e134d;hp=992922a2b15f2d9e8081a5ad1438758dd1716e6e;hpb=0dc3f6fe0bca19b271b3f34263e216ad9aba8d0a;p=pazpar2-moved-to-github.git diff --git a/src/test_icu_I18N.c b/src/test_icu_I18N.c index 992922a..bc55d7d 100644 --- a/src/test_icu_I18N.c +++ b/src/test_icu_I18N.c @@ -1,29 +1,27 @@ -/* $Id: test_icu_I18N.c,v 1.11 2007-05-09 14:01:21 marc Exp $ - Copyright (c) 2006-2007, Index Data. +/* This file is part of Pazpar2. + Copyright (C) 2006-2008 Index Data - This file is part of Pazpar2. +Pazpar2 is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. - Pazpar2 is free software; you can redistribute it and/or modify it under - the terms of the GNU General Public License as published by the Free - Software Foundation; either version 2, or (at your option) any later - version. +Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. - Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY - WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - for more details. +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - You should have received a copy of the GNU General Public License - along with Pazpar2; see the file LICENSE. If not, write to the - Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA - 02111-1307, USA. */ // DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 #if HAVE_CONFIG_H -#include "cconfig.h" +#include #endif #define USE_TIMING 0 @@ -267,14 +265,17 @@ int test_icu_sortmap(const char * locale, int src_list_len, printf("\n"); } - + + + for( i = 0; i < src_list_len; i++) + free(list[i]); + + ucol_close(coll); icu_buf_utf8_destroy(buf8); icu_buf_utf16_destroy(buf16); - - return success; } @@ -314,27 +315,107 @@ void test_icu_I18N_sortmap(int argc, char **argv) // DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 -void test_icu_I18N_normmap(int argc, char **argv) + + + +int test_icu_normalizer(const char * rules8cstr, + const char * src8cstr, + const char * chk8cstr) { + int success = 0; + + UErrorCode status = U_ZERO_ERROR; + struct icu_buf_utf16 * src16 = icu_buf_utf16_create(0); + struct icu_buf_utf16 * dest16 = icu_buf_utf16_create(0); + struct icu_buf_utf8 * dest8 = icu_buf_utf8_create(0); + struct icu_normalizer * normalizer + = icu_normalizer_create(rules8cstr, 'f', &status); + icu_check_status(status); + + icu_utf16_from_utf8_cstr(src16, src8cstr, &status); + icu_check_status(status); -} + icu_normalizer_normalize(normalizer, dest16, src16, &status); + icu_check_status(status); + + icu_utf16_to_utf8(dest8, dest16, &status); + icu_check_status(status); + + + if(!strcmp((const char *) dest8->utf8, + (const char *) chk8cstr)) + success = 1; + else { + success = 0; + printf("Normalization\n"); + printf("Rules: '%s'\n", rules8cstr); + printf("Input: '%s'\n", src8cstr); + printf("Normalized: '%s'\n", dest8->utf8); + printf("Expected: '%s'\n", chk8cstr); + } + + + icu_normalizer_destroy(normalizer); + icu_buf_utf16_destroy(src16); + icu_buf_utf16_destroy(dest16); + icu_buf_utf8_destroy(dest8); + + return success; +}; // DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 -void test_icu_I18N_tokenizer(int argc, char **argv) +void test_icu_I18N_normalizer(int argc, char **argv) { - const char * src8cstr - = "Though I am not naturally honest, I am so sometimes by chance."; + YAZ_CHECK(test_icu_normalizer("[:Punctuation:] Any-Remove", + "Don't shoot!", + "Dont shoot")); + + YAZ_CHECK(test_icu_normalizer("[:Control:] Any-Remove", + "Don't\n shoot!", + "Don't shoot!")); + + YAZ_CHECK(test_icu_normalizer("[:Decimal_Number:] Any-Remove", + "This is 4 you!", + "This is you!")); + + YAZ_CHECK(test_icu_normalizer("Lower; [:^Letter:] Remove", + "Don't shoot!", + "dontshoot")); + + YAZ_CHECK(test_icu_normalizer("[:^Number:] Remove", + "Monday 15th of April", + "15")); + + YAZ_CHECK(test_icu_normalizer("Lower;" + "[[:WhiteSpace:][:Punctuation:]] Remove", + " word4you? ", + "word4you")); + + + YAZ_CHECK(test_icu_normalizer("NFD; [:Nonspacing Mark:] Remove; NFC", + "à côté de l'alcôve ovoïde", + "a cote de l'alcove ovoide")); + +} + + +// DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 + +int test_icu_tokenizer(const char * locale, char action, + const char * src8cstr, int count) +{ + int success = 1; UErrorCode status = U_ZERO_ERROR; struct icu_buf_utf16 * src16 = icu_buf_utf16_create(0); struct icu_buf_utf16 * tkn16 = icu_buf_utf16_create(0); struct icu_buf_utf8 * tkn8 = icu_buf_utf8_create(0); - printf("Input: '%s'\n", src8cstr); + //printf("Input: '%s'\n", src8cstr); // transforming to UTF16 icu_utf16_from_utf8_cstr(src16, src8cstr, &status); @@ -342,7 +423,7 @@ void test_icu_I18N_tokenizer(int argc, char **argv) // set up tokenizer struct icu_tokenizer * tokenizer - = icu_tokenizer_create("en", 's', &status); + = icu_tokenizer_create(locale, action, &status); icu_check_status(status); YAZ_CHECK(tokenizer); @@ -352,35 +433,219 @@ void test_icu_I18N_tokenizer(int argc, char **argv) YAZ_CHECK(tokenizer->bi); // perform work on tokens - printf("Tokens: "); + //printf("Tokens: "); while(icu_tokenizer_next_token(tokenizer, tkn16, &status)){ icu_check_status(status); // converting to UTF8 icu_utf16_to_utf8(tkn8, tkn16, &status); - printf("'%s' ", tkn8->utf8); - //printf("token %d %d %d %d '%s'\n", - // icu_tokenizer_token_id(tokenizer), + // // icu_tokenizer_token_start(tokenizer), // icu_tokenizer_token_end(tokenizer), // icu_tokenizer_token_length(tokenizer), // tkn8->utf8); } - printf(" (%d)(%d)\n", icu_tokenizer_token_id(tokenizer), - icu_tokenizer_token_count(tokenizer)); + + if (count != icu_tokenizer_token_count(tokenizer)){ + success = 0; + printf("\nTokenizer '%s:%c' Error: \n", locale, action); + printf("Input: '%s'\n", src8cstr); + printf("Tokens: %d", icu_tokenizer_token_count(tokenizer)); + printf(", expected: %d\n", count); + } icu_tokenizer_destroy(tokenizer); icu_buf_utf16_destroy(src16); icu_buf_utf16_destroy(tkn16); icu_buf_utf8_destroy(tkn8); + + return success; } +// DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 +void test_icu_I18N_tokenizer(int argc, char **argv) +{ + + + const char * en_str + = "O Romeo, Romeo! wherefore art thou Romeo?"; + + YAZ_CHECK(test_icu_tokenizer("en", 's', en_str, 2)); + YAZ_CHECK(test_icu_tokenizer("en", 'l', en_str, 7)); + YAZ_CHECK(test_icu_tokenizer("en", 'w', en_str, 16)); + YAZ_CHECK(test_icu_tokenizer("en", 'c', en_str, 41)); + + const char * da_str + = "Blåbærtærte. Denne kage stammer fra Finland. " + "Den er med blåbær, men alle sommerens forskellige bær kan bruges."; + + YAZ_CHECK(test_icu_tokenizer("da", 's', da_str, 3)); + YAZ_CHECK(test_icu_tokenizer("dar", 'l', da_str, 17)); + YAZ_CHECK(test_icu_tokenizer("da", 'w', da_str, 37)); + YAZ_CHECK(test_icu_tokenizer("da", 'c', da_str, 110)); + +} + + +void test_icu_I18N_chain(int argc, char **argv) +{ + const char * en_str + = "O Romeo, Romeo! wherefore art thou\t Romeo?"; + + printf("ICU chain:\ninput: '%s'\n", en_str); + + UErrorCode status = U_ZERO_ERROR; + //struct icu_chain_step * step = 0; + struct icu_chain * chain = 0; + + + const char * xml_str = "" + "" + "" + "" + "" + "" + "" + "" + ""; + + + xmlDoc *doc = xmlParseMemory(xml_str, strlen(xml_str)); + xmlNode *xml_node = xmlDocGetRootElement(doc); + YAZ_CHECK(xml_node); + + + chain = icu_chain_xml_config(xml_node, &status); + +#if 0 + chain = icu_chain_create((uint8_t *) "en:word", (uint8_t *) "en"); + step = icu_chain_insert_step(chain, ICU_chain_step_type_normalize, + (const uint8_t *) "[:Control:] Any-Remove", + &status); + step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize, + (const uint8_t *) "s", + &status); + step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize, + (const uint8_t *) "l", + &status); + step = icu_chain_insert_step(chain, ICU_chain_step_type_normalize, + (const uint8_t *) + "[[:WhiteSpace:][:Punctuation:]] Any-Remove", + &status); + step = icu_chain_insert_step(chain, ICU_chain_step_type_display, + (const uint8_t *)"", + &status); +/* step = icu_chain_insert_step(chain, ICU_chain_step_type_normalize, */ +/* (const uint8_t *) "Lower", */ +/* &status); */ + step = icu_chain_insert_step(chain, ICU_chain_step_type_casemap, + (const uint8_t *) "l", + &status); + step = icu_chain_insert_step(chain, ICU_chain_step_type_index, + (const uint8_t *)"", + &status); +/* step = icu_chain_insert_step(chain, ICU_chain_step_type_sortkey, */ +/* (const uint8_t *)"", */ +/* &status); */ + +#endif + + xmlFreeDoc(doc); + YAZ_CHECK(chain); + + YAZ_CHECK(icu_chain_assign_cstr(chain, en_str, &status)); + + while (icu_chain_next_token(chain, &status)){ + printf("%d '%s' '%s'\n", + icu_chain_get_token_count(chain), + icu_chain_get_norm(chain), + icu_chain_get_display(chain)); + } + + YAZ_CHECK_EQ(icu_chain_get_token_count(chain), 7); + + + YAZ_CHECK(icu_chain_assign_cstr(chain, "what is this?", &status)); + + while (icu_chain_next_token(chain, &status)){ + printf("%d '%s' '%s'\n", + icu_chain_get_token_count(chain), + icu_chain_get_norm(chain), + icu_chain_get_display(chain)); + } + + + YAZ_CHECK_EQ(icu_chain_get_token_count(chain), 3); + + icu_chain_destroy(chain); +} + + +void test_bug_1140(void) +{ + const char * en_str + = "O Romeo, Romeo! wherefore art thou\t Romeo?"; + + printf("ICU chain:\ninput: '%s'\n", en_str); + + UErrorCode status = U_ZERO_ERROR; + //struct icu_chain_step * step = 0; + struct icu_chain * chain = 0; + + const char * xml_str = "" + + /* if the first rule is normalize instead. Then it works */ +#if 0 + "" +#endif + "" + "" + "" + "" + "" + "" + ""; + + + xmlDoc *doc = xmlParseMemory(xml_str, strlen(xml_str)); + xmlNode *xml_node = xmlDocGetRootElement(doc); + YAZ_CHECK(xml_node); + + chain = icu_chain_xml_config(xml_node, &status); + + xmlFreeDoc(doc); + YAZ_CHECK(chain); + + YAZ_CHECK(icu_chain_assign_cstr( + chain, "O Romeo, Romeo! wherefore art thou\t Romeo?", + &status)); + + while (icu_chain_next_token(chain, &status)) + ; + + YAZ_CHECK_EQ(icu_chain_get_token_count(chain), 7); + + YAZ_CHECK(icu_chain_assign_cstr(chain, "what is this?", &status)); + + while (icu_chain_next_token(chain, &status)){ + printf("%d '%s' '%s'\n", + icu_chain_get_token_count(chain), + icu_chain_get_norm(chain), + icu_chain_get_display(chain)); + } + + /* we expect 'what' 'is' 'this', i.e. 3 tokens */ + YAZ_CHECK_EQ(icu_chain_get_token_count(chain), 3); + + icu_chain_destroy(chain); +} + #endif // HAVE_ICU // DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 @@ -396,8 +661,10 @@ int main(int argc, char **argv) //test_icu_I18N_casemap_failures(argc, argv); test_icu_I18N_casemap(argc, argv); test_icu_I18N_sortmap(argc, argv); - test_icu_I18N_normmap(argc, argv); + test_icu_I18N_normalizer(argc, argv); test_icu_I18N_tokenizer(argc, argv); + test_icu_I18N_chain(argc, argv); + test_bug_1140(); #else // HAVE_ICU @@ -415,67 +682,6 @@ int main(int argc, char **argv) -// CRAP to follow -#if 0 - -// DO NOT EDIT THIS FILE IF YOUR EDITOR DOES NOT SUPPORT UTF-8 - -void test_icu_I18N_casemap_failures(int argc, char **argv) -{ - - size_t buf_cap = 128; - char buf[buf_cap]; - size_t dest8_len = 0; - NMEM nmem = nmem_create(); - char * dest8 = 0; - - const char * src8 = "A ReD fOx hunTS sQUirriLs"; - //size_t src8_len = strlen(src8); - - //printf("original string: '%s' (%d)\n", src8, (int) src8_len); - - // some calling error needs investigation - dest8 = icu_casemap(nmem, buf, buf_cap, &dest8_len, - src8, "en", 't'); - YAZ_CHECK(0 == dest8_len); - //printf("icu_casemap 'en:t' '%s' (%d)\n", dest8, (int) dest8_len); - - - // attention: does not fail even if no locale 'xy_zz' defined - // it seems to default to english locale - dest8 = icu_casemap(nmem, buf, buf_cap, &dest8_len, - src8, "zz_abc", 'l'); - YAZ_CHECK(dest8_len); - //printf("icu_casemap 'zz:l' '%s' (%d)\n", dest8, (int) dest8_len); - - - // shall fail - no buf buffer defined - dest8 = icu_casemap(nmem, 0, buf_cap, &dest8_len, - src8, "en", 'l'); - YAZ_CHECK(0 == dest8_len); - //printf("icu_casemap 'en:l' '%s' (%d)\n", dest8, (int) dest8_len); - - // shall fail - no buf_cap defined - dest8 = icu_casemap(nmem, buf, 0, &dest8_len, - src8, "en", 'l'); - YAZ_CHECK(0 == dest8_len); - //printf("icu_casemap 'en:l' '%s' (%d)\n", dest8, (int) dest8_len); - - // shall fail - no action 'x' defined - dest8 = icu_casemap(nmem, buf, buf_cap, &dest8_len, - src8, "en", 'x'); - YAZ_CHECK(0 == dest8_len); - //printf("icu_casemap 'en:x' '%s' (%d)\n", dest8, (int) dest8_len); - - nmem_destroy(nmem); -} - - - -#endif - - - /* * Local variables: * c-basic-offset: 4