From 3a98cd4b508df847d4aa2b7ed7c1a7213766bad3 Mon Sep 17 00:00:00 2001 From: Adam Dickmeiss Date: Tue, 1 Dec 2009 15:30:16 +0100 Subject: [PATCH] ICU: iterator type --- include/yaz/icu_I18N.h | 10 +++- src/icu_chain.c | 149 ++++++++++++++++++++++++++++++++++++++++++++++++ src/icu_tokenizer.c | 9 ++- src/icu_transform.c | 4 +- test/tst_icu_I18N.c | 94 ++++++++++++++++++++++++++++++ 5 files changed, 260 insertions(+), 6 deletions(-) diff --git a/include/yaz/icu_I18N.h b/include/yaz/icu_I18N.h index f50b60a..c409bf9 100644 --- a/include/yaz/icu_I18N.h +++ b/include/yaz/icu_I18N.h @@ -163,7 +163,7 @@ void icu_transform_destroy(struct icu_transform * transform); int icu_transform_trans(struct icu_transform * transform, struct icu_buf_utf16 * dest16, - struct icu_buf_utf16 * src16, + const struct icu_buf_utf16 * src16, UErrorCode *status); struct icu_chain_step; @@ -175,6 +175,14 @@ yaz_icu_chain_t icu_chain_create(const char * locale, UErrorCode * status); +struct icu_iter; +struct icu_iter *icu_iter_create(struct icu_chain *chain, + const char *src8cstr); +void icu_iter_destroy(struct icu_iter *iter); +int icu_iter_next(struct icu_iter *iter, struct icu_buf_utf8 *result); +const char *icu_iter_get_sortkey(struct icu_iter *iter); +const char *icu_iter_get_display(struct icu_iter *iter); + #endif /* ICU_I18NL_H */ /* diff --git a/src/icu_chain.c b/src/icu_chain.c index 73a7674..a215e5e 100644 --- a/src/icu_chain.c +++ b/src/icu_chain.c @@ -22,6 +22,7 @@ #include #include #include +#include #include /* some more string fcns*/ #include /* char names */ @@ -440,6 +441,154 @@ static int icu_chain_step_next_token(struct icu_chain * chain, return 1; } +struct icu_iter { + struct icu_chain *chain; + struct icu_buf_utf16 *next; + UErrorCode status; + struct icu_buf_utf8 *display; + struct icu_buf_utf8 *sort8; +}; + +static void utf16_print(struct icu_buf_utf16 *src16) +{ + UErrorCode status = U_ZERO_ERROR; + const char *p; + struct icu_buf_utf8 *dst8 = icu_buf_utf8_create(0); + icu_utf16_to_utf8(dst8, src16, &status); + + assert(status != 1234); + if (U_FAILURE(status)) + { + printf("utf8:failure\n"); + } + else + { + p = icu_buf_utf8_to_cstr(dst8); + printf("utf8:%s\n", p); + } + icu_buf_utf8_destroy(dst8); +} + +struct icu_buf_utf16 *icu_iter_invoke(struct icu_iter *iter, + struct icu_chain_step *step, + struct icu_buf_utf16 *src) +{ + if (!step) + return src; + else + { + struct icu_buf_utf16 *dst = icu_iter_invoke(iter, step->previous, src); + + switch (step->type) + { + case ICU_chain_step_type_casemap: + if (dst) + { + struct icu_buf_utf16 *src = dst; + + dst = icu_buf_utf16_create(0); + icu_casemap_casemap(step->u.casemap, dst, src, &iter->status, + iter->chain->locale); + icu_buf_utf16_destroy(src); + } + break; + case ICU_chain_step_type_tokenize: + if (dst) + { + struct icu_buf_utf16 *src = dst; + + icu_tokenizer_attach(step->u.tokenizer, src, &iter->status); + icu_buf_utf16_destroy(src); + } + dst = icu_buf_utf16_create(0); + iter->status = U_ZERO_ERROR; + if (!icu_tokenizer_next_token(step->u.tokenizer, dst, &iter->status)) + { + icu_buf_utf16_destroy(dst); + dst = 0; + } + break; + case ICU_chain_step_type_transform: + case ICU_chain_step_type_transliterate: + if (dst) + { + struct icu_buf_utf16 *src = dst; + dst = icu_buf_utf16_create(0); + icu_transform_trans(step->u.transform, dst, src, &iter->status); + icu_buf_utf16_destroy(src); + } + break; + case ICU_chain_step_type_display: + if (dst) + icu_utf16_to_utf8(iter->display, dst, &iter->status); + break; + default: + assert(0); + } + return dst; + } +} + +struct icu_iter *icu_iter_create(struct icu_chain *chain, + const char *src8cstr) +{ + if (!src8cstr) + return 0; + else + { + struct icu_buf_utf16 *src16 = icu_buf_utf16_create(0); + struct icu_iter *iter = xmalloc(sizeof(*iter)); + iter->chain = chain; + iter->status = U_ZERO_ERROR; + iter->display = icu_buf_utf8_create(0); + iter->sort8 = icu_buf_utf8_create(0); + + icu_utf16_from_utf8_cstr(src16, src8cstr, &iter->status); + iter->next = icu_iter_invoke(iter, chain->steps, src16); + return iter; + } +} + +void icu_iter_destroy(struct icu_iter *iter) +{ + if (iter) + { + icu_buf_utf8_destroy(iter->display); + icu_buf_utf8_destroy(iter->sort8); + xfree(iter); + } +} + +int icu_iter_next(struct icu_iter *iter, struct icu_buf_utf8 *result) +{ + struct icu_buf_utf16 *last = iter->next; + if (!last) + return 0; + else + { + if (iter->chain->sort) + { + icu_sortkey8_from_utf16(iter->chain->coll, + iter->sort8, last, + &iter->status); + } + icu_utf16_to_utf8(result, last, &iter->status); + iter->next = icu_iter_invoke(iter, iter->chain->steps, 0); + icu_buf_utf16_destroy(last); + return 1; + } +} + +const char *icu_iter_get_sortkey(struct icu_iter *iter) +{ + return icu_buf_utf8_to_cstr(iter->sort8); +} + +const char *icu_iter_get_display(struct icu_iter *iter) +{ + return icu_buf_utf8_to_cstr(iter->display); +} + int icu_chain_assign_cstr(struct icu_chain * chain, const char * src8cstr, UErrorCode *status) { diff --git a/src/icu_tokenizer.c b/src/icu_tokenizer.c index c7bba7f..a79fd62 100644 --- a/src/icu_tokenizer.c +++ b/src/icu_tokenizer.c @@ -34,7 +34,7 @@ struct icu_tokenizer *icu_tokenizer_create(const char *locale, char action, tokenizer->action = action; tokenizer->bi = 0; - tokenizer->buf16 = 0; + tokenizer->buf16 = icu_buf_utf16_create(0); tokenizer->token_count = 0; tokenizer->token_id = 0; tokenizer->token_start = 0; @@ -81,6 +81,7 @@ void icu_tokenizer_destroy(struct icu_tokenizer * tokenizer) { if (tokenizer) { + icu_buf_utf16_destroy(tokenizer->buf16); if (tokenizer->bi) ubrk_close(tokenizer->bi); xfree(tokenizer); @@ -94,13 +95,15 @@ int icu_tokenizer_attach(struct icu_tokenizer * tokenizer, if (!tokenizer || !tokenizer->bi || !src16) return 0; - tokenizer->buf16 = src16; + icu_buf_utf16_copy(tokenizer->buf16, src16); + tokenizer->token_count = 0; tokenizer->token_id = 0; tokenizer->token_start = 0; tokenizer->token_end = 0; - ubrk_setText(tokenizer->bi, src16->utf16, src16->utf16_len, status); + ubrk_setText(tokenizer->bi, + tokenizer->buf16->utf16, tokenizer->buf16->utf16_len, status); if (U_FAILURE(*status)) return 0; diff --git a/src/icu_transform.c b/src/icu_transform.c index 10ace94..77a9673 100644 --- a/src/icu_transform.c +++ b/src/icu_transform.c @@ -99,7 +99,7 @@ void icu_transform_destroy(struct icu_transform * transform) int icu_transform_trans(struct icu_transform * transform, struct icu_buf_utf16 * dest16, - struct icu_buf_utf16 * src16, + const struct icu_buf_utf16 * src16, UErrorCode *status) { if (!transform || !transform->trans @@ -118,7 +118,7 @@ int icu_transform_trans(struct icu_transform * transform, utrans_transUChars (transform->trans, dest16->utf16, &(dest16->utf16_len), dest16->utf16_cap, - 0, &(src16->utf16_len), status); + 0, &(dest16->utf16_len), status); if (U_FAILURE(*status)) icu_buf_utf16_clear(dest16); diff --git a/test/tst_icu_I18N.c b/test/tst_icu_I18N.c index bbdc993..fcb438d 100644 --- a/test/tst_icu_I18N.c +++ b/test/tst_icu_I18N.c @@ -616,6 +616,97 @@ void test_chain_empty_chain(void) icu_chain_destroy(chain); } +void test_icu_iter1(void) +{ + UErrorCode status = U_ZERO_ERROR; + struct icu_chain * chain = 0; + xmlNode *xml_node; + struct icu_iter *iter; + struct icu_buf_utf8 *token; + + const char * xml_str = "" + "" + "" + ""; + + xmlDoc *doc = xmlParseMemory(xml_str, strlen(xml_str)); + YAZ_CHECK(doc); + if (!doc) + return; + xml_node = xmlDocGetRootElement(doc); + YAZ_CHECK(xml_node); + if (!xml_node) + return ; + + chain = icu_chain_xml_config(xml_node, 0, &status); + + xmlFreeDoc(doc); + YAZ_CHECK(chain); + + iter = icu_iter_create(chain, "a string with 15 tokens and 8 displays"); + YAZ_CHECK(iter); + if (!iter) + return; +#if 1 + token = icu_buf_utf8_create(0); + while (icu_iter_next(iter, token)) + { + printf("[%.*s]", (int) token->utf8_len, token->utf8); + } + icu_buf_utf8_destroy(token); +#endif + + icu_iter_destroy(iter); + icu_chain_destroy(chain); +} + + +void test_icu_iter2(void) +{ + UErrorCode status = U_ZERO_ERROR; + struct icu_chain * chain = 0; + xmlNode *xml_node; + struct icu_iter *iter; + struct icu_buf_utf8 *token; + + const char * xml_str = "" + "" + "" + "" + "" + "" + "" + ""; + + xmlDoc *doc = xmlParseMemory(xml_str, strlen(xml_str)); + YAZ_CHECK(doc); + if (!doc) + return; + xml_node = xmlDocGetRootElement(doc); + YAZ_CHECK(xml_node); + if (!xml_node) + return ; + + chain = icu_chain_xml_config(xml_node, 0, &status); + + xmlFreeDoc(doc); + YAZ_CHECK(chain); + + iter = icu_iter_create(chain, "Adobe Acrobat Reader, 1991-1999."); + YAZ_CHECK(iter); + if (!iter) + return; + token = icu_buf_utf8_create(0); + while (icu_iter_next(iter, token)) + { + printf("[%.*s]", (int) token->utf8_len, token->utf8); + } + icu_buf_utf8_destroy(token); + + icu_iter_destroy(iter); + icu_chain_destroy(chain); +} + #endif /* YAZ_HAVE_ICU */ int main(int argc, char **argv) @@ -632,6 +723,9 @@ int main(int argc, char **argv) test_icu_I18N_chain(argc, argv); test_chain_empty_token(); test_chain_empty_chain(); + test_icu_iter1(); + test_icu_iter2(); + test_bug_1140(); #else /* YAZ_HAVE_ICU */ -- 1.7.10.4