X-Git-Url: http://git.indexdata.com/?p=yaz-moved-to-github.git;a=blobdiff_plain;f=src%2Ficu_I18N.c;h=b2119b6804cf7ebeb5e999a2958d4e947fe2fb48;hp=9e6cdd9640e758004d6f1b549cfec43c31cc0fd4;hb=68398183c3b25120b484681482f461a1da4c6a36;hpb=c168c38b0beb83bca23c72d9203cc796a374bfac diff --git a/src/icu_I18N.c b/src/icu_I18N.c index 9e6cdd9..b2119b6 100644 --- a/src/icu_I18N.c +++ b/src/icu_I18N.c @@ -1,8 +1,11 @@ -/* - * Copyright (C) 1995-2007, Index Data ApS +/* This file is part of the YAZ toolkit. + * Copyright (C) 1995-2008 Index Data * See the file LICENSE for details. - * - * $Id: icu_I18N.c,v 1.2 2007-10-22 17:32:07 adam Exp $ + */ + +/** + * \file icu_I18N.c + * \brief ICU utilities */ #if HAVE_CONFIG_H @@ -14,8 +17,9 @@ #include #endif +#if YAZ_HAVE_ICU +#include -#if HAVE_ICU #include #include @@ -33,9 +37,9 @@ int icu_check_status (UErrorCode status) { - if(U_FAILURE(status)){ - yaz_log(YLOG_WARN, - "ICU: %d %s\n", status, u_errorName(status)); + if (U_FAILURE(status)) + { + yaz_log(YLOG_WARN, "ICU: %d %s\n", status, u_errorName(status)); return 0; } return 1; @@ -47,41 +51,51 @@ int icu_check_status (UErrorCode status) struct icu_buf_utf16 * icu_buf_utf16_create(size_t capacity) { struct icu_buf_utf16 * buf16 - = (struct icu_buf_utf16 *) malloc(sizeof(struct icu_buf_utf16)); + = (struct icu_buf_utf16 *) xmalloc(sizeof(struct icu_buf_utf16)); buf16->utf16 = 0; buf16->utf16_len = 0; buf16->utf16_cap = 0; if (capacity > 0){ - buf16->utf16 = (UChar *) malloc(sizeof(UChar) * capacity); + buf16->utf16 = (UChar *) xmalloc(sizeof(UChar) * capacity); buf16->utf16[0] = (UChar) 0; buf16->utf16_cap = capacity; } return buf16; } -struct icu_buf_utf16 * icu_buf_utf16_resize(struct icu_buf_utf16 * buf16, - size_t capacity) +struct icu_buf_utf16 * icu_buf_utf16_clear(struct icu_buf_utf16 * buf16) { if (buf16){ - if (capacity > 0){ - if (0 == buf16->utf16) - buf16->utf16 = (UChar *) malloc(sizeof(UChar) * capacity); - else - buf16->utf16 - = (UChar *) realloc(buf16->utf16, sizeof(UChar) * capacity); + if (buf16->utf16) buf16->utf16[0] = (UChar) 0; - buf16->utf16_len = 0; - buf16->utf16_cap = capacity; - } - else { - if (buf16->utf16) - free(buf16->utf16); - buf16->utf16 = 0; - buf16->utf16_len = 0; - buf16->utf16_cap = 0; - } + buf16->utf16_len = 0; + } + return buf16; +} + +struct icu_buf_utf16 * icu_buf_utf16_resize(struct icu_buf_utf16 * buf16, + size_t capacity) +{ + if (!buf16) + return 0; + + if (capacity > 0){ + if (0 == buf16->utf16) + buf16->utf16 = (UChar *) xmalloc(sizeof(UChar) * capacity); + else + buf16->utf16 + = (UChar *) xrealloc(buf16->utf16, sizeof(UChar) * capacity); + + icu_buf_utf16_clear(buf16); + buf16->utf16_cap = capacity; + } + else { + xfree(buf16->utf16); + buf16->utf16 = 0; + buf16->utf16_len = 0; + buf16->utf16_cap = 0; } return buf16; @@ -107,29 +121,24 @@ struct icu_buf_utf16 * icu_buf_utf16_copy(struct icu_buf_utf16 * dest16, void icu_buf_utf16_destroy(struct icu_buf_utf16 * buf16) { - if (buf16){ - if (buf16->utf16) - free(buf16->utf16); - free(buf16); - } + if (buf16) + xfree(buf16->utf16); + xfree(buf16); } - - - struct icu_buf_utf8 * icu_buf_utf8_create(size_t capacity) { struct icu_buf_utf8 * buf8 - = (struct icu_buf_utf8 *) malloc(sizeof(struct icu_buf_utf8)); + = (struct icu_buf_utf8 *) xmalloc(sizeof(struct icu_buf_utf8)); buf8->utf8 = 0; buf8->utf8_len = 0; buf8->utf8_cap = 0; if (capacity > 0){ - buf8->utf8 = (uint8_t *) malloc(sizeof(uint8_t) * capacity); + buf8->utf8 = (uint8_t *) xmalloc(sizeof(uint8_t) * capacity); buf8->utf8[0] = (uint8_t) 0; buf8->utf8_cap = capacity; } @@ -137,47 +146,40 @@ struct icu_buf_utf8 * icu_buf_utf8_create(size_t capacity) } - -struct icu_buf_utf8 * icu_buf_utf8_resize(struct icu_buf_utf8 * buf8, - size_t capacity) +struct icu_buf_utf8 * icu_buf_utf8_clear(struct icu_buf_utf8 * buf8) { if (buf8){ - if (capacity > 0){ - if (0 == buf8->utf8) - buf8->utf8 = (uint8_t *) malloc(sizeof(uint8_t) * capacity); - else - buf8->utf8 - = (uint8_t *) realloc(buf8->utf8, - sizeof(uint8_t) * capacity); - buf8->utf8_cap = capacity; - } - else { - if (buf8->utf8) - free(buf8->utf8); - buf8->utf8 = 0; - buf8->utf8_len = 0; - buf8->utf8_cap = 0; - } + if (buf8->utf8) + buf8->utf8[0] = (uint8_t) 0; + buf8->utf8_len = 0; } - return buf8; } -struct icu_buf_utf8 * icu_buf_utf8_copy(struct icu_buf_utf8 * dest8, - struct icu_buf_utf8 * src8) +struct icu_buf_utf8 * icu_buf_utf8_resize(struct icu_buf_utf8 * buf8, + size_t capacity) { - if(!dest8 || !src8 - || dest8 == src8) + if (!buf8) return 0; - - - if (dest8->utf8_cap < src8->utf8_len) - icu_buf_utf8_resize(dest8, src8->utf8_len * 2); - - strncpy((char*) dest8->utf8, (char*) src8->utf8, src8->utf8_len); - return dest8; + if (capacity > 0){ + if (0 == buf8->utf8) + buf8->utf8 = (uint8_t *) xmalloc(sizeof(uint8_t) * capacity); + else + buf8->utf8 + = (uint8_t *) xrealloc(buf8->utf8, sizeof(uint8_t) * capacity); + + buf8->utf8_cap = capacity; + } + else { + xfree(buf8->utf8); + buf8->utf8 = 0; + buf8->utf8_len = 0; + buf8->utf8_cap = 0; + } + + return buf8; } @@ -185,20 +187,21 @@ const char *icu_buf_utf8_to_cstr(struct icu_buf_utf8 *src8) { if (!src8 || src8->utf8_len == 0) return ""; + if (src8->utf8_len == src8->utf8_cap) src8 = icu_buf_utf8_resize(src8, src8->utf8_len * 2 + 1); + src8->utf8[src8->utf8_len] = '\0'; + return (const char *) src8->utf8; } void icu_buf_utf8_destroy(struct icu_buf_utf8 * buf8) { - if (buf8){ - if (buf8->utf8) - free(buf8->utf8); - free(buf8); - } + if (buf8) + xfree(buf8->utf8); + xfree(buf8); } @@ -226,10 +229,8 @@ UErrorCode icu_utf16_from_utf8(struct icu_buf_utf16 * dest16, if (U_SUCCESS(*status) && utf16_len <= dest16->utf16_cap) dest16->utf16_len = utf16_len; - else { - dest16->utf16[0] = (UChar) 0; - dest16->utf16_len = 0; - } + else + icu_buf_utf16_clear(dest16); return *status; } @@ -243,6 +244,7 @@ UErrorCode icu_utf16_from_utf8_cstr(struct icu_buf_utf16 * dest16, size_t src8cstr_len = 0; int32_t utf16_len = 0; + *status = U_ZERO_ERROR; src8cstr_len = strlen(src8cstr); u_strFromUTF8(dest16->utf16, dest16->utf16_cap, @@ -262,10 +264,8 @@ UErrorCode icu_utf16_from_utf8_cstr(struct icu_buf_utf16 * dest16, if (U_SUCCESS(*status) && utf16_len <= dest16->utf16_cap) dest16->utf16_len = utf16_len; - else { - dest16->utf16[0] = (UChar) 0; - dest16->utf16_len = 0; - } + else + icu_buf_utf16_clear(dest16); return *status; } @@ -297,32 +297,29 @@ UErrorCode icu_utf16_to_utf8(struct icu_buf_utf8 * dest8, if (U_SUCCESS(*status) && utf8_len <= dest8->utf8_cap) dest8->utf8_len = utf8_len; - else { - dest8->utf8[0] = (uint8_t) 0; - dest8->utf8_len = 0; - } + else + icu_buf_utf8_clear(dest8); return *status; } -struct icu_casemap * icu_casemap_create(const char *locale, char action, - UErrorCode *status) +struct icu_casemap * icu_casemap_create(char action, UErrorCode *status) { struct icu_casemap * casemap - = (struct icu_casemap *) malloc(sizeof(struct icu_casemap)); - strcpy(casemap->locale, locale); + = (struct icu_casemap *) xmalloc(sizeof(struct icu_casemap)); casemap->action = action; switch(casemap->action) { case 'l': - break; + case 'L': case 'u': - break; + case 'U': case 't': - break; + case 'T': case 'f': + case 'F': break; default: icu_casemap_destroy(casemap); @@ -334,21 +331,21 @@ struct icu_casemap * icu_casemap_create(const char *locale, char action, void icu_casemap_destroy(struct icu_casemap * casemap) { - if (casemap) - free(casemap); + xfree(casemap); } int icu_casemap_casemap(struct icu_casemap * casemap, struct icu_buf_utf16 * dest16, struct icu_buf_utf16 * src16, - UErrorCode *status) + UErrorCode *status, + const char *locale) { if(!casemap) return 0; - return icu_utf16_casemap(dest16, src16, - casemap->locale, casemap->action, status); + return icu_utf16_casemap(dest16, src16, locale, + casemap->action, status); } @@ -358,24 +355,37 @@ int icu_utf16_casemap(struct icu_buf_utf16 * dest16, UErrorCode *status) { int32_t dest16_len = 0; + + + if (!src16->utf16_len){ /* guarding for empty source string */ + if (dest16->utf16) + dest16->utf16[0] = (UChar) 0; + dest16->utf16_len = 0; + return U_ZERO_ERROR; + } + switch(action) { case 'l': + case 'L': dest16_len = u_strToLower(dest16->utf16, dest16->utf16_cap, src16->utf16, src16->utf16_len, locale, status); break; case 'u': + case 'U': dest16_len = u_strToUpper(dest16->utf16, dest16->utf16_cap, src16->utf16, src16->utf16_len, locale, status); break; case 't': + case 'T': dest16_len = u_strToTitle(dest16->utf16, dest16->utf16_cap, src16->utf16, src16->utf16_len, 0, locale, status); break; case 'f': + case 'F': dest16_len = u_strFoldCase(dest16->utf16, dest16->utf16_cap, src16->utf16, src16->utf16_len, U_FOLD_CASE_DEFAULT, status); @@ -396,21 +406,25 @@ int icu_utf16_casemap(struct icu_buf_utf16 * dest16, switch(action) { case 'l': + case 'L': dest16_len = u_strToLower(dest16->utf16, dest16->utf16_cap, src16->utf16, src16->utf16_len, locale, status); break; case 'u': + case 'U': dest16_len = u_strToUpper(dest16->utf16, dest16->utf16_cap, src16->utf16, src16->utf16_len, locale, status); break; case 't': + case 'T': dest16_len = u_strToTitle(dest16->utf16, dest16->utf16_cap, src16->utf16, src16->utf16_len, 0, locale, status); break; case 'f': + case 'F': dest16_len = u_strFoldCase(dest16->utf16, dest16->utf16_cap, src16->utf16, src16->utf16_len, U_FOLD_CASE_DEFAULT, status); @@ -426,7 +440,8 @@ int icu_utf16_casemap(struct icu_buf_utf16 * dest16, && dest16_len <= dest16->utf16_cap) dest16->utf16_len = dest16_len; else { - dest16->utf16[0] = (UChar) 0; + if (dest16->utf16) + dest16->utf16[0] = (UChar) 0; dest16->utf16_len = 0; } @@ -435,10 +450,10 @@ int icu_utf16_casemap(struct icu_buf_utf16 * dest16, -UErrorCode icu_sortkey8_from_utf16(UCollator *coll, - struct icu_buf_utf8 * dest8, - struct icu_buf_utf16 * src16, - UErrorCode * status) +void icu_sortkey8_from_utf16(UCollator *coll, + struct icu_buf_utf8 * dest8, + struct icu_buf_utf16 * src16, + UErrorCode * status) { int32_t sortkey_len = 0; @@ -456,12 +471,8 @@ UErrorCode icu_sortkey8_from_utf16(UCollator *coll, if (U_SUCCESS(*status) && sortkey_len > 0) dest8->utf8_len = sortkey_len; - else { - dest8->utf8[0] = (UChar) 0; - dest8->utf8_len = 0; - } - - return sortkey_len; + else + icu_buf_utf8_clear(dest8); } @@ -470,9 +481,8 @@ struct icu_tokenizer * icu_tokenizer_create(const char *locale, char action, UErrorCode *status) { struct icu_tokenizer * tokenizer - = (struct icu_tokenizer *) malloc(sizeof(struct icu_tokenizer)); + = (struct icu_tokenizer *) xmalloc(sizeof(struct icu_tokenizer)); - strcpy(tokenizer->locale, locale); tokenizer->action = action; tokenizer->bi = 0; tokenizer->buf16 = 0; @@ -484,29 +494,24 @@ struct icu_tokenizer * icu_tokenizer_create(const char *locale, char action, switch(tokenizer->action) { case 'l': - tokenizer->bi - = ubrk_open(UBRK_LINE, tokenizer->locale, - 0, 0, status); + case 'L': + tokenizer->bi = ubrk_open(UBRK_LINE, locale, 0, 0, status); break; case 's': - tokenizer->bi - = ubrk_open(UBRK_SENTENCE, tokenizer->locale, - 0, 0, status); + case 'S': + tokenizer->bi = ubrk_open(UBRK_SENTENCE, locale, 0, 0, status); break; case 'w': - tokenizer->bi - = ubrk_open(UBRK_WORD, tokenizer->locale, - 0, 0, status); + case 'W': + tokenizer->bi = ubrk_open(UBRK_WORD, locale, 0, 0, status); break; case 'c': - tokenizer->bi - = ubrk_open(UBRK_CHARACTER, tokenizer->locale, - 0, 0, status); + case 'C': + tokenizer->bi = ubrk_open(UBRK_CHARACTER, locale, 0, 0, status); break; case 't': - tokenizer->bi - = ubrk_open(UBRK_TITLE, tokenizer->locale, - 0, 0, status); + case 'T': + tokenizer->bi = ubrk_open(UBRK_TITLE, locale, 0, 0, status); break; default: *status = U_UNSUPPORTED_ERROR; @@ -528,7 +533,7 @@ void icu_tokenizer_destroy(struct icu_tokenizer * tokenizer) if (tokenizer) { if (tokenizer->bi) ubrk_close(tokenizer->bi); - free(tokenizer); + xfree(tokenizer); } } @@ -652,7 +657,7 @@ struct icu_normalizer * icu_normalizer_create(const char *rules, char action, { struct icu_normalizer * normalizer - = (struct icu_normalizer *) malloc(sizeof(struct icu_normalizer)); + = (struct icu_normalizer *) xmalloc(sizeof(struct icu_normalizer)); normalizer->action = action; normalizer->trans = 0; @@ -661,20 +666,22 @@ struct icu_normalizer * icu_normalizer_create(const char *rules, char action, switch(normalizer->action) { case 'f': + case 'F': normalizer->trans = utrans_openU(normalizer->rules16->utf16, normalizer->rules16->utf16_len, UTRANS_FORWARD, 0, 0, - normalizer->parse_error, status); + &normalizer->parse_error, status); break; case 'r': + case 'R': normalizer->trans = utrans_openU(normalizer->rules16->utf16, normalizer->rules16->utf16_len, UTRANS_REVERSE , 0, 0, - normalizer->parse_error, status); + &normalizer->parse_error, status); break; default: *status = U_UNSUPPORTED_ERROR; @@ -696,10 +703,8 @@ void icu_normalizer_destroy(struct icu_normalizer * normalizer){ if (normalizer->rules16) icu_buf_utf16_destroy(normalizer->rules16); if (normalizer->trans) - { utrans_close(normalizer->trans); - } - free(normalizer); + xfree(normalizer); } } @@ -710,21 +715,27 @@ int icu_normalizer_normalize(struct icu_normalizer * normalizer, struct icu_buf_utf16 * src16, UErrorCode *status) { - if (!normalizer || !normalizer->trans || !src16 || !dest16) + if (!normalizer || !normalizer->trans + || !src16 + || !dest16) return 0; + if (!src16->utf16_len){ /* guarding for empty source string */ + icu_buf_utf16_clear(dest16); + return 0; + } + if (!icu_buf_utf16_copy(dest16, src16)) return 0; + utrans_transUChars (normalizer->trans, dest16->utf16, &(dest16->utf16_len), dest16->utf16_cap, 0, &(src16->utf16_len), status); - if (U_FAILURE(*status)){ - dest16->utf16[0] = (UChar) 0; - dest16->utf16_len = 0; - } + if (U_FAILURE(*status)) + icu_buf_utf16_clear(dest16); return dest16->utf16_len; } @@ -743,7 +754,7 @@ struct icu_chain_step * icu_chain_step_create(struct icu_chain * chain, if(!chain || !type || !rule) return 0; - step = (struct icu_chain_step *) malloc(sizeof(struct icu_chain_step)); + step = (struct icu_chain_step *) xmalloc(sizeof(struct icu_chain_step)); step->type = type; @@ -753,13 +764,8 @@ struct icu_chain_step * icu_chain_step_create(struct icu_chain * chain, switch(step->type) { case ICU_chain_step_type_display: break; - case ICU_chain_step_type_index: - break; - case ICU_chain_step_type_sortkey: - break; case ICU_chain_step_type_casemap: - step->u.casemap = icu_casemap_create((char *) chain->locale, - (char) rule[0], status); + step->u.casemap = icu_casemap_create(rule[0], status); break; case ICU_chain_step_type_normalize: step->u.normalizer = icu_normalizer_create((char *) rule, 'f', status); @@ -786,10 +792,6 @@ void icu_chain_step_destroy(struct icu_chain_step * step){ switch(step->type) { case ICU_chain_step_type_display: break; - case ICU_chain_step_type_index: - break; - case ICU_chain_step_type_sortkey: - break; case ICU_chain_step_type_casemap: icu_casemap_destroy(step->u.casemap); icu_buf_utf16_destroy(step->buf16); @@ -805,25 +807,32 @@ void icu_chain_step_destroy(struct icu_chain_step * step){ default: break; } - free(step); + xfree(step); } -struct icu_chain * icu_chain_create(const uint8_t * identifier, - const uint8_t * locale) +struct icu_chain * icu_chain_create(const char *locale, int sort, + UErrorCode * status) { - struct icu_chain * chain - = (struct icu_chain *) malloc(sizeof(struct icu_chain)); + = (struct icu_chain *) xmalloc(sizeof(struct icu_chain)); + + *status = U_ZERO_ERROR; - strncpy((char *) chain->identifier, (const char *) identifier, 128); - chain->identifier[128 - 1] = '\0'; - strncpy((char *) chain->locale, (const char *) locale, 16); - chain->locale[16 - 1] = '\0'; + chain->locale = xstrdup(locale); + + chain->sort = sort; + + chain->coll = ucol_open((const char *) chain->locale, status); + + if (U_FAILURE(*status)) + return 0; chain->token_count = 0; + chain->src8cstr = 0; + chain->display8 = icu_buf_utf8_create(0); chain->norm8 = icu_buf_utf8_create(0); chain->sort8 = icu_buf_utf8_create(0); @@ -838,7 +847,11 @@ struct icu_chain * icu_chain_create(const uint8_t * identifier, void icu_chain_destroy(struct icu_chain * chain) { - if (chain){ + if (chain) + { + if (chain->coll) + ucol_close(chain->coll); + icu_buf_utf8_destroy(chain->display8); icu_buf_utf8_destroy(chain->norm8); icu_buf_utf8_destroy(chain->sort8); @@ -846,41 +859,39 @@ void icu_chain_destroy(struct icu_chain * chain) icu_buf_utf16_destroy(chain->src16); icu_chain_step_destroy(chain->steps); - free(chain); + xfree(chain->locale); + xfree(chain); } } -struct icu_chain * icu_chain_xml_config(xmlNode *xml_node, - UErrorCode * status){ - +struct icu_chain * icu_chain_xml_config(const xmlNode *xml_node, + int sort, + UErrorCode * status) +{ xmlNode *node = 0; struct icu_chain * chain = 0; - if (!xml_node - ||xml_node->type != XML_ELEMENT_NODE - || strcmp((const char *) xml_node->name, "icu_chain")) + *status = U_ZERO_ERROR; + if (!xml_node ||xml_node->type != XML_ELEMENT_NODE) return 0; - - { - xmlChar *xml_id = xmlGetProp(xml_node, (xmlChar *) "id"); - xmlChar *xml_locale = xmlGetProp(xml_node, (xmlChar *) "locale"); - - if (!xml_id || !strlen((const char *) xml_id) - || !xml_locale || !strlen((const char *) xml_locale)) - return 0; + + { + xmlChar * xml_locale = xmlGetProp((xmlNode *) xml_node, + (xmlChar *) "locale"); - chain = icu_chain_create((const uint8_t *) xml_id, - (const uint8_t *) xml_locale); + if (xml_locale) + { + chain = icu_chain_create((const char *) xml_locale, sort, status); + xmlFree(xml_locale); + } - xmlFree(xml_id); - xmlFree(xml_locale); } if (!chain) return 0; - + for (node = xml_node->children; node; node = node->next) { xmlChar *xml_rule; @@ -891,46 +902,44 @@ struct icu_chain * icu_chain_xml_config(xmlNode *xml_node, xml_rule = xmlGetProp(node, (xmlChar *) "rule"); - if (!strcmp((const char *) node->name, - (const char *) "casemap")){ + if (!strcmp((const char *) node->name, "casemap")) step = icu_chain_insert_step(chain, ICU_chain_step_type_casemap, (const uint8_t *) xml_rule, status); - } - else if (!strcmp((const char *) node->name, - (const char *) "normalize")){ + else if (!strcmp((const char *) node->name, "transform")) step = icu_chain_insert_step(chain, ICU_chain_step_type_normalize, (const uint8_t *) xml_rule, status); - } - else if (!strcmp((const char *) node->name, - (const char *) "tokenize")){ + else if (!strcmp((const char *) node->name, "tokenize")) step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize, (const uint8_t *) xml_rule, status); - } - else if (!strcmp((const char *) node->name, - (const char *) "display")){ + else if (!strcmp((const char *) node->name, "display")) step = icu_chain_insert_step(chain, ICU_chain_step_type_display, (const uint8_t *) "", status); + else if (!strcmp((const char *) node->name, "normalize")) + { + yaz_log(YLOG_WARN, "Element %s is deprecated. " + "Use transform instead", node->name); + step = icu_chain_insert_step(chain, ICU_chain_step_type_normalize, + (const uint8_t *) xml_rule, status); } - else if (!strcmp((const char *) node->name, - (const char *) "index")){ - step = icu_chain_insert_step(chain, ICU_chain_step_type_index, - (const uint8_t *) "", status); + else if (!strcmp((const char *) node->name, "index") + || !strcmp((const char *) node->name, "sortkey")) + { + yaz_log(YLOG_WARN, "Element %s is no longer needed. " + "Remove it from the configuration", node->name); } - else if (!strcmp((const char *) node->name, - (const char *) "sortkey")){ - step = icu_chain_insert_step(chain, ICU_chain_step_type_sortkey, - (const uint8_t *) "", status); + else + { + yaz_log(YLOG_WARN, "Unknown element %s", node->name); + icu_chain_destroy(chain); + return 0; } - xmlFree(xml_rule); - if (!step || U_FAILURE(*status)){ + if (step && U_FAILURE(*status)) + { icu_chain_destroy(chain); return 0; } - - } - return chain; } @@ -958,16 +967,11 @@ struct icu_chain_step * icu_chain_insert_step(struct icu_chain * chain, /* create utf16 destination buffers as needed, or */ - switch(type) { + switch(type) + { case ICU_chain_step_type_display: buf16 = src16; break; - case ICU_chain_step_type_index: - buf16 = src16; - break; - case ICU_chain_step_type_sortkey: - buf16 = src16; - break; case ICU_chain_step_type_casemap: buf16 = icu_buf_utf16_create(0); break; @@ -996,45 +1000,55 @@ int icu_chain_step_next_token(struct icu_chain * chain, UErrorCode *status) { struct icu_buf_utf16 * src16 = 0; - + int got_new_token = 0; + if (!chain || !chain->src16 || !step || !step->more_tokens) return 0; /* assign utf16 src buffers as neeed, advance in previous steps - tokens until non-zero token met, and setting stop condition - */ - if (step->previous){ + tokens until non-zero token met, and setting stop condition */ + + if (step->previous) + { src16 = step->previous->buf16; - if (step->need_new_token) - step->more_tokens + /* tokens might be killed in previous steps, therefore looping */ + + while (step->need_new_token + && step->previous->more_tokens + && !got_new_token) + got_new_token = icu_chain_step_next_token(chain, step->previous, status); } - else { /* first step can only work once on chain->src16 input buffer */ + else + { /* first step can only work once on chain->src16 input buffer */ src16 = chain->src16; - step->more_tokens = 1; + step->more_tokens = 0; + got_new_token = 1; } - /* stop if nothing to process - i.e new token source was not properly assigned - */ - if (!step->more_tokens || !src16) + if (!src16) return 0; - /* perform the work, eventually put this steps output in + /* stop if nothing to process */ + if (step->need_new_token && !got_new_token) + { + step->more_tokens = 0; + return 0; + } + + /* either an old token not finished yet, or a new token, thus + perform the work, eventually put this steps output in step->buf16 or the chains UTF8 output buffers */ - switch(step->type) { + + switch(step->type) + { case ICU_chain_step_type_display: icu_utf16_to_utf8(chain->display8, src16, status); break; - case ICU_chain_step_type_index: - icu_utf16_to_utf8(chain->norm8, src16, status); - break; - case ICU_chain_step_type_sortkey: - icu_utf16_to_utf8(chain->sort8, src16, status); - break; case ICU_chain_step_type_casemap: icu_casemap_casemap(step->u.casemap, - step->buf16, src16, status); + step->buf16, src16, status, + chain->locale); break; case ICU_chain_step_type_normalize: icu_normalizer_normalize(step->u.normalizer, @@ -1042,48 +1056,42 @@ int icu_chain_step_next_token(struct icu_chain * chain, break; case ICU_chain_step_type_tokenize: /* attach to new src16 token only first time during splitting */ - if (step->need_new_token){ + if (step->need_new_token) + { icu_tokenizer_attach(step->u.tokenizer, src16, status); step->need_new_token = 0; } + /* splitting one src16 token into multiple buf16 tokens */ step->more_tokens = icu_tokenizer_next_token(step->u.tokenizer, step->buf16, status); - /* make sure to get new previous token if this one had been used up */ - if (step->previous && !step->more_tokens){ - if (icu_chain_step_next_token(chain, step->previous, status)){ - icu_tokenizer_attach(step->u.tokenizer, src16, status); - step->need_new_token = 0; - step->more_tokens - = icu_tokenizer_next_token(step->u.tokenizer, - step->buf16, status); - } + + /* make sure to get new previous token if this one had been used up + by recursive call to _same_ step */ + + if (!step->more_tokens) + { + step->more_tokens = icu_chain_step_next_token(chain, step, status); + return step->more_tokens; /* avoid one token count too much! */ } - if (0 == step->more_tokens) - return 0; break; default: return 0; break; } - - - /* stop further token processing if last step and - new tokens are needed from previous (non-existing) step - */ - if (!step->previous && step->need_new_token) - step->more_tokens = 0; - if (U_FAILURE(*status)) return 0; + /* if token disappered into thin air, tell caller */ + /* if (!step->buf16->utf16_len && !step->more_tokens) */ + /* return 0; */ + return 1; } - int icu_chain_assign_cstr(struct icu_chain * chain, const char * src8cstr, UErrorCode *status) @@ -1093,20 +1101,24 @@ int icu_chain_assign_cstr(struct icu_chain * chain, if (!chain || !src8cstr) return 0; + chain->src8cstr = src8cstr; + stp = chain->steps; /* clear token count */ chain->token_count = 0; /* clear all steps stop states */ - while (stp){ + while (stp) + { stp->more_tokens = 1; stp->need_new_token = 1; stp = stp->previous; } - /* finally convert UTF8 to UTF16 string */ - icu_utf16_from_utf8_cstr(chain->src16, src8cstr, status); + /* finally convert UTF8 to UTF16 string if needed */ + if (chain->steps || chain->sort) + icu_utf16_from_utf8_cstr(chain->src16, chain->src8cstr, status); if (U_FAILURE(*status)) return 0; @@ -1119,22 +1131,54 @@ int icu_chain_assign_cstr(struct icu_chain * chain, int icu_chain_next_token(struct icu_chain * chain, UErrorCode *status) { - int success = 0; + int got_token = 0; - if (!chain || !chain->steps) + *status = U_ZERO_ERROR; + + if (!chain) return 0; - success = icu_chain_step_next_token(chain, chain->steps, status); - - if (success){ - chain->token_count++; - return chain->token_count; + /* special case with no steps - same as index type binary */ + if (!chain->steps) + { + if (chain->token_count) + return 0; + else + { + chain->token_count++; + + if (chain->sort) + icu_sortkey8_from_utf16(chain->coll, + chain->sort8, chain->steps->buf16, + status); + return chain->token_count; + } } + /* usual case, one or more icu chain steps existing */ + else + { + while(!got_token && chain->steps && chain->steps->more_tokens) + got_token = icu_chain_step_next_token(chain, chain->steps, status); + + if (got_token) + { + chain->token_count++; + + icu_utf16_to_utf8(chain->norm8, chain->steps->buf16, status); + + if (chain->sort) + icu_sortkey8_from_utf16(chain->coll, + chain->sort8, chain->steps->buf16, + status); + return chain->token_count; + } + } + return 0; } -int icu_chain_get_token_count(struct icu_chain * chain) +int icu_chain_token_number(struct icu_chain * chain) { if (!chain) return 0; @@ -1143,8 +1187,7 @@ int icu_chain_get_token_count(struct icu_chain * chain) } - -const char * icu_chain_get_display(struct icu_chain * chain) +const char * icu_chain_token_display(struct icu_chain * chain) { if (chain->display8) return icu_buf_utf8_to_cstr(chain->display8); @@ -1152,15 +1195,18 @@ const char * icu_chain_get_display(struct icu_chain * chain) return 0; } -const char * icu_chain_get_norm(struct icu_chain * chain) +const char * icu_chain_token_norm(struct icu_chain * chain) { + if (!chain->steps) + return chain->src8cstr; + if (chain->norm8) return icu_buf_utf8_to_cstr(chain->norm8); return 0; } -const char * icu_chain_get_sort(struct icu_chain * chain) +const char * icu_chain_token_sortkey(struct icu_chain * chain) { if (chain->sort8) return icu_buf_utf8_to_cstr(chain->sort8); @@ -1168,11 +1214,12 @@ const char * icu_chain_get_sort(struct icu_chain * chain) return 0; } +const UCollator * icu_chain_get_coll(struct icu_chain * chain) +{ + return chain->coll; +} -#endif /* HAVE_ICU */ - - - +#endif /* YAZ_HAVE_ICU */ /* * Local variables: