X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=src%2Ficu_I18N.c;h=a978b2159677ba45686d3acd0b0c2de28ec1808a;hb=def0d51c189bc8a2e9f5f9a67a58833897edecb7;hp=a085caaaa113f1e0759a8a09b825dfd0fc92cbd6;hpb=0ffb25b0bf78f53399267664b71206c88980c4da;p=yaz-moved-to-github.git diff --git a/src/icu_I18N.c b/src/icu_I18N.c index a085caa..a978b21 100644 --- a/src/icu_I18N.c +++ b/src/icu_I18N.c @@ -1,26 +1,12 @@ -/* $Id: icu_I18N.c,v 1.1 2007-10-22 12:21:39 adam Exp $ - Copyright (c) 2006-2007, Index Data. - - This file is part of Pazpar2. - - Pazpar2 is free software; you can redistribute it and/or modify it under - the terms of the GNU General Public License as published by the Free - Software Foundation; either version 2, or (at your option) any later - version. - - Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY - WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - for more details. - - You should have received a copy of the GNU General Public License - along with Pazpar2; see the file LICENSE. If not, write to the - Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA - 02111-1307, USA. -*/ +/* + * Copyright (C) 1995-2007, Index Data ApS + * See the file LICENSE for details. + * + * $Id: icu_I18N.c,v 1.12 2007-11-07 10:19:12 adam Exp $ + */ #if HAVE_CONFIG_H -#include "cconfig.h" +#include "config.h" #endif #define USE_TIMING 0 @@ -29,7 +15,7 @@ #endif -#ifdef HAVE_ICU +#if HAVE_ICU #include #include @@ -42,15 +28,7 @@ #include /* char names */ -//#include -//#include /* Basic ICU data types */ #include -//#include /* C Converter API */ -//#include -//#include -/* #include */ - - int icu_check_status (UErrorCode status) @@ -81,33 +59,44 @@ struct icu_buf_utf16 * icu_buf_utf16_create(size_t capacity) buf16->utf16_cap = capacity; } return buf16; -}; +} -struct icu_buf_utf16 * icu_buf_utf16_resize(struct icu_buf_utf16 * buf16, - size_t capacity) +struct icu_buf_utf16 * icu_buf_utf16_clear(struct icu_buf_utf16 * buf16) { if (buf16){ - if (capacity > 0){ - if (0 == buf16->utf16) - buf16->utf16 = (UChar *) malloc(sizeof(UChar) * capacity); - else - buf16->utf16 - = (UChar *) realloc(buf16->utf16, sizeof(UChar) * capacity); + if (buf16->utf16) buf16->utf16[0] = (UChar) 0; - buf16->utf16_len = 0; - buf16->utf16_cap = capacity; - } - else { - if (buf16->utf16) - free(buf16->utf16); - buf16->utf16 = 0; - buf16->utf16_len = 0; - buf16->utf16_cap = 0; - } + buf16->utf16_len = 0; + } + return buf16; +} + +struct icu_buf_utf16 * icu_buf_utf16_resize(struct icu_buf_utf16 * buf16, + size_t capacity) +{ + if (!buf16) + return 0; + + if (capacity > 0){ + if (0 == buf16->utf16) + buf16->utf16 = (UChar *) malloc(sizeof(UChar) * capacity); + else + buf16->utf16 + = (UChar *) realloc(buf16->utf16, sizeof(UChar) * capacity); + + icu_buf_utf16_clear(buf16); + buf16->utf16_cap = capacity; + } + else { + if (buf16->utf16) + free(buf16->utf16); + buf16->utf16 = 0; + buf16->utf16_len = 0; + buf16->utf16_cap = 0; } return buf16; -}; +} struct icu_buf_utf16 * icu_buf_utf16_copy(struct icu_buf_utf16 * dest16, @@ -124,7 +113,7 @@ struct icu_buf_utf16 * icu_buf_utf16_copy(struct icu_buf_utf16 * dest16, dest16->utf16_len = src16->utf16_len; return dest16; -}; +} void icu_buf_utf16_destroy(struct icu_buf_utf16 * buf16) @@ -134,7 +123,7 @@ void icu_buf_utf16_destroy(struct icu_buf_utf16 * buf16) free(buf16->utf16); free(buf16); } -}; +} @@ -156,34 +145,46 @@ struct icu_buf_utf8 * icu_buf_utf8_create(size_t capacity) buf8->utf8_cap = capacity; } return buf8; -}; +} +struct icu_buf_utf8 * icu_buf_utf8_clear(struct icu_buf_utf8 * buf8) +{ + if (buf8){ + if (buf8->utf8) + buf8->utf8[0] = (uint8_t) 0; + buf8->utf8_len = 0; + } + return buf8; +} + struct icu_buf_utf8 * icu_buf_utf8_resize(struct icu_buf_utf8 * buf8, size_t capacity) { - if (buf8){ - if (capacity > 0){ - if (0 == buf8->utf8) - buf8->utf8 = (uint8_t *) malloc(sizeof(uint8_t) * capacity); - else - buf8->utf8 - = (uint8_t *) realloc(buf8->utf8, - sizeof(uint8_t) * capacity); - buf8->utf8_cap = capacity; - } - else { - if (buf8->utf8) - free(buf8->utf8); - buf8->utf8 = 0; - buf8->utf8_len = 0; - buf8->utf8_cap = 0; - } - } + if (!buf8) + return 0; + if (capacity > 0){ + if (0 == buf8->utf8) + buf8->utf8 = (uint8_t *) malloc(sizeof(uint8_t) * capacity); + else + buf8->utf8 + = (uint8_t *) realloc(buf8->utf8, sizeof(uint8_t) * capacity); + + icu_buf_utf8_clear(buf8); + buf8->utf8_cap = capacity; + } + else { + if (buf8->utf8) + free(buf8->utf8); + buf8->utf8 = 0; + buf8->utf8_len = 0; + buf8->utf8_cap = 0; + } + return buf8; -}; +} struct icu_buf_utf8 * icu_buf_utf8_copy(struct icu_buf_utf8 * dest8, @@ -200,16 +201,19 @@ struct icu_buf_utf8 * icu_buf_utf8_copy(struct icu_buf_utf8 * dest8, strncpy((char*) dest8->utf8, (char*) src8->utf8, src8->utf8_len); return dest8; -}; +} const char *icu_buf_utf8_to_cstr(struct icu_buf_utf8 *src8) { if (!src8 || src8->utf8_len == 0) return ""; + if (src8->utf8_len == src8->utf8_cap) src8 = icu_buf_utf8_resize(src8, src8->utf8_len * 2 + 1); + src8->utf8[src8->utf8_len] = '\0'; + return (const char *) src8->utf8; } @@ -221,7 +225,7 @@ void icu_buf_utf8_destroy(struct icu_buf_utf8 * buf8) free(buf8->utf8); free(buf8); } -}; +} @@ -235,10 +239,9 @@ UErrorCode icu_utf16_from_utf8(struct icu_buf_utf16 * dest16, &utf16_len, (const char *) src8->utf8, src8->utf8_len, status); - // check for buffer overflow, resize and retry - if (*status == U_BUFFER_OVERFLOW_ERROR - //|| dest16->utf16_len > dest16->utf16_cap - ){ + /* check for buffer overflow, resize and retry */ + if (*status == U_BUFFER_OVERFLOW_ERROR) + { icu_buf_utf16_resize(dest16, utf16_len * 2); *status = U_ZERO_ERROR; u_strFromUTF8(dest16->utf16, dest16->utf16_cap, @@ -246,17 +249,14 @@ UErrorCode icu_utf16_from_utf8(struct icu_buf_utf16 * dest16, (const char *) src8->utf8, src8->utf8_len, status); } - //if (*status != U_BUFFER_OVERFLOW_ERROR if (U_SUCCESS(*status) && utf16_len <= dest16->utf16_cap) dest16->utf16_len = utf16_len; - else { - dest16->utf16[0] = (UChar) 0; - dest16->utf16_len = 0; - } + else + icu_buf_utf16_clear(dest16); return *status; -}; +} @@ -267,16 +267,16 @@ UErrorCode icu_utf16_from_utf8_cstr(struct icu_buf_utf16 * dest16, size_t src8cstr_len = 0; int32_t utf16_len = 0; + *status = U_ZERO_ERROR; src8cstr_len = strlen(src8cstr); u_strFromUTF8(dest16->utf16, dest16->utf16_cap, &utf16_len, src8cstr, src8cstr_len, status); - // check for buffer overflow, resize and retry - if (*status == U_BUFFER_OVERFLOW_ERROR - //|| dest16->utf16_len > dest16->utf16_cap - ){ + /* check for buffer overflow, resize and retry */ + if (*status == U_BUFFER_OVERFLOW_ERROR) + { icu_buf_utf16_resize(dest16, utf16_len * 2); *status = U_ZERO_ERROR; u_strFromUTF8(dest16->utf16, dest16->utf16_cap, @@ -284,17 +284,14 @@ UErrorCode icu_utf16_from_utf8_cstr(struct icu_buf_utf16 * dest16, src8cstr, src8cstr_len, status); } - // if (*status != U_BUFFER_OVERFLOW_ERROR if (U_SUCCESS(*status) && utf16_len <= dest16->utf16_cap) dest16->utf16_len = utf16_len; - else { - dest16->utf16[0] = (UChar) 0; - dest16->utf16_len = 0; - } + else + icu_buf_utf16_clear(dest16); return *status; -}; +} @@ -309,10 +306,9 @@ UErrorCode icu_utf16_to_utf8(struct icu_buf_utf8 * dest8, &utf8_len, src16->utf16, src16->utf16_len, status); - // check for buffer overflow, resize and retry - if (*status == U_BUFFER_OVERFLOW_ERROR - //|| dest8->utf8_len > dest8->utf8_cap - ){ + /* check for buffer overflow, resize and retry */ + if (*status == U_BUFFER_OVERFLOW_ERROR) + { icu_buf_utf8_resize(dest8, utf8_len * 2); *status = U_ZERO_ERROR; u_strToUTF8((char *) dest8->utf8, dest8->utf8_cap, @@ -321,17 +317,14 @@ UErrorCode icu_utf16_to_utf8(struct icu_buf_utf8 * dest8, } - //if (*status != U_BUFFER_OVERFLOW_ERROR if (U_SUCCESS(*status) && utf8_len <= dest8->utf8_cap) dest8->utf8_len = utf8_len; - else { - dest8->utf8[0] = (uint8_t) 0; - dest8->utf8_len = 0; - } + else + icu_buf_utf8_clear(dest8); return *status; -}; +} @@ -345,12 +338,13 @@ struct icu_casemap * icu_casemap_create(const char *locale, char action, switch(casemap->action) { case 'l': - break; + case 'L': case 'u': - break; + case 'U': case 't': - break; + case 'T': case 'f': + case 'F': break; default: icu_casemap_destroy(casemap); @@ -358,13 +352,13 @@ struct icu_casemap * icu_casemap_create(const char *locale, char action, } return casemap; -}; +} void icu_casemap_destroy(struct icu_casemap * casemap) { if (casemap) free(casemap); -}; +} int icu_casemap_casemap(struct icu_casemap * casemap, @@ -377,7 +371,7 @@ int icu_casemap_casemap(struct icu_casemap * casemap, return icu_utf16_casemap(dest16, src16, casemap->locale, casemap->action, status); -}; +} int icu_utf16_casemap(struct icu_buf_utf16 * dest16, @@ -386,24 +380,37 @@ int icu_utf16_casemap(struct icu_buf_utf16 * dest16, UErrorCode *status) { int32_t dest16_len = 0; + + + if (!src16->utf16_len){ //guarding for empty source string + if (dest16->utf16) + dest16->utf16[0] = (UChar) 0; + dest16->utf16_len = 0; + return U_ZERO_ERROR; + } + switch(action) { case 'l': + case 'L': dest16_len = u_strToLower(dest16->utf16, dest16->utf16_cap, src16->utf16, src16->utf16_len, locale, status); break; case 'u': + case 'U': dest16_len = u_strToUpper(dest16->utf16, dest16->utf16_cap, src16->utf16, src16->utf16_len, locale, status); break; case 't': + case 'T': dest16_len = u_strToTitle(dest16->utf16, dest16->utf16_cap, src16->utf16, src16->utf16_len, 0, locale, status); break; case 'f': + case 'F': dest16_len = u_strFoldCase(dest16->utf16, dest16->utf16_cap, src16->utf16, src16->utf16_len, U_FOLD_CASE_DEFAULT, status); @@ -414,10 +421,9 @@ int icu_utf16_casemap(struct icu_buf_utf16 * dest16, break; } - // check for buffer overflow, resize and retry + /* check for buffer overflow, resize and retry */ if (*status == U_BUFFER_OVERFLOW_ERROR - && dest16 != src16 // do not resize if in-place conversion - //|| dest16_len > dest16->utf16_cap + && dest16 != src16 /* do not resize if in-place conversion */ ){ icu_buf_utf16_resize(dest16, dest16_len * 2); *status = U_ZERO_ERROR; @@ -425,21 +431,25 @@ int icu_utf16_casemap(struct icu_buf_utf16 * dest16, switch(action) { case 'l': + case 'L': dest16_len = u_strToLower(dest16->utf16, dest16->utf16_cap, src16->utf16, src16->utf16_len, locale, status); break; case 'u': + case 'U': dest16_len = u_strToUpper(dest16->utf16, dest16->utf16_cap, src16->utf16, src16->utf16_len, locale, status); break; case 't': + case 'T': dest16_len = u_strToTitle(dest16->utf16, dest16->utf16_cap, src16->utf16, src16->utf16_len, 0, locale, status); break; case 'f': + case 'F': dest16_len = u_strFoldCase(dest16->utf16, dest16->utf16_cap, src16->utf16, src16->utf16_len, U_FOLD_CASE_DEFAULT, status); @@ -455,12 +465,13 @@ int icu_utf16_casemap(struct icu_buf_utf16 * dest16, && dest16_len <= dest16->utf16_cap) dest16->utf16_len = dest16_len; else { - dest16->utf16[0] = (UChar) 0; + if (dest16->utf16) + dest16->utf16[0] = (UChar) 0; dest16->utf16_len = 0; } return *status; -}; +} @@ -475,7 +486,7 @@ UErrorCode icu_sortkey8_from_utf16(UCollator *coll, sortkey_len = ucol_getSortKey(coll, src16->utf16, src16->utf16_len, dest8->utf8, dest8->utf8_cap); - // check for buffer overflow, resize and retry + /* check for buffer overflow, resize and retry */ if (sortkey_len > dest8->utf8_cap) { icu_buf_utf8_resize(dest8, sortkey_len * 2); sortkey_len = ucol_getSortKey(coll, src16->utf16, src16->utf16_len, @@ -485,13 +496,11 @@ UErrorCode icu_sortkey8_from_utf16(UCollator *coll, if (U_SUCCESS(*status) && sortkey_len > 0) dest8->utf8_len = sortkey_len; - else { - dest8->utf8[0] = (UChar) 0; - dest8->utf8_len = 0; - } - + else + icu_buf_utf8_clear(dest8); + return sortkey_len; -}; +} @@ -513,26 +522,31 @@ struct icu_tokenizer * icu_tokenizer_create(const char *locale, char action, switch(tokenizer->action) { case 'l': + case 'L': tokenizer->bi = ubrk_open(UBRK_LINE, tokenizer->locale, 0, 0, status); break; case 's': + case 'S': tokenizer->bi = ubrk_open(UBRK_SENTENCE, tokenizer->locale, 0, 0, status); break; case 'w': + case 'W': tokenizer->bi = ubrk_open(UBRK_WORD, tokenizer->locale, 0, 0, status); break; case 'c': + case 'C': tokenizer->bi = ubrk_open(UBRK_CHARACTER, tokenizer->locale, 0, 0, status); break; case 't': + case 'T': tokenizer->bi = ubrk_open(UBRK_TITLE, tokenizer->locale, 0, 0, status); @@ -543,14 +557,14 @@ struct icu_tokenizer * icu_tokenizer_create(const char *locale, char action, break; } - // ICU error stuff is a very funny business + /* ICU error stuff is a very funny business */ if (U_SUCCESS(*status)) return tokenizer; - // freeing if failed + /* freeing if failed */ icu_tokenizer_destroy(tokenizer); return 0; -}; +} void icu_tokenizer_destroy(struct icu_tokenizer * tokenizer) { @@ -559,7 +573,7 @@ void icu_tokenizer_destroy(struct icu_tokenizer * tokenizer) ubrk_close(tokenizer->bi); free(tokenizer); } -}; +} int icu_tokenizer_attach(struct icu_tokenizer * tokenizer, struct icu_buf_utf16 * src16, @@ -597,29 +611,31 @@ int32_t icu_tokenizer_next_token(struct icu_tokenizer * tokenizer, || !tokenizer->buf16 || !tokenizer->buf16->utf16_len) return 0; - // never change tokenizer->buf16 and keep always invariant - // 0 <= tokenizer->token_start - // <= tokenizer->token_end - // <= tokenizer->buf16->utf16_len - // returns length of token + /* + never change tokenizer->buf16 and keep always invariant + 0 <= tokenizer->token_start + <= tokenizer->token_end + <= tokenizer->buf16->utf16_len + returns length of token + */ - if (0 == tokenizer->token_end) // first call + if (0 == tokenizer->token_end) /* first call */ tkn_start = ubrk_first(tokenizer->bi); - else //successive calls + else /* successive calls */ tkn_start = tokenizer->token_end; - // get next position + /* get next position */ tkn_end = ubrk_next(tokenizer->bi); - // repairing invariant at end of ubrk, which is UBRK_DONE = -1 + /* repairing invariant at end of ubrk, which is UBRK_DONE = -1 */ if (UBRK_DONE == tkn_end) tkn_end = tokenizer->buf16->utf16_len; - // copy out if everything is well + /* copy out if everything is well */ if(U_FAILURE(*status)) return 0; - // everything OK, now update internal state + /* everything OK, now update internal state */ tkn_len = tkn_end - tkn_start; if (0 < tkn_len){ @@ -632,7 +648,7 @@ int32_t icu_tokenizer_next_token(struct icu_tokenizer * tokenizer, tokenizer->token_end = tkn_end; - // copying into token buffer if it exists + /* copying into token buffer if it exists */ if (tkn16){ if (tkn16->utf16_cap < tkn_len) icu_buf_utf16_resize(tkn16, (size_t) tkn_len * 2); @@ -650,27 +666,27 @@ int32_t icu_tokenizer_next_token(struct icu_tokenizer * tokenizer, int32_t icu_tokenizer_token_id(struct icu_tokenizer * tokenizer) { return tokenizer->token_id; -}; +} int32_t icu_tokenizer_token_start(struct icu_tokenizer * tokenizer) { return tokenizer->token_start; -}; +} int32_t icu_tokenizer_token_end(struct icu_tokenizer * tokenizer) { return tokenizer->token_end; -}; +} int32_t icu_tokenizer_token_length(struct icu_tokenizer * tokenizer) { return (tokenizer->token_end - tokenizer->token_start); -}; +} int32_t icu_tokenizer_token_count(struct icu_tokenizer * tokenizer) { return tokenizer->token_count; -}; +} @@ -688,22 +704,22 @@ struct icu_normalizer * icu_normalizer_create(const char *rules, char action, switch(normalizer->action) { case 'f': + case 'F': normalizer->trans = utrans_openU(normalizer->rules16->utf16, normalizer->rules16->utf16_len, UTRANS_FORWARD, 0, 0, normalizer->parse_error, status); - // yaz_log(YLOG_LOG, "utrans_open %p", normalizer->trans); break; case 'r': + case 'R': normalizer->trans = utrans_openU(normalizer->rules16->utf16, normalizer->rules16->utf16_len, UTRANS_REVERSE , 0, 0, normalizer->parse_error, status); - // yaz_log(YLOG_LOG, "utrans_open %p", normalizer->trans); break; default: *status = U_UNSUPPORTED_ERROR; @@ -714,10 +730,10 @@ struct icu_normalizer * icu_normalizer_create(const char *rules, char action, if (U_SUCCESS(*status)) return normalizer; - // freeing if failed + /* freeing if failed */ icu_normalizer_destroy(normalizer); return 0; -}; +} void icu_normalizer_destroy(struct icu_normalizer * normalizer){ @@ -725,13 +741,10 @@ void icu_normalizer_destroy(struct icu_normalizer * normalizer){ if (normalizer->rules16) icu_buf_utf16_destroy(normalizer->rules16); if (normalizer->trans) - { - // yaz_log(YLOG_LOG, "utrans_close %p", normalizer->trans); utrans_close(normalizer->trans); - } free(normalizer); } -}; +} @@ -740,21 +753,27 @@ int icu_normalizer_normalize(struct icu_normalizer * normalizer, struct icu_buf_utf16 * src16, UErrorCode *status) { - if (!normalizer || !normalizer->trans || !src16 || !dest16) + if (!normalizer || !normalizer->trans + || !src16 + || !dest16) return 0; + if (!src16->utf16_len){ //guarding for empty source string + icu_buf_utf16_clear(dest16); + return 0; + } + if (!icu_buf_utf16_copy(dest16, src16)) return 0; + utrans_transUChars (normalizer->trans, dest16->utf16, &(dest16->utf16_len), dest16->utf16_cap, 0, &(src16->utf16_len), status); - if (U_FAILURE(*status)){ - dest16->utf16[0] = (UChar) 0; - dest16->utf16_len = 0; - } + if (U_FAILURE(*status)) + icu_buf_utf16_clear(dest16); return dest16->utf16_len; } @@ -779,14 +798,10 @@ struct icu_chain_step * icu_chain_step_create(struct icu_chain * chain, step->buf16 = buf16; - // create auxilary objects + /* create auxilary objects */ switch(step->type) { case ICU_chain_step_type_display: break; - case ICU_chain_step_type_index: - break; - case ICU_chain_step_type_sortkey: - break; case ICU_chain_step_type_casemap: step->u.casemap = icu_casemap_create((char *) chain->locale, (char) rule[0], status); @@ -803,7 +818,7 @@ struct icu_chain_step * icu_chain_step_create(struct icu_chain * chain, } return step; -}; +} void icu_chain_step_destroy(struct icu_chain_step * step){ @@ -816,10 +831,6 @@ void icu_chain_step_destroy(struct icu_chain_step * step){ switch(step->type) { case ICU_chain_step_type_display: break; - case ICU_chain_step_type_index: - break; - case ICU_chain_step_type_sortkey: - break; case ICU_chain_step_type_casemap: icu_casemap_destroy(step->u.casemap); icu_buf_utf16_destroy(step->buf16); @@ -836,24 +847,35 @@ void icu_chain_step_destroy(struct icu_chain_step * step){ break; } free(step); -}; +} -struct icu_chain * icu_chain_create(const uint8_t * identifier, - const uint8_t * locale) +struct icu_chain * icu_chain_create(const char *locale, + int sort, + UErrorCode * status) { struct icu_chain * chain = (struct icu_chain *) malloc(sizeof(struct icu_chain)); - strncpy((char *) chain->identifier, (const char *) identifier, 128); - chain->identifier[128 - 1] = '\0'; + *status = U_ZERO_ERROR; + strncpy((char *) chain->locale, (const char *) locale, 16); chain->locale[16 - 1] = '\0'; + chain->sort = sort; + + chain->coll = ucol_open((const char *) chain->locale, status); + + if (U_FAILURE(*status)) + return 0; + + chain->token_count = 0; + chain->src8cstr = 0; + chain->display8 = icu_buf_utf8_create(0); chain->norm8 = icu_buf_utf8_create(0); chain->sort8 = icu_buf_utf8_create(0); @@ -863,12 +885,16 @@ struct icu_chain * icu_chain_create(const uint8_t * identifier, chain->steps = 0; return chain; -}; +} void icu_chain_destroy(struct icu_chain * chain) { if (chain){ + + if (chain->coll) + ucol_close(chain->coll); + icu_buf_utf8_destroy(chain->display8); icu_buf_utf8_destroy(chain->norm8); icu_buf_utf8_destroy(chain->sort8); @@ -878,44 +904,42 @@ void icu_chain_destroy(struct icu_chain * chain) icu_chain_step_destroy(chain->steps); free(chain); } -}; +} -struct icu_chain * icu_chain_xml_config(xmlNode *xml_node, +struct icu_chain * icu_chain_xml_config(const xmlNode *xml_node, + const char *locale, + int sort, UErrorCode * status){ xmlNode *node = 0; struct icu_chain * chain = 0; + *status = U_ZERO_ERROR; + if (!xml_node ||xml_node->type != XML_ELEMENT_NODE - || strcmp((const char *) xml_node->name, "icu_chain")) + // || strcmp((const char *) xml_node->name, "icu_chain") + ) return 0; - - xmlChar *xml_id = xmlGetProp(xml_node, (xmlChar *) "id"); - xmlChar *xml_locale = xmlGetProp(xml_node, (xmlChar *) "locale"); - if (!xml_id || !strlen((const char *) xml_id) - || !xml_locale || !strlen((const char *) xml_locale)) - return 0; + chain = icu_chain_create(locale, sort, status); - chain = icu_chain_create((const uint8_t *) xml_id, - (const uint8_t *) xml_locale); - - xmlFree(xml_id); - xmlFree(xml_locale); if (!chain) return 0; + for (node = xml_node->children; node; node = node->next) { + xmlChar *xml_rule; + struct icu_chain_step * step = 0; + if (node->type != XML_ELEMENT_NODE) continue; - xmlChar *xml_rule = xmlGetProp(node, (xmlChar *) "rule"); - struct icu_chain_step * step = 0; + xml_rule = xmlGetProp(node, (xmlChar *) "rule"); if (!strcmp((const char *) node->name, (const char *) "casemap")){ @@ -937,17 +961,6 @@ struct icu_chain * icu_chain_xml_config(xmlNode *xml_node, step = icu_chain_insert_step(chain, ICU_chain_step_type_display, (const uint8_t *) "", status); } - else if (!strcmp((const char *) node->name, - (const char *) "index")){ - step = icu_chain_insert_step(chain, ICU_chain_step_type_index, - (const uint8_t *) "", status); - } - else if (!strcmp((const char *) node->name, - (const char *) "sortkey")){ - step = icu_chain_insert_step(chain, ICU_chain_step_type_sortkey, - (const uint8_t *) "", status); - } - xmlFree(xml_rule); if (!step || U_FAILURE(*status)){ icu_chain_destroy(chain); @@ -958,7 +971,7 @@ struct icu_chain * icu_chain_xml_config(xmlNode *xml_node, } return chain; -}; +} @@ -974,7 +987,7 @@ struct icu_chain_step * icu_chain_insert_step(struct icu_chain * chain, if (!chain || !type || !rule) return 0; - // assign utf16 src buffers as needed + /* assign utf16 src buffers as needed */ if (chain->steps && chain->steps->buf16) src16 = chain->steps->buf16; else if (chain->src16) @@ -983,17 +996,11 @@ struct icu_chain_step * icu_chain_insert_step(struct icu_chain * chain, return 0; - // create utf16 destination buffers as needed, or + /* create utf16 destination buffers as needed, or */ switch(type) { case ICU_chain_step_type_display: buf16 = src16; break; - case ICU_chain_step_type_index: - buf16 = src16; - break; - case ICU_chain_step_type_sortkey: - buf16 = src16; - break; case ICU_chain_step_type_casemap: buf16 = icu_buf_utf16_create(0); break; @@ -1007,14 +1014,14 @@ struct icu_chain_step * icu_chain_insert_step(struct icu_chain * chain, break; } - // create actual chain step with this buffer + /* create actual chain step with this buffer */ step = icu_chain_step_create(chain, type, rule, buf16, status); step->previous = chain->steps; chain->steps = step; return step; -}; +} int icu_chain_step_next_token(struct icu_chain * chain, @@ -1022,46 +1029,47 @@ int icu_chain_step_next_token(struct icu_chain * chain, UErrorCode *status) { struct icu_buf_utf16 * src16 = 0; - - //printf("icu_chain_step_next_token %d\n", (int) step); + int got_new_token = 0; if (!chain || !chain->src16 || !step || !step->more_tokens) return 0; - // assign utf16 src buffers as neeed, advance in previous steps - // tokens until non-zero token met, and setting stop condition + /* assign utf16 src buffers as neeed, advance in previous steps + tokens until non-zero token met, and setting stop condition */ + if (step->previous){ src16 = step->previous->buf16; - if (step->need_new_token) - //while (step->more_tokens && !src16->utf16_len) - step->more_tokens - = icu_chain_step_next_token(chain, step->previous, status); + /* tokens might be killed in previous steps, therefore looping */ + + while (step->need_new_token + && step->previous->more_tokens + && !got_new_token) + got_new_token + = icu_chain_step_next_token(chain, step->previous, status); } - else { // first step can only work once on chain->src16 input buffer + else { /* first step can only work once on chain->src16 input buffer */ src16 = chain->src16; - step->more_tokens = 1; + step->more_tokens = 0; + got_new_token = 1; } - // stop if nothing to process - // i.e new token source was not properly assigned - if (!step->more_tokens || !src16) // || !src16->utf16_len + if (!src16) return 0; - //printf("icu_chain_step_next_token %d working\n", (int) step); + /* stop if nothing to process */ + if (step->need_new_token && !got_new_token){ + step->more_tokens = 0; + return 0; + } + /* either an old token not finished yet, or a new token, thus + perform the work, eventually put this steps output in + step->buf16 or the chains UTF8 output buffers */ - // perform the work, eventually put this steps output in - // step->buf16 or the chains UTF8 output buffers switch(step->type) { case ICU_chain_step_type_display: icu_utf16_to_utf8(chain->display8, src16, status); break; - case ICU_chain_step_type_index: - icu_utf16_to_utf8(chain->norm8, src16, status); - break; - case ICU_chain_step_type_sortkey: - icu_utf16_to_utf8(chain->sort8, src16, status); - break; case ICU_chain_step_type_casemap: icu_casemap_casemap(step->u.casemap, step->buf16, src16, status); @@ -1071,50 +1079,41 @@ int icu_chain_step_next_token(struct icu_chain * chain, step->buf16, src16, status); break; case ICU_chain_step_type_tokenize: - // attach to new src16 token only first time during splitting + /* attach to new src16 token only first time during splitting */ if (step->need_new_token){ icu_tokenizer_attach(step->u.tokenizer, src16, status); step->need_new_token = 0; } - // splitting one src16 token into multiple buf16 tokens + + + /* splitting one src16 token into multiple buf16 tokens */ step->more_tokens = icu_tokenizer_next_token(step->u.tokenizer, step->buf16, status); - // make sure to get new previous token if this one had been used up - if (step->previous && !step->more_tokens){ - if (icu_chain_step_next_token(chain, step->previous, status)){ - icu_tokenizer_attach(step->u.tokenizer, src16, status); - step->need_new_token = 0; - step->more_tokens - = icu_tokenizer_next_token(step->u.tokenizer, - step->buf16, status); - } + + /* make sure to get new previous token if this one had been used up + by recursive call to _same_ step */ + + if (!step->more_tokens){ + step->more_tokens = icu_chain_step_next_token(chain, step, status); + return step->more_tokens; // avoid one token count too much! } - if (0 == step->more_tokens) - return 0; + break; default: return 0; break; } - - - // stop further token processing if last step and - // new tokens are needed from previous (non-existing) step - if (!step->previous && step->need_new_token) - step->more_tokens = 0; - - //printf("%d %d %d\n", - // step->more_tokens, src16->utf16_len, step->buf16->utf16_len); - - if (U_FAILURE(*status)) return 0; - return 1; -}; + /* if token disappered into thin air, tell caller */ + /* if (!step->buf16->utf16_len && !step->more_tokens) */ + /* return 0; */ + return 1; +} int icu_chain_assign_cstr(struct icu_chain * chain, @@ -1126,86 +1125,124 @@ int icu_chain_assign_cstr(struct icu_chain * chain, if (!chain || !src8cstr) return 0; + chain->src8cstr = src8cstr; + stp = chain->steps; - // clear token count + /* clear token count */ chain->token_count = 0; - // clear all steps stop states - + /* clear all steps stop states */ while (stp){ stp->more_tokens = 1; stp->need_new_token = 1; stp = stp->previous; } - // finally convert UTF8 to UTF16 string - icu_utf16_from_utf8_cstr(chain->src16, src8cstr, status); + /* finally convert UTF8 to UTF16 string if needed */ + if (chain->steps || chain->sort) + icu_utf16_from_utf8_cstr(chain->src16, chain->src8cstr, status); if (U_FAILURE(*status)) return 0; return 1; -}; +} int icu_chain_next_token(struct icu_chain * chain, UErrorCode *status) { - int success = 0; + int got_token = 0; - if (!chain || !chain->steps) + *status = U_ZERO_ERROR; + + if (!chain) return 0; - success = icu_chain_step_next_token(chain, chain->steps, status); - - if (success){ - chain->token_count++; - return chain->token_count; + /* special case with no steps - same as index type binary */ + if (!chain->steps){ + if (chain->token_count) + return 0; + else { + chain->token_count++; + + if (chain->sort) + icu_sortkey8_from_utf16(chain->coll, + chain->sort8, chain->steps->buf16, + status); + return chain->token_count; + } } + /* usual case, one or more icu chain steps existing */ + else { + + while(!got_token && chain->steps && chain->steps->more_tokens) + got_token = icu_chain_step_next_token(chain, chain->steps, status); + if (got_token){ + chain->token_count++; + + icu_utf16_to_utf8(chain->norm8, chain->steps->buf16, status); + + if (chain->sort) + icu_sortkey8_from_utf16(chain->coll, + chain->sort8, chain->steps->buf16, + status); + + return chain->token_count; + } + } + return 0; -}; +} -int icu_chain_get_token_count(struct icu_chain * chain) +int icu_chain_token_number(struct icu_chain * chain) { if (!chain) return 0; return chain->token_count; -}; - +} -const char * icu_chain_get_display(struct icu_chain * chain) +const char * icu_chain_token_display(struct icu_chain * chain) { if (chain->display8) return icu_buf_utf8_to_cstr(chain->display8); return 0; -}; +} -const char * icu_chain_get_norm(struct icu_chain * chain) +const char * icu_chain_token_norm(struct icu_chain * chain) { + if (!chain->steps) + return chain->src8cstr; + if (chain->norm8) return icu_buf_utf8_to_cstr(chain->norm8); return 0; -}; +} -const char * icu_chain_get_sort(struct icu_chain * chain) +const char * icu_chain_token_sortkey(struct icu_chain * chain) { if (chain->sort8) return icu_buf_utf8_to_cstr(chain->sort8); return 0; -}; +} + +const UCollator * icu_chain_get_coll(struct icu_chain * chain) +{ + return chain->coll; +} -#endif // HAVE_ICU +#endif /* HAVE_ICU */