X-Git-Url: http://git.indexdata.com/?p=yaz-moved-to-github.git;a=blobdiff_plain;f=src%2Ficu_I18N.c;h=9f1d13c3de1a4f492278b8dd022f56a9f9eac3b9;hp=b4d6b859dd26502a6958b643b7ce4ae95c4a067f;hb=173f78f50536784bdc6a59d4586ba409b2b69182;hpb=c262e2c4e953e90016aaa2c4d3bbb190a8d29b99 diff --git a/src/icu_I18N.c b/src/icu_I18N.c index b4d6b85..9f1d13c 100644 --- a/src/icu_I18N.c +++ b/src/icu_I18N.c @@ -1,8 +1,11 @@ -/* - * Copyright (C) 1995-2007, Index Data ApS +/* This file is part of the YAZ toolkit. + * Copyright (C) 1995-2009 Index Data * See the file LICENSE for details. - * - * $Id: icu_I18N.c,v 1.16 2007-11-08 17:15:13 adam Exp $ + */ + +/** + * \file icu_I18N.c + * \brief ICU utilities */ #if HAVE_CONFIG_H @@ -14,8 +17,9 @@ #include #endif - #if YAZ_HAVE_ICU +#include + #include #include @@ -33,9 +37,9 @@ int icu_check_status (UErrorCode status) { - if(U_FAILURE(status)){ - yaz_log(YLOG_WARN, - "ICU: %d %s\n", status, u_errorName(status)); + if (U_FAILURE(status)) + { + yaz_log(YLOG_WARN, "ICU: %d %s\n", status, u_errorName(status)); return 0; } return 1; @@ -47,14 +51,14 @@ int icu_check_status (UErrorCode status) struct icu_buf_utf16 * icu_buf_utf16_create(size_t capacity) { struct icu_buf_utf16 * buf16 - = (struct icu_buf_utf16 *) malloc(sizeof(struct icu_buf_utf16)); + = (struct icu_buf_utf16 *) xmalloc(sizeof(struct icu_buf_utf16)); buf16->utf16 = 0; buf16->utf16_len = 0; buf16->utf16_cap = 0; if (capacity > 0){ - buf16->utf16 = (UChar *) malloc(sizeof(UChar) * capacity); + buf16->utf16 = (UChar *) xmalloc(sizeof(UChar) * capacity); buf16->utf16[0] = (UChar) 0; buf16->utf16_cap = capacity; } @@ -79,17 +83,16 @@ struct icu_buf_utf16 * icu_buf_utf16_resize(struct icu_buf_utf16 * buf16, if (capacity > 0){ if (0 == buf16->utf16) - buf16->utf16 = (UChar *) malloc(sizeof(UChar) * capacity); + buf16->utf16 = (UChar *) xmalloc(sizeof(UChar) * capacity); else buf16->utf16 - = (UChar *) realloc(buf16->utf16, sizeof(UChar) * capacity); + = (UChar *) xrealloc(buf16->utf16, sizeof(UChar) * capacity); icu_buf_utf16_clear(buf16); buf16->utf16_cap = capacity; } else { - if (buf16->utf16) - free(buf16->utf16); + xfree(buf16->utf16); buf16->utf16 = 0; buf16->utf16_len = 0; buf16->utf16_cap = 0; @@ -118,29 +121,24 @@ struct icu_buf_utf16 * icu_buf_utf16_copy(struct icu_buf_utf16 * dest16, void icu_buf_utf16_destroy(struct icu_buf_utf16 * buf16) { - if (buf16){ - if (buf16->utf16) - free(buf16->utf16); - free(buf16); - } + if (buf16) + xfree(buf16->utf16); + xfree(buf16); } - - - struct icu_buf_utf8 * icu_buf_utf8_create(size_t capacity) { struct icu_buf_utf8 * buf8 - = (struct icu_buf_utf8 *) malloc(sizeof(struct icu_buf_utf8)); + = (struct icu_buf_utf8 *) xmalloc(sizeof(struct icu_buf_utf8)); buf8->utf8 = 0; buf8->utf8_len = 0; buf8->utf8_cap = 0; if (capacity > 0){ - buf8->utf8 = (uint8_t *) malloc(sizeof(uint8_t) * capacity); + buf8->utf8 = (uint8_t *) xmalloc(sizeof(uint8_t) * capacity); buf8->utf8[0] = (uint8_t) 0; buf8->utf8_cap = capacity; } @@ -167,17 +165,15 @@ struct icu_buf_utf8 * icu_buf_utf8_resize(struct icu_buf_utf8 * buf8, if (capacity > 0){ if (0 == buf8->utf8) - buf8->utf8 = (uint8_t *) malloc(sizeof(uint8_t) * capacity); + buf8->utf8 = (uint8_t *) xmalloc(sizeof(uint8_t) * capacity); else buf8->utf8 - = (uint8_t *) realloc(buf8->utf8, sizeof(uint8_t) * capacity); - - icu_buf_utf8_clear(buf8); + = (uint8_t *) xrealloc(buf8->utf8, sizeof(uint8_t) * capacity); + buf8->utf8_cap = capacity; } else { - if (buf8->utf8) - free(buf8->utf8); + xfree(buf8->utf8); buf8->utf8 = 0; buf8->utf8_len = 0; buf8->utf8_cap = 0; @@ -187,23 +183,6 @@ struct icu_buf_utf8 * icu_buf_utf8_resize(struct icu_buf_utf8 * buf8, } -struct icu_buf_utf8 * icu_buf_utf8_copy(struct icu_buf_utf8 * dest8, - struct icu_buf_utf8 * src8) -{ - if(!dest8 || !src8 - || dest8 == src8) - return 0; - - - if (dest8->utf8_cap < src8->utf8_len) - icu_buf_utf8_resize(dest8, src8->utf8_len * 2); - - strncpy((char*) dest8->utf8, (char*) src8->utf8, src8->utf8_len); - - return dest8; -} - - const char *icu_buf_utf8_to_cstr(struct icu_buf_utf8 *src8) { if (!src8 || src8->utf8_len == 0) @@ -220,11 +199,9 @@ const char *icu_buf_utf8_to_cstr(struct icu_buf_utf8 *src8) void icu_buf_utf8_destroy(struct icu_buf_utf8 * buf8) { - if (buf8){ - if (buf8->utf8) - free(buf8->utf8); - free(buf8); - } + if (buf8) + xfree(buf8->utf8); + xfree(buf8); } @@ -331,7 +308,7 @@ UErrorCode icu_utf16_to_utf8(struct icu_buf_utf8 * dest8, struct icu_casemap * icu_casemap_create(char action, UErrorCode *status) { struct icu_casemap * casemap - = (struct icu_casemap *) malloc(sizeof(struct icu_casemap)); + = (struct icu_casemap *) xmalloc(sizeof(struct icu_casemap)); casemap->action = action; switch(casemap->action) { @@ -354,8 +331,7 @@ struct icu_casemap * icu_casemap_create(char action, UErrorCode *status) void icu_casemap_destroy(struct icu_casemap * casemap) { - if (casemap) - free(casemap); + xfree(casemap); } @@ -381,7 +357,7 @@ int icu_utf16_casemap(struct icu_buf_utf16 * dest16, int32_t dest16_len = 0; - if (!src16->utf16_len){ //guarding for empty source string + if (!src16->utf16_len){ /* guarding for empty source string */ if (dest16->utf16) dest16->utf16[0] = (UChar) 0; dest16->utf16_len = 0; @@ -474,10 +450,10 @@ int icu_utf16_casemap(struct icu_buf_utf16 * dest16, -UErrorCode icu_sortkey8_from_utf16(UCollator *coll, - struct icu_buf_utf8 * dest8, - struct icu_buf_utf16 * src16, - UErrorCode * status) +void icu_sortkey8_from_utf16(UCollator *coll, + struct icu_buf_utf8 * dest8, + struct icu_buf_utf16 * src16, + UErrorCode * status) { int32_t sortkey_len = 0; @@ -497,8 +473,6 @@ UErrorCode icu_sortkey8_from_utf16(UCollator *coll, dest8->utf8_len = sortkey_len; else icu_buf_utf8_clear(dest8); - - return sortkey_len; } @@ -507,7 +481,7 @@ struct icu_tokenizer * icu_tokenizer_create(const char *locale, char action, UErrorCode *status) { struct icu_tokenizer * tokenizer - = (struct icu_tokenizer *) malloc(sizeof(struct icu_tokenizer)); + = (struct icu_tokenizer *) xmalloc(sizeof(struct icu_tokenizer)); tokenizer->action = action; tokenizer->bi = 0; @@ -559,7 +533,7 @@ void icu_tokenizer_destroy(struct icu_tokenizer * tokenizer) if (tokenizer) { if (tokenizer->bi) ubrk_close(tokenizer->bi); - free(tokenizer); + xfree(tokenizer); } } @@ -678,75 +652,83 @@ int32_t icu_tokenizer_token_count(struct icu_tokenizer * tokenizer) -struct icu_normalizer * icu_normalizer_create(const char *rules, char action, - UErrorCode *status) +struct icu_transform * icu_transform_create(const char *id, char action, + const char *rules, + UErrorCode *status) { + struct icu_buf_utf16 *id16 = icu_buf_utf16_create(0); + struct icu_buf_utf16 *rules16 = icu_buf_utf16_create(0); + + struct icu_transform * transform + = (struct icu_transform *) xmalloc(sizeof(struct icu_transform)); - struct icu_normalizer * normalizer - = (struct icu_normalizer *) malloc(sizeof(struct icu_normalizer)); + transform->action = action; + transform->trans = 0; - normalizer->action = action; - normalizer->trans = 0; - normalizer->rules16 = icu_buf_utf16_create(0); - icu_utf16_from_utf8_cstr(normalizer->rules16, rules, status); - - switch(normalizer->action) { + if (id) + icu_utf16_from_utf8_cstr(id16, id, status); + if (rules) + icu_utf16_from_utf8_cstr(rules16, rules, status); + + switch(transform->action) + { case 'f': case 'F': - normalizer->trans - = utrans_openU(normalizer->rules16->utf16, - normalizer->rules16->utf16_len, + transform->trans + = utrans_openU(id16->utf16, + id16->utf16_len, UTRANS_FORWARD, - 0, 0, - normalizer->parse_error, status); + rules16->utf16, + rules16->utf16_len, + &transform->parse_error, status); break; case 'r': case 'R': - normalizer->trans - = utrans_openU(normalizer->rules16->utf16, - normalizer->rules16->utf16_len, + transform->trans + = utrans_openU(id16->utf16, + id16->utf16_len, UTRANS_REVERSE , - 0, 0, - normalizer->parse_error, status); + rules16->utf16, + rules16->utf16_len, + &transform->parse_error, status); break; default: *status = U_UNSUPPORTED_ERROR; - return 0; break; } + icu_buf_utf16_destroy(rules16); + icu_buf_utf16_destroy(id16); if (U_SUCCESS(*status)) - return normalizer; + return transform; /* freeing if failed */ - icu_normalizer_destroy(normalizer); + icu_transform_destroy(transform); return 0; } -void icu_normalizer_destroy(struct icu_normalizer * normalizer){ - if (normalizer) { - if (normalizer->rules16) - icu_buf_utf16_destroy(normalizer->rules16); - if (normalizer->trans) - utrans_close(normalizer->trans); - free(normalizer); +void icu_transform_destroy(struct icu_transform * transform){ + if (transform) { + if (transform->trans) + utrans_close(transform->trans); + xfree(transform); } } -int icu_normalizer_normalize(struct icu_normalizer * normalizer, - struct icu_buf_utf16 * dest16, - struct icu_buf_utf16 * src16, - UErrorCode *status) +int icu_transform_trans(struct icu_transform * transform, + struct icu_buf_utf16 * dest16, + struct icu_buf_utf16 * src16, + UErrorCode *status) { - if (!normalizer || !normalizer->trans + if (!transform || !transform->trans || !src16 || !dest16) return 0; - if (!src16->utf16_len){ //guarding for empty source string + if (!src16->utf16_len){ /* guarding for empty source string */ icu_buf_utf16_clear(dest16); return 0; } @@ -755,7 +737,7 @@ int icu_normalizer_normalize(struct icu_normalizer * normalizer, return 0; - utrans_transUChars (normalizer->trans, + utrans_transUChars (transform->trans, dest16->utf16, &(dest16->utf16_len), dest16->utf16_cap, 0, &(src16->utf16_len), status); @@ -780,7 +762,7 @@ struct icu_chain_step * icu_chain_step_create(struct icu_chain * chain, if(!chain || !type || !rule) return 0; - step = (struct icu_chain_step *) malloc(sizeof(struct icu_chain_step)); + step = (struct icu_chain_step *) xmalloc(sizeof(struct icu_chain_step)); step->type = type; @@ -793,13 +775,20 @@ struct icu_chain_step * icu_chain_step_create(struct icu_chain * chain, case ICU_chain_step_type_casemap: step->u.casemap = icu_casemap_create(rule[0], status); break; - case ICU_chain_step_type_normalize: - step->u.normalizer = icu_normalizer_create((char *) rule, 'f', status); + case ICU_chain_step_type_transform: + /* rule omitted. Only ID used */ + step->u.transform = icu_transform_create((const char *) rule, 'f', + 0, status); break; case ICU_chain_step_type_tokenize: step->u.tokenizer = icu_tokenizer_create((char *) chain->locale, (char) rule[0], status); break; + case ICU_chain_step_type_transliterate: + /* we pass a dummy ID to utrans_openU.. */ + step->u.transform = icu_transform_create("custom", 'f', + (const char *) rule, status); + break; default: break; } @@ -822,8 +811,9 @@ void icu_chain_step_destroy(struct icu_chain_step * step){ icu_casemap_destroy(step->u.casemap); icu_buf_utf16_destroy(step->buf16); break; - case ICU_chain_step_type_normalize: - icu_normalizer_destroy(step->u.normalizer); + case ICU_chain_step_type_transform: + case ICU_chain_step_type_transliterate: + icu_transform_destroy(step->u.transform); icu_buf_utf16_destroy(step->buf16); break; case ICU_chain_step_type_tokenize: @@ -833,23 +823,20 @@ void icu_chain_step_destroy(struct icu_chain_step * step){ default: break; } - free(step); + xfree(step); } -struct icu_chain * icu_chain_create(const char *locale, - int sort, +struct icu_chain * icu_chain_create(const char *locale, int sort, UErrorCode * status) { - struct icu_chain * chain - = (struct icu_chain *) malloc(sizeof(struct icu_chain)); + = (struct icu_chain *) xmalloc(sizeof(struct icu_chain)); *status = U_ZERO_ERROR; - strncpy((char *) chain->locale, (const char *) locale, 16); - chain->locale[16 - 1] = '\0'; + chain->locale = xstrdup(locale); chain->sort = sort; @@ -858,7 +845,6 @@ struct icu_chain * icu_chain_create(const char *locale, if (U_FAILURE(*status)) return 0; - chain->token_count = 0; chain->src8cstr = 0; @@ -877,8 +863,8 @@ struct icu_chain * icu_chain_create(const char *locale, void icu_chain_destroy(struct icu_chain * chain) { - if (chain){ - + if (chain) + { if (chain->coll) ucol_close(chain->coll); @@ -889,7 +875,8 @@ void icu_chain_destroy(struct icu_chain * chain) icu_buf_utf16_destroy(chain->src16); icu_chain_step_destroy(chain->steps); - free(chain); + xfree(chain->locale); + xfree(chain); } } @@ -931,35 +918,47 @@ struct icu_chain * icu_chain_xml_config(const xmlNode *xml_node, xml_rule = xmlGetProp(node, (xmlChar *) "rule"); - if (!strcmp((const char *) node->name, - (const char *) "casemap")){ + if (!strcmp((const char *) node->name, "casemap")) step = icu_chain_insert_step(chain, ICU_chain_step_type_casemap, (const uint8_t *) xml_rule, status); - } - else if (!strcmp((const char *) node->name, - (const char *) "normalize")){ - step = icu_chain_insert_step(chain, ICU_chain_step_type_normalize, + else if (!strcmp((const char *) node->name, "transform")) + step = icu_chain_insert_step(chain, ICU_chain_step_type_transform, (const uint8_t *) xml_rule, status); - } - else if (!strcmp((const char *) node->name, - (const char *) "tokenize")){ + else if (!strcmp((const char *) node->name, "transliterate")) + step = icu_chain_insert_step(chain, ICU_chain_step_type_transliterate, + (const uint8_t *) xml_rule, status); + else if (!strcmp((const char *) node->name, "tokenize")) step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize, (const uint8_t *) xml_rule, status); - } - else if (!strcmp((const char *) node->name, - (const char *) "display")){ + else if (!strcmp((const char *) node->name, "display")) step = icu_chain_insert_step(chain, ICU_chain_step_type_display, (const uint8_t *) "", status); + else if (!strcmp((const char *) node->name, "normalize")) + { + yaz_log(YLOG_WARN, "Element %s is deprecated. " + "Use transform instead", node->name); + step = icu_chain_insert_step(chain, ICU_chain_step_type_transform, + (const uint8_t *) xml_rule, status); + } + else if (!strcmp((const char *) node->name, "index") + || !strcmp((const char *) node->name, "sortkey")) + { + yaz_log(YLOG_WARN, "Element %s is no longer needed. " + "Remove it from the configuration", node->name); + } + else + { + yaz_log(YLOG_WARN, "Unknown element %s", node->name); + icu_chain_destroy(chain); + return 0; } xmlFree(xml_rule); - if (!step || U_FAILURE(*status)){ + if (step && U_FAILURE(*status)) + { icu_chain_destroy(chain); return 0; } - - } - return chain; } @@ -987,19 +986,22 @@ struct icu_chain_step * icu_chain_insert_step(struct icu_chain * chain, /* create utf16 destination buffers as needed, or */ - switch(type) { + switch(type) + { case ICU_chain_step_type_display: buf16 = src16; break; case ICU_chain_step_type_casemap: buf16 = icu_buf_utf16_create(0); break; - case ICU_chain_step_type_normalize: + case ICU_chain_step_type_transform: + case ICU_chain_step_type_transliterate: buf16 = icu_buf_utf16_create(0); break; case ICU_chain_step_type_tokenize: buf16 = icu_buf_utf16_create(0); break; + break; default: break; } @@ -1027,7 +1029,8 @@ int icu_chain_step_next_token(struct icu_chain * chain, /* assign utf16 src buffers as neeed, advance in previous steps tokens until non-zero token met, and setting stop condition */ - if (step->previous){ + if (step->previous) + { src16 = step->previous->buf16; /* tokens might be killed in previous steps, therefore looping */ @@ -1037,7 +1040,8 @@ int icu_chain_step_next_token(struct icu_chain * chain, got_new_token = icu_chain_step_next_token(chain, step->previous, status); } - else { /* first step can only work once on chain->src16 input buffer */ + else + { /* first step can only work once on chain->src16 input buffer */ src16 = chain->src16; step->more_tokens = 0; got_new_token = 1; @@ -1047,7 +1051,8 @@ int icu_chain_step_next_token(struct icu_chain * chain, return 0; /* stop if nothing to process */ - if (step->need_new_token && !got_new_token){ + if (step->need_new_token && !got_new_token) + { step->more_tokens = 0; return 0; } @@ -1056,7 +1061,8 @@ int icu_chain_step_next_token(struct icu_chain * chain, perform the work, eventually put this steps output in step->buf16 or the chains UTF8 output buffers */ - switch(step->type) { + switch(step->type) + { case ICU_chain_step_type_display: icu_utf16_to_utf8(chain->display8, src16, status); break; @@ -1065,18 +1071,19 @@ int icu_chain_step_next_token(struct icu_chain * chain, step->buf16, src16, status, chain->locale); break; - case ICU_chain_step_type_normalize: - icu_normalizer_normalize(step->u.normalizer, - step->buf16, src16, status); + case ICU_chain_step_type_transform: + case ICU_chain_step_type_transliterate: + icu_transform_trans(step->u.transform, + step->buf16, src16, status); break; case ICU_chain_step_type_tokenize: /* attach to new src16 token only first time during splitting */ - if (step->need_new_token){ + if (step->need_new_token) + { icu_tokenizer_attach(step->u.tokenizer, src16, status); step->need_new_token = 0; } - /* splitting one src16 token into multiple buf16 tokens */ step->more_tokens = icu_tokenizer_next_token(step->u.tokenizer, @@ -1085,11 +1092,11 @@ int icu_chain_step_next_token(struct icu_chain * chain, /* make sure to get new previous token if this one had been used up by recursive call to _same_ step */ - if (!step->more_tokens){ + if (!step->more_tokens) + { step->more_tokens = icu_chain_step_next_token(chain, step, status); - return step->more_tokens; // avoid one token count too much! + return step->more_tokens; /* avoid one token count too much! */ } - break; default: return 0; @@ -1124,7 +1131,8 @@ int icu_chain_assign_cstr(struct icu_chain * chain, chain->token_count = 0; /* clear all steps stop states */ - while (stp){ + while (stp) + { stp->more_tokens = 1; stp->need_new_token = 1; stp = stp->previous; @@ -1153,10 +1161,12 @@ int icu_chain_next_token(struct icu_chain * chain, return 0; /* special case with no steps - same as index type binary */ - if (!chain->steps){ + if (!chain->steps) + { if (chain->token_count) return 0; - else { + else + { chain->token_count++; if (chain->sort) @@ -1167,12 +1177,13 @@ int icu_chain_next_token(struct icu_chain * chain, } } /* usual case, one or more icu chain steps existing */ - else { - + else + { while(!got_token && chain->steps && chain->steps->more_tokens) got_token = icu_chain_step_next_token(chain, chain->steps, status); - if (got_token){ + if (got_token) + { chain->token_count++; icu_utf16_to_utf8(chain->norm8, chain->steps->buf16, status); @@ -1181,7 +1192,7 @@ int icu_chain_next_token(struct icu_chain * chain, icu_sortkey8_from_utf16(chain->coll, chain->sort8, chain->steps->buf16, status); - + return chain->token_count; } } @@ -1230,16 +1241,14 @@ const UCollator * icu_chain_get_coll(struct icu_chain * chain) return chain->coll; } - #endif /* YAZ_HAVE_ICU */ - - - /* * Local variables: * c-basic-offset: 4 + * c-file-style: "Stroustrup" * indent-tabs-mode: nil * End: * vim: shiftwidth=4 tabstop=8 expandtab */ +