X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=src%2Ficu_I18N.c;h=67867778e6a4717cdaf4ed3c575a4b2118b9085c;hb=e107b0011a295ccc61502d6e5ea79d9125a3fbb4;hp=d791abadd5f4b6e7e4c439439dc185327e82f83f;hpb=a8b19203ad695ea4f749b8d460dc363f8a7085cd;p=pazpar2-moved-to-github.git diff --git a/src/icu_I18N.c b/src/icu_I18N.c index d791aba..6786777 100644 --- a/src/icu_I18N.c +++ b/src/icu_I18N.c @@ -1,22 +1,20 @@ -/* $Id: icu_I18N.c,v 1.16 2007-05-16 19:50:01 marc Exp $ - Copyright (c) 2006-2007, Index Data. +/* This file is part of Pazpar2. + Copyright (C) 2006-2008 Index Data - This file is part of Pazpar2. +Pazpar2 is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2, or (at your option) any later +version. - Pazpar2 is free software; you can redistribute it and/or modify it under - the terms of the GNU General Public License as published by the Free - Software Foundation; either version 2, or (at your option) any later - version. +Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. - Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY - WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - for more details. +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - You should have received a copy of the GNU General Public License - along with Pazpar2; see the file LICENSE. If not, write to the - Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA - 02111-1307, USA. */ #if HAVE_CONFIG_H @@ -83,7 +81,6 @@ struct icu_buf_utf16 * icu_buf_utf16_create(size_t capacity) return buf16; }; - struct icu_buf_utf16 * icu_buf_utf16_resize(struct icu_buf_utf16 * buf16, size_t capacity) { @@ -172,8 +169,6 @@ struct icu_buf_utf8 * icu_buf_utf8_resize(struct icu_buf_utf8 * buf8, buf8->utf8 = (uint8_t *) realloc(buf8->utf8, sizeof(uint8_t) * capacity); - buf8->utf8[0] = (uint8_t) 0; - buf8->utf8_len = 0; buf8->utf8_cap = capacity; } else { @@ -206,6 +201,16 @@ struct icu_buf_utf8 * icu_buf_utf8_copy(struct icu_buf_utf8 * dest8, }; +const char *icu_buf_utf8_to_cstr(struct icu_buf_utf8 *src8) +{ + if (!src8 || src8->utf8_len == 0) + return ""; + if (src8->utf8_len == src8->utf8_cap) + src8 = icu_buf_utf8_resize(src8, src8->utf8_len * 2 + 1); + src8->utf8[src8->utf8_len] = '\0'; + return (const char *) src8->utf8; +} + void icu_buf_utf8_destroy(struct icu_buf_utf8 * buf8) { @@ -241,7 +246,7 @@ UErrorCode icu_utf16_from_utf8(struct icu_buf_utf16 * dest16, //if (*status != U_BUFFER_OVERFLOW_ERROR if (U_SUCCESS(*status) - && utf16_len < dest16->utf16_cap) + && utf16_len <= dest16->utf16_cap) dest16->utf16_len = utf16_len; else { dest16->utf16[0] = (UChar) 0; @@ -279,7 +284,7 @@ UErrorCode icu_utf16_from_utf8_cstr(struct icu_buf_utf16 * dest16, // if (*status != U_BUFFER_OVERFLOW_ERROR if (U_SUCCESS(*status) - && utf16_len < dest16->utf16_cap) + && utf16_len <= dest16->utf16_cap) dest16->utf16_len = utf16_len; else { dest16->utf16[0] = (UChar) 0; @@ -316,7 +321,7 @@ UErrorCode icu_utf16_to_utf8(struct icu_buf_utf8 * dest8, //if (*status != U_BUFFER_OVERFLOW_ERROR if (U_SUCCESS(*status) - && utf8_len < dest8->utf8_cap) + && utf8_len <= dest8->utf8_cap) dest8->utf8_len = utf8_len; else { dest8->utf8[0] = (uint8_t) 0; @@ -328,6 +333,51 @@ UErrorCode icu_utf16_to_utf8(struct icu_buf_utf8 * dest8, +struct icu_casemap * icu_casemap_create(const char *locale, char action, + UErrorCode *status) +{ + struct icu_casemap * casemap + = (struct icu_casemap *) malloc(sizeof(struct icu_casemap)); + strcpy(casemap->locale, locale); + casemap->action = action; + + switch(casemap->action) { + case 'l': + break; + case 'u': + break; + case 't': + break; + case 'f': + break; + default: + icu_casemap_destroy(casemap); + return 0; + } + + return casemap; +}; + +void icu_casemap_destroy(struct icu_casemap * casemap) +{ + if (casemap) + free(casemap); +}; + + +int icu_casemap_casemap(struct icu_casemap * casemap, + struct icu_buf_utf16 * dest16, + struct icu_buf_utf16 * src16, + UErrorCode *status) +{ + if(!casemap) + return 0; + + return icu_utf16_casemap(dest16, src16, + casemap->locale, casemap->action, status); +}; + + int icu_utf16_casemap(struct icu_buf_utf16 * dest16, struct icu_buf_utf16 * src16, const char *locale, char action, @@ -400,7 +450,7 @@ int icu_utf16_casemap(struct icu_buf_utf16 * dest16, } if (U_SUCCESS(*status) - && dest16_len < dest16->utf16_cap) + && dest16_len <= dest16->utf16_cap) dest16->utf16_len = dest16_len; else { dest16->utf16[0] = (UChar) 0; @@ -622,15 +672,6 @@ int32_t icu_tokenizer_token_count(struct icu_tokenizer * tokenizer) -//struct icu_normalizer -//{ -// char action; -// struct icu_buf_utf16 * rules16; -// UParseError parse_error[256]; -// UTransliterator * trans; -//}; - - struct icu_normalizer * icu_normalizer_create(const char *rules, char action, UErrorCode *status) { @@ -651,6 +692,7 @@ struct icu_normalizer * icu_normalizer_create(const char *rules, char action, UTRANS_FORWARD, 0, 0, normalizer->parse_error, status); + // yaz_log(YLOG_LOG, "utrans_open %p", normalizer->trans); break; case 'r': normalizer->trans @@ -659,6 +701,7 @@ struct icu_normalizer * icu_normalizer_create(const char *rules, char action, UTRANS_REVERSE , 0, 0, normalizer->parse_error, status); + // yaz_log(YLOG_LOG, "utrans_open %p", normalizer->trans); break; default: *status = U_UNSUPPORTED_ERROR; @@ -680,7 +723,10 @@ void icu_normalizer_destroy(struct icu_normalizer * normalizer){ if (normalizer->rules16) icu_buf_utf16_destroy(normalizer->rules16); if (normalizer->trans) + { + // yaz_log(YLOG_LOG, "utrans_close %p", normalizer->trans); utrans_close(normalizer->trans); + } free(normalizer); } }; @@ -728,22 +774,20 @@ struct icu_chain_step * icu_chain_step_create(struct icu_chain * chain, step = (struct icu_chain_step *) malloc(sizeof(struct icu_chain_step)); step->type = type; - step->more_tokens = 0; - if (buf16) - step->buf16 = buf16; - else - step->buf16 = 0; + step->buf16 = buf16; // create auxilary objects switch(step->type) { case ICU_chain_step_type_display: break; - case ICU_chain_step_type_norm: + case ICU_chain_step_type_index: break; - case ICU_chain_step_type_sort: + case ICU_chain_step_type_sortkey: break; - case ICU_chain_step_type_charmap: + case ICU_chain_step_type_casemap: + step->u.casemap = icu_casemap_create((char *) chain->locale, + (char) rule[0], status); break; case ICU_chain_step_type_normalize: step->u.normalizer = icu_normalizer_create((char *) rule, 'f', status); @@ -770,11 +814,12 @@ void icu_chain_step_destroy(struct icu_chain_step * step){ switch(step->type) { case ICU_chain_step_type_display: break; - case ICU_chain_step_type_norm: + case ICU_chain_step_type_index: break; - case ICU_chain_step_type_sort: + case ICU_chain_step_type_sortkey: break; - case ICU_chain_step_type_charmap: + case ICU_chain_step_type_casemap: + icu_casemap_destroy(step->u.casemap); icu_buf_utf16_destroy(step->buf16); break; case ICU_chain_step_type_normalize: @@ -788,8 +833,7 @@ void icu_chain_step_destroy(struct icu_chain_step * step){ default: break; } - - + free(step); }; @@ -822,16 +866,100 @@ struct icu_chain * icu_chain_create(const uint8_t * identifier, void icu_chain_destroy(struct icu_chain * chain) { - icu_buf_utf8_destroy(chain->display8); - icu_buf_utf8_destroy(chain->norm8); - icu_buf_utf8_destroy(chain->sort8); + if (chain){ + icu_buf_utf8_destroy(chain->display8); + icu_buf_utf8_destroy(chain->norm8); + icu_buf_utf8_destroy(chain->sort8); + + icu_buf_utf16_destroy(chain->src16); + + icu_chain_step_destroy(chain->steps); + free(chain); + } +}; + + + +struct icu_chain * icu_chain_xml_config(xmlNode *xml_node, + UErrorCode * status){ + + xmlNode *node = 0; + struct icu_chain * chain = 0; + + if (!xml_node + ||xml_node->type != XML_ELEMENT_NODE + || strcmp((const char *) xml_node->name, "icu_chain")) + + return 0; + + xmlChar *xml_id = xmlGetProp(xml_node, (xmlChar *) "id"); + xmlChar *xml_locale = xmlGetProp(xml_node, (xmlChar *) "locale"); - icu_buf_utf16_destroy(chain->src16); + if (!xml_id || !strlen((const char *) xml_id) + || !xml_locale || !strlen((const char *) xml_locale)) + return 0; - icu_chain_step_destroy(chain->steps); + chain = icu_chain_create((const uint8_t *) xml_id, + (const uint8_t *) xml_locale); + + xmlFree(xml_id); + xmlFree(xml_locale); + if (!chain) + return 0; + + for (node = xml_node->children; node; node = node->next) + { + if (node->type != XML_ELEMENT_NODE) + continue; + + xmlChar *xml_rule = xmlGetProp(node, (xmlChar *) "rule"); + struct icu_chain_step * step = 0; + + if (!strcmp((const char *) node->name, + (const char *) "casemap")){ + step = icu_chain_insert_step(chain, ICU_chain_step_type_casemap, + (const uint8_t *) xml_rule, status); + } + else if (!strcmp((const char *) node->name, + (const char *) "normalize")){ + step = icu_chain_insert_step(chain, ICU_chain_step_type_normalize, + (const uint8_t *) xml_rule, status); + } + else if (!strcmp((const char *) node->name, + (const char *) "tokenize")){ + step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize, + (const uint8_t *) xml_rule, status); + } + else if (!strcmp((const char *) node->name, + (const char *) "display")){ + step = icu_chain_insert_step(chain, ICU_chain_step_type_display, + (const uint8_t *) "", status); + } + else if (!strcmp((const char *) node->name, + (const char *) "index")){ + step = icu_chain_insert_step(chain, ICU_chain_step_type_index, + (const uint8_t *) "", status); + } + else if (!strcmp((const char *) node->name, + (const char *) "sortkey")){ + step = icu_chain_insert_step(chain, ICU_chain_step_type_sortkey, + (const uint8_t *) "", status); + } + + xmlFree(xml_rule); + if (!step || U_FAILURE(*status)){ + icu_chain_destroy(chain); + return 0; + } + + + } + + return chain; }; + struct icu_chain_step * icu_chain_insert_step(struct icu_chain * chain, enum icu_chain_step_type type, const uint8_t * rule, @@ -853,19 +981,18 @@ struct icu_chain_step * icu_chain_insert_step(struct icu_chain * chain, return 0; - // assign utf16 destination buffers as needed, or - // re-use previous uft18 buffer if this step does not touch it + // create utf16 destination buffers as needed, or switch(type) { case ICU_chain_step_type_display: buf16 = src16; break; - case ICU_chain_step_type_norm: + case ICU_chain_step_type_index: buf16 = src16; break; - case ICU_chain_step_type_sort: + case ICU_chain_step_type_sortkey: buf16 = src16; break; - case ICU_chain_step_type_charmap: + case ICU_chain_step_type_casemap: buf16 = icu_buf_utf16_create(0); break; case ICU_chain_step_type_normalize: @@ -894,17 +1021,19 @@ int icu_chain_step_next_token(struct icu_chain * chain, { struct icu_buf_utf16 * src16 = 0; - printf("icu_chain_step_next_token %d\n", (int) step); + //printf("icu_chain_step_next_token %d\n", (int) step); if (!chain || !chain->src16 || !step || !step->more_tokens) return 0; // assign utf16 src buffers as neeed, advance in previous steps - // tokens, and setting stop condition + // tokens until non-zero token met, and setting stop condition if (step->previous){ src16 = step->previous->buf16; - step->more_tokens - = icu_chain_step_next_token(chain, step->previous, status); + if (step->need_new_token) + //while (step->more_tokens && !src16->utf16_len) + step->more_tokens + = icu_chain_step_next_token(chain, step->previous, status); } else { // first step can only work once on chain->src16 input buffer src16 = chain->src16; @@ -913,10 +1042,10 @@ int icu_chain_step_next_token(struct icu_chain * chain, // stop if nothing to process // i.e new token source was not properly assigned - if (!step->more_tokens || !src16 || !src16->utf16_len) // + if (!step->more_tokens || !src16) // || !src16->utf16_len return 0; - printf("icu_chain_step_next_token %d working\n", (int) step); + //printf("icu_chain_step_next_token %d working\n", (int) step); // perform the work, eventually put this steps output in @@ -925,34 +1054,58 @@ int icu_chain_step_next_token(struct icu_chain * chain, case ICU_chain_step_type_display: icu_utf16_to_utf8(chain->display8, src16, status); break; - case ICU_chain_step_type_norm: + case ICU_chain_step_type_index: icu_utf16_to_utf8(chain->norm8, src16, status); break; - case ICU_chain_step_type_sort: + case ICU_chain_step_type_sortkey: icu_utf16_to_utf8(chain->sort8, src16, status); break; - case ICU_chain_step_type_charmap: + case ICU_chain_step_type_casemap: + icu_casemap_casemap(step->u.casemap, + step->buf16, src16, status); break; case ICU_chain_step_type_normalize: icu_normalizer_normalize(step->u.normalizer, step->buf16, src16, status); break; case ICU_chain_step_type_tokenize: - icu_tokenizer_attach(step->u.tokenizer, src16, status); + // attach to new src16 token only first time during splitting + if (step->need_new_token){ + icu_tokenizer_attach(step->u.tokenizer, src16, status); + step->need_new_token = 0; + } + // splitting one src16 token into multiple buf16 tokens step->more_tokens = icu_tokenizer_next_token(step->u.tokenizer, step->buf16, status); + // make sure to get new previous token if this one had been used up + if (step->previous && !step->more_tokens){ + if (icu_chain_step_next_token(chain, step->previous, status)){ + icu_tokenizer_attach(step->u.tokenizer, src16, status); + step->need_new_token = 0; + step->more_tokens + = icu_tokenizer_next_token(step->u.tokenizer, + step->buf16, status); + } + } + if (0 == step->more_tokens) + return 0; break; default: return 0; break; } - - // stop further token processing if last step - if (!step->previous) + + + // stop further token processing if last step and + // new tokens are needed from previous (non-existing) step + if (!step->previous && step->need_new_token) step->more_tokens = 0; + //printf("%d %d %d\n", + // step->more_tokens, src16->utf16_len, step->buf16->utf16_len); + if (U_FAILURE(*status)) return 0; @@ -966,10 +1119,12 @@ int icu_chain_assign_cstr(struct icu_chain * chain, const char * src8cstr, UErrorCode *status) { - struct icu_chain_step * stp = chain->steps; + struct icu_chain_step * stp = 0; if (!chain || !src8cstr) return 0; + + stp = chain->steps; // clear token count chain->token_count = 0; @@ -978,6 +1133,7 @@ int icu_chain_assign_cstr(struct icu_chain * chain, while (stp){ stp->more_tokens = 1; + stp->need_new_token = 1; stp = stp->previous; } @@ -1023,7 +1179,7 @@ int icu_chain_get_token_count(struct icu_chain * chain) const char * icu_chain_get_display(struct icu_chain * chain) { if (chain->display8) - return (const char *) chain->display8->utf8; + return icu_buf_utf8_to_cstr(chain->display8); return 0; }; @@ -1031,7 +1187,7 @@ const char * icu_chain_get_display(struct icu_chain * chain) const char * icu_chain_get_norm(struct icu_chain * chain) { if (chain->norm8) - return (const char *) chain->norm8->utf8; + return icu_buf_utf8_to_cstr(chain->norm8); return 0; }; @@ -1039,7 +1195,7 @@ const char * icu_chain_get_norm(struct icu_chain * chain) const char * icu_chain_get_sort(struct icu_chain * chain) { if (chain->sort8) - return (const char *) chain->sort8->utf8; + return icu_buf_utf8_to_cstr(chain->sort8); return 0; };