src/icu_chain.c

   1 /* This file is part of the YAZ toolkit.
   2  * Copyright (C) 1995-2009 Index Data
   3  * See the file LICENSE for details.
   4  */
   5
   6 /**
   7  * \file
   8  * \brief ICU chain
   9  */
  10
  11 #if HAVE_CONFIG_H
  12 #include "config.h"
  13 #endif
  14
  15 #if YAZ_HAVE_ICU
  16 #include <yaz/xmalloc.h>
  17
  18 #include <yaz/icu_I18N.h>
  19
  20 #include <yaz/log.h>
  21
  22 #include <string.h>
  23 #include <stdlib.h>
  24 #include <stdio.h>
  25 #include <assert.h>
  26
  27 #include <unicode/ustring.h>  /* some more string fcns*/
  28 #include <unicode/uchar.h>    /* char names           */
  29
  30 enum icu_chain_step_type {
  31     ICU_chain_step_type_none,
  32     ICU_chain_step_type_display,   /* convert to utf8 display format */
  33     ICU_chain_step_type_casemap,   /* apply utf16 charmap */
  34     ICU_chain_step_type_transform, /* apply utf16 transform */
  35     ICU_chain_step_type_tokenize,  /* apply utf16 tokenization */
  36     ICU_chain_step_type_transliterate  /* apply utf16 tokenization */
  37 };
  38
  39 struct icu_chain_step
  40 {
  41     /* type and action object */
  42     enum icu_chain_step_type type;
  43     union {
  44         struct icu_casemap * casemap;
  45         struct icu_transform * transform;
  46         struct icu_tokenizer * tokenizer;
  47     } u;
  48     struct icu_chain_step * previous;
  49 };
  50
  51 struct icu_chain
  52 {
  53     struct icu_iter *iter;
  54     char *locale;
  55     int sort;
  56
  57     UCollator * coll;
  58
  59     /* utf8 output buffers */
  60     struct icu_buf_utf8 * norm8;
  61
  62     /* linked list of chain steps */
  63     struct icu_chain_step * steps;
  64 };
  65
  66 int icu_check_status(UErrorCode status)
  67 {
  68     if (U_FAILURE(status))
  69     {
  70         yaz_log(YLOG_WARN, "ICU: %d %s\n", status, u_errorName(status));
  71         return 0;
  72     }
  73     return 1;
  74 }
  75
  76 static struct icu_chain_step *icu_chain_step_create(
  77     struct icu_chain * chain,  enum icu_chain_step_type type,
  78     const uint8_t * rule,
  79     UErrorCode *status)
  80 {
  81     struct icu_chain_step * step = 0;
  82
  83     if(!chain || !type || !rule)
  84         return 0;
  85
  86     step = (struct icu_chain_step *) xmalloc(sizeof(struct icu_chain_step));
  87
  88     step->type = type;
  89     /* create auxilary objects */
  90     switch (step->type)
  91     {
  92     case ICU_chain_step_type_display:
  93         break;
  94     case ICU_chain_step_type_casemap:
  95         step->u.casemap = icu_casemap_create(rule[0], status);
  96         break;
  97     case ICU_chain_step_type_transform:
  98         /* rule omitted. Only ID used */
  99         step->u.transform = icu_transform_create((const char *) rule, 'f',
 100                                                  0, status);
 101         break;
 102     case ICU_chain_step_type_tokenize:
 103         step->u.tokenizer = icu_tokenizer_create((char *) chain->locale,
 104                                                  (char) rule[0], status);
 105         break;
 106     case ICU_chain_step_type_transliterate:
 107         /* we pass a dummy ID to utrans_openU.. */
 108         step->u.transform = icu_transform_create("custom", 'f',
 109                                                  (const char *) rule, status);
 110         break;
 111     default:
 112         break;
 113     }
 114     return step;
 115 }
 116
 117
 118 static void icu_chain_step_destroy(struct icu_chain_step * step)
 119 {
 120     if (!step)
 121         return;
 122
 123     icu_chain_step_destroy(step->previous);
 124
 125     switch (step->type)
 126     {
 127     case ICU_chain_step_type_display:
 128         break;
 129     case ICU_chain_step_type_casemap:
 130         icu_casemap_destroy(step->u.casemap);
 131         break;
 132     case ICU_chain_step_type_transform:
 133     case ICU_chain_step_type_transliterate:
 134         icu_transform_destroy(step->u.transform);
 135         break;
 136     case ICU_chain_step_type_tokenize:
 137         icu_tokenizer_destroy(step->u.tokenizer);
 138         break;
 139     default:
 140         break;
 141     }
 142     xfree(step);
 143 }
 144
 145 struct icu_chain *icu_chain_create(const char *locale, int sort,
 146                                    UErrorCode * status)
 147 {
 148     struct icu_chain * chain
 149         = (struct icu_chain *) xmalloc(sizeof(struct icu_chain));
 150
 151     *status = U_ZERO_ERROR;
 152
 153     chain->iter = 0;
 154     chain->locale = xstrdup(locale);
 155
 156     chain->sort = sort;
 157
 158     chain->coll = ucol_open((const char *) chain->locale, status);
 159
 160     if (U_FAILURE(*status))
 161         return 0;
 162
 163     chain->norm8 = icu_buf_utf8_create(0);
 164     chain->steps = 0;
 165
 166     return chain;
 167 }
 168
 169 void icu_chain_destroy(struct icu_chain * chain)
 170 {
 171     if (chain)
 172     {
 173         if (chain->coll)
 174             ucol_close(chain->coll);
 175
 176         icu_buf_utf8_destroy(chain->norm8);
 177         if (chain->iter)
 178             icu_iter_destroy(chain->iter);
 179         icu_chain_step_destroy(chain->steps);
 180         xfree(chain->locale);
 181         xfree(chain);
 182     }
 183 }
 184
 185 static struct icu_chain_step *icu_chain_insert_step(
 186     struct icu_chain * chain, enum icu_chain_step_type type,
 187     const uint8_t * rule, UErrorCode *status);
 188
 189 struct icu_chain * icu_chain_xml_config(const xmlNode *xml_node,
 190                                         int sort,
 191                                         UErrorCode * status)
 192 {
 193     xmlNode *node = 0;
 194     struct icu_chain * chain = 0;
 195
 196     *status = U_ZERO_ERROR;
 197
 198     if (!xml_node ||xml_node->type != XML_ELEMENT_NODE)
 199         return 0;
 200
 201     {
 202         xmlChar * xml_locale = xmlGetProp((xmlNode *) xml_node,
 203                                           (xmlChar *) "locale");
 204
 205         if (xml_locale)
 206         {
 207             chain = icu_chain_create((const char *) xml_locale, sort, status);
 208             xmlFree(xml_locale);
 209         }
 210
 211     }
 212     if (!chain)
 213         return 0;
 214
 215     for (node = xml_node->children; node; node = node->next)
 216     {
 217         xmlChar *xml_rule;
 218         struct icu_chain_step * step = 0;
 219
 220         if (node->type != XML_ELEMENT_NODE)
 221             continue;
 222
 223         xml_rule = xmlGetProp(node, (xmlChar *) "rule");
 224
 225         if (!strcmp((const char *) node->name, "casemap"))
 226             step = icu_chain_insert_step(chain, ICU_chain_step_type_casemap,
 227                                          (const uint8_t *) xml_rule, status);
 228         else if (!strcmp((const char *) node->name, "transform"))
 229             step = icu_chain_insert_step(chain, ICU_chain_step_type_transform,
 230                                          (const uint8_t *) xml_rule, status);
 231         else if (!strcmp((const char *) node->name, "transliterate"))
 232             step = icu_chain_insert_step(chain, ICU_chain_step_type_transliterate,
 233                                          (const uint8_t *) xml_rule, status);
 234         else if (!strcmp((const char *) node->name, "tokenize"))
 235             step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize,
 236                                          (const uint8_t *) xml_rule, status);
 237         else if (!strcmp((const char *) node->name, "display"))
 238             step = icu_chain_insert_step(chain, ICU_chain_step_type_display,
 239                                          (const uint8_t *) "", status);
 240         else if (!strcmp((const char *) node->name, "normalize"))
 241         {
 242             yaz_log(YLOG_WARN, "Element %s is deprecated. "
 243                     "Use transform instead", node->name);
 244             step = icu_chain_insert_step(chain, ICU_chain_step_type_transform,
 245                                          (const uint8_t *) xml_rule, status);
 246         }
 247         else if (!strcmp((const char *) node->name, "index")
 248                  || !strcmp((const char *) node->name, "sortkey"))
 249         {
 250             yaz_log(YLOG_WARN, "Element %s is no longer needed. "
 251                     "Remove it from the configuration", node->name);
 252         }
 253         else
 254         {
 255             yaz_log(YLOG_WARN, "Unknown element %s", node->name);
 256             icu_chain_destroy(chain);
 257             return 0;
 258         }
 259         xmlFree(xml_rule);
 260         if (step && U_FAILURE(*status))
 261         {
 262             icu_chain_destroy(chain);
 263             return 0;
 264         }
 265     }
 266     return chain;
 267 }
 268
 269 static struct icu_chain_step *icu_chain_insert_step(
 270     struct icu_chain * chain, enum icu_chain_step_type type,
 271     const uint8_t * rule, UErrorCode *status)
 272 {
 273     struct icu_chain_step * step = 0;
 274     if (!chain || !type || !rule)
 275         return 0;
 276
 277     /* create actual chain step with this buffer */
 278     step = icu_chain_step_create(chain, type, rule,
 279                                  status);
 280
 281     step->previous = chain->steps;
 282     chain->steps = step;
 283
 284     return step;
 285 }
 286
 287 struct icu_iter {
 288     struct icu_chain *chain;
 289     struct icu_buf_utf16 *last;
 290     UErrorCode status;
 291     struct icu_buf_utf8 *display;
 292     struct icu_buf_utf8 *sort8;
 293     struct icu_buf_utf16 *input;
 294     int token_count;
 295 };
 296
 297 void icu_utf16_print(struct icu_buf_utf16 *src16)
 298 {
 299     UErrorCode status = U_ZERO_ERROR;
 300     const char *p;
 301     struct icu_buf_utf8 *dst8 = icu_buf_utf8_create(0);
 302     icu_utf16_to_utf8(dst8, src16, &status);
 303
 304     assert(status != 1234);
 305     if (U_FAILURE(status))
 306     {
 307         printf("failure");
 308     }
 309     else
 310     {
 311         p = icu_buf_utf8_to_cstr(dst8);
 312         printf("%s", p);
 313     }
 314     icu_buf_utf8_destroy(dst8);
 315 }
 316
 317 struct icu_buf_utf16 *icu_iter_invoke(struct icu_iter *iter,
 318                                       struct icu_chain_step *step,
 319                                       struct icu_buf_utf16 *src)
 320 {
 321     if (!step)
 322         return src;
 323     else
 324     {
 325         struct icu_buf_utf16 *dst = icu_iter_invoke(iter, step->previous, src);
 326
 327         switch (step->type)
 328         {
 329         case ICU_chain_step_type_casemap:
 330             if (dst)
 331             {
 332                 struct icu_buf_utf16 *src = dst;
 333
 334                 dst = icu_buf_utf16_create(0);
 335                 icu_casemap_casemap(step->u.casemap, dst, src, &iter->status,
 336                                     iter->chain->locale);
 337                 icu_buf_utf16_destroy(src);
 338             }
 339             break;
 340         case ICU_chain_step_type_tokenize:
 341             if (dst)
 342             {
 343                 struct icu_buf_utf16 *src = dst;
 344
 345                 icu_tokenizer_attach(step->u.tokenizer, src, &iter->status);
 346                 icu_buf_utf16_destroy(src);
 347             }
 348             dst = icu_buf_utf16_create(0);
 349             iter->status = U_ZERO_ERROR;
 350             if (!icu_tokenizer_next_token(step->u.tokenizer, dst, &iter->status))
 351             {
 352                 icu_buf_utf16_destroy(dst);
 353                 dst = 0;
 354             }
 355             break;
 356         case ICU_chain_step_type_transform:
 357         case ICU_chain_step_type_transliterate:
 358             if (dst)
 359             {
 360                 struct icu_buf_utf16 *src = dst;
 361                 dst = icu_buf_utf16_create(0);
 362                 icu_transform_trans(step->u.transform, dst, src, &iter->status);
 363                 icu_buf_utf16_destroy(src);
 364             }
 365             break;
 366         case ICU_chain_step_type_display:
 367             if (dst)
 368                 icu_utf16_to_utf8(iter->display, dst, &iter->status);
 369             break;
 370         default:
 371             assert(0);
 372         }
 373         return dst;
 374     }
 375 }
 376
 377 struct icu_iter *icu_iter_create(struct icu_chain *chain,
 378                                  const char *src8cstr)
 379 {
 380     if (!src8cstr)
 381         return 0;
 382     else
 383     {
 384         struct icu_iter *iter = xmalloc(sizeof(*iter));
 385         iter->chain = chain;
 386         iter->status = U_ZERO_ERROR;
 387         iter->display = icu_buf_utf8_create(0);
 388         iter->sort8 = icu_buf_utf8_create(0);
 389         iter->token_count = 0;
 390         iter->last = 0; /* no last returned string (yet) */
 391
 392         /* fill and assign input string.. It will be 0 after
 393            first iteration */
 394         iter->input =  icu_buf_utf16_create(0);
 395         icu_utf16_from_utf8_cstr(iter->input, src8cstr, &iter->status);
 396         return iter;
 397
 398     }
 399 }
 400
 401 void icu_iter_destroy(struct icu_iter *iter)
 402 {
 403     if (iter)
 404     {
 405         icu_buf_utf8_destroy(iter->display);
 406         icu_buf_utf8_destroy(iter->sort8);
 407         if (iter->input)
 408             icu_buf_utf16_destroy(iter->input);
 409         xfree(iter);
 410     }
 411 }
 412
 413 int icu_iter_next(struct icu_iter *iter, struct icu_buf_utf8 *result)
 414 {
 415     if (!iter->input && iter->last == 0)
 416         return 0;
 417     else
 418     {
 419         /* on first call, iter->input is the input string. Thereafter: 0. */
 420         iter->last = icu_iter_invoke(iter, iter->chain->steps, iter->input);
 421         iter->input = 0;
 422
 423         if (!iter->last)
 424             return 0;
 425
 426         iter->token_count++;
 427
 428         if (iter->chain->sort)
 429         {
 430             icu_sortkey8_from_utf16(iter->chain->coll,
 431                                     iter->sort8, iter->last,
 432                                     &iter->status);
 433         }
 434         icu_utf16_to_utf8(result, iter->last, &iter->status);
 435         icu_buf_utf16_destroy(iter->last);
 436
 437         return 1;
 438     }
 439 }
 440
 441 const char *icu_iter_get_sortkey(struct icu_iter *iter)
 442 {
 443     return icu_buf_utf8_to_cstr(iter->sort8);
 444 }
 445
 446 const char *icu_iter_get_display(struct icu_iter *iter)
 447 {
 448     return icu_buf_utf8_to_cstr(iter->display);
 449 }
 450
 451 int icu_chain_assign_cstr(struct icu_chain * chain, const char * src8cstr,
 452                           UErrorCode *status)
 453 {
 454     if (chain->iter)
 455         icu_iter_destroy(chain->iter);
 456     chain->iter = icu_iter_create(chain, src8cstr);
 457     return 1;
 458 }
 459
 460 int icu_chain_next_token(struct icu_chain * chain, UErrorCode *status)
 461 {
 462     *status = U_ZERO_ERROR;
 463     return icu_iter_next(chain->iter, chain->norm8);
 464 }
 465
 466 int icu_chain_token_number(struct icu_chain * chain)
 467 {
 468     if (chain && chain->iter)
 469         return chain->iter->token_count;
 470     return 0;
 471 }
 472
 473 const char * icu_chain_token_display(struct icu_chain * chain)
 474 {
 475     if (chain->iter)
 476         return icu_iter_get_display(chain->iter);
 477     return 0;
 478 }
 479
 480 const char * icu_chain_token_norm(struct icu_chain * chain)
 481 {
 482     if (chain->norm8)
 483         return icu_buf_utf8_to_cstr(chain->norm8);
 484     return 0;
 485 }
 486
 487 const char * icu_chain_token_sortkey(struct icu_chain * chain)
 488 {
 489     if (chain->iter)
 490         return icu_iter_get_sortkey(chain->iter);
 491     return 0;
 492 }
 493
 494 #endif /* YAZ_HAVE_ICU */
 495
 496 /*
 497  * Local variables:
 498  * c-basic-offset: 4
 499  * c-file-style: "Stroustrup"
 500  * indent-tabs-mode: nil
 501  * End:
 502  * vim: shiftwidth=4 tabstop=8 expandtab
 503  */
 504