src/icu_chain.c

   1 /* This file is part of the YAZ toolkit.
   2  * Copyright (C) 1995-2010 Index Data
   3  * See the file LICENSE for details.
   4  */
   5
   6 /**
   7  * \file
   8  * \brief ICU chain
   9  */
  10
  11 #if HAVE_CONFIG_H
  12 #include "config.h"
  13 #endif
  14
  15 #if YAZ_HAVE_ICU
  16 #include <yaz/xmalloc.h>
  17
  18 #include <yaz/icu_I18N.h>
  19
  20 #include <yaz/log.h>
  21
  22 #include <string.h>
  23 #include <stdlib.h>
  24 #include <stdio.h>
  25 #include <assert.h>
  26
  27 #include <unicode/ustring.h>  /* some more string fcns*/
  28 #include <unicode/uchar.h>    /* char names           */
  29
  30 enum icu_chain_step_type {
  31     ICU_chain_step_type_none,
  32     ICU_chain_step_type_display,   /* convert to utf8 display format */
  33     ICU_chain_step_type_casemap,   /* apply utf16 charmap */
  34     ICU_chain_step_type_transform, /* apply utf16 transform */
  35     ICU_chain_step_type_tokenize,  /* apply utf16 tokenization */
  36     ICU_chain_step_type_transliterate  /* apply utf16 tokenization */
  37 };
  38
  39 struct icu_chain_step
  40 {
  41     /* type and action object */
  42     enum icu_chain_step_type type;
  43     union {
  44         struct icu_casemap * casemap;
  45         struct icu_transform * transform;
  46         struct icu_tokenizer * tokenizer;
  47     } u;
  48     struct icu_chain_step * previous;
  49 };
  50
  51 struct icu_chain
  52 {
  53     struct icu_iter *iter;
  54     char *locale;
  55     int sort;
  56
  57     UCollator * coll;
  58
  59     /* utf8 output buffers */
  60     struct icu_buf_utf8 * norm8;
  61
  62     /* linked list of chain steps */
  63     struct icu_chain_step * csteps;
  64 };
  65
  66 int icu_check_status(UErrorCode status)
  67 {
  68     if (U_FAILURE(status))
  69     {
  70         yaz_log(YLOG_WARN, "ICU: %d %s\n", status, u_errorName(status));
  71         return 0;
  72     }
  73     return 1;
  74 }
  75
  76 static struct icu_chain_step *icu_chain_step_create(
  77     struct icu_chain * chain,  enum icu_chain_step_type type,
  78     const uint8_t * rule,
  79     UErrorCode *status)
  80 {
  81     struct icu_chain_step * step = 0;
  82
  83     if (!chain || !type || !rule)
  84         return 0;
  85
  86     step = (struct icu_chain_step *) xmalloc(sizeof(struct icu_chain_step));
  87
  88     step->type = type;
  89     /* create auxilary objects */
  90     switch (step->type)
  91     {
  92     case ICU_chain_step_type_display:
  93         break;
  94     case ICU_chain_step_type_casemap:
  95         step->u.casemap = icu_casemap_create(rule[0], status);
  96         break;
  97     case ICU_chain_step_type_transform:
  98         /* rule omitted. Only ID used */
  99         step->u.transform = icu_transform_create((const char *) rule, 'f',
 100                                                  0, status);
 101         break;
 102     case ICU_chain_step_type_tokenize:
 103         step->u.tokenizer = icu_tokenizer_create((char *) chain->locale,
 104                                                  (char) rule[0], status);
 105         break;
 106     case ICU_chain_step_type_transliterate:
 107         /* we pass a dummy ID to utrans_openU.. */
 108         step->u.transform = icu_transform_create("custom", 'f',
 109                                                  (const char *) rule, status);
 110         break;
 111     default:
 112         break;
 113     }
 114     return step;
 115 }
 116
 117
 118 static void icu_chain_step_destroy(struct icu_chain_step * step)
 119 {
 120     if (!step)
 121         return;
 122
 123     icu_chain_step_destroy(step->previous);
 124
 125     switch (step->type)
 126     {
 127     case ICU_chain_step_type_display:
 128         break;
 129     case ICU_chain_step_type_casemap:
 130         icu_casemap_destroy(step->u.casemap);
 131         break;
 132     case ICU_chain_step_type_transform:
 133     case ICU_chain_step_type_transliterate:
 134         icu_transform_destroy(step->u.transform);
 135         break;
 136     case ICU_chain_step_type_tokenize:
 137         icu_tokenizer_destroy(step->u.tokenizer);
 138         break;
 139     default:
 140         break;
 141     }
 142     xfree(step);
 143 }
 144
 145 struct icu_chain_step *icu_chain_step_clone(struct icu_chain_step *old)
 146 {
 147     struct icu_chain_step *step = 0;
 148     struct icu_chain_step **sp = &step;
 149     while (old)
 150     {
 151         *sp = (struct icu_chain_step *) xmalloc(sizeof(**sp));
 152         (*sp)->type = old->type;
 153
 154         switch ((*sp)->type)
 155         {
 156         case ICU_chain_step_type_display:
 157             break;
 158         case ICU_chain_step_type_casemap:
 159             (*sp)->u.casemap = icu_casemap_clone(old->u.casemap);
 160             break;
 161         case ICU_chain_step_type_transform:
 162         case ICU_chain_step_type_transliterate:
 163             (*sp)->u.transform = icu_transform_clone(old->u.transform);
 164             break;
 165         case ICU_chain_step_type_tokenize:
 166             (*sp)->u.tokenizer = icu_tokenizer_clone(old->u.tokenizer);
 167             break;
 168         case ICU_chain_step_type_none:
 169             break;
 170         }
 171         old = old->previous;
 172         sp = &(*sp)->previous;
 173     }
 174     *sp = 0;
 175     return step;
 176 }
 177
 178 struct icu_chain *icu_chain_create(const char *locale, int sort,
 179                                    UErrorCode * status)
 180 {
 181     struct icu_chain * chain
 182         = (struct icu_chain *) xmalloc(sizeof(struct icu_chain));
 183
 184     *status = U_ZERO_ERROR;
 185
 186     chain->iter = 0;
 187     chain->locale = xstrdup(locale);
 188
 189     chain->sort = sort;
 190
 191     chain->coll = ucol_open((const char *) chain->locale, status);
 192
 193     if (U_FAILURE(*status))
 194         return 0;
 195
 196     chain->norm8 = icu_buf_utf8_create(0);
 197     chain->csteps = 0;
 198
 199     return chain;
 200 }
 201
 202 void icu_chain_destroy(struct icu_chain * chain)
 203 {
 204     if (chain)
 205     {
 206         if (chain->coll)
 207             ucol_close(chain->coll);
 208
 209         icu_buf_utf8_destroy(chain->norm8);
 210         if (chain->iter)
 211             icu_iter_destroy(chain->iter);
 212         icu_chain_step_destroy(chain->csteps);
 213         xfree(chain->locale);
 214         xfree(chain);
 215     }
 216 }
 217
 218 static struct icu_chain_step *icu_chain_insert_step(
 219     struct icu_chain * chain, enum icu_chain_step_type type,
 220     const uint8_t * rule, UErrorCode *status);
 221
 222 struct icu_chain * icu_chain_xml_config(const xmlNode *xml_node,
 223                                         int sort,
 224                                         UErrorCode * status)
 225 {
 226     xmlNode *node = 0;
 227     struct icu_chain * chain = 0;
 228
 229     *status = U_ZERO_ERROR;
 230
 231     if (!xml_node ||xml_node->type != XML_ELEMENT_NODE)
 232         return 0;
 233
 234     {
 235         xmlChar * xml_locale = xmlGetProp((xmlNode *) xml_node,
 236                                           (xmlChar *) "locale");
 237
 238         if (xml_locale)
 239         {
 240             chain = icu_chain_create((const char *) xml_locale, sort, status);
 241             xmlFree(xml_locale);
 242         }
 243
 244     }
 245     if (!chain)
 246         return 0;
 247
 248     for (node = xml_node->children; node; node = node->next)
 249     {
 250         xmlChar *xml_rule;
 251         struct icu_chain_step * step = 0;
 252
 253         if (node->type != XML_ELEMENT_NODE)
 254             continue;
 255
 256         xml_rule = xmlGetProp(node, (xmlChar *) "rule");
 257
 258         if (!strcmp((const char *) node->name, "casemap"))
 259             step = icu_chain_insert_step(chain, ICU_chain_step_type_casemap,
 260                                          (const uint8_t *) xml_rule, status);
 261         else if (!strcmp((const char *) node->name, "transform"))
 262             step = icu_chain_insert_step(chain, ICU_chain_step_type_transform,
 263                                          (const uint8_t *) xml_rule, status);
 264         else if (!strcmp((const char *) node->name, "transliterate"))
 265             step = icu_chain_insert_step(chain, ICU_chain_step_type_transliterate,
 266                                          (const uint8_t *) xml_rule, status);
 267         else if (!strcmp((const char *) node->name, "tokenize"))
 268             step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize,
 269                                          (const uint8_t *) xml_rule, status);
 270         else if (!strcmp((const char *) node->name, "display"))
 271             step = icu_chain_insert_step(chain, ICU_chain_step_type_display,
 272                                          (const uint8_t *) "", status);
 273         else if (!strcmp((const char *) node->name, "normalize"))
 274         {
 275             yaz_log(YLOG_WARN, "Element %s is deprecated. "
 276                     "Use transform instead", node->name);
 277             step = icu_chain_insert_step(chain, ICU_chain_step_type_transform,
 278                                          (const uint8_t *) xml_rule, status);
 279         }
 280         else if (!strcmp((const char *) node->name, "index")
 281                  || !strcmp((const char *) node->name, "sortkey"))
 282         {
 283             yaz_log(YLOG_WARN, "Element %s is no longer needed. "
 284                     "Remove it from the configuration", node->name);
 285         }
 286         else
 287         {
 288             yaz_log(YLOG_WARN, "Unknown element %s", node->name);
 289             icu_chain_destroy(chain);
 290             return 0;
 291         }
 292         xmlFree(xml_rule);
 293         if (step && U_FAILURE(*status))
 294         {
 295             icu_chain_destroy(chain);
 296             return 0;
 297         }
 298     }
 299     return chain;
 300 }
 301
 302
 303 static struct icu_chain_step *icu_chain_insert_step(
 304     struct icu_chain * chain, enum icu_chain_step_type type,
 305     const uint8_t * rule, UErrorCode *status)
 306 {
 307     struct icu_chain_step * step = 0;
 308     if (!chain || !type || !rule)
 309         return 0;
 310
 311     /* create actual chain step with this buffer */
 312     step = icu_chain_step_create(chain, type, rule,
 313                                  status);
 314
 315     step->previous = chain->csteps;
 316     chain->csteps = step;
 317
 318     return step;
 319 }
 320
 321 struct icu_iter {
 322     struct icu_chain *chain;
 323     struct icu_buf_utf16 *last;
 324     UErrorCode status;
 325     struct icu_buf_utf8 *display;
 326     struct icu_buf_utf8 *sort8;
 327     struct icu_buf_utf16 *input;
 328     int token_count;
 329     struct icu_chain_step *steps;
 330 };
 331
 332 void icu_utf16_print(struct icu_buf_utf16 *src16)
 333 {
 334     UErrorCode status = U_ZERO_ERROR;
 335     const char *p;
 336     struct icu_buf_utf8 *dst8 = icu_buf_utf8_create(0);
 337     icu_utf16_to_utf8(dst8, src16, &status);
 338
 339     assert(status != 1234);
 340     if (U_FAILURE(status))
 341     {
 342         printf("failure");
 343     }
 344     else
 345     {
 346         p = icu_buf_utf8_to_cstr(dst8);
 347         printf("%s", p);
 348     }
 349     icu_buf_utf8_destroy(dst8);
 350 }
 351
 352 struct icu_buf_utf16 *icu_iter_invoke(struct icu_iter *iter,
 353                                       struct icu_chain_step *step,
 354                                       struct icu_buf_utf16 *src)
 355 {
 356     if (!step)
 357         return src;
 358     else
 359     {
 360         struct icu_buf_utf16 *dst = icu_iter_invoke(iter, step->previous, src);
 361
 362         switch (step->type)
 363         {
 364         case ICU_chain_step_type_casemap:
 365             if (dst)
 366             {
 367                 struct icu_buf_utf16 *src = dst;
 368
 369                 dst = icu_buf_utf16_create(0);
 370                 icu_casemap_casemap(step->u.casemap, dst, src, &iter->status,
 371                                     iter->chain->locale);
 372                 icu_buf_utf16_destroy(src);
 373             }
 374             break;
 375         case ICU_chain_step_type_tokenize:
 376             if (dst)
 377             {
 378                 struct icu_buf_utf16 *src = dst;
 379
 380                 icu_tokenizer_attach(step->u.tokenizer, src, &iter->status);
 381                 icu_buf_utf16_destroy(src);
 382             }
 383             dst = icu_buf_utf16_create(0);
 384             iter->status = U_ZERO_ERROR;
 385             if (!icu_tokenizer_next_token(step->u.tokenizer, dst, &iter->status))
 386             {
 387                 icu_buf_utf16_destroy(dst);
 388                 dst = 0;
 389             }
 390             break;
 391         case ICU_chain_step_type_transform:
 392         case ICU_chain_step_type_transliterate:
 393             if (dst)
 394             {
 395                 struct icu_buf_utf16 *src = dst;
 396                 dst = icu_buf_utf16_create(0);
 397                 icu_transform_trans(step->u.transform, dst, src, &iter->status);
 398                 icu_buf_utf16_destroy(src);
 399             }
 400             break;
 401         case ICU_chain_step_type_display:
 402             if (dst)
 403                 icu_utf16_to_utf8(iter->display, dst, &iter->status);
 404             break;
 405         default:
 406             assert(0);
 407         }
 408         return dst;
 409     }
 410 }
 411
 412 struct icu_iter *icu_iter_create(struct icu_chain *chain)
 413 {
 414     struct icu_iter *iter = xmalloc(sizeof(*iter));
 415     iter->chain = chain;
 416     iter->status = U_ZERO_ERROR;
 417     iter->display = icu_buf_utf8_create(0);
 418     iter->sort8 = icu_buf_utf8_create(0);
 419     iter->last = 0; /* no last returned string (yet) */
 420     iter->steps = icu_chain_step_clone(chain->csteps);
 421     iter->input = 0;
 422
 423     return iter;
 424 }
 425
 426 void icu_iter_first(struct icu_iter *iter, const char *src8cstr)
 427 {
 428     if (iter->input)
 429         icu_buf_utf16_destroy(iter->input);
 430     iter->input = icu_buf_utf16_create(0);
 431     iter->token_count = 0;
 432     /* fill and assign input string.. It will be 0 after
 433        first iteration */
 434     icu_utf16_from_utf8_cstr(iter->input, src8cstr, &iter->status);
 435 }
 436
 437 void icu_iter_destroy(struct icu_iter *iter)
 438 {
 439     if (iter)
 440     {
 441         icu_buf_utf8_destroy(iter->display);
 442         icu_buf_utf8_destroy(iter->sort8);
 443         if (iter->input)
 444             icu_buf_utf16_destroy(iter->input);
 445         icu_chain_step_destroy(iter->steps);
 446         xfree(iter);
 447     }
 448 }
 449
 450 int icu_iter_next(struct icu_iter *iter, struct icu_buf_utf8 *result)
 451 {
 452     if (!iter->input && iter->last == 0)
 453         return 0;
 454     else
 455     {
 456         /* on first call, iter->input is the input string. Thereafter: 0. */
 457         iter->last = icu_iter_invoke(iter, iter->steps ?
 458                                      iter->steps : iter->chain->csteps,
 459                                      iter->input);
 460         iter->input = 0;
 461
 462         if (!iter->last)
 463             return 0;
 464
 465         iter->token_count++;
 466
 467         if (iter->chain->sort)
 468         {
 469             icu_sortkey8_from_utf16(iter->chain->coll,
 470                                     iter->sort8, iter->last,
 471                                     &iter->status);
 472         }
 473         icu_utf16_to_utf8(result, iter->last, &iter->status);
 474         icu_buf_utf16_destroy(iter->last);
 475
 476         return 1;
 477     }
 478 }
 479
 480 const char *icu_iter_get_sortkey(struct icu_iter *iter)
 481 {
 482     return icu_buf_utf8_to_cstr(iter->sort8);
 483 }
 484
 485 const char *icu_iter_get_display(struct icu_iter *iter)
 486 {
 487     return icu_buf_utf8_to_cstr(iter->display);
 488 }
 489
 490 int icu_chain_assign_cstr(struct icu_chain * chain, const char * src8cstr,
 491                           UErrorCode *status)
 492 {
 493     if (chain->iter)
 494         icu_iter_destroy(chain->iter);
 495     chain->iter = icu_iter_create(chain);
 496     icu_iter_first(chain->iter, src8cstr);
 497     return 1;
 498 }
 499
 500 int icu_chain_next_token(struct icu_chain * chain, UErrorCode *status)
 501 {
 502     *status = U_ZERO_ERROR;
 503     return icu_iter_next(chain->iter, chain->norm8);
 504 }
 505
 506 int icu_chain_token_number(struct icu_chain * chain)
 507 {
 508     if (chain && chain->iter)
 509         return chain->iter->token_count;
 510     return 0;
 511 }
 512
 513 const char * icu_chain_token_display(struct icu_chain * chain)
 514 {
 515     if (chain->iter)
 516         return icu_iter_get_display(chain->iter);
 517     return 0;
 518 }
 519
 520 const char * icu_chain_token_norm(struct icu_chain * chain)
 521 {
 522     if (chain->norm8)
 523         return icu_buf_utf8_to_cstr(chain->norm8);
 524     return 0;
 525 }
 526
 527 const char * icu_chain_token_sortkey(struct icu_chain * chain)
 528 {
 529     if (chain->iter)
 530         return icu_iter_get_sortkey(chain->iter);
 531     return 0;
 532 }
 533
 534 #endif /* YAZ_HAVE_ICU */
 535
 536 /*
 537  * Local variables:
 538  * c-basic-offset: 4
 539  * c-file-style: "Stroustrup"
 540  * indent-tabs-mode: nil
 541  * End:
 542  * vim: shiftwidth=4 tabstop=8 expandtab
 543  */
 544