src/icu_chain.c

   1 /* This file is part of the YAZ toolkit.
   2  * Copyright (C) 1995-2010 Index Data
   3  * See the file LICENSE for details.
   4  */
   5
   6 /**
   7  * \file
   8  * \brief ICU chain
   9  */
  10
  11 #if HAVE_CONFIG_H
  12 #include "config.h"
  13 #endif
  14
  15 #if YAZ_HAVE_ICU
  16 #include <yaz/xmalloc.h>
  17
  18 #include <yaz/icu_I18N.h>
  19
  20 #include <yaz/log.h>
  21
  22 #include <string.h>
  23 #include <stdlib.h>
  24 #include <stdio.h>
  25 #include <assert.h>
  26
  27 #include <unicode/ustring.h>  /* some more string fcns*/
  28 #include <unicode/uchar.h>    /* char names           */
  29
  30 enum icu_chain_step_type {
  31     ICU_chain_step_type_none,
  32     ICU_chain_step_type_display,   /* convert to utf8 display format */
  33     ICU_chain_step_type_casemap,   /* apply utf16 charmap */
  34     ICU_chain_step_type_transform, /* apply utf16 transform */
  35     ICU_chain_step_type_tokenize,  /* apply utf16 tokenization */
  36     ICU_chain_step_type_transliterate  /* apply utf16 tokenization */
  37 };
  38
  39 struct icu_chain_step
  40 {
  41     /* type and action object */
  42     enum icu_chain_step_type type;
  43     union {
  44         struct icu_casemap *casemap;
  45         struct icu_transform *transform;
  46         struct icu_tokenizer *tokenizer;
  47     } u;
  48     struct icu_chain_step *previous;
  49 };
  50
  51 struct icu_chain
  52 {
  53     yaz_icu_iter_t iter;
  54     char *locale;
  55     int sort;
  56
  57     UCollator *coll;
  58
  59     /* linked list of chain steps */
  60     struct icu_chain_step *csteps;
  61 };
  62
  63 int icu_check_status(UErrorCode status)
  64 {
  65     if (U_FAILURE(status))
  66     {
  67         yaz_log(YLOG_WARN, "ICU: %d %s\n", status, u_errorName(status));
  68         return 0;
  69     }
  70     return 1;
  71 }
  72
  73 static struct icu_chain_step *icu_chain_step_create(
  74     struct icu_chain *chain,  enum icu_chain_step_type type,
  75     const uint8_t *rule, UErrorCode *status)
  76 {
  77     struct icu_chain_step *step = 0;
  78
  79     if (!chain || !type || !rule)
  80         return 0;
  81
  82     step = (struct icu_chain_step *) xmalloc(sizeof(*step));
  83
  84     step->type = type;
  85     /* create auxilary objects */
  86     switch (step->type)
  87     {
  88     case ICU_chain_step_type_display:
  89         break;
  90     case ICU_chain_step_type_casemap:
  91         step->u.casemap = icu_casemap_create(rule[0], status);
  92         break;
  93     case ICU_chain_step_type_transform:
  94         /* rule omitted. Only ID used */
  95         step->u.transform = icu_transform_create((const char *) rule, 'f',
  96                                                  0, status);
  97         break;
  98     case ICU_chain_step_type_tokenize:
  99         step->u.tokenizer = icu_tokenizer_create((char *) chain->locale,
 100                                                  (char) rule[0], status);
 101         break;
 102     case ICU_chain_step_type_transliterate:
 103         /* we pass a dummy ID to utrans_openU.. */
 104         step->u.transform = icu_transform_create("custom", 'f',
 105                                                  (const char *) rule, status);
 106         break;
 107     default:
 108         break;
 109     }
 110     return step;
 111 }
 112
 113
 114 static void icu_chain_step_destroy(struct icu_chain_step *step)
 115 {
 116     if (!step)
 117         return;
 118
 119     icu_chain_step_destroy(step->previous);
 120
 121     switch (step->type)
 122     {
 123     case ICU_chain_step_type_display:
 124         break;
 125     case ICU_chain_step_type_casemap:
 126         icu_casemap_destroy(step->u.casemap);
 127         break;
 128     case ICU_chain_step_type_transform:
 129     case ICU_chain_step_type_transliterate:
 130         icu_transform_destroy(step->u.transform);
 131         break;
 132     case ICU_chain_step_type_tokenize:
 133         icu_tokenizer_destroy(step->u.tokenizer);
 134         break;
 135     default:
 136         break;
 137     }
 138     xfree(step);
 139 }
 140
 141 struct icu_chain_step *icu_chain_step_clone(struct icu_chain_step *old)
 142 {
 143     struct icu_chain_step *step = 0;
 144     struct icu_chain_step **sp = &step;
 145     while (old)
 146     {
 147         *sp = (struct icu_chain_step *) xmalloc(sizeof(**sp));
 148         (*sp)->type = old->type;
 149
 150         switch ((*sp)->type)
 151         {
 152         case ICU_chain_step_type_display:
 153             break;
 154         case ICU_chain_step_type_casemap:
 155             (*sp)->u.casemap = icu_casemap_clone(old->u.casemap);
 156             break;
 157         case ICU_chain_step_type_transform:
 158         case ICU_chain_step_type_transliterate:
 159             (*sp)->u.transform = icu_transform_clone(old->u.transform);
 160             break;
 161         case ICU_chain_step_type_tokenize:
 162             (*sp)->u.tokenizer = icu_tokenizer_clone(old->u.tokenizer);
 163             break;
 164         case ICU_chain_step_type_none:
 165             break;
 166         }
 167         old = old->previous;
 168         sp = &(*sp)->previous;
 169     }
 170     *sp = 0;
 171     return step;
 172 }
 173
 174 struct icu_chain *icu_chain_create(const char *locale, int sort,
 175                                    UErrorCode *status)
 176 {
 177     struct icu_chain *chain
 178         = (struct icu_chain *) xmalloc(sizeof(*chain));
 179
 180     *status = U_ZERO_ERROR;
 181
 182     chain->iter = 0;
 183     chain->locale = xstrdup(locale);
 184
 185     chain->sort = sort;
 186
 187     chain->coll = ucol_open((const char *) chain->locale, status);
 188
 189     if (U_FAILURE(*status))
 190         return 0;
 191
 192     chain->csteps = 0;
 193
 194     return chain;
 195 }
 196
 197 void icu_chain_destroy(struct icu_chain *chain)
 198 {
 199     if (chain)
 200     {
 201         if (chain->coll)
 202             ucol_close(chain->coll);
 203
 204         if (chain->iter)
 205             icu_iter_destroy(chain->iter);
 206         icu_chain_step_destroy(chain->csteps);
 207         xfree(chain->locale);
 208         xfree(chain);
 209     }
 210 }
 211
 212 static struct icu_chain_step *icu_chain_insert_step(
 213     struct icu_chain *chain, enum icu_chain_step_type type,
 214     const uint8_t *rule, UErrorCode *status);
 215
 216 struct icu_chain *icu_chain_xml_config(const xmlNode *xml_node,
 217                                        int sort,
 218                                        UErrorCode *status)
 219 {
 220     xmlNode *node = 0;
 221     struct icu_chain *chain = 0;
 222
 223     *status = U_ZERO_ERROR;
 224
 225     if (!xml_node ||xml_node->type != XML_ELEMENT_NODE)
 226         return 0;
 227
 228     {
 229         xmlChar *xml_locale = xmlGetProp((xmlNode *) xml_node,
 230                                          (xmlChar *) "locale");
 231
 232         if (xml_locale)
 233         {
 234             chain = icu_chain_create((const char *) xml_locale, sort, status);
 235             xmlFree(xml_locale);
 236         }
 237
 238     }
 239     if (!chain)
 240         return 0;
 241
 242     for (node = xml_node->children; node; node = node->next)
 243     {
 244         xmlChar *xml_rule;
 245         struct icu_chain_step *step = 0;
 246
 247         if (node->type != XML_ELEMENT_NODE)
 248             continue;
 249
 250         xml_rule = xmlGetProp(node, (xmlChar *) "rule");
 251
 252         yaz_log(YLOG_LOG, "rule=%s", xml_rule);
 253
 254         if (!strcmp((const char *) node->name, "casemap"))
 255             step = icu_chain_insert_step(chain, ICU_chain_step_type_casemap,
 256                                          (const uint8_t *) xml_rule, status);
 257         else if (!strcmp((const char *) node->name, "transform"))
 258             step = icu_chain_insert_step(chain, ICU_chain_step_type_transform,
 259                                          (const uint8_t *) xml_rule, status);
 260         else if (!strcmp((const char *) node->name, "transliterate"))
 261             step = icu_chain_insert_step(chain, ICU_chain_step_type_transliterate,
 262                                          (const uint8_t *) xml_rule, status);
 263         else if (!strcmp((const char *) node->name, "tokenize"))
 264             step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize,
 265                                          (const uint8_t *) xml_rule, status);
 266         else if (!strcmp((const char *) node->name, "display"))
 267             step = icu_chain_insert_step(chain, ICU_chain_step_type_display,
 268                                          (const uint8_t *) "", status);
 269         else if (!strcmp((const char *) node->name, "normalize"))
 270         {
 271             yaz_log(YLOG_WARN, "Element %s is deprecated. "
 272                     "Use transform instead", node->name);
 273             step = icu_chain_insert_step(chain, ICU_chain_step_type_transform,
 274                                          (const uint8_t *) xml_rule, status);
 275         }
 276         else if (!strcmp((const char *) node->name, "index")
 277                  || !strcmp((const char *) node->name, "sortkey"))
 278         {
 279             yaz_log(YLOG_WARN, "Element %s is no longer needed. "
 280                     "Remove it from the configuration", node->name);
 281         }
 282         else
 283         {
 284             yaz_log(YLOG_WARN, "Unknown element %s", node->name);
 285             icu_chain_destroy(chain);
 286             return 0;
 287         }
 288         xmlFree(xml_rule);
 289         if (step && U_FAILURE(*status))
 290         {
 291             icu_chain_destroy(chain);
 292             return 0;
 293         }
 294     }
 295     return chain;
 296 }
 297
 298
 299 static struct icu_chain_step *icu_chain_insert_step(
 300     struct icu_chain *chain, enum icu_chain_step_type type,
 301     const uint8_t *rule, UErrorCode *status)
 302 {
 303     struct icu_chain_step *step = 0;
 304     if (!chain || !type || !rule)
 305         return 0;
 306
 307     /* create actual chain step with this buffer */
 308     step = icu_chain_step_create(chain, type, rule,
 309                                  status);
 310
 311     step->previous = chain->csteps;
 312     chain->csteps = step;
 313
 314     return step;
 315 }
 316
 317 struct icu_iter {
 318     struct icu_chain *chain;
 319     struct icu_buf_utf16 *last;
 320     UErrorCode status;
 321     struct icu_buf_utf8 *display;
 322     struct icu_buf_utf8 *sort8;
 323     struct icu_buf_utf8 *result;
 324     struct icu_buf_utf16 *input;
 325     int token_count;
 326     struct icu_chain_step *steps;
 327 };
 328
 329 void icu_utf16_print(struct icu_buf_utf16 *src16)
 330 {
 331     UErrorCode status = U_ZERO_ERROR;
 332     const char *p;
 333     struct icu_buf_utf8 *dst8 = icu_buf_utf8_create(0);
 334     icu_utf16_to_utf8(dst8, src16, &status);
 335
 336     assert(status != 1234);
 337     if (U_FAILURE(status))
 338     {
 339         printf("failure");
 340     }
 341     else
 342     {
 343         p = icu_buf_utf8_to_cstr(dst8);
 344         printf("%s", p);
 345     }
 346     icu_buf_utf8_destroy(dst8);
 347 }
 348
 349 struct icu_buf_utf16 *icu_iter_invoke(yaz_icu_iter_t iter,
 350                                       struct icu_chain_step *step,
 351                                       struct icu_buf_utf16 *src)
 352 {
 353     if (!step)
 354         return src;
 355     else
 356     {
 357         struct icu_buf_utf16 *dst = icu_iter_invoke(iter, step->previous, src);
 358
 359         switch (step->type)
 360         {
 361         case ICU_chain_step_type_casemap:
 362             if (dst)
 363             {
 364                 struct icu_buf_utf16 *src = dst;
 365
 366                 dst = icu_buf_utf16_create(0);
 367                 icu_casemap_casemap(step->u.casemap, dst, src, &iter->status,
 368                                     iter->chain->locale);
 369                 icu_buf_utf16_destroy(src);
 370             }
 371             break;
 372         case ICU_chain_step_type_tokenize:
 373             if (dst)
 374             {
 375                 struct icu_buf_utf16 *src = dst;
 376
 377                 icu_tokenizer_attach(step->u.tokenizer, src, &iter->status);
 378                 icu_buf_utf16_destroy(src);
 379             }
 380             dst = icu_buf_utf16_create(0);
 381             iter->status = U_ZERO_ERROR;
 382             if (!icu_tokenizer_next_token(step->u.tokenizer, dst, &iter->status))
 383             {
 384                 icu_buf_utf16_destroy(dst);
 385                 dst = 0;
 386             }
 387             break;
 388         case ICU_chain_step_type_transform:
 389         case ICU_chain_step_type_transliterate:
 390             if (dst)
 391             {
 392                 struct icu_buf_utf16 *src = dst;
 393                 dst = icu_buf_utf16_create(0);
 394                 icu_transform_trans(step->u.transform, dst, src, &iter->status);
 395                 icu_buf_utf16_destroy(src);
 396             }
 397             break;
 398         case ICU_chain_step_type_display:
 399             if (dst)
 400                 icu_utf16_to_utf8(iter->display, dst, &iter->status);
 401             break;
 402         default:
 403             assert(0);
 404         }
 405         return dst;
 406     }
 407 }
 408
 409 yaz_icu_iter_t icu_iter_create(struct icu_chain *chain)
 410 {
 411     yaz_icu_iter_t iter = xmalloc(sizeof(*iter));
 412     iter->chain = chain;
 413     iter->status = U_ZERO_ERROR;
 414     iter->display = icu_buf_utf8_create(0);
 415     iter->sort8 = icu_buf_utf8_create(0);
 416     iter->result = icu_buf_utf8_create(0);
 417     iter->last = 0; /* no last returned string (yet) */
 418     iter->steps = icu_chain_step_clone(chain->csteps);
 419     iter->input = 0;
 420
 421     return iter;
 422 }
 423
 424 void icu_iter_first(yaz_icu_iter_t iter, const char *src8cstr)
 425 {
 426     if (iter->input)
 427         icu_buf_utf16_destroy(iter->input);
 428     iter->input = icu_buf_utf16_create(0);
 429     iter->token_count = 0;
 430     /* fill and assign input string.. It will be 0 after
 431        first iteration */
 432     icu_utf16_from_utf8_cstr(iter->input, src8cstr, &iter->status);
 433 }
 434
 435 void icu_iter_destroy(yaz_icu_iter_t iter)
 436 {
 437     if (iter)
 438     {
 439         icu_buf_utf8_destroy(iter->display);
 440         icu_buf_utf8_destroy(iter->sort8);
 441         icu_buf_utf8_destroy(iter->result);
 442         if (iter->input)
 443             icu_buf_utf16_destroy(iter->input);
 444         icu_chain_step_destroy(iter->steps);
 445         xfree(iter);
 446     }
 447 }
 448
 449 int icu_iter_next(yaz_icu_iter_t iter)
 450 {
 451     if (!iter->input && iter->last == 0)
 452         return 0;
 453     else
 454     {
 455         /* on first call, iter->input is the input string. Thereafter: 0. */
 456         iter->last = icu_iter_invoke(iter, iter->steps ?
 457                                      iter->steps : iter->chain->csteps,
 458                                      iter->input);
 459         iter->input = 0;
 460
 461         if (!iter->last)
 462             return 0;
 463
 464         iter->token_count++;
 465
 466         if (iter->chain->sort)
 467         {
 468             icu_sortkey8_from_utf16(iter->chain->coll,
 469                                     iter->sort8, iter->last,
 470                                     &iter->status);
 471         }
 472         icu_utf16_to_utf8(iter->result, iter->last, &iter->status);
 473         icu_buf_utf16_destroy(iter->last);
 474
 475         return 1;
 476     }
 477 }
 478
 479 const char *icu_iter_get_norm(yaz_icu_iter_t iter)
 480 {
 481     return icu_buf_utf8_to_cstr(iter->result);
 482 }
 483
 484 const char *icu_iter_get_sortkey(yaz_icu_iter_t iter)
 485 {
 486     return icu_buf_utf8_to_cstr(iter->sort8);
 487 }
 488
 489 const char *icu_iter_get_display(yaz_icu_iter_t iter)
 490 {
 491     return icu_buf_utf8_to_cstr(iter->display);
 492 }
 493
 494 int icu_iter_get_token_number(yaz_icu_iter_t iter)
 495 {
 496     return iter->token_count;
 497 }
 498
 499 int icu_chain_assign_cstr(struct icu_chain *chain, const char *src8cstr,
 500                           UErrorCode *status)
 501 {
 502     if (chain->iter)
 503         icu_iter_destroy(chain->iter);
 504     chain->iter = icu_iter_create(chain);
 505     icu_iter_first(chain->iter, src8cstr);
 506     return 1;
 507 }
 508
 509 int icu_chain_next_token(struct icu_chain *chain, UErrorCode *status)
 510 {
 511     *status = U_ZERO_ERROR;
 512     return icu_iter_next(chain->iter);
 513 }
 514
 515 int icu_chain_token_number(struct icu_chain *chain)
 516 {
 517     if (chain && chain->iter)
 518         return chain->iter->token_count;
 519     return 0;
 520 }
 521
 522 const char *icu_chain_token_display(struct icu_chain *chain)
 523 {
 524     if (chain->iter)
 525         return icu_iter_get_display(chain->iter);
 526     return 0;
 527 }
 528
 529 const char *icu_chain_token_norm(struct icu_chain *chain)
 530 {
 531     if (chain->iter)
 532         return icu_iter_get_norm(chain->iter);
 533     return 0;
 534 }
 535
 536 const char *icu_chain_token_sortkey(struct icu_chain *chain)
 537 {
 538     if (chain->iter)
 539         return icu_iter_get_sortkey(chain->iter);
 540     return 0;
 541 }
 542
 543 #endif /* YAZ_HAVE_ICU */
 544
 545 /*
 546  * Local variables:
 547  * c-basic-offset: 4
 548  * c-file-style: "Stroustrup"
 549  * indent-tabs-mode: nil
 550  * End:
 551  * vim: shiftwidth=4 tabstop=8 expandtab
 552  */
 553