src/icu_chain.c

   1 /* This file is part of the YAZ toolkit.
   2  * Copyright (C) 1995-2010 Index Data
   3  * See the file LICENSE for details.
   4  */
   5
   6 /**
   7  * \file
   8  * \brief ICU chain
   9  */
  10
  11 #if HAVE_CONFIG_H
  12 #include "config.h"
  13 #endif
  14
  15 #if YAZ_HAVE_ICU
  16 #include <yaz/xmalloc.h>
  17
  18 #include <yaz/icu_I18N.h>
  19
  20 #include <yaz/log.h>
  21 #include <yaz/nmem.h>
  22 #include <yaz/nmem_xml.h>
  23 #include <string.h>
  24 #include <stdlib.h>
  25 #include <stdio.h>
  26 #include <assert.h>
  27
  28 #include <unicode/ustring.h>  /* some more string fcns*/
  29 #include <unicode/uchar.h>    /* char names           */
  30
  31 enum icu_chain_step_type {
  32     ICU_chain_step_type_none,
  33     ICU_chain_step_type_display,   /* convert to utf8 display format */
  34     ICU_chain_step_type_casemap,   /* apply utf16 charmap */
  35     ICU_chain_step_type_transform, /* apply utf16 transform */
  36     ICU_chain_step_type_tokenize,  /* apply utf16 tokenization */
  37     ICU_chain_step_type_transliterate  /* apply utf16 tokenization */
  38 };
  39
  40 struct icu_chain_step
  41 {
  42     /* type and action object */
  43     enum icu_chain_step_type type;
  44     union {
  45         struct icu_casemap *casemap;
  46         struct icu_transform *transform;
  47         struct icu_tokenizer *tokenizer;
  48     } u;
  49     struct icu_chain_step *previous;
  50 };
  51
  52 struct icu_chain
  53 {
  54     yaz_icu_iter_t iter;
  55     char *locale;
  56     int sort;
  57
  58     UCollator *coll;
  59
  60     /* linked list of chain steps */
  61     struct icu_chain_step *csteps;
  62 };
  63
  64 int icu_check_status(UErrorCode status)
  65 {
  66     if (U_FAILURE(status))
  67     {
  68         yaz_log(YLOG_WARN, "ICU: %d %s\n", status, u_errorName(status));
  69         return 0;
  70     }
  71     return 1;
  72 }
  73
  74 static struct icu_chain_step *icu_chain_step_create(
  75     struct icu_chain *chain,  enum icu_chain_step_type type,
  76     const uint8_t *rule, UErrorCode *status)
  77 {
  78     struct icu_chain_step *step = 0;
  79
  80     if (!chain || !type || !rule)
  81         return 0;
  82
  83     step = (struct icu_chain_step *) xmalloc(sizeof(*step));
  84
  85     step->type = type;
  86     /* create auxilary objects */
  87     switch (step->type)
  88     {
  89     case ICU_chain_step_type_display:
  90         break;
  91     case ICU_chain_step_type_casemap:
  92         step->u.casemap = icu_casemap_create(rule[0], status);
  93         break;
  94     case ICU_chain_step_type_transform:
  95         /* rule omitted. Only ID used */
  96         step->u.transform = icu_transform_create((const char *) rule, 'f',
  97                                                  0, status);
  98         break;
  99     case ICU_chain_step_type_tokenize:
 100         step->u.tokenizer = icu_tokenizer_create((char *) chain->locale,
 101                                                  (char) rule[0], status);
 102         break;
 103     case ICU_chain_step_type_transliterate:
 104         /* we pass a dummy ID to utrans_openU.. */
 105         step->u.transform = icu_transform_create("custom", 'f',
 106                                                  (const char *) rule, status);
 107         break;
 108     default:
 109         break;
 110     }
 111     return step;
 112 }
 113
 114
 115 static void icu_chain_step_destroy(struct icu_chain_step *step)
 116 {
 117     if (!step)
 118         return;
 119
 120     icu_chain_step_destroy(step->previous);
 121
 122     switch (step->type)
 123     {
 124     case ICU_chain_step_type_display:
 125         break;
 126     case ICU_chain_step_type_casemap:
 127         icu_casemap_destroy(step->u.casemap);
 128         break;
 129     case ICU_chain_step_type_transform:
 130     case ICU_chain_step_type_transliterate:
 131         icu_transform_destroy(step->u.transform);
 132         break;
 133     case ICU_chain_step_type_tokenize:
 134         icu_tokenizer_destroy(step->u.tokenizer);
 135         break;
 136     default:
 137         break;
 138     }
 139     xfree(step);
 140 }
 141
 142 struct icu_chain_step *icu_chain_step_clone(struct icu_chain_step *old)
 143 {
 144     struct icu_chain_step *step = 0;
 145     struct icu_chain_step **sp = &step;
 146     while (old)
 147     {
 148         *sp = (struct icu_chain_step *) xmalloc(sizeof(**sp));
 149         (*sp)->type = old->type;
 150
 151         switch ((*sp)->type)
 152         {
 153         case ICU_chain_step_type_display:
 154             break;
 155         case ICU_chain_step_type_casemap:
 156             (*sp)->u.casemap = icu_casemap_clone(old->u.casemap);
 157             break;
 158         case ICU_chain_step_type_transform:
 159         case ICU_chain_step_type_transliterate:
 160             (*sp)->u.transform = icu_transform_clone(old->u.transform);
 161             break;
 162         case ICU_chain_step_type_tokenize:
 163             (*sp)->u.tokenizer = icu_tokenizer_clone(old->u.tokenizer);
 164             break;
 165         case ICU_chain_step_type_none:
 166             break;
 167         }
 168         old = old->previous;
 169         sp = &(*sp)->previous;
 170     }
 171     *sp = 0;
 172     return step;
 173 }
 174
 175 struct icu_chain *icu_chain_create(const char *locale, int sort,
 176                                    UErrorCode *status)
 177 {
 178     struct icu_chain *chain
 179         = (struct icu_chain *) xmalloc(sizeof(*chain));
 180
 181     *status = U_ZERO_ERROR;
 182
 183     chain->iter = 0;
 184     chain->locale = xstrdup(locale);
 185
 186     chain->sort = sort;
 187
 188     chain->coll = ucol_open((const char *) chain->locale, status);
 189
 190     if (U_FAILURE(*status))
 191         return 0;
 192
 193     chain->csteps = 0;
 194
 195     return chain;
 196 }
 197
 198 void icu_chain_destroy(struct icu_chain *chain)
 199 {
 200     if (chain)
 201     {
 202         if (chain->coll)
 203             ucol_close(chain->coll);
 204
 205         if (chain->iter)
 206             icu_iter_destroy(chain->iter);
 207         icu_chain_step_destroy(chain->csteps);
 208         xfree(chain->locale);
 209         xfree(chain);
 210     }
 211 }
 212
 213 static struct icu_chain_step *icu_chain_insert_step(
 214     struct icu_chain *chain, enum icu_chain_step_type type,
 215     const uint8_t *rule, UErrorCode *status);
 216
 217 struct icu_chain *icu_chain_xml_config(const xmlNode *xml_node,
 218                                        int sort,
 219                                        UErrorCode *status)
 220 {
 221     xmlNode *node = 0;
 222     int no_errors = 0;
 223     struct icu_chain *chain = 0;
 224     NMEM nmem = 0;
 225
 226     *status = U_ZERO_ERROR;
 227
 228     if (!xml_node ||xml_node->type != XML_ELEMENT_NODE)
 229         return 0;
 230
 231     {
 232         xmlChar *xml_locale = xmlGetProp((xmlNode *) xml_node,
 233                                          (xmlChar *) "locale");
 234
 235         if (xml_locale)
 236         {
 237             chain = icu_chain_create((const char *) xml_locale, sort, status);
 238             xmlFree(xml_locale);
 239         }
 240
 241     }
 242     if (!chain)
 243         return 0;
 244
 245     nmem = nmem_create();
 246     for (node = xml_node->children; node; node = node->next)
 247     {
 248         char *rule = 0;
 249         struct icu_chain_step *step = 0;
 250         struct _xmlAttr *attr;
 251
 252         nmem_reset(nmem);
 253         if (node->type != XML_ELEMENT_NODE)
 254             continue;
 255
 256         for (attr = node->properties; attr; attr = attr->next)
 257         {
 258             if (!strcmp((const char *) attr->name, "rule"))
 259             {
 260                 rule = nmem_text_node_cdata(attr->children, nmem);
 261             }
 262             else
 263             {
 264                 yaz_log(YLOG_WARN, "Unsupported attribute '%s' for "
 265                         "element '%s'", attr->name, node->name);
 266                 no_errors++;
 267                 continue;
 268             }
 269         }
 270         if (!rule && node->children)
 271             rule = nmem_text_node_cdata(node->children, nmem);
 272
 273         if (!rule || !*rule)
 274         {
 275             yaz_log(YLOG_WARN, "Missing rule for element '%s'", node->name);
 276             no_errors++;
 277             continue;
 278         }
 279
 280         if (!strcmp((const char *) node->name, "casemap"))
 281             step = icu_chain_insert_step(chain, ICU_chain_step_type_casemap,
 282                                          (const uint8_t *) rule, status);
 283         else if (!strcmp((const char *) node->name, "transform"))
 284             step = icu_chain_insert_step(chain, ICU_chain_step_type_transform,
 285                                          (const uint8_t *) rule, status);
 286         else if (!strcmp((const char *) node->name, "transliterate"))
 287             step = icu_chain_insert_step(chain, ICU_chain_step_type_transliterate,
 288                                          (const uint8_t *) rule, status);
 289         else if (!strcmp((const char *) node->name, "tokenize"))
 290             step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize,
 291                                          (const uint8_t *) rule, status);
 292         else if (!strcmp((const char *) node->name, "display"))
 293             step = icu_chain_insert_step(chain, ICU_chain_step_type_display,
 294                                          (const uint8_t *) "", status);
 295         else if (!strcmp((const char *) node->name, "normalize"))
 296         {
 297             yaz_log(YLOG_WARN, "Element %s is deprecated. "
 298                     "Use transform instead", node->name);
 299             step = icu_chain_insert_step(chain, ICU_chain_step_type_transform,
 300                                          (const uint8_t *) rule, status);
 301         }
 302         else if (!strcmp((const char *) node->name, "index")
 303                  || !strcmp((const char *) node->name, "sortkey"))
 304         {
 305             yaz_log(YLOG_WARN, "Element %s is no longer needed. "
 306                     "Remove it from the configuration", node->name);
 307         }
 308         else
 309         {
 310             yaz_log(YLOG_WARN, "Unknown element %s", node->name);
 311             no_errors++;
 312             continue;
 313         }
 314         if (step && U_FAILURE(*status))
 315         {
 316             no_errors++;
 317             break;
 318         }
 319     }
 320     nmem_destroy(nmem);
 321     if (no_errors)
 322     {
 323         icu_chain_destroy(chain);
 324         return 0;
 325     }
 326     return chain;
 327 }
 328
 329
 330 static struct icu_chain_step *icu_chain_insert_step(
 331     struct icu_chain *chain, enum icu_chain_step_type type,
 332     const uint8_t *rule, UErrorCode *status)
 333 {
 334     struct icu_chain_step *step = 0;
 335     if (!chain || !type || !rule)
 336         return 0;
 337
 338     /* create actual chain step with this buffer */
 339     step = icu_chain_step_create(chain, type, rule,
 340                                  status);
 341
 342     step->previous = chain->csteps;
 343     chain->csteps = step;
 344
 345     return step;
 346 }
 347
 348 struct icu_iter {
 349     struct icu_chain *chain;
 350     struct icu_buf_utf16 *last;
 351     UErrorCode status;
 352     struct icu_buf_utf8 *display;
 353     struct icu_buf_utf8 *sort8;
 354     struct icu_buf_utf8 *result;
 355     struct icu_buf_utf16 *input;
 356     int token_count;
 357     struct icu_chain_step *steps;
 358 };
 359
 360 void icu_utf16_print(struct icu_buf_utf16 *src16)
 361 {
 362     UErrorCode status = U_ZERO_ERROR;
 363     const char *p;
 364     struct icu_buf_utf8 *dst8 = icu_buf_utf8_create(0);
 365     icu_utf16_to_utf8(dst8, src16, &status);
 366
 367     assert(status != 1234);
 368     if (U_FAILURE(status))
 369     {
 370         printf("failure");
 371     }
 372     else
 373     {
 374         p = icu_buf_utf8_to_cstr(dst8);
 375         printf("%s", p);
 376     }
 377     icu_buf_utf8_destroy(dst8);
 378 }
 379
 380 struct icu_buf_utf16 *icu_iter_invoke(yaz_icu_iter_t iter,
 381                                       struct icu_chain_step *step,
 382                                       struct icu_buf_utf16 *src)
 383 {
 384     if (!step)
 385         return src;
 386     else
 387     {
 388         struct icu_buf_utf16 *dst = icu_iter_invoke(iter, step->previous, src);
 389
 390         switch (step->type)
 391         {
 392         case ICU_chain_step_type_casemap:
 393             if (dst)
 394             {
 395                 struct icu_buf_utf16 *src = dst;
 396
 397                 dst = icu_buf_utf16_create(0);
 398                 icu_casemap_casemap(step->u.casemap, dst, src, &iter->status,
 399                                     iter->chain->locale);
 400                 icu_buf_utf16_destroy(src);
 401             }
 402             break;
 403         case ICU_chain_step_type_tokenize:
 404             if (dst)
 405             {
 406                 struct icu_buf_utf16 *src = dst;
 407
 408                 icu_tokenizer_attach(step->u.tokenizer, src, &iter->status);
 409                 icu_buf_utf16_destroy(src);
 410             }
 411             dst = icu_buf_utf16_create(0);
 412             iter->status = U_ZERO_ERROR;
 413             if (!icu_tokenizer_next_token(step->u.tokenizer, dst, &iter->status))
 414             {
 415                 icu_buf_utf16_destroy(dst);
 416                 dst = 0;
 417             }
 418             break;
 419         case ICU_chain_step_type_transform:
 420         case ICU_chain_step_type_transliterate:
 421             if (dst)
 422             {
 423                 struct icu_buf_utf16 *src = dst;
 424                 dst = icu_buf_utf16_create(0);
 425                 icu_transform_trans(step->u.transform, dst, src, &iter->status);
 426                 icu_buf_utf16_destroy(src);
 427             }
 428             break;
 429         case ICU_chain_step_type_display:
 430             if (dst)
 431                 icu_utf16_to_utf8(iter->display, dst, &iter->status);
 432             break;
 433         default:
 434             assert(0);
 435         }
 436         return dst;
 437     }
 438 }
 439
 440 yaz_icu_iter_t icu_iter_create(struct icu_chain *chain)
 441 {
 442     yaz_icu_iter_t iter = xmalloc(sizeof(*iter));
 443     iter->chain = chain;
 444     iter->status = U_ZERO_ERROR;
 445     iter->display = icu_buf_utf8_create(0);
 446     iter->sort8 = icu_buf_utf8_create(0);
 447     iter->result = icu_buf_utf8_create(0);
 448     iter->last = 0; /* no last returned string (yet) */
 449     iter->steps = icu_chain_step_clone(chain->csteps);
 450     iter->input = 0;
 451
 452     return iter;
 453 }
 454
 455 void icu_iter_first(yaz_icu_iter_t iter, const char *src8cstr)
 456 {
 457     if (iter->input)
 458         icu_buf_utf16_destroy(iter->input);
 459     iter->input = icu_buf_utf16_create(0);
 460     iter->token_count = 0;
 461     /* fill and assign input string.. It will be 0 after
 462        first iteration */
 463     icu_utf16_from_utf8_cstr(iter->input, src8cstr, &iter->status);
 464 }
 465
 466 void icu_iter_destroy(yaz_icu_iter_t iter)
 467 {
 468     if (iter)
 469     {
 470         icu_buf_utf8_destroy(iter->display);
 471         icu_buf_utf8_destroy(iter->sort8);
 472         icu_buf_utf8_destroy(iter->result);
 473         if (iter->input)
 474             icu_buf_utf16_destroy(iter->input);
 475         icu_chain_step_destroy(iter->steps);
 476         xfree(iter);
 477     }
 478 }
 479
 480 int icu_iter_next(yaz_icu_iter_t iter)
 481 {
 482     if (!iter->input && iter->last == 0)
 483         return 0;
 484     else
 485     {
 486         /* on first call, iter->input is the input string. Thereafter: 0. */
 487         iter->last = icu_iter_invoke(iter, iter->steps ?
 488                                      iter->steps : iter->chain->csteps,
 489                                      iter->input);
 490         iter->input = 0;
 491
 492         if (!iter->last)
 493             return 0;
 494
 495         iter->token_count++;
 496
 497         if (iter->chain->sort)
 498         {
 499             icu_sortkey8_from_utf16(iter->chain->coll,
 500                                     iter->sort8, iter->last,
 501                                     &iter->status);
 502         }
 503         icu_utf16_to_utf8(iter->result, iter->last, &iter->status);
 504         icu_buf_utf16_destroy(iter->last);
 505
 506         return 1;
 507     }
 508 }
 509
 510 const char *icu_iter_get_norm(yaz_icu_iter_t iter)
 511 {
 512     return icu_buf_utf8_to_cstr(iter->result);
 513 }
 514
 515 const char *icu_iter_get_sortkey(yaz_icu_iter_t iter)
 516 {
 517     return icu_buf_utf8_to_cstr(iter->sort8);
 518 }
 519
 520 const char *icu_iter_get_display(yaz_icu_iter_t iter)
 521 {
 522     return icu_buf_utf8_to_cstr(iter->display);
 523 }
 524
 525 int icu_iter_get_token_number(yaz_icu_iter_t iter)
 526 {
 527     return iter->token_count;
 528 }
 529
 530 int icu_chain_assign_cstr(struct icu_chain *chain, const char *src8cstr,
 531                           UErrorCode *status)
 532 {
 533     if (chain->iter)
 534         icu_iter_destroy(chain->iter);
 535     chain->iter = icu_iter_create(chain);
 536     icu_iter_first(chain->iter, src8cstr);
 537     return 1;
 538 }
 539
 540 int icu_chain_next_token(struct icu_chain *chain, UErrorCode *status)
 541 {
 542     *status = U_ZERO_ERROR;
 543     return icu_iter_next(chain->iter);
 544 }
 545
 546 int icu_chain_token_number(struct icu_chain *chain)
 547 {
 548     if (chain && chain->iter)
 549         return chain->iter->token_count;
 550     return 0;
 551 }
 552
 553 const char *icu_chain_token_display(struct icu_chain *chain)
 554 {
 555     if (chain->iter)
 556         return icu_iter_get_display(chain->iter);
 557     return 0;
 558 }
 559
 560 const char *icu_chain_token_norm(struct icu_chain *chain)
 561 {
 562     if (chain->iter)
 563         return icu_iter_get_norm(chain->iter);
 564     return 0;
 565 }
 566
 567 const char *icu_chain_token_sortkey(struct icu_chain *chain)
 568 {
 569     if (chain->iter)
 570         return icu_iter_get_sortkey(chain->iter);
 571     return 0;
 572 }
 573
 574 #endif /* YAZ_HAVE_ICU */
 575
 576 /*
 577  * Local variables:
 578  * c-basic-offset: 4
 579  * c-file-style: "Stroustrup"
 580  * indent-tabs-mode: nil
 581  * End:
 582  * vim: shiftwidth=4 tabstop=8 expandtab
 583  */
 584