src/icu_chain.c

   1 /* This file is part of the YAZ toolkit.
   2  * Copyright (C) 1995-2010 Index Data
   3  * See the file LICENSE for details.
   4  */
   5
   6 /**
   7  * \file
   8  * \brief ICU chain
   9  */
  10
  11 #if HAVE_CONFIG_H
  12 #include "config.h"
  13 #endif
  14
  15 #if YAZ_HAVE_ICU
  16 #include <yaz/xmalloc.h>
  17
  18 #include <yaz/icu_I18N.h>
  19
  20 #include <yaz/log.h>
  21 #include <yaz/nmem.h>
  22 #include <yaz/nmem_xml.h>
  23 #include <string.h>
  24 #include <stdlib.h>
  25 #include <stdio.h>
  26 #include <assert.h>
  27
  28 #include <unicode/ustring.h>  /* some more string fcns*/
  29 #include <unicode/uchar.h>    /* char names           */
  30
  31 enum icu_chain_step_type {
  32     ICU_chain_step_type_none,
  33     ICU_chain_step_type_display,   /* convert to utf8 display format */
  34     ICU_chain_step_type_casemap,   /* apply utf16 charmap */
  35     ICU_chain_step_type_transform, /* apply utf16 transform */
  36     ICU_chain_step_type_tokenize,  /* apply utf16 tokenization */
  37     ICU_chain_step_type_transliterate  /* apply utf16 tokenization */
  38 };
  39
  40 struct icu_chain_step
  41 {
  42     /* type and action object */
  43     enum icu_chain_step_type type;
  44     union {
  45         struct icu_casemap *casemap;
  46         struct icu_transform *transform;
  47         struct icu_tokenizer *tokenizer;
  48     } u;
  49     struct icu_chain_step *previous;
  50 };
  51
  52 struct icu_chain
  53 {
  54     yaz_icu_iter_t iter;
  55     char *locale;
  56     int sort;
  57
  58     UCollator *coll;
  59
  60     /* linked list of chain steps */
  61     struct icu_chain_step *csteps;
  62 };
  63
  64 int icu_check_status(UErrorCode status)
  65 {
  66     if (U_FAILURE(status))
  67     {
  68         yaz_log(YLOG_WARN, "ICU: %d %s\n", status, u_errorName(status));
  69         return 0;
  70     }
  71     return 1;
  72 }
  73
  74 static struct icu_chain_step *icu_chain_insert_step(
  75     struct icu_chain *chain, enum icu_chain_step_type type,
  76     const uint8_t *rule, UErrorCode *status)
  77 {
  78     struct icu_chain_step *step = 0;
  79
  80     if (!chain || !type || !rule)
  81         return 0;
  82
  83     step = (struct icu_chain_step *) xmalloc(sizeof(*step));
  84
  85     step->type = type;
  86     /* create auxilary objects */
  87     switch (step->type)
  88     {
  89     case ICU_chain_step_type_display:
  90         break;
  91     case ICU_chain_step_type_casemap:
  92         step->u.casemap = icu_casemap_create(rule[0], status);
  93         break;
  94     case ICU_chain_step_type_transform:
  95         /* rule omitted. Only ID used */
  96         step->u.transform = icu_transform_create((const char *) rule, 'f',
  97                                                  0, status);
  98         break;
  99     case ICU_chain_step_type_tokenize:
 100         step->u.tokenizer = icu_tokenizer_create((char *) chain->locale,
 101                                                  (char) rule[0], status);
 102         break;
 103     case ICU_chain_step_type_transliterate:
 104         /* we pass a dummy ID to utrans_openU.. */
 105         step->u.transform = icu_transform_create("custom", 'f',
 106                                                  (const char *) rule, status);
 107         break;
 108     default:
 109         break;
 110     }
 111     step->previous = chain->csteps;
 112     chain->csteps = step;
 113
 114     return step;
 115 }
 116
 117
 118 static void icu_chain_step_destroy(struct icu_chain_step *step)
 119 {
 120     if (!step)
 121         return;
 122
 123     icu_chain_step_destroy(step->previous);
 124
 125     switch (step->type)
 126     {
 127     case ICU_chain_step_type_display:
 128         break;
 129     case ICU_chain_step_type_casemap:
 130         icu_casemap_destroy(step->u.casemap);
 131         break;
 132     case ICU_chain_step_type_transform:
 133     case ICU_chain_step_type_transliterate:
 134         icu_transform_destroy(step->u.transform);
 135         break;
 136     case ICU_chain_step_type_tokenize:
 137         icu_tokenizer_destroy(step->u.tokenizer);
 138         break;
 139     default:
 140         break;
 141     }
 142     xfree(step);
 143 }
 144
 145 struct icu_chain_step *icu_chain_step_clone(struct icu_chain_step *old)
 146 {
 147     struct icu_chain_step *step = 0;
 148     struct icu_chain_step **sp = &step;
 149     while (old)
 150     {
 151         *sp = (struct icu_chain_step *) xmalloc(sizeof(**sp));
 152         (*sp)->type = old->type;
 153
 154         switch ((*sp)->type)
 155         {
 156         case ICU_chain_step_type_display:
 157             break;
 158         case ICU_chain_step_type_casemap:
 159             (*sp)->u.casemap = icu_casemap_clone(old->u.casemap);
 160             break;
 161         case ICU_chain_step_type_transform:
 162         case ICU_chain_step_type_transliterate:
 163             (*sp)->u.transform = icu_transform_clone(old->u.transform);
 164             break;
 165         case ICU_chain_step_type_tokenize:
 166             (*sp)->u.tokenizer = icu_tokenizer_clone(old->u.tokenizer);
 167             break;
 168         case ICU_chain_step_type_none:
 169             break;
 170         }
 171         old = old->previous;
 172         sp = &(*sp)->previous;
 173     }
 174     *sp = 0;
 175     return step;
 176 }
 177
 178 struct icu_chain *icu_chain_create(const char *locale, int sort,
 179                                    UErrorCode *status)
 180 {
 181     struct icu_chain *chain
 182         = (struct icu_chain *) xmalloc(sizeof(*chain));
 183
 184     *status = U_ZERO_ERROR;
 185
 186     chain->iter = 0;
 187     chain->locale = xstrdup(locale);
 188
 189     chain->sort = sort;
 190
 191     chain->coll = ucol_open((const char *) chain->locale, status);
 192
 193     if (U_FAILURE(*status))
 194         return 0;
 195
 196     chain->csteps = 0;
 197
 198     return chain;
 199 }
 200
 201 void icu_chain_destroy(struct icu_chain *chain)
 202 {
 203     if (chain)
 204     {
 205         if (chain->coll)
 206             ucol_close(chain->coll);
 207
 208         if (chain->iter)
 209             icu_iter_destroy(chain->iter);
 210         icu_chain_step_destroy(chain->csteps);
 211         xfree(chain->locale);
 212         xfree(chain);
 213     }
 214 }
 215
 216 static struct icu_chain_step *icu_chain_insert_step(
 217     struct icu_chain *chain, enum icu_chain_step_type type,
 218     const uint8_t *rule, UErrorCode *status);
 219
 220 struct icu_chain *icu_chain_xml_config(const xmlNode *xml_node,
 221                                        int sort,
 222                                        UErrorCode *status)
 223 {
 224     xmlNode *node = 0;
 225     int no_errors = 0;
 226     struct icu_chain *chain = 0;
 227     NMEM nmem = 0;
 228
 229     *status = U_ZERO_ERROR;
 230
 231     if (!xml_node ||xml_node->type != XML_ELEMENT_NODE)
 232         return 0;
 233
 234     {
 235         xmlChar *xml_locale = xmlGetProp((xmlNode *) xml_node,
 236                                          (xmlChar *) "locale");
 237
 238         if (xml_locale)
 239         {
 240             chain = icu_chain_create((const char *) xml_locale, sort, status);
 241             xmlFree(xml_locale);
 242         }
 243
 244     }
 245     if (!chain)
 246         return 0;
 247
 248     nmem = nmem_create();
 249     for (node = xml_node->children; node; node = node->next)
 250     {
 251         char *rule = 0;
 252         struct icu_chain_step *step = 0;
 253         struct _xmlAttr *attr;
 254
 255         nmem_reset(nmem);
 256         if (node->type != XML_ELEMENT_NODE)
 257             continue;
 258
 259         for (attr = node->properties; attr; attr = attr->next)
 260         {
 261             if (!strcmp((const char *) attr->name, "rule"))
 262             {
 263                 rule = nmem_text_node_cdata(attr->children, nmem);
 264             }
 265             else
 266             {
 267                 yaz_log(YLOG_WARN, "Unsupported attribute '%s' for "
 268                         "element '%s'", attr->name, node->name);
 269                 no_errors++;
 270                 continue;
 271             }
 272         }
 273         if (!rule && node->children)
 274             rule = nmem_text_node_cdata(node->children, nmem);
 275
 276         if (!strcmp((const char *) node->name, "casemap"))
 277             step = icu_chain_insert_step(chain, ICU_chain_step_type_casemap,
 278                                          (const uint8_t *) rule, status);
 279         else if (!strcmp((const char *) node->name, "transform"))
 280             step = icu_chain_insert_step(chain, ICU_chain_step_type_transform,
 281                                          (const uint8_t *) rule, status);
 282         else if (!strcmp((const char *) node->name, "transliterate"))
 283             step = icu_chain_insert_step(chain, ICU_chain_step_type_transliterate,
 284                                          (const uint8_t *) rule, status);
 285         else if (!strcmp((const char *) node->name, "tokenize"))
 286             step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize,
 287                                          (const uint8_t *) rule, status);
 288         else if (!strcmp((const char *) node->name, "display"))
 289             step = icu_chain_insert_step(chain, ICU_chain_step_type_display,
 290                                          (const uint8_t *) "", status);
 291         else if (!strcmp((const char *) node->name, "normalize"))
 292         {
 293             yaz_log(YLOG_WARN, "Element %s is deprecated. "
 294                     "Use transform instead", node->name);
 295             step = icu_chain_insert_step(chain, ICU_chain_step_type_transform,
 296                                          (const uint8_t *) rule, status);
 297         }
 298         else if (!strcmp((const char *) node->name, "index")
 299                  || !strcmp((const char *) node->name, "sortkey"))
 300         {
 301             yaz_log(YLOG_WARN, "Element %s is no longer needed. "
 302                     "Remove it from the configuration", node->name);
 303         }
 304         else
 305         {
 306             yaz_log(YLOG_WARN, "Unknown element %s", node->name);
 307             no_errors++;
 308             continue;
 309         }
 310         if (step && U_FAILURE(*status))
 311         {
 312             no_errors++;
 313             break;
 314         }
 315     }
 316     nmem_destroy(nmem);
 317     if (no_errors)
 318     {
 319         icu_chain_destroy(chain);
 320         return 0;
 321     }
 322     return chain;
 323 }
 324
 325 struct icu_iter {
 326     struct icu_chain *chain;
 327     struct icu_buf_utf16 *last;
 328     UErrorCode status;
 329     struct icu_buf_utf8 *display;
 330     struct icu_buf_utf8 *sort8;
 331     struct icu_buf_utf8 *result;
 332     struct icu_buf_utf16 *input;
 333     int token_count;
 334     struct icu_chain_step *steps;
 335 };
 336
 337 void icu_utf16_print(struct icu_buf_utf16 *src16)
 338 {
 339     UErrorCode status = U_ZERO_ERROR;
 340     const char *p;
 341     struct icu_buf_utf8 *dst8 = icu_buf_utf8_create(0);
 342     icu_utf16_to_utf8(dst8, src16, &status);
 343
 344     assert(status != 1234);
 345     if (U_FAILURE(status))
 346     {
 347         printf("failure");
 348     }
 349     else
 350     {
 351         p = icu_buf_utf8_to_cstr(dst8);
 352         printf("%s", p);
 353     }
 354     icu_buf_utf8_destroy(dst8);
 355 }
 356
 357 struct icu_buf_utf16 *icu_iter_invoke(yaz_icu_iter_t iter,
 358                                       struct icu_chain_step *step,
 359                                       struct icu_buf_utf16 *src)
 360 {
 361     if (!step)
 362         return src;
 363     else
 364     {
 365         struct icu_buf_utf16 *dst = icu_iter_invoke(iter, step->previous, src);
 366
 367         switch (step->type)
 368         {
 369         case ICU_chain_step_type_casemap:
 370             if (dst)
 371             {
 372                 struct icu_buf_utf16 *src = dst;
 373
 374                 dst = icu_buf_utf16_create(0);
 375                 icu_casemap_casemap(step->u.casemap, dst, src, &iter->status,
 376                                     iter->chain->locale);
 377                 icu_buf_utf16_destroy(src);
 378             }
 379             break;
 380         case ICU_chain_step_type_tokenize:
 381             if (dst)
 382             {
 383                 struct icu_buf_utf16 *src = dst;
 384
 385                 icu_tokenizer_attach(step->u.tokenizer, src, &iter->status);
 386                 icu_buf_utf16_destroy(src);
 387             }
 388             dst = icu_buf_utf16_create(0);
 389             iter->status = U_ZERO_ERROR;
 390             if (!icu_tokenizer_next_token(step->u.tokenizer, dst, &iter->status))
 391             {
 392                 icu_buf_utf16_destroy(dst);
 393                 dst = 0;
 394             }
 395             break;
 396         case ICU_chain_step_type_transform:
 397         case ICU_chain_step_type_transliterate:
 398             if (dst)
 399             {
 400                 struct icu_buf_utf16 *src = dst;
 401                 dst = icu_buf_utf16_create(0);
 402                 icu_transform_trans(step->u.transform, dst, src, &iter->status);
 403                 icu_buf_utf16_destroy(src);
 404             }
 405             break;
 406         case ICU_chain_step_type_display:
 407             if (dst)
 408                 icu_utf16_to_utf8(iter->display, dst, &iter->status);
 409             break;
 410         default:
 411             assert(0);
 412         }
 413         return dst;
 414     }
 415 }
 416
 417 yaz_icu_iter_t icu_iter_create(struct icu_chain *chain)
 418 {
 419     yaz_icu_iter_t iter = xmalloc(sizeof(*iter));
 420     iter->chain = chain;
 421     iter->status = U_ZERO_ERROR;
 422     iter->display = icu_buf_utf8_create(0);
 423     iter->sort8 = icu_buf_utf8_create(0);
 424     iter->result = icu_buf_utf8_create(0);
 425     iter->last = 0; /* no last returned string (yet) */
 426     iter->steps = icu_chain_step_clone(chain->csteps);
 427     iter->input = 0;
 428
 429     return iter;
 430 }
 431
 432 void icu_iter_first(yaz_icu_iter_t iter, const char *src8cstr)
 433 {
 434     if (iter->input)
 435         icu_buf_utf16_destroy(iter->input);
 436     iter->input = icu_buf_utf16_create(0);
 437     iter->token_count = 0;
 438     /* fill and assign input string.. It will be 0 after
 439        first iteration */
 440     icu_utf16_from_utf8_cstr(iter->input, src8cstr, &iter->status);
 441 }
 442
 443 void icu_iter_destroy(yaz_icu_iter_t iter)
 444 {
 445     if (iter)
 446     {
 447         icu_buf_utf8_destroy(iter->display);
 448         icu_buf_utf8_destroy(iter->sort8);
 449         icu_buf_utf8_destroy(iter->result);
 450         if (iter->input)
 451             icu_buf_utf16_destroy(iter->input);
 452         icu_chain_step_destroy(iter->steps);
 453         xfree(iter);
 454     }
 455 }
 456
 457 int icu_iter_next(yaz_icu_iter_t iter)
 458 {
 459     if (!iter->input && iter->last == 0)
 460         return 0;
 461     else
 462     {
 463         /* on first call, iter->input is the input string. Thereafter: 0. */
 464         iter->last = icu_iter_invoke(iter, iter->steps ?
 465                                      iter->steps : iter->chain->csteps,
 466                                      iter->input);
 467         iter->input = 0;
 468
 469         if (!iter->last)
 470             return 0;
 471
 472         iter->token_count++;
 473
 474         if (iter->chain->sort)
 475         {
 476             icu_sortkey8_from_utf16(iter->chain->coll,
 477                                     iter->sort8, iter->last,
 478                                     &iter->status);
 479         }
 480         icu_utf16_to_utf8(iter->result, iter->last, &iter->status);
 481         icu_buf_utf16_destroy(iter->last);
 482
 483         return 1;
 484     }
 485 }
 486
 487 const char *icu_iter_get_norm(yaz_icu_iter_t iter)
 488 {
 489     return icu_buf_utf8_to_cstr(iter->result);
 490 }
 491
 492 const char *icu_iter_get_sortkey(yaz_icu_iter_t iter)
 493 {
 494     return icu_buf_utf8_to_cstr(iter->sort8);
 495 }
 496
 497 const char *icu_iter_get_display(yaz_icu_iter_t iter)
 498 {
 499     return icu_buf_utf8_to_cstr(iter->display);
 500 }
 501
 502 int icu_iter_get_token_number(yaz_icu_iter_t iter)
 503 {
 504     return iter->token_count;
 505 }
 506
 507 int icu_chain_assign_cstr(struct icu_chain *chain, const char *src8cstr,
 508                           UErrorCode *status)
 509 {
 510     if (chain->iter)
 511         icu_iter_destroy(chain->iter);
 512     chain->iter = icu_iter_create(chain);
 513     icu_iter_first(chain->iter, src8cstr);
 514     return 1;
 515 }
 516
 517 int icu_chain_next_token(struct icu_chain *chain, UErrorCode *status)
 518 {
 519     *status = U_ZERO_ERROR;
 520     return icu_iter_next(chain->iter);
 521 }
 522
 523 int icu_chain_token_number(struct icu_chain *chain)
 524 {
 525     if (chain && chain->iter)
 526         return chain->iter->token_count;
 527     return 0;
 528 }
 529
 530 const char *icu_chain_token_display(struct icu_chain *chain)
 531 {
 532     if (chain->iter)
 533         return icu_iter_get_display(chain->iter);
 534     return 0;
 535 }
 536
 537 const char *icu_chain_token_norm(struct icu_chain *chain)
 538 {
 539     if (chain->iter)
 540         return icu_iter_get_norm(chain->iter);
 541     return 0;
 542 }
 543
 544 const char *icu_chain_token_sortkey(struct icu_chain *chain)
 545 {
 546     if (chain->iter)
 547         return icu_iter_get_sortkey(chain->iter);
 548     return 0;
 549 }
 550
 551 #endif /* YAZ_HAVE_ICU */
 552
 553 /*
 554  * Local variables:
 555  * c-basic-offset: 4
 556  * c-file-style: "Stroustrup"
 557  * indent-tabs-mode: nil
 558  * End:
 559  * vim: shiftwidth=4 tabstop=8 expandtab
 560  */
 561