src/icu_chain.c

   1 /* This file is part of the YAZ toolkit.
   2  * Copyright (C) 1995-2010 Index Data
   3  * See the file LICENSE for details.
   4  */
   5
   6 /**
   7  * \file
   8  * \brief ICU chain
   9  */
  10
  11 #if HAVE_CONFIG_H
  12 #include "config.h"
  13 #endif
  14
  15 #if YAZ_HAVE_ICU
  16 #include <yaz/xmalloc.h>
  17
  18 #include <yaz/icu_I18N.h>
  19
  20 #include <yaz/log.h>
  21
  22 #include <string.h>
  23 #include <stdlib.h>
  24 #include <stdio.h>
  25 #include <assert.h>
  26
  27 #include <unicode/ustring.h>  /* some more string fcns*/
  28 #include <unicode/uchar.h>    /* char names           */
  29
  30 enum icu_chain_step_type {
  31     ICU_chain_step_type_none,
  32     ICU_chain_step_type_display,   /* convert to utf8 display format */
  33     ICU_chain_step_type_casemap,   /* apply utf16 charmap */
  34     ICU_chain_step_type_transform, /* apply utf16 transform */
  35     ICU_chain_step_type_tokenize,  /* apply utf16 tokenization */
  36     ICU_chain_step_type_transliterate  /* apply utf16 tokenization */
  37 };
  38
  39 struct icu_chain_step
  40 {
  41     /* type and action object */
  42     enum icu_chain_step_type type;
  43     union {
  44         struct icu_casemap * casemap;
  45         struct icu_transform * transform;
  46         struct icu_tokenizer * tokenizer;
  47     } u;
  48     struct icu_chain_step * previous;
  49 };
  50
  51 struct icu_chain
  52 {
  53     yaz_icu_iter_t iter;
  54     char *locale;
  55     int sort;
  56
  57     UCollator * coll;
  58
  59     /* linked list of chain steps */
  60     struct icu_chain_step * csteps;
  61 };
  62
  63 int icu_check_status(UErrorCode status)
  64 {
  65     if (U_FAILURE(status))
  66     {
  67         yaz_log(YLOG_WARN, "ICU: %d %s\n", status, u_errorName(status));
  68         return 0;
  69     }
  70     return 1;
  71 }
  72
  73 static struct icu_chain_step *icu_chain_step_create(
  74     struct icu_chain * chain,  enum icu_chain_step_type type,
  75     const uint8_t * rule,
  76     UErrorCode *status)
  77 {
  78     struct icu_chain_step * step = 0;
  79
  80     if (!chain || !type || !rule)
  81         return 0;
  82
  83     step = (struct icu_chain_step *) xmalloc(sizeof(*step));
  84
  85     step->type = type;
  86     /* create auxilary objects */
  87     switch (step->type)
  88     {
  89     case ICU_chain_step_type_display:
  90         break;
  91     case ICU_chain_step_type_casemap:
  92         step->u.casemap = icu_casemap_create(rule[0], status);
  93         break;
  94     case ICU_chain_step_type_transform:
  95         /* rule omitted. Only ID used */
  96         step->u.transform = icu_transform_create((const char *) rule, 'f',
  97                                                  0, status);
  98         break;
  99     case ICU_chain_step_type_tokenize:
 100         step->u.tokenizer = icu_tokenizer_create((char *) chain->locale,
 101                                                  (char) rule[0], status);
 102         break;
 103     case ICU_chain_step_type_transliterate:
 104         /* we pass a dummy ID to utrans_openU.. */
 105         step->u.transform = icu_transform_create("custom", 'f',
 106                                                  (const char *) rule, status);
 107         break;
 108     default:
 109         break;
 110     }
 111     return step;
 112 }
 113
 114
 115 static void icu_chain_step_destroy(struct icu_chain_step * step)
 116 {
 117     if (!step)
 118         return;
 119
 120     icu_chain_step_destroy(step->previous);
 121
 122     switch (step->type)
 123     {
 124     case ICU_chain_step_type_display:
 125         break;
 126     case ICU_chain_step_type_casemap:
 127         icu_casemap_destroy(step->u.casemap);
 128         break;
 129     case ICU_chain_step_type_transform:
 130     case ICU_chain_step_type_transliterate:
 131         icu_transform_destroy(step->u.transform);
 132         break;
 133     case ICU_chain_step_type_tokenize:
 134         icu_tokenizer_destroy(step->u.tokenizer);
 135         break;
 136     default:
 137         break;
 138     }
 139     xfree(step);
 140 }
 141
 142 struct icu_chain_step *icu_chain_step_clone(struct icu_chain_step *old)
 143 {
 144     struct icu_chain_step *step = 0;
 145     struct icu_chain_step **sp = &step;
 146     while (old)
 147     {
 148         *sp = (struct icu_chain_step *) xmalloc(sizeof(**sp));
 149         (*sp)->type = old->type;
 150
 151         switch ((*sp)->type)
 152         {
 153         case ICU_chain_step_type_display:
 154             break;
 155         case ICU_chain_step_type_casemap:
 156             (*sp)->u.casemap = icu_casemap_clone(old->u.casemap);
 157             break;
 158         case ICU_chain_step_type_transform:
 159         case ICU_chain_step_type_transliterate:
 160             (*sp)->u.transform = icu_transform_clone(old->u.transform);
 161             break;
 162         case ICU_chain_step_type_tokenize:
 163             (*sp)->u.tokenizer = icu_tokenizer_clone(old->u.tokenizer);
 164             break;
 165         case ICU_chain_step_type_none:
 166             break;
 167         }
 168         old = old->previous;
 169         sp = &(*sp)->previous;
 170     }
 171     *sp = 0;
 172     return step;
 173 }
 174
 175 struct icu_chain *icu_chain_create(const char *locale, int sort,
 176                                    UErrorCode * status)
 177 {
 178     struct icu_chain * chain
 179         = (struct icu_chain *) xmalloc(sizeof(*chain));
 180
 181     *status = U_ZERO_ERROR;
 182
 183     chain->iter = 0;
 184     chain->locale = xstrdup(locale);
 185
 186     chain->sort = sort;
 187
 188     chain->coll = ucol_open((const char *) chain->locale, status);
 189
 190     if (U_FAILURE(*status))
 191         return 0;
 192
 193     chain->csteps = 0;
 194
 195     return chain;
 196 }
 197
 198 void icu_chain_destroy(struct icu_chain * chain)
 199 {
 200     if (chain)
 201     {
 202         if (chain->coll)
 203             ucol_close(chain->coll);
 204
 205         if (chain->iter)
 206             icu_iter_destroy(chain->iter);
 207         icu_chain_step_destroy(chain->csteps);
 208         xfree(chain->locale);
 209         xfree(chain);
 210     }
 211 }
 212
 213 static struct icu_chain_step *icu_chain_insert_step(
 214     struct icu_chain * chain, enum icu_chain_step_type type,
 215     const uint8_t * rule, UErrorCode *status);
 216
 217 struct icu_chain * icu_chain_xml_config(const xmlNode *xml_node,
 218                                         int sort,
 219                                         UErrorCode * status)
 220 {
 221     xmlNode *node = 0;
 222     struct icu_chain * chain = 0;
 223
 224     *status = U_ZERO_ERROR;
 225
 226     if (!xml_node ||xml_node->type != XML_ELEMENT_NODE)
 227         return 0;
 228
 229     {
 230         xmlChar * xml_locale = xmlGetProp((xmlNode *) xml_node,
 231                                           (xmlChar *) "locale");
 232
 233         if (xml_locale)
 234         {
 235             chain = icu_chain_create((const char *) xml_locale, sort, status);
 236             xmlFree(xml_locale);
 237         }
 238
 239     }
 240     if (!chain)
 241         return 0;
 242
 243     for (node = xml_node->children; node; node = node->next)
 244     {
 245         xmlChar *xml_rule;
 246         struct icu_chain_step * step = 0;
 247
 248         if (node->type != XML_ELEMENT_NODE)
 249             continue;
 250
 251         xml_rule = xmlGetProp(node, (xmlChar *) "rule");
 252
 253         if (!strcmp((const char *) node->name, "casemap"))
 254             step = icu_chain_insert_step(chain, ICU_chain_step_type_casemap,
 255                                          (const uint8_t *) xml_rule, status);
 256         else if (!strcmp((const char *) node->name, "transform"))
 257             step = icu_chain_insert_step(chain, ICU_chain_step_type_transform,
 258                                          (const uint8_t *) xml_rule, status);
 259         else if (!strcmp((const char *) node->name, "transliterate"))
 260             step = icu_chain_insert_step(chain, ICU_chain_step_type_transliterate,
 261                                          (const uint8_t *) xml_rule, status);
 262         else if (!strcmp((const char *) node->name, "tokenize"))
 263             step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize,
 264                                          (const uint8_t *) xml_rule, status);
 265         else if (!strcmp((const char *) node->name, "display"))
 266             step = icu_chain_insert_step(chain, ICU_chain_step_type_display,
 267                                          (const uint8_t *) "", status);
 268         else if (!strcmp((const char *) node->name, "normalize"))
 269         {
 270             yaz_log(YLOG_WARN, "Element %s is deprecated. "
 271                     "Use transform instead", node->name);
 272             step = icu_chain_insert_step(chain, ICU_chain_step_type_transform,
 273                                          (const uint8_t *) xml_rule, status);
 274         }
 275         else if (!strcmp((const char *) node->name, "index")
 276                  || !strcmp((const char *) node->name, "sortkey"))
 277         {
 278             yaz_log(YLOG_WARN, "Element %s is no longer needed. "
 279                     "Remove it from the configuration", node->name);
 280         }
 281         else
 282         {
 283             yaz_log(YLOG_WARN, "Unknown element %s", node->name);
 284             icu_chain_destroy(chain);
 285             return 0;
 286         }
 287         xmlFree(xml_rule);
 288         if (step && U_FAILURE(*status))
 289         {
 290             icu_chain_destroy(chain);
 291             return 0;
 292         }
 293     }
 294     return chain;
 295 }
 296
 297
 298 static struct icu_chain_step *icu_chain_insert_step(
 299     struct icu_chain * chain, enum icu_chain_step_type type,
 300     const uint8_t * rule, UErrorCode *status)
 301 {
 302     struct icu_chain_step * step = 0;
 303     if (!chain || !type || !rule)
 304         return 0;
 305
 306     /* create actual chain step with this buffer */
 307     step = icu_chain_step_create(chain, type, rule,
 308                                  status);
 309
 310     step->previous = chain->csteps;
 311     chain->csteps = step;
 312
 313     return step;
 314 }
 315
 316 struct icu_iter {
 317     struct icu_chain *chain;
 318     struct icu_buf_utf16 *last;
 319     UErrorCode status;
 320     struct icu_buf_utf8 *display;
 321     struct icu_buf_utf8 *sort8;
 322     struct icu_buf_utf8 *result;
 323     struct icu_buf_utf16 *input;
 324     int token_count;
 325     struct icu_chain_step *steps;
 326 };
 327
 328 void icu_utf16_print(struct icu_buf_utf16 *src16)
 329 {
 330     UErrorCode status = U_ZERO_ERROR;
 331     const char *p;
 332     struct icu_buf_utf8 *dst8 = icu_buf_utf8_create(0);
 333     icu_utf16_to_utf8(dst8, src16, &status);
 334
 335     assert(status != 1234);
 336     if (U_FAILURE(status))
 337     {
 338         printf("failure");
 339     }
 340     else
 341     {
 342         p = icu_buf_utf8_to_cstr(dst8);
 343         printf("%s", p);
 344     }
 345     icu_buf_utf8_destroy(dst8);
 346 }
 347
 348 struct icu_buf_utf16 *icu_iter_invoke(yaz_icu_iter_t iter,
 349                                       struct icu_chain_step *step,
 350                                       struct icu_buf_utf16 *src)
 351 {
 352     if (!step)
 353         return src;
 354     else
 355     {
 356         struct icu_buf_utf16 *dst = icu_iter_invoke(iter, step->previous, src);
 357
 358         switch (step->type)
 359         {
 360         case ICU_chain_step_type_casemap:
 361             if (dst)
 362             {
 363                 struct icu_buf_utf16 *src = dst;
 364
 365                 dst = icu_buf_utf16_create(0);
 366                 icu_casemap_casemap(step->u.casemap, dst, src, &iter->status,
 367                                     iter->chain->locale);
 368                 icu_buf_utf16_destroy(src);
 369             }
 370             break;
 371         case ICU_chain_step_type_tokenize:
 372             if (dst)
 373             {
 374                 struct icu_buf_utf16 *src = dst;
 375
 376                 icu_tokenizer_attach(step->u.tokenizer, src, &iter->status);
 377                 icu_buf_utf16_destroy(src);
 378             }
 379             dst = icu_buf_utf16_create(0);
 380             iter->status = U_ZERO_ERROR;
 381             if (!icu_tokenizer_next_token(step->u.tokenizer, dst, &iter->status))
 382             {
 383                 icu_buf_utf16_destroy(dst);
 384                 dst = 0;
 385             }
 386             break;
 387         case ICU_chain_step_type_transform:
 388         case ICU_chain_step_type_transliterate:
 389             if (dst)
 390             {
 391                 struct icu_buf_utf16 *src = dst;
 392                 dst = icu_buf_utf16_create(0);
 393                 icu_transform_trans(step->u.transform, dst, src, &iter->status);
 394                 icu_buf_utf16_destroy(src);
 395             }
 396             break;
 397         case ICU_chain_step_type_display:
 398             if (dst)
 399                 icu_utf16_to_utf8(iter->display, dst, &iter->status);
 400             break;
 401         default:
 402             assert(0);
 403         }
 404         return dst;
 405     }
 406 }
 407
 408 yaz_icu_iter_t icu_iter_create(struct icu_chain *chain)
 409 {
 410     yaz_icu_iter_t iter = xmalloc(sizeof(*iter));
 411     iter->chain = chain;
 412     iter->status = U_ZERO_ERROR;
 413     iter->display = icu_buf_utf8_create(0);
 414     iter->sort8 = icu_buf_utf8_create(0);
 415     iter->result = icu_buf_utf8_create(0);
 416     iter->last = 0; /* no last returned string (yet) */
 417     iter->steps = icu_chain_step_clone(chain->csteps);
 418     iter->input = 0;
 419
 420     return iter;
 421 }
 422
 423 void icu_iter_first(yaz_icu_iter_t iter, const char *src8cstr)
 424 {
 425     if (iter->input)
 426         icu_buf_utf16_destroy(iter->input);
 427     iter->input = icu_buf_utf16_create(0);
 428     iter->token_count = 0;
 429     /* fill and assign input string.. It will be 0 after
 430        first iteration */
 431     icu_utf16_from_utf8_cstr(iter->input, src8cstr, &iter->status);
 432 }
 433
 434 void icu_iter_destroy(yaz_icu_iter_t iter)
 435 {
 436     if (iter)
 437     {
 438         icu_buf_utf8_destroy(iter->display);
 439         icu_buf_utf8_destroy(iter->sort8);
 440         icu_buf_utf8_destroy(iter->result);
 441         if (iter->input)
 442             icu_buf_utf16_destroy(iter->input);
 443         icu_chain_step_destroy(iter->steps);
 444         xfree(iter);
 445     }
 446 }
 447
 448 int icu_iter_next(yaz_icu_iter_t iter)
 449 {
 450     if (!iter->input && iter->last == 0)
 451         return 0;
 452     else
 453     {
 454         /* on first call, iter->input is the input string. Thereafter: 0. */
 455         iter->last = icu_iter_invoke(iter, iter->steps ?
 456                                      iter->steps : iter->chain->csteps,
 457                                      iter->input);
 458         iter->input = 0;
 459
 460         if (!iter->last)
 461             return 0;
 462
 463         iter->token_count++;
 464
 465         if (iter->chain->sort)
 466         {
 467             icu_sortkey8_from_utf16(iter->chain->coll,
 468                                     iter->sort8, iter->last,
 469                                     &iter->status);
 470         }
 471         icu_utf16_to_utf8(iter->result, iter->last, &iter->status);
 472         icu_buf_utf16_destroy(iter->last);
 473
 474         return 1;
 475     }
 476 }
 477
 478 const char *icu_iter_get_norm(yaz_icu_iter_t iter)
 479 {
 480     return icu_buf_utf8_to_cstr(iter->result);
 481 }
 482
 483 const char *icu_iter_get_sortkey(yaz_icu_iter_t iter)
 484 {
 485     return icu_buf_utf8_to_cstr(iter->sort8);
 486 }
 487
 488 const char *icu_iter_get_display(yaz_icu_iter_t iter)
 489 {
 490     return icu_buf_utf8_to_cstr(iter->display);
 491 }
 492
 493 const char *icu_iter_get_token_number(yaz_icu_iter_t iter)
 494 {
 495     return iter->token_count;
 496 }
 497
 498 int icu_chain_assign_cstr(struct icu_chain * chain, const char * src8cstr,
 499                           UErrorCode *status)
 500 {
 501     if (chain->iter)
 502         icu_iter_destroy(chain->iter);
 503     chain->iter = icu_iter_create(chain);
 504     icu_iter_first(chain->iter, src8cstr);
 505     return 1;
 506 }
 507
 508 int icu_chain_next_token(struct icu_chain * chain, UErrorCode *status)
 509 {
 510     *status = U_ZERO_ERROR;
 511     return icu_iter_next(chain->iter);
 512 }
 513
 514 int icu_chain_token_number(struct icu_chain * chain)
 515 {
 516     if (chain && chain->iter)
 517         return chain->iter->token_count;
 518     return 0;
 519 }
 520
 521 const char * icu_chain_token_display(struct icu_chain * chain)
 522 {
 523     if (chain->iter)
 524         return icu_iter_get_display(chain->iter);
 525     return 0;
 526 }
 527
 528 const char * icu_chain_token_norm(struct icu_chain * chain)
 529 {
 530     if (chain->iter)
 531         return icu_iter_get_norm(chain->iter);
 532     return 0;
 533 }
 534
 535 const char * icu_chain_token_sortkey(struct icu_chain * chain)
 536 {
 537     if (chain->iter)
 538         return icu_iter_get_sortkey(chain->iter);
 539     return 0;
 540 }
 541
 542 #endif /* YAZ_HAVE_ICU */
 543
 544 /*
 545  * Local variables:
 546  * c-basic-offset: 4
 547  * c-file-style: "Stroustrup"
 548  * indent-tabs-mode: nil
 549  * End:
 550  * vim: shiftwidth=4 tabstop=8 expandtab
 551  */
 552