src/icu_chain.c

   1 /* This file is part of the YAZ toolkit.
   2  * Copyright (C) 1995-2010 Index Data
   3  * See the file LICENSE for details.
   4  */
   5
   6 /**
   7  * \file
   8  * \brief ICU chain
   9  */
  10
  11 #if HAVE_CONFIG_H
  12 #include "config.h"
  13 #endif
  14
  15 #if YAZ_HAVE_ICU
  16 #include <yaz/xmalloc.h>
  17
  18 #include <yaz/icu_I18N.h>
  19
  20 #include <yaz/stemmer.h>
  21
  22 #include <yaz/log.h>
  23
  24 #include <string.h>
  25 #include <stdlib.h>
  26 #include <stdio.h>
  27 #include <assert.h>
  28
  29 #include <unicode/ustring.h>  /* some more string fcns*/
  30 #include <unicode/uchar.h>    /* char names           */
  31
  32 enum icu_chain_step_type {
  33     ICU_chain_step_type_none,
  34     ICU_chain_step_type_display,        /* convert to utf8 display format */
  35     ICU_chain_step_type_casemap,        /* apply utf16 charmap */
  36     ICU_chain_step_type_transform,      /* apply utf16 transform */
  37     ICU_chain_step_type_tokenize,       /* apply utf16 tokenization */
  38     ICU_chain_step_type_transliterate,  /* apply utf16 tokenization */
  39     YAZ_chain_step_type_stemming        /* apply utf16 stemming (YAZ) */
  40 };
  41
  42 struct icu_chain_step
  43 {
  44     /* type and action object */
  45     enum icu_chain_step_type type;
  46     union {
  47         struct icu_casemap   * casemap;
  48         struct icu_transform * transform;
  49         struct icu_tokenizer * tokenizer;
  50         yaz_stemmer_p          stemmer;
  51     } u;
  52     struct icu_chain_step * previous;
  53 };
  54
  55 struct icu_chain
  56 {
  57     yaz_icu_iter_t iter;
  58     char *locale;
  59     int sort;
  60
  61     UCollator * coll;
  62
  63     /* linked list of chain steps */
  64     struct icu_chain_step * csteps;
  65 };
  66
  67 int icu_check_status(UErrorCode status)
  68 {
  69     if (U_FAILURE(status))
  70     {
  71         yaz_log(YLOG_WARN, "ICU: %d %s\n", status, u_errorName(status));
  72         return 0;
  73     }
  74     return 1;
  75 }
  76
  77 static struct icu_chain_step *icu_chain_step_create(
  78     struct icu_chain * chain,  enum icu_chain_step_type type,
  79     const uint8_t * rule,
  80     UErrorCode *status)
  81 {
  82     struct icu_chain_step * step = 0;
  83
  84     if (!chain || !type || !rule)
  85         return 0;
  86
  87     step = (struct icu_chain_step *) xmalloc(sizeof(*step));
  88
  89     step->type = type;
  90     /* create auxilary objects */
  91     switch (step->type)
  92     {
  93     case ICU_chain_step_type_display:
  94         break;
  95     case ICU_chain_step_type_casemap:
  96         step->u.casemap = icu_casemap_create(rule[0], status);
  97         break;
  98     case ICU_chain_step_type_transform:
  99         /* rule omitted. Only ID used */
 100         step->u.transform = icu_transform_create((const char *) rule, 'f',
 101                                                  0, status);
 102         break;
 103     case ICU_chain_step_type_tokenize:
 104         step->u.tokenizer = icu_tokenizer_create((char *) chain->locale,
 105                                                  (char) rule[0], status);
 106         break;
 107     case ICU_chain_step_type_transliterate:
 108         /* we pass a dummy ID to utrans_openU.. */
 109         step->u.transform = icu_transform_create("custom", 'f',
 110                                                  (const char *) rule, status);
 111         break;
 112     case YAZ_chain_step_type_stemming:
 113         step->u.stemmer = yaz_stemmer_create((char *) chain->locale, (const char *) rule, status);
 114         break;
 115     default:
 116         break;
 117     }
 118     return step;
 119 }
 120
 121
 122 static void icu_chain_step_destroy(struct icu_chain_step * step)
 123 {
 124     if (!step)
 125         return;
 126
 127     icu_chain_step_destroy(step->previous);
 128
 129     switch (step->type)
 130     {
 131     case ICU_chain_step_type_display:
 132         break;
 133     case ICU_chain_step_type_casemap:
 134         icu_casemap_destroy(step->u.casemap);
 135         break;
 136     case ICU_chain_step_type_transform:
 137     case ICU_chain_step_type_transliterate:
 138         icu_transform_destroy(step->u.transform);
 139         break;
 140     case ICU_chain_step_type_tokenize:
 141         icu_tokenizer_destroy(step->u.tokenizer);
 142         break;
 143     case YAZ_chain_step_type_stemming:
 144         yaz_stemmer_destroy(step->u.stemmer);
 145         break;
 146     default:
 147         break;
 148     }
 149     xfree(step);
 150 }
 151
 152 struct icu_chain_step *icu_chain_step_clone(struct icu_chain_step *old)
 153 {
 154     struct icu_chain_step *step = 0;
 155     struct icu_chain_step **sp = &step;
 156     while (old)
 157     {
 158         *sp = (struct icu_chain_step *) xmalloc(sizeof(**sp));
 159         (*sp)->type = old->type;
 160
 161         switch ((*sp)->type)
 162         {
 163         case ICU_chain_step_type_display:
 164             break;
 165         case ICU_chain_step_type_casemap:
 166             (*sp)->u.casemap = icu_casemap_clone(old->u.casemap);
 167             break;
 168         case ICU_chain_step_type_transform:
 169         case ICU_chain_step_type_transliterate:
 170             (*sp)->u.transform = icu_transform_clone(old->u.transform);
 171             break;
 172         case ICU_chain_step_type_tokenize:
 173             (*sp)->u.tokenizer = icu_tokenizer_clone(old->u.tokenizer);
 174             break;
 175         case YAZ_chain_step_type_stemming:
 176             yaz_stemmer_clone(step->u.stemmer);
 177             break;
 178         case ICU_chain_step_type_none:
 179             break;
 180         }
 181         old = old->previous;
 182         sp = &(*sp)->previous;
 183     }
 184     *sp = 0;
 185     return step;
 186 }
 187
 188 struct icu_chain *icu_chain_create(const char *locale, int sort,
 189                                    UErrorCode * status)
 190 {
 191     struct icu_chain * chain
 192         = (struct icu_chain *) xmalloc(sizeof(*chain));
 193
 194     *status = U_ZERO_ERROR;
 195
 196     chain->iter = 0;
 197     chain->locale = xstrdup(locale);
 198
 199     chain->sort = sort;
 200
 201     chain->coll = ucol_open((const char *) chain->locale, status);
 202
 203     if (U_FAILURE(*status))
 204         return 0;
 205
 206     chain->csteps = 0;
 207
 208     return chain;
 209 }
 210
 211 void icu_chain_destroy(struct icu_chain * chain)
 212 {
 213     if (chain)
 214     {
 215         if (chain->coll)
 216             ucol_close(chain->coll);
 217
 218         if (chain->iter)
 219             icu_iter_destroy(chain->iter);
 220         icu_chain_step_destroy(chain->csteps);
 221         xfree(chain->locale);
 222         xfree(chain);
 223     }
 224 }
 225
 226 static struct icu_chain_step *icu_chain_insert_step(
 227     struct icu_chain * chain, enum icu_chain_step_type type,
 228     const uint8_t * rule, UErrorCode *status);
 229
 230 struct icu_chain * icu_chain_xml_config(const xmlNode *xml_node,
 231                                         int sort,
 232                                         UErrorCode * status)
 233 {
 234     xmlNode *node = 0;
 235     struct icu_chain * chain = 0;
 236
 237     *status = U_ZERO_ERROR;
 238
 239     if (!xml_node ||xml_node->type != XML_ELEMENT_NODE)
 240         return 0;
 241
 242     {
 243         xmlChar * xml_locale = xmlGetProp((xmlNode *) xml_node,
 244                                           (xmlChar *) "locale");
 245
 246         if (xml_locale)
 247         {
 248             chain = icu_chain_create((const char *) xml_locale, sort, status);
 249             xmlFree(xml_locale);
 250         }
 251
 252     }
 253     if (!chain)
 254         return 0;
 255
 256     for (node = xml_node->children; node; node = node->next)
 257     {
 258         xmlChar *xml_rule;
 259         struct icu_chain_step * step = 0;
 260
 261         if (node->type != XML_ELEMENT_NODE)
 262             continue;
 263
 264         xml_rule = xmlGetProp(node, (xmlChar *) "rule");
 265
 266         if (!strcmp((const char *) node->name, "casemap"))
 267             step = icu_chain_insert_step(chain, ICU_chain_step_type_casemap,
 268                                          (const uint8_t *) xml_rule, status);
 269         else if (!strcmp((const char *) node->name, "transform"))
 270             step = icu_chain_insert_step(chain, ICU_chain_step_type_transform,
 271                                          (const uint8_t *) xml_rule, status);
 272         else if (!strcmp((const char *) node->name, "transliterate"))
 273             step = icu_chain_insert_step(chain, ICU_chain_step_type_transliterate,
 274                                          (const uint8_t *) xml_rule, status);
 275         else if (!strcmp((const char *) node->name, "tokenize"))
 276             step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize,
 277                                          (const uint8_t *) xml_rule, status);
 278         else if (!strcmp((const char *) node->name, "display"))
 279             step = icu_chain_insert_step(chain, ICU_chain_step_type_display,
 280                                          (const uint8_t *) "", status);
 281         else if (!strcmp((const char *) node->name, "stemming"))
 282             step = yaz_chain_insert_step(chain, YAZ_chain_step_type_stemming,
 283                                          (const uint8_t *) xml_rule, status);
 284         else if (!strcmp((const char *) node->name, "normalize"))
 285         {
 286             yaz_log(YLOG_WARN, "Element %s is deprecated. "
 287                     "Use transform instead", node->name);
 288             step = icu_chain_insert_step(chain, ICU_chain_step_type_transform,
 289                                          (const uint8_t *) xml_rule, status);
 290         }
 291         else if (!strcmp((const char *) node->name, "index")
 292                  || !strcmp((const char *) node->name, "sortkey"))
 293         {
 294             yaz_log(YLOG_WARN, "Element %s is no longer needed. "
 295                     "Remove it from the configuration", node->name);
 296         }
 297         else
 298         {
 299             yaz_log(YLOG_WARN, "Unknown element %s", node->name);
 300             icu_chain_destroy(chain);
 301             return 0;
 302         }
 303         xmlFree(xml_rule);
 304         if (step && U_FAILURE(*status))
 305         {
 306             icu_chain_destroy(chain);
 307             return 0;
 308         }
 309     }
 310     return chain;
 311 }
 312
 313
 314 static struct icu_chain_step *icu_chain_insert_step(
 315     struct icu_chain * chain, enum icu_chain_step_type type,
 316     const uint8_t * rule, UErrorCode *status)
 317 {
 318     struct icu_chain_step * step = 0;
 319     if (!chain || !type || !rule)
 320         return 0;
 321
 322     /* create actual chain step with this buffer */
 323     step = icu_chain_step_create(chain, type, rule,
 324                                  status);
 325
 326     step->previous = chain->csteps;
 327     chain->csteps = step;
 328
 329     return step;
 330 }
 331
 332 struct icu_iter {
 333     struct icu_chain *chain;
 334     struct icu_buf_utf16 *last;
 335     UErrorCode status;
 336     struct icu_buf_utf8 *display;
 337     struct icu_buf_utf8 *sort8;
 338     struct icu_buf_utf8 *result;
 339     struct icu_buf_utf16 *input;
 340     int token_count;
 341     struct icu_chain_step *steps;
 342 };
 343
 344 void icu_utf16_print(struct icu_buf_utf16 *src16)
 345 {
 346     UErrorCode status = U_ZERO_ERROR;
 347     const char *p;
 348     struct icu_buf_utf8 *dst8 = icu_buf_utf8_create(0);
 349     icu_utf16_to_utf8(dst8, src16, &status);
 350
 351     assert(status != 1234);
 352     if (U_FAILURE(status))
 353     {
 354         printf("failure");
 355     }
 356     else
 357     {
 358         p = icu_buf_utf8_to_cstr(dst8);
 359         printf("%s", p);
 360     }
 361     icu_buf_utf8_destroy(dst8);
 362 }
 363
 364 struct icu_buf_utf16 *icu_iter_invoke(yaz_icu_iter_t iter,
 365                                       struct icu_chain_step *step,
 366                                       struct icu_buf_utf16 *src)
 367 {
 368     if (!step)
 369         return src;
 370     else
 371     {
 372         struct icu_buf_utf16 *dst = icu_iter_invoke(iter, step->previous, src);
 373
 374         switch (step->type)
 375         {
 376         case ICU_chain_step_type_casemap:
 377             if (dst)
 378             {
 379                 struct icu_buf_utf16 *src = dst;
 380
 381                 dst = icu_buf_utf16_create(0);
 382                 icu_casemap_casemap(step->u.casemap, dst, src, &iter->status,
 383                                     iter->chain->locale);
 384                 icu_buf_utf16_destroy(src);
 385             }
 386             break;
 387         case ICU_chain_step_type_tokenize:
 388             if (dst)
 389             {
 390                 struct icu_buf_utf16 *src = dst;
 391
 392                 icu_tokenizer_attach(step->u.tokenizer, src, &iter->status);
 393                 icu_buf_utf16_destroy(src);
 394             }
 395             dst = icu_buf_utf16_create(0);
 396             iter->status = U_ZERO_ERROR;
 397             if (!icu_tokenizer_next_token(step->u.tokenizer, dst, &iter->status))
 398             {
 399                 icu_buf_utf16_destroy(dst);
 400                 dst = 0;
 401             }
 402             break;
 403         case ICU_chain_step_type_transform:
 404         case ICU_chain_step_type_transliterate:
 405             if (dst)
 406             {
 407                 struct icu_buf_utf16 *src = dst;
 408                 dst = icu_buf_utf16_create(0);
 409                 icu_transform_trans(step->u.transform, dst, src, &iter->status);
 410                 icu_buf_utf16_destroy(src);
 411             }
 412             break;
 413         case ICU_chain_step_type_display:
 414             if (dst)
 415                 icu_utf16_to_utf8(iter->display, dst, &iter->status);
 416             break;
 417         case YAZ_chain_step_type_stemming:
 418             if (dst)
 419             {
 420                 struct icu_buf_utf16 *src = dst;
 421                 dst = icu_buf_utf16_create(0);
 422                 yaz_stemmer_stem(step->u.stemmer, dst, src, &iter->status);
 423                 icu_buf_utf16_destroy(src);
 424             }
 425             break;
 426         default:
 427             assert(0);
 428         }
 429         return dst;
 430     }
 431 }
 432
 433 yaz_icu_iter_t icu_iter_create(struct icu_chain *chain)
 434 {
 435     yaz_icu_iter_t iter = xmalloc(sizeof(*iter));
 436     iter->chain = chain;
 437     iter->status = U_ZERO_ERROR;
 438     iter->display = icu_buf_utf8_create(0);
 439     iter->sort8 = icu_buf_utf8_create(0);
 440     iter->result = icu_buf_utf8_create(0);
 441     iter->last = 0; /* no last returned string (yet) */
 442     iter->steps = icu_chain_step_clone(chain->csteps);
 443     iter->input = 0;
 444
 445     return iter;
 446 }
 447
 448 void icu_iter_first(yaz_icu_iter_t iter, const char *src8cstr)
 449 {
 450     if (iter->input)
 451         icu_buf_utf16_destroy(iter->input);
 452     iter->input = icu_buf_utf16_create(0);
 453     iter->token_count = 0;
 454     /* fill and assign input string.. It will be 0 after
 455        first iteration */
 456     icu_utf16_from_utf8_cstr(iter->input, src8cstr, &iter->status);
 457 }
 458
 459 void icu_iter_destroy(yaz_icu_iter_t iter)
 460 {
 461     if (iter)
 462     {
 463         icu_buf_utf8_destroy(iter->display);
 464         icu_buf_utf8_destroy(iter->sort8);
 465         icu_buf_utf8_destroy(iter->result);
 466         if (iter->input)
 467             icu_buf_utf16_destroy(iter->input);
 468         icu_chain_step_destroy(iter->steps);
 469         xfree(iter);
 470     }
 471 }
 472
 473 int icu_iter_next(yaz_icu_iter_t iter)
 474 {
 475     if (!iter->input && iter->last == 0)
 476         return 0;
 477     else
 478     {
 479         /* on first call, iter->input is the input string. Thereafter: 0. */
 480         iter->last = icu_iter_invoke(iter, iter->steps ?
 481                                      iter->steps : iter->chain->csteps,
 482                                      iter->input);
 483         iter->input = 0;
 484
 485         if (!iter->last)
 486             return 0;
 487
 488         iter->token_count++;
 489
 490         if (iter->chain->sort)
 491         {
 492             icu_sortkey8_from_utf16(iter->chain->coll,
 493                                     iter->sort8, iter->last,
 494                                     &iter->status);
 495         }
 496         icu_utf16_to_utf8(iter->result, iter->last, &iter->status);
 497         icu_buf_utf16_destroy(iter->last);
 498
 499         return 1;
 500     }
 501 }
 502
 503 const char *icu_iter_get_norm(yaz_icu_iter_t iter)
 504 {
 505     return icu_buf_utf8_to_cstr(iter->result);
 506 }
 507
 508 const char *icu_iter_get_sortkey(yaz_icu_iter_t iter)
 509 {
 510     return icu_buf_utf8_to_cstr(iter->sort8);
 511 }
 512
 513 const char *icu_iter_get_display(yaz_icu_iter_t iter)
 514 {
 515     return icu_buf_utf8_to_cstr(iter->display);
 516 }
 517
 518 int icu_iter_get_token_number(yaz_icu_iter_t iter)
 519 {
 520     return iter->token_count;
 521 }
 522
 523 int icu_chain_assign_cstr(struct icu_chain * chain, const char * src8cstr,
 524                           UErrorCode *status)
 525 {
 526     if (chain->iter)
 527         icu_iter_destroy(chain->iter);
 528     chain->iter = icu_iter_create(chain);
 529     icu_iter_first(chain->iter, src8cstr);
 530     return 1;
 531 }
 532
 533 int icu_chain_next_token(struct icu_chain * chain, UErrorCode *status)
 534 {
 535     *status = U_ZERO_ERROR;
 536     return icu_iter_next(chain->iter);
 537 }
 538
 539 int icu_chain_token_number(struct icu_chain * chain)
 540 {
 541     if (chain && chain->iter)
 542         return chain->iter->token_count;
 543     return 0;
 544 }
 545
 546 const char * icu_chain_token_display(struct icu_chain * chain)
 547 {
 548     if (chain->iter)
 549         return icu_iter_get_display(chain->iter);
 550     return 0;
 551 }
 552
 553 const char * icu_chain_token_norm(struct icu_chain * chain)
 554 {
 555     if (chain->iter)
 556         return icu_iter_get_norm(chain->iter);
 557     return 0;
 558 }
 559
 560 const char * icu_chain_token_sortkey(struct icu_chain * chain)
 561 {
 562     if (chain->iter)
 563         return icu_iter_get_sortkey(chain->iter);
 564     return 0;
 565 }
 566
 567 #endif /* YAZ_HAVE_ICU */
 568
 569 /*
 570  * Local variables:
 571  * c-basic-offset: 4
 572  * c-file-style: "Stroustrup"
 573  * indent-tabs-mode: nil
 574  * End:
 575  * vim: shiftwidth=4 tabstop=8 expandtab
 576  */
 577