src/icu_chain.c

   1 /* This file is part of the YAZ toolkit.
   2  * Copyright (C) 1995-2011 Index Data
   3  * See the file LICENSE for details.
   4  */
   5
   6 /**
   7  * \file
   8  * \brief ICU chain
   9  */
  10
  11 #if HAVE_CONFIG_H
  12 #include "config.h"
  13 #endif
  14
  15 #if YAZ_HAVE_ICU
  16 #include <yaz/xmalloc.h>
  17
  18 #include <yaz/icu_I18N.h>
  19
  20 #include <yaz/stemmer.h>
  21
  22 #include <yaz/log.h>
  23 #include <yaz/nmem.h>
  24 #include <yaz/nmem_xml.h>
  25 #include <string.h>
  26 #include <stdlib.h>
  27 #include <stdio.h>
  28 #include <assert.h>
  29
  30 #include <unicode/ustring.h>  /* some more string fcns*/
  31 #include <unicode/uchar.h>    /* char names           */
  32
  33 enum icu_chain_step_type {
  34     ICU_chain_step_type_none,
  35     ICU_chain_step_type_display,        /* convert to utf8 display format */
  36     ICU_chain_step_type_casemap,        /* apply utf16 charmap */
  37     ICU_chain_step_type_transform,      /* apply utf16 transform */
  38     ICU_chain_step_type_tokenize,       /* apply utf16 tokenization */
  39     ICU_chain_step_type_transliterate,  /* apply utf16 tokenization */
  40     YAZ_chain_step_type_stemming        /* apply utf16 stemming (YAZ) */
  41 };
  42
  43 struct icu_chain_step
  44 {
  45     /* type and action object */
  46     enum icu_chain_step_type type;
  47     union {
  48         struct icu_casemap   *casemap;
  49         struct icu_transform *transform;
  50         struct icu_tokenizer *tokenizer;
  51         yaz_stemmer_p         stemmer;
  52     } u;
  53     struct icu_chain_step *previous;
  54 };
  55
  56 struct icu_chain
  57 {
  58     yaz_icu_iter_t iter;
  59     char *locale;
  60     int sort;
  61
  62     UCollator *coll;
  63
  64     /* linked list of chain steps */
  65     struct icu_chain_step *csteps;
  66 };
  67
  68 int icu_check_status(UErrorCode status)
  69 {
  70     if (U_FAILURE(status))
  71     {
  72         yaz_log(YLOG_WARN, "ICU: %d %s\n", status, u_errorName(status));
  73         return 0;
  74     }
  75     return 1;
  76 }
  77
  78 static struct icu_chain_step *icu_chain_insert_step(
  79     struct icu_chain *chain, enum icu_chain_step_type type,
  80     const uint8_t *rule, UErrorCode *status)
  81 {
  82     struct icu_chain_step *step = 0;
  83
  84     if (!chain || !type || !rule)
  85         return 0;
  86
  87     step = (struct icu_chain_step *) xmalloc(sizeof(*step));
  88
  89     step->type = type;
  90     /* create auxilary objects */
  91     switch (step->type)
  92     {
  93     case ICU_chain_step_type_display:
  94         break;
  95     case ICU_chain_step_type_casemap:
  96         step->u.casemap = icu_casemap_create(rule[0], status);
  97         break;
  98     case ICU_chain_step_type_transform:
  99         /* rule omitted. Only ID used */
 100         step->u.transform = icu_transform_create((const char *) rule, 'f',
 101                                                  0, status);
 102         break;
 103     case ICU_chain_step_type_tokenize:
 104         step->u.tokenizer = icu_tokenizer_create(chain->locale,
 105                                                  (char) rule[0], status);
 106         break;
 107     case ICU_chain_step_type_transliterate:
 108         /* we pass a dummy ID to utrans_openU.. */
 109         step->u.transform = icu_transform_create("custom", 'f',
 110                                                  (const char *) rule, status);
 111         break;
 112     case YAZ_chain_step_type_stemming:
 113         step->u.stemmer = yaz_stemmer_create(chain->locale,
 114                                              (const char *) rule, status);
 115         break;
 116     default:
 117         break;
 118     }
 119     step->previous = chain->csteps;
 120     chain->csteps = step;
 121
 122     return step;
 123 }
 124
 125
 126 static void icu_chain_step_destroy(struct icu_chain_step *step)
 127 {
 128     if (!step)
 129         return;
 130
 131     icu_chain_step_destroy(step->previous);
 132
 133     switch (step->type)
 134     {
 135     case ICU_chain_step_type_display:
 136         break;
 137     case ICU_chain_step_type_casemap:
 138         icu_casemap_destroy(step->u.casemap);
 139         break;
 140     case ICU_chain_step_type_transform:
 141     case ICU_chain_step_type_transliterate:
 142         icu_transform_destroy(step->u.transform);
 143         break;
 144     case ICU_chain_step_type_tokenize:
 145         icu_tokenizer_destroy(step->u.tokenizer);
 146         break;
 147     case YAZ_chain_step_type_stemming:
 148         yaz_stemmer_destroy(step->u.stemmer);
 149         break;
 150     default:
 151         break;
 152     }
 153     xfree(step);
 154 }
 155
 156 struct icu_chain_step *icu_chain_step_clone(struct icu_chain_step *old)
 157 {
 158     struct icu_chain_step *step = 0;
 159     struct icu_chain_step **sp = &step;
 160     while (old)
 161     {
 162         *sp = (struct icu_chain_step *) xmalloc(sizeof(**sp));
 163         (*sp)->type = old->type;
 164
 165         switch ((*sp)->type)
 166         {
 167         case ICU_chain_step_type_display:
 168             break;
 169         case ICU_chain_step_type_casemap:
 170             (*sp)->u.casemap = icu_casemap_clone(old->u.casemap);
 171             break;
 172         case ICU_chain_step_type_transform:
 173         case ICU_chain_step_type_transliterate:
 174             (*sp)->u.transform = icu_transform_clone(old->u.transform);
 175             break;
 176         case ICU_chain_step_type_tokenize:
 177             (*sp)->u.tokenizer = icu_tokenizer_clone(old->u.tokenizer);
 178             break;
 179         case YAZ_chain_step_type_stemming:
 180             (*sp)->u.stemmer = yaz_stemmer_clone(old->u.stemmer);
 181             break;
 182         case ICU_chain_step_type_none:
 183             break;
 184         }
 185         old = old->previous;
 186         sp = &(*sp)->previous;
 187     }
 188     *sp = 0;
 189     return step;
 190 }
 191
 192 struct icu_chain *icu_chain_create(const char *locale, int sort,
 193                                    UErrorCode *status)
 194 {
 195     struct icu_chain *chain
 196         = (struct icu_chain *) xmalloc(sizeof(*chain));
 197
 198     *status = U_ZERO_ERROR;
 199
 200     chain->iter = 0;
 201     chain->locale = xstrdup(locale);
 202
 203     chain->sort = sort;
 204
 205     chain->coll = ucol_open(chain->locale, status);
 206
 207     if (U_FAILURE(*status))
 208         return 0;
 209
 210     chain->csteps = 0;
 211
 212     return chain;
 213 }
 214
 215 void icu_chain_destroy(struct icu_chain *chain)
 216 {
 217     if (chain)
 218     {
 219         if (chain->coll)
 220             ucol_close(chain->coll);
 221
 222         if (chain->iter)
 223             icu_iter_destroy(chain->iter);
 224         icu_chain_step_destroy(chain->csteps);
 225         xfree(chain->locale);
 226         xfree(chain);
 227     }
 228 }
 229
 230 struct icu_chain *icu_chain_xml_config(const xmlNode *xml_node,
 231                                        int sort,
 232                                        UErrorCode *status)
 233 {
 234     xmlNode *node = 0;
 235     int no_errors = 0;
 236     struct icu_chain *chain = 0;
 237     NMEM nmem = 0;
 238
 239     *status = U_ZERO_ERROR;
 240
 241     if (xml_node && xml_node->type == XML_ELEMENT_NODE)
 242     {
 243         xmlChar *xml_locale = xmlGetProp((xmlNode *) xml_node,
 244                                          (xmlChar *) "locale");
 245         if (xml_locale)
 246         {
 247             chain = icu_chain_create((const char *) xml_locale, sort, status);
 248             xmlFree(xml_locale);
 249         }
 250     }
 251
 252     if (!chain)
 253         return 0;
 254
 255     nmem = nmem_create();
 256     for (node = xml_node->children; node; node = node->next)
 257     {
 258         char *rule = 0;
 259         struct icu_chain_step *step = 0;
 260         struct _xmlAttr *attr;
 261
 262         nmem_reset(nmem);
 263         if (node->type != XML_ELEMENT_NODE)
 264             continue;
 265
 266         for (attr = node->properties; attr; attr = attr->next)
 267         {
 268             if (!strcmp((const char *) attr->name, "rule"))
 269             {
 270                 rule = nmem_text_node_cdata(attr->children, nmem);
 271             }
 272             else
 273             {
 274                 yaz_log(YLOG_WARN, "Unsupported attribute '%s' for "
 275                         "element '%s'", attr->name, node->name);
 276                 no_errors++;
 277                 continue;
 278             }
 279         }
 280         if (!rule && node->children)
 281             rule = nmem_text_node_cdata(node->children, nmem);
 282
 283         if (!strcmp((const char *) node->name, "casemap"))
 284             step = icu_chain_insert_step(chain, ICU_chain_step_type_casemap,
 285                                          (const uint8_t *) rule, status);
 286         else if (!strcmp((const char *) node->name, "transform"))
 287             step = icu_chain_insert_step(chain, ICU_chain_step_type_transform,
 288                                          (const uint8_t *) rule, status);
 289         else if (!strcmp((const char *) node->name, "transliterate"))
 290             step = icu_chain_insert_step(chain, ICU_chain_step_type_transliterate,
 291                                          (const uint8_t *) rule, status);
 292         else if (!strcmp((const char *) node->name, "tokenize"))
 293             step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize,
 294                                          (const uint8_t *) rule, status);
 295         else if (!strcmp((const char *) node->name, "display"))
 296             step = icu_chain_insert_step(chain, ICU_chain_step_type_display,
 297                                          (const uint8_t *) "", status);
 298         else if (!strcmp((const char *) node->name, "stemming"))
 299             step = icu_chain_insert_step(chain, YAZ_chain_step_type_stemming,
 300                                          (const uint8_t *) rule, status);
 301         else if (!strcmp((const char *) node->name, "normalize"))
 302         {
 303             yaz_log(YLOG_WARN, "Element %s is deprecated. "
 304                     "Use transform instead", node->name);
 305             step = icu_chain_insert_step(chain, ICU_chain_step_type_transform,
 306                                          (const uint8_t *) rule, status);
 307         }
 308         else if (!strcmp((const char *) node->name, "index")
 309                  || !strcmp((const char *) node->name, "sortkey"))
 310         {
 311             yaz_log(YLOG_WARN, "Element %s is no longer needed. "
 312                     "Remove it from the configuration", node->name);
 313         }
 314         else
 315         {
 316             yaz_log(YLOG_WARN, "Unknown element %s", node->name);
 317             no_errors++;
 318             continue;
 319         }
 320         if (step && U_FAILURE(*status))
 321         {
 322             no_errors++;
 323             break;
 324         }
 325     }
 326     nmem_destroy(nmem);
 327     if (no_errors)
 328     {
 329         icu_chain_destroy(chain);
 330         return 0;
 331     }
 332     return chain;
 333 }
 334
 335 struct icu_iter {
 336     struct icu_chain *chain;
 337     struct icu_buf_utf16 *last;
 338     UErrorCode status;
 339     struct icu_buf_utf8 *display;
 340     struct icu_buf_utf8 *sort8;
 341     struct icu_buf_utf8 *result;
 342     struct icu_buf_utf16 *input;
 343     int token_count;
 344     struct icu_chain_step *steps;
 345 };
 346
 347 void icu_utf16_print(struct icu_buf_utf16 *src16)
 348 {
 349     UErrorCode status = U_ZERO_ERROR;
 350     const char *p;
 351     struct icu_buf_utf8 *dst8 = icu_buf_utf8_create(0);
 352     icu_utf16_to_utf8(dst8, src16, &status);
 353
 354     assert(status != 1234);
 355     if (U_FAILURE(status))
 356     {
 357         printf("failure");
 358     }
 359     else
 360     {
 361         p = icu_buf_utf8_to_cstr(dst8);
 362         printf("%s", p);
 363     }
 364     icu_buf_utf8_destroy(dst8);
 365 }
 366
 367 struct icu_buf_utf16 *icu_iter_invoke(yaz_icu_iter_t iter,
 368                                       struct icu_chain_step *step,
 369                                       struct icu_buf_utf16 *src)
 370 {
 371     if (!step)
 372         return src;
 373     else
 374     {
 375         struct icu_buf_utf16 *dst = icu_iter_invoke(iter, step->previous, src);
 376
 377         switch (step->type)
 378         {
 379         case ICU_chain_step_type_casemap:
 380             if (dst)
 381             {
 382                 struct icu_buf_utf16 *src = dst;
 383
 384                 dst = icu_buf_utf16_create(0);
 385                 icu_casemap_casemap(step->u.casemap, dst, src, &iter->status,
 386                                     iter->chain->locale);
 387                 icu_buf_utf16_destroy(src);
 388             }
 389             break;
 390         case ICU_chain_step_type_tokenize:
 391             if (dst)
 392             {
 393                 struct icu_buf_utf16 *src = dst;
 394
 395                 icu_tokenizer_attach(step->u.tokenizer, src, &iter->status);
 396                 icu_buf_utf16_destroy(src);
 397             }
 398             dst = icu_buf_utf16_create(0);
 399             iter->status = U_ZERO_ERROR;
 400             if (!icu_tokenizer_next_token(step->u.tokenizer, dst, &iter->status))
 401             {
 402                 icu_buf_utf16_destroy(dst);
 403                 dst = 0;
 404             }
 405             break;
 406         case ICU_chain_step_type_transform:
 407         case ICU_chain_step_type_transliterate:
 408             if (dst)
 409             {
 410                 struct icu_buf_utf16 *src = dst;
 411                 dst = icu_buf_utf16_create(0);
 412                 icu_transform_trans(step->u.transform, dst, src, &iter->status);
 413                 icu_buf_utf16_destroy(src);
 414             }
 415             break;
 416         case ICU_chain_step_type_display:
 417             if (dst)
 418                 icu_utf16_to_utf8(iter->display, dst, &iter->status);
 419             break;
 420         case YAZ_chain_step_type_stemming:
 421             if (dst)
 422             {
 423                 struct icu_buf_utf16 *src = dst;
 424                 dst = icu_buf_utf16_create(0);
 425                 yaz_stemmer_stem(step->u.stemmer, dst, src, &iter->status);
 426                 icu_buf_utf16_destroy(src);
 427             }
 428             break;
 429         default:
 430             assert(0);
 431         }
 432         return dst;
 433     }
 434 }
 435
 436 yaz_icu_iter_t icu_iter_create(struct icu_chain *chain)
 437 {
 438     yaz_icu_iter_t iter = xmalloc(sizeof(*iter));
 439     iter->chain = chain;
 440     iter->status = U_ZERO_ERROR;
 441     iter->display = icu_buf_utf8_create(0);
 442     iter->sort8 = icu_buf_utf8_create(0);
 443     iter->result = icu_buf_utf8_create(0);
 444     iter->last = 0; /* no last returned string (yet) */
 445     iter->steps = icu_chain_step_clone(chain->csteps);
 446     iter->input = 0;
 447
 448     return iter;
 449 }
 450
 451 void icu_iter_first(yaz_icu_iter_t iter, const char *src8cstr)
 452 {
 453     if (iter->input)
 454         icu_buf_utf16_destroy(iter->input);
 455     iter->input = icu_buf_utf16_create(0);
 456     iter->token_count = 0;
 457     /* fill and assign input string.. It will be 0 after
 458        first iteration */
 459     icu_utf16_from_utf8_cstr(iter->input, src8cstr, &iter->status);
 460 }
 461
 462 void icu_iter_destroy(yaz_icu_iter_t iter)
 463 {
 464     if (iter)
 465     {
 466         icu_buf_utf8_destroy(iter->display);
 467         icu_buf_utf8_destroy(iter->sort8);
 468         icu_buf_utf8_destroy(iter->result);
 469         if (iter->input)
 470             icu_buf_utf16_destroy(iter->input);
 471         icu_chain_step_destroy(iter->steps);
 472         xfree(iter);
 473     }
 474 }
 475
 476 int icu_iter_next(yaz_icu_iter_t iter)
 477 {
 478     if (!iter->input && iter->last == 0)
 479         return 0;
 480     else
 481     {
 482         /* on first call, iter->input is the input string. Thereafter: 0. */
 483         iter->last = icu_iter_invoke(iter, iter->steps ?
 484                                      iter->steps : iter->chain->csteps,
 485                                      iter->input);
 486         iter->input = 0;
 487
 488         if (!iter->last)
 489             return 0;
 490
 491         iter->token_count++;
 492
 493         if (iter->chain->sort)
 494         {
 495             icu_sortkey8_from_utf16(iter->chain->coll,
 496                                     iter->sort8, iter->last,
 497                                     &iter->status);
 498         }
 499         icu_utf16_to_utf8(iter->result, iter->last, &iter->status);
 500         icu_buf_utf16_destroy(iter->last);
 501
 502         return 1;
 503     }
 504 }
 505
 506 const char *icu_iter_get_norm(yaz_icu_iter_t iter)
 507 {
 508     return icu_buf_utf8_to_cstr(iter->result);
 509 }
 510
 511 const char *icu_iter_get_sortkey(yaz_icu_iter_t iter)
 512 {
 513     return icu_buf_utf8_to_cstr(iter->sort8);
 514 }
 515
 516 const char *icu_iter_get_display(yaz_icu_iter_t iter)
 517 {
 518     return icu_buf_utf8_to_cstr(iter->display);
 519 }
 520
 521 int icu_iter_get_token_number(yaz_icu_iter_t iter)
 522 {
 523     return iter->token_count;
 524 }
 525
 526 int icu_chain_assign_cstr(struct icu_chain *chain, const char *src8cstr,
 527                           UErrorCode *status)
 528 {
 529     if (chain->iter)
 530         icu_iter_destroy(chain->iter);
 531     chain->iter = icu_iter_create(chain);
 532     icu_iter_first(chain->iter, src8cstr);
 533     return 1;
 534 }
 535
 536 int icu_chain_next_token(struct icu_chain *chain, UErrorCode *status)
 537 {
 538     *status = U_ZERO_ERROR;
 539     return icu_iter_next(chain->iter);
 540 }
 541
 542 int icu_chain_token_number(struct icu_chain *chain)
 543 {
 544     if (chain && chain->iter)
 545         return chain->iter->token_count;
 546     return 0;
 547 }
 548
 549 const char *icu_chain_token_display(struct icu_chain *chain)
 550 {
 551     if (chain->iter)
 552         return icu_iter_get_display(chain->iter);
 553     return 0;
 554 }
 555
 556 const char *icu_chain_token_norm(struct icu_chain *chain)
 557 {
 558     if (chain->iter)
 559         return icu_iter_get_norm(chain->iter);
 560     return 0;
 561 }
 562
 563 const char *icu_chain_token_sortkey(struct icu_chain *chain)
 564 {
 565     if (chain->iter)
 566         return icu_iter_get_sortkey(chain->iter);
 567     return 0;
 568 }
 569
 570 #endif /* YAZ_HAVE_ICU */
 571
 572 /*
 573  * Local variables:
 574  * c-basic-offset: 4
 575  * c-file-style: "Stroustrup"
 576  * indent-tabs-mode: nil
 577  * End:
 578  * vim: shiftwidth=4 tabstop=8 expandtab
 579  */
 580