src/icu_chain.c

   1 /* This file is part of the YAZ toolkit.
   2  * Copyright (C) 1995-2011 Index Data
   3  * See the file LICENSE for details.
   4  */
   5
   6 /**
   7  * \file
   8  * \brief ICU chain
   9  */
  10
  11 #if HAVE_CONFIG_H
  12 #include "config.h"
  13 #endif
  14
  15 #if YAZ_HAVE_ICU
  16 #include <yaz/xmalloc.h>
  17
  18 #include <yaz/icu_I18N.h>
  19
  20 #include <yaz/stemmer.h>
  21
  22 #include <yaz/log.h>
  23 #include <yaz/nmem.h>
  24 #include <yaz/nmem_xml.h>
  25 #include <string.h>
  26 #include <stdlib.h>
  27 #include <stdio.h>
  28 #include <assert.h>
  29
  30 #include <unicode/ustring.h>  /* some more string fcns*/
  31 #include <unicode/uchar.h>    /* char names           */
  32
  33 enum icu_chain_step_type {
  34     ICU_chain_step_type_none,
  35     ICU_chain_step_type_display,        /* convert to utf8 display format */
  36     ICU_chain_step_type_casemap,        /* apply utf16 charmap */
  37     ICU_chain_step_type_transform,      /* apply utf16 transform */
  38     ICU_chain_step_type_tokenize,       /* apply utf16 tokenization */
  39     ICU_chain_step_type_transliterate,  /* apply utf16 tokenization */
  40     YAZ_chain_step_type_stemming        /* apply utf16 stemming (YAZ) */
  41 };
  42
  43 struct icu_chain_step
  44 {
  45     /* type and action object */
  46     enum icu_chain_step_type type;
  47     union {
  48         struct icu_casemap   *casemap;
  49         struct icu_transform *transform;
  50         struct icu_tokenizer *tokenizer;
  51         yaz_stemmer_p         stemmer;
  52     } u;
  53     struct icu_chain_step *previous;
  54 };
  55
  56 struct icu_chain
  57 {
  58     yaz_icu_iter_t iter;
  59     char *locale;
  60     int sort;
  61
  62     UCollator *coll;
  63
  64     /* linked list of chain steps */
  65     struct icu_chain_step *csteps;
  66 };
  67
  68 int icu_check_status(UErrorCode status)
  69 {
  70     if (U_FAILURE(status))
  71     {
  72         yaz_log(YLOG_WARN, "ICU: %d %s\n", status, u_errorName(status));
  73         return 0;
  74     }
  75     return 1;
  76 }
  77
  78 static struct icu_chain_step *icu_chain_insert_step(
  79     struct icu_chain *chain, enum icu_chain_step_type type,
  80     const uint8_t *rule, UErrorCode *status)
  81 {
  82     struct icu_chain_step *step = 0;
  83
  84     if (!chain || !type || !rule)
  85         return 0;
  86
  87     step = (struct icu_chain_step *) xmalloc(sizeof(*step));
  88
  89     step->type = type;
  90     /* create auxilary objects */
  91     switch (step->type)
  92     {
  93     case ICU_chain_step_type_display:
  94         break;
  95     case ICU_chain_step_type_casemap:
  96         step->u.casemap = icu_casemap_create(rule[0], status);
  97         break;
  98     case ICU_chain_step_type_transform:
  99         /* rule omitted. Only ID used */
 100         step->u.transform = icu_transform_create((const char *) rule, 'f',
 101                                                  0, status);
 102         break;
 103     case ICU_chain_step_type_tokenize:
 104         step->u.tokenizer = icu_tokenizer_create((char *) chain->locale,
 105                                                  (char) rule[0], status);
 106         break;
 107     case ICU_chain_step_type_transliterate:
 108         /* we pass a dummy ID to utrans_openU.. */
 109         step->u.transform = icu_transform_create("custom", 'f',
 110                                                  (const char *) rule, status);
 111         break;
 112     case YAZ_chain_step_type_stemming:
 113         step->u.stemmer = yaz_stemmer_create((char *) chain->locale, (const char *) rule, status);
 114         break;
 115     default:
 116         break;
 117     }
 118     step->previous = chain->csteps;
 119     chain->csteps = step;
 120
 121     return step;
 122 }
 123
 124
 125 static void icu_chain_step_destroy(struct icu_chain_step *step)
 126 {
 127     if (!step)
 128         return;
 129
 130     icu_chain_step_destroy(step->previous);
 131
 132     switch (step->type)
 133     {
 134     case ICU_chain_step_type_display:
 135         break;
 136     case ICU_chain_step_type_casemap:
 137         icu_casemap_destroy(step->u.casemap);
 138         break;
 139     case ICU_chain_step_type_transform:
 140     case ICU_chain_step_type_transliterate:
 141         icu_transform_destroy(step->u.transform);
 142         break;
 143     case ICU_chain_step_type_tokenize:
 144         icu_tokenizer_destroy(step->u.tokenizer);
 145         break;
 146     case YAZ_chain_step_type_stemming:
 147         yaz_stemmer_destroy(step->u.stemmer);
 148         break;
 149     default:
 150         break;
 151     }
 152     xfree(step);
 153 }
 154
 155 struct icu_chain_step *icu_chain_step_clone(struct icu_chain_step *old)
 156 {
 157     struct icu_chain_step *step = 0;
 158     struct icu_chain_step **sp = &step;
 159     while (old)
 160     {
 161         *sp = (struct icu_chain_step *) xmalloc(sizeof(**sp));
 162         (*sp)->type = old->type;
 163
 164         switch ((*sp)->type)
 165         {
 166         case ICU_chain_step_type_display:
 167             break;
 168         case ICU_chain_step_type_casemap:
 169             (*sp)->u.casemap = icu_casemap_clone(old->u.casemap);
 170             break;
 171         case ICU_chain_step_type_transform:
 172         case ICU_chain_step_type_transliterate:
 173             (*sp)->u.transform = icu_transform_clone(old->u.transform);
 174             break;
 175         case ICU_chain_step_type_tokenize:
 176             (*sp)->u.tokenizer = icu_tokenizer_clone(old->u.tokenizer);
 177             break;
 178         case YAZ_chain_step_type_stemming:
 179             (*sp)->u.stemmer = yaz_stemmer_clone(old->u.stemmer);
 180             break;
 181         case ICU_chain_step_type_none:
 182             break;
 183         }
 184         old = old->previous;
 185         sp = &(*sp)->previous;
 186     }
 187     *sp = 0;
 188     return step;
 189 }
 190
 191 struct icu_chain *icu_chain_create(const char *locale, int sort,
 192                                    UErrorCode *status)
 193 {
 194     struct icu_chain *chain
 195         = (struct icu_chain *) xmalloc(sizeof(*chain));
 196
 197     *status = U_ZERO_ERROR;
 198
 199     chain->iter = 0;
 200     chain->locale = xstrdup(locale);
 201
 202     chain->sort = sort;
 203
 204     chain->coll = ucol_open((const char *) chain->locale, status);
 205
 206     if (U_FAILURE(*status))
 207         return 0;
 208
 209     chain->csteps = 0;
 210
 211     return chain;
 212 }
 213
 214 void icu_chain_destroy(struct icu_chain *chain)
 215 {
 216     if (chain)
 217     {
 218         if (chain->coll)
 219             ucol_close(chain->coll);
 220
 221         if (chain->iter)
 222             icu_iter_destroy(chain->iter);
 223         icu_chain_step_destroy(chain->csteps);
 224         xfree(chain->locale);
 225         xfree(chain);
 226     }
 227 }
 228
 229 static struct icu_chain_step *icu_chain_insert_step(
 230     struct icu_chain *chain, enum icu_chain_step_type type,
 231     const uint8_t *rule, UErrorCode *status);
 232
 233 struct icu_chain *icu_chain_xml_config(const xmlNode *xml_node,
 234                                        int sort,
 235                                        UErrorCode *status)
 236 {
 237     xmlNode *node = 0;
 238     int no_errors = 0;
 239     struct icu_chain *chain = 0;
 240     NMEM nmem = 0;
 241
 242     *status = U_ZERO_ERROR;
 243
 244     if (!xml_node ||xml_node->type != XML_ELEMENT_NODE)
 245         return 0;
 246
 247     {
 248         xmlChar *xml_locale = xmlGetProp((xmlNode *) xml_node,
 249                                          (xmlChar *) "locale");
 250
 251         if (xml_locale)
 252         {
 253             chain = icu_chain_create((const char *) xml_locale, sort, status);
 254             xmlFree(xml_locale);
 255         }
 256
 257     }
 258     if (!chain)
 259         return 0;
 260
 261     nmem = nmem_create();
 262     for (node = xml_node->children; node; node = node->next)
 263     {
 264         char *rule = 0;
 265         struct icu_chain_step *step = 0;
 266         struct _xmlAttr *attr;
 267
 268         nmem_reset(nmem);
 269         if (node->type != XML_ELEMENT_NODE)
 270             continue;
 271
 272         for (attr = node->properties; attr; attr = attr->next)
 273         {
 274             if (!strcmp((const char *) attr->name, "rule"))
 275             {
 276                 rule = nmem_text_node_cdata(attr->children, nmem);
 277             }
 278             else
 279             {
 280                 yaz_log(YLOG_WARN, "Unsupported attribute '%s' for "
 281                         "element '%s'", attr->name, node->name);
 282                 no_errors++;
 283                 continue;
 284             }
 285         }
 286         if (!rule && node->children)
 287             rule = nmem_text_node_cdata(node->children, nmem);
 288
 289         if (!strcmp((const char *) node->name, "casemap"))
 290             step = icu_chain_insert_step(chain, ICU_chain_step_type_casemap,
 291                                          (const uint8_t *) rule, status);
 292         else if (!strcmp((const char *) node->name, "transform"))
 293             step = icu_chain_insert_step(chain, ICU_chain_step_type_transform,
 294                                          (const uint8_t *) rule, status);
 295         else if (!strcmp((const char *) node->name, "transliterate"))
 296             step = icu_chain_insert_step(chain, ICU_chain_step_type_transliterate,
 297                                          (const uint8_t *) rule, status);
 298         else if (!strcmp((const char *) node->name, "tokenize"))
 299             step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize,
 300                                          (const uint8_t *) rule, status);
 301         else if (!strcmp((const char *) node->name, "display"))
 302             step = icu_chain_insert_step(chain, ICU_chain_step_type_display,
 303                                          (const uint8_t *) "", status);
 304         else if (!strcmp((const char *) node->name, "stemming"))
 305             step = icu_chain_insert_step(chain, YAZ_chain_step_type_stemming,
 306                                          (const uint8_t *) rule, status);
 307         else if (!strcmp((const char *) node->name, "normalize"))
 308         {
 309             yaz_log(YLOG_WARN, "Element %s is deprecated. "
 310                     "Use transform instead", node->name);
 311             step = icu_chain_insert_step(chain, ICU_chain_step_type_transform,
 312                                          (const uint8_t *) rule, status);
 313         }
 314         else if (!strcmp((const char *) node->name, "index")
 315                  || !strcmp((const char *) node->name, "sortkey"))
 316         {
 317             yaz_log(YLOG_WARN, "Element %s is no longer needed. "
 318                     "Remove it from the configuration", node->name);
 319         }
 320         else
 321         {
 322             yaz_log(YLOG_WARN, "Unknown element %s", node->name);
 323             no_errors++;
 324             continue;
 325         }
 326         if (step && U_FAILURE(*status))
 327         {
 328             no_errors++;
 329             break;
 330         }
 331     }
 332     nmem_destroy(nmem);
 333     if (no_errors)
 334     {
 335         icu_chain_destroy(chain);
 336         return 0;
 337     }
 338     return chain;
 339 }
 340
 341 struct icu_iter {
 342     struct icu_chain *chain;
 343     struct icu_buf_utf16 *last;
 344     UErrorCode status;
 345     struct icu_buf_utf8 *display;
 346     struct icu_buf_utf8 *sort8;
 347     struct icu_buf_utf8 *result;
 348     struct icu_buf_utf16 *input;
 349     int token_count;
 350     struct icu_chain_step *steps;
 351 };
 352
 353 void icu_utf16_print(struct icu_buf_utf16 *src16)
 354 {
 355     UErrorCode status = U_ZERO_ERROR;
 356     const char *p;
 357     struct icu_buf_utf8 *dst8 = icu_buf_utf8_create(0);
 358     icu_utf16_to_utf8(dst8, src16, &status);
 359
 360     assert(status != 1234);
 361     if (U_FAILURE(status))
 362     {
 363         printf("failure");
 364     }
 365     else
 366     {
 367         p = icu_buf_utf8_to_cstr(dst8);
 368         printf("%s", p);
 369     }
 370     icu_buf_utf8_destroy(dst8);
 371 }
 372
 373 struct icu_buf_utf16 *icu_iter_invoke(yaz_icu_iter_t iter,
 374                                       struct icu_chain_step *step,
 375                                       struct icu_buf_utf16 *src)
 376 {
 377     if (!step)
 378         return src;
 379     else
 380     {
 381         struct icu_buf_utf16 *dst = icu_iter_invoke(iter, step->previous, src);
 382
 383         switch (step->type)
 384         {
 385         case ICU_chain_step_type_casemap:
 386             if (dst)
 387             {
 388                 struct icu_buf_utf16 *src = dst;
 389
 390                 dst = icu_buf_utf16_create(0);
 391                 icu_casemap_casemap(step->u.casemap, dst, src, &iter->status,
 392                                     iter->chain->locale);
 393                 icu_buf_utf16_destroy(src);
 394             }
 395             break;
 396         case ICU_chain_step_type_tokenize:
 397             if (dst)
 398             {
 399                 struct icu_buf_utf16 *src = dst;
 400
 401                 icu_tokenizer_attach(step->u.tokenizer, src, &iter->status);
 402                 icu_buf_utf16_destroy(src);
 403             }
 404             dst = icu_buf_utf16_create(0);
 405             iter->status = U_ZERO_ERROR;
 406             if (!icu_tokenizer_next_token(step->u.tokenizer, dst, &iter->status))
 407             {
 408                 icu_buf_utf16_destroy(dst);
 409                 dst = 0;
 410             }
 411             break;
 412         case ICU_chain_step_type_transform:
 413         case ICU_chain_step_type_transliterate:
 414             if (dst)
 415             {
 416                 struct icu_buf_utf16 *src = dst;
 417                 dst = icu_buf_utf16_create(0);
 418                 icu_transform_trans(step->u.transform, dst, src, &iter->status);
 419                 icu_buf_utf16_destroy(src);
 420             }
 421             break;
 422         case ICU_chain_step_type_display:
 423             if (dst)
 424                 icu_utf16_to_utf8(iter->display, dst, &iter->status);
 425             break;
 426         case YAZ_chain_step_type_stemming:
 427             if (dst)
 428             {
 429                 struct icu_buf_utf16 *src = dst;
 430                 dst = icu_buf_utf16_create(0);
 431                 yaz_stemmer_stem(step->u.stemmer, dst, src, &iter->status);
 432                 icu_buf_utf16_destroy(src);
 433             }
 434             break;
 435         default:
 436             assert(0);
 437         }
 438         return dst;
 439     }
 440 }
 441
 442 yaz_icu_iter_t icu_iter_create(struct icu_chain *chain)
 443 {
 444     yaz_icu_iter_t iter = xmalloc(sizeof(*iter));
 445     iter->chain = chain;
 446     iter->status = U_ZERO_ERROR;
 447     iter->display = icu_buf_utf8_create(0);
 448     iter->sort8 = icu_buf_utf8_create(0);
 449     iter->result = icu_buf_utf8_create(0);
 450     iter->last = 0; /* no last returned string (yet) */
 451     iter->steps = icu_chain_step_clone(chain->csteps);
 452     iter->input = 0;
 453
 454     return iter;
 455 }
 456
 457 void icu_iter_first(yaz_icu_iter_t iter, const char *src8cstr)
 458 {
 459     if (iter->input)
 460         icu_buf_utf16_destroy(iter->input);
 461     iter->input = icu_buf_utf16_create(0);
 462     iter->token_count = 0;
 463     /* fill and assign input string.. It will be 0 after
 464        first iteration */
 465     icu_utf16_from_utf8_cstr(iter->input, src8cstr, &iter->status);
 466 }
 467
 468 void icu_iter_destroy(yaz_icu_iter_t iter)
 469 {
 470     if (iter)
 471     {
 472         icu_buf_utf8_destroy(iter->display);
 473         icu_buf_utf8_destroy(iter->sort8);
 474         icu_buf_utf8_destroy(iter->result);
 475         if (iter->input)
 476             icu_buf_utf16_destroy(iter->input);
 477         icu_chain_step_destroy(iter->steps);
 478         xfree(iter);
 479     }
 480 }
 481
 482 int icu_iter_next(yaz_icu_iter_t iter)
 483 {
 484     if (!iter->input && iter->last == 0)
 485         return 0;
 486     else
 487     {
 488         /* on first call, iter->input is the input string. Thereafter: 0. */
 489         iter->last = icu_iter_invoke(iter, iter->steps ?
 490                                      iter->steps : iter->chain->csteps,
 491                                      iter->input);
 492         iter->input = 0;
 493
 494         if (!iter->last)
 495             return 0;
 496
 497         iter->token_count++;
 498
 499         if (iter->chain->sort)
 500         {
 501             icu_sortkey8_from_utf16(iter->chain->coll,
 502                                     iter->sort8, iter->last,
 503                                     &iter->status);
 504         }
 505         icu_utf16_to_utf8(iter->result, iter->last, &iter->status);
 506         icu_buf_utf16_destroy(iter->last);
 507
 508         return 1;
 509     }
 510 }
 511
 512 const char *icu_iter_get_norm(yaz_icu_iter_t iter)
 513 {
 514     return icu_buf_utf8_to_cstr(iter->result);
 515 }
 516
 517 const char *icu_iter_get_sortkey(yaz_icu_iter_t iter)
 518 {
 519     return icu_buf_utf8_to_cstr(iter->sort8);
 520 }
 521
 522 const char *icu_iter_get_display(yaz_icu_iter_t iter)
 523 {
 524     return icu_buf_utf8_to_cstr(iter->display);
 525 }
 526
 527 int icu_iter_get_token_number(yaz_icu_iter_t iter)
 528 {
 529     return iter->token_count;
 530 }
 531
 532 int icu_chain_assign_cstr(struct icu_chain *chain, const char *src8cstr,
 533                           UErrorCode *status)
 534 {
 535     if (chain->iter)
 536         icu_iter_destroy(chain->iter);
 537     chain->iter = icu_iter_create(chain);
 538     icu_iter_first(chain->iter, src8cstr);
 539     return 1;
 540 }
 541
 542 int icu_chain_next_token(struct icu_chain *chain, UErrorCode *status)
 543 {
 544     *status = U_ZERO_ERROR;
 545     return icu_iter_next(chain->iter);
 546 }
 547
 548 int icu_chain_token_number(struct icu_chain *chain)
 549 {
 550     if (chain && chain->iter)
 551         return chain->iter->token_count;
 552     return 0;
 553 }
 554
 555 const char *icu_chain_token_display(struct icu_chain *chain)
 556 {
 557     if (chain->iter)
 558         return icu_iter_get_display(chain->iter);
 559     return 0;
 560 }
 561
 562 const char *icu_chain_token_norm(struct icu_chain *chain)
 563 {
 564     if (chain->iter)
 565         return icu_iter_get_norm(chain->iter);
 566     return 0;
 567 }
 568
 569 const char *icu_chain_token_sortkey(struct icu_chain *chain)
 570 {
 571     if (chain->iter)
 572         return icu_iter_get_sortkey(chain->iter);
 573     return 0;
 574 }
 575
 576 #endif /* YAZ_HAVE_ICU */
 577
 578 /*
 579  * Local variables:
 580  * c-basic-offset: 4
 581  * c-file-style: "Stroustrup"
 582  * indent-tabs-mode: nil
 583  * End:
 584  * vim: shiftwidth=4 tabstop=8 expandtab
 585  */
 586