src/icu_I18N.c

   1 /* This file is part of the YAZ toolkit.
   2  * Copyright (C) 1995-2008 Index Data
   3  * See the file LICENSE for details.
   4  */
   5
   6 /**
   7  * \file icu_I18N.c
   8  * \brief ICU utilities
   9  */
  10
  11 #if HAVE_CONFIG_H
  12 #include "config.h"
  13 #endif
  14
  15 #define USE_TIMING 0
  16 #if USE_TIMING
  17 #include <yaz/timing.h>
  18 #endif
  19
  20 #if YAZ_HAVE_ICU
  21 #include <yaz/xmalloc.h>
  22
  23 #include <yaz/icu_I18N.h>
  24
  25 #include <yaz/log.h>
  26
  27 #include <string.h>
  28 #include <stdlib.h>
  29 #include <stdio.h>
  30
  31 #include <unicode/ustring.h>  /* some more string fcns*/
  32 #include <unicode/uchar.h>    /* char names           */
  33
  34
  35 #include <unicode/ucol.h>
  36
  37
  38 int icu_check_status (UErrorCode status)
  39 {
  40     if (U_FAILURE(status))
  41     {
  42         yaz_log(YLOG_WARN, "ICU: %d %s\n", status, u_errorName(status));
  43         return 0;
  44     }
  45     return 1;
  46
  47 }
  48
  49
  50
  51 struct icu_buf_utf16 * icu_buf_utf16_create(size_t capacity)
  52 {
  53     struct icu_buf_utf16 * buf16
  54         = (struct icu_buf_utf16 *) xmalloc(sizeof(struct icu_buf_utf16));
  55
  56     buf16->utf16 = 0;
  57     buf16->utf16_len = 0;
  58     buf16->utf16_cap = 0;
  59
  60     if (capacity > 0){
  61         buf16->utf16 = (UChar *) xmalloc(sizeof(UChar) * capacity);
  62         buf16->utf16[0] = (UChar) 0;
  63         buf16->utf16_cap = capacity;
  64     }
  65     return buf16;
  66 }
  67
  68 struct icu_buf_utf16 * icu_buf_utf16_clear(struct icu_buf_utf16 * buf16)
  69 {
  70     if (buf16){
  71         if (buf16->utf16)
  72             buf16->utf16[0] = (UChar) 0;
  73         buf16->utf16_len = 0;
  74     }
  75     return buf16;
  76 }
  77
  78 struct icu_buf_utf16 * icu_buf_utf16_resize(struct icu_buf_utf16 * buf16,
  79                                             size_t capacity)
  80 {
  81     if (!buf16)
  82         return 0;
  83
  84     if (capacity >  0){
  85         if (0 == buf16->utf16)
  86             buf16->utf16 = (UChar *) xmalloc(sizeof(UChar) * capacity);
  87         else
  88             buf16->utf16
  89                 = (UChar *) xrealloc(buf16->utf16, sizeof(UChar) * capacity);
  90
  91         icu_buf_utf16_clear(buf16);
  92         buf16->utf16_cap = capacity;
  93     }
  94     else {
  95         xfree(buf16->utf16);
  96         buf16->utf16 = 0;
  97         buf16->utf16_len = 0;
  98         buf16->utf16_cap = 0;
  99     }
 100
 101     return buf16;
 102 }
 103
 104
 105 struct icu_buf_utf16 * icu_buf_utf16_copy(struct icu_buf_utf16 * dest16,
 106                                           struct icu_buf_utf16 * src16)
 107 {
 108     if(!dest16 || !src16
 109        || dest16 == src16)
 110         return 0;
 111
 112     if (dest16->utf16_cap < src16->utf16_len)
 113         icu_buf_utf16_resize(dest16, src16->utf16_len * 2);
 114
 115     u_strncpy(dest16->utf16, src16->utf16, src16->utf16_len);
 116     dest16->utf16_len = src16->utf16_len;
 117
 118     return dest16;
 119 }
 120
 121
 122 void icu_buf_utf16_destroy(struct icu_buf_utf16 * buf16)
 123 {
 124     if (buf16)
 125         xfree(buf16->utf16);
 126     xfree(buf16);
 127 }
 128
 129
 130
 131 struct icu_buf_utf8 * icu_buf_utf8_create(size_t capacity)
 132 {
 133     struct icu_buf_utf8 * buf8
 134         = (struct icu_buf_utf8 *) xmalloc(sizeof(struct icu_buf_utf8));
 135
 136     buf8->utf8 = 0;
 137     buf8->utf8_len = 0;
 138     buf8->utf8_cap = 0;
 139
 140     if (capacity > 0){
 141         buf8->utf8 = (uint8_t *) xmalloc(sizeof(uint8_t) * capacity);
 142         buf8->utf8[0] = (uint8_t) 0;
 143         buf8->utf8_cap = capacity;
 144     }
 145     return buf8;
 146 }
 147
 148
 149 struct icu_buf_utf8 * icu_buf_utf8_clear(struct icu_buf_utf8 * buf8)
 150 {
 151     if (buf8){
 152         if (buf8->utf8)
 153             buf8->utf8[0] = (uint8_t) 0;
 154         buf8->utf8_len = 0;
 155     }
 156     return buf8;
 157 }
 158
 159
 160 struct icu_buf_utf8 * icu_buf_utf8_resize(struct icu_buf_utf8 * buf8,
 161                                           size_t capacity)
 162 {
 163     if (!buf8)
 164         return 0;
 165
 166     if (capacity >  0){
 167         if (0 == buf8->utf8)
 168             buf8->utf8 = (uint8_t *) xmalloc(sizeof(uint8_t) * capacity);
 169         else
 170             buf8->utf8
 171                 = (uint8_t *) xrealloc(buf8->utf8, sizeof(uint8_t) * capacity);
 172
 173         buf8->utf8_cap = capacity;
 174     }
 175     else {
 176         xfree(buf8->utf8);
 177         buf8->utf8 = 0;
 178         buf8->utf8_len = 0;
 179         buf8->utf8_cap = 0;
 180     }
 181
 182     return buf8;
 183 }
 184
 185
 186 const char *icu_buf_utf8_to_cstr(struct icu_buf_utf8 *src8)
 187 {
 188     if (!src8 || src8->utf8_len == 0)
 189         return "";
 190
 191     if (src8->utf8_len == src8->utf8_cap)
 192         src8 = icu_buf_utf8_resize(src8, src8->utf8_len * 2 + 1);
 193
 194     src8->utf8[src8->utf8_len] = '\0';
 195
 196     return (const char *) src8->utf8;
 197 }
 198
 199
 200 void icu_buf_utf8_destroy(struct icu_buf_utf8 * buf8)
 201 {
 202     if (buf8)
 203         xfree(buf8->utf8);
 204     xfree(buf8);
 205 }
 206
 207
 208
 209 UErrorCode icu_utf16_from_utf8(struct icu_buf_utf16 * dest16,
 210                                struct icu_buf_utf8 * src8,
 211                                UErrorCode * status)
 212 {
 213     int32_t utf16_len = 0;
 214
 215     u_strFromUTF8(dest16->utf16, dest16->utf16_cap,
 216                   &utf16_len,
 217                   (const char *) src8->utf8, src8->utf8_len, status);
 218
 219     /* check for buffer overflow, resize and retry */
 220     if (*status == U_BUFFER_OVERFLOW_ERROR)
 221     {
 222         icu_buf_utf16_resize(dest16, utf16_len * 2);
 223         *status = U_ZERO_ERROR;
 224         u_strFromUTF8(dest16->utf16, dest16->utf16_cap,
 225                       &utf16_len,
 226                       (const char *) src8->utf8, src8->utf8_len, status);
 227     }
 228
 229     if (U_SUCCESS(*status)
 230         && utf16_len <= dest16->utf16_cap)
 231         dest16->utf16_len = utf16_len;
 232     else
 233         icu_buf_utf16_clear(dest16);
 234
 235     return *status;
 236 }
 237
 238
 239
 240 UErrorCode icu_utf16_from_utf8_cstr(struct icu_buf_utf16 * dest16,
 241                                     const char * src8cstr,
 242                                     UErrorCode * status)
 243 {
 244     size_t src8cstr_len = 0;
 245     int32_t utf16_len = 0;
 246
 247     *status = U_ZERO_ERROR;
 248     src8cstr_len = strlen(src8cstr);
 249
 250     u_strFromUTF8(dest16->utf16, dest16->utf16_cap,
 251                   &utf16_len,
 252                   src8cstr, src8cstr_len, status);
 253
 254     /* check for buffer overflow, resize and retry */
 255     if (*status == U_BUFFER_OVERFLOW_ERROR)
 256     {
 257         icu_buf_utf16_resize(dest16, utf16_len * 2);
 258         *status = U_ZERO_ERROR;
 259         u_strFromUTF8(dest16->utf16, dest16->utf16_cap,
 260                       &utf16_len,
 261                       src8cstr, src8cstr_len, status);
 262     }
 263
 264     if (U_SUCCESS(*status)
 265         && utf16_len <= dest16->utf16_cap)
 266         dest16->utf16_len = utf16_len;
 267     else
 268         icu_buf_utf16_clear(dest16);
 269
 270     return *status;
 271 }
 272
 273
 274
 275
 276 UErrorCode icu_utf16_to_utf8(struct icu_buf_utf8 * dest8,
 277                              struct icu_buf_utf16 * src16,
 278                              UErrorCode * status)
 279 {
 280     int32_t utf8_len = 0;
 281
 282     u_strToUTF8((char *) dest8->utf8, dest8->utf8_cap,
 283                 &utf8_len,
 284                 src16->utf16, src16->utf16_len, status);
 285
 286     /* check for buffer overflow, resize and retry */
 287     if (*status == U_BUFFER_OVERFLOW_ERROR)
 288     {
 289         icu_buf_utf8_resize(dest8, utf8_len * 2);
 290         *status = U_ZERO_ERROR;
 291         u_strToUTF8((char *) dest8->utf8, dest8->utf8_cap,
 292                     &utf8_len,
 293                     src16->utf16, src16->utf16_len, status);
 294
 295     }
 296
 297     if (U_SUCCESS(*status)
 298         && utf8_len <= dest8->utf8_cap)
 299         dest8->utf8_len = utf8_len;
 300     else
 301         icu_buf_utf8_clear(dest8);
 302
 303     return *status;
 304 }
 305
 306
 307
 308 struct icu_casemap * icu_casemap_create(char action, UErrorCode *status)
 309 {
 310     struct icu_casemap * casemap
 311         = (struct icu_casemap *) xmalloc(sizeof(struct icu_casemap));
 312     casemap->action = action;
 313
 314     switch(casemap->action) {
 315     case 'l':
 316     case 'L':
 317     case 'u':
 318     case 'U':
 319     case 't':
 320     case 'T':
 321     case 'f':
 322     case 'F':
 323         break;
 324     default:
 325         icu_casemap_destroy(casemap);
 326         return 0;
 327     }
 328
 329     return casemap;
 330 }
 331
 332 void icu_casemap_destroy(struct icu_casemap * casemap)
 333 {
 334     xfree(casemap);
 335 }
 336
 337
 338 int icu_casemap_casemap(struct icu_casemap * casemap,
 339                         struct icu_buf_utf16 * dest16,
 340                         struct icu_buf_utf16 * src16,
 341                         UErrorCode *status,
 342                         const char *locale)
 343 {
 344     if(!casemap)
 345         return 0;
 346
 347     return icu_utf16_casemap(dest16, src16, locale,
 348                              casemap->action, status);
 349 }
 350
 351
 352 int icu_utf16_casemap(struct icu_buf_utf16 * dest16,
 353                       struct icu_buf_utf16 * src16,
 354                       const char *locale, char action,
 355                       UErrorCode *status)
 356 {
 357     int32_t dest16_len = 0;
 358
 359
 360     if (!src16->utf16_len){           /* guarding for empty source string */
 361         if (dest16->utf16)
 362             dest16->utf16[0] = (UChar) 0;
 363         dest16->utf16_len = 0;
 364         return U_ZERO_ERROR;
 365     }
 366
 367
 368     switch(action) {
 369     case 'l':
 370     case 'L':
 371         dest16_len = u_strToLower(dest16->utf16, dest16->utf16_cap,
 372                                   src16->utf16, src16->utf16_len,
 373                                   locale, status);
 374         break;
 375     case 'u':
 376     case 'U':
 377         dest16_len = u_strToUpper(dest16->utf16, dest16->utf16_cap,
 378                                   src16->utf16, src16->utf16_len,
 379                                   locale, status);
 380         break;
 381     case 't':
 382     case 'T':
 383         dest16_len = u_strToTitle(dest16->utf16, dest16->utf16_cap,
 384                                   src16->utf16, src16->utf16_len,
 385                                   0, locale, status);
 386         break;
 387     case 'f':
 388     case 'F':
 389         dest16_len = u_strFoldCase(dest16->utf16, dest16->utf16_cap,
 390                                    src16->utf16, src16->utf16_len,
 391                                    U_FOLD_CASE_DEFAULT, status);
 392         break;
 393
 394     default:
 395         return U_UNSUPPORTED_ERROR;
 396         break;
 397     }
 398
 399     /* check for buffer overflow, resize and retry */
 400     if (*status == U_BUFFER_OVERFLOW_ERROR
 401         && dest16 != src16        /* do not resize if in-place conversion */
 402         ){
 403         icu_buf_utf16_resize(dest16, dest16_len * 2);
 404         *status = U_ZERO_ERROR;
 405
 406
 407         switch(action) {
 408         case 'l':
 409         case 'L':
 410             dest16_len = u_strToLower(dest16->utf16, dest16->utf16_cap,
 411                                       src16->utf16, src16->utf16_len,
 412                                       locale, status);
 413             break;
 414         case 'u':
 415         case 'U':
 416             dest16_len = u_strToUpper(dest16->utf16, dest16->utf16_cap,
 417                                       src16->utf16, src16->utf16_len,
 418                                       locale, status);
 419             break;
 420         case 't':
 421         case 'T':
 422             dest16_len = u_strToTitle(dest16->utf16, dest16->utf16_cap,
 423                                       src16->utf16, src16->utf16_len,
 424                                       0, locale, status);
 425             break;
 426         case 'f':
 427         case 'F':
 428             dest16_len = u_strFoldCase(dest16->utf16, dest16->utf16_cap,
 429                                        src16->utf16, src16->utf16_len,
 430                                        U_FOLD_CASE_DEFAULT, status);
 431             break;
 432
 433         default:
 434             return U_UNSUPPORTED_ERROR;
 435             break;
 436         }
 437     }
 438
 439     if (U_SUCCESS(*status)
 440         && dest16_len <= dest16->utf16_cap)
 441         dest16->utf16_len = dest16_len;
 442     else {
 443         if (dest16->utf16)
 444             dest16->utf16[0] = (UChar) 0;
 445         dest16->utf16_len = 0;
 446     }
 447
 448     return *status;
 449 }
 450
 451
 452
 453 void icu_sortkey8_from_utf16(UCollator *coll,
 454                              struct icu_buf_utf8 * dest8,
 455                              struct icu_buf_utf16 * src16,
 456                              UErrorCode * status)
 457 {
 458
 459     int32_t sortkey_len = 0;
 460
 461     sortkey_len = ucol_getSortKey(coll, src16->utf16, src16->utf16_len,
 462                                   dest8->utf8, dest8->utf8_cap);
 463
 464     /* check for buffer overflow, resize and retry */
 465     if (sortkey_len > dest8->utf8_cap) {
 466         icu_buf_utf8_resize(dest8, sortkey_len * 2);
 467         sortkey_len = ucol_getSortKey(coll, src16->utf16, src16->utf16_len,
 468                                       dest8->utf8, dest8->utf8_cap);
 469     }
 470
 471     if (U_SUCCESS(*status)
 472         && sortkey_len > 0)
 473         dest8->utf8_len = sortkey_len;
 474     else
 475         icu_buf_utf8_clear(dest8);
 476 }
 477
 478
 479
 480 struct icu_tokenizer * icu_tokenizer_create(const char *locale, char action,
 481                                             UErrorCode *status)
 482 {
 483     struct icu_tokenizer * tokenizer
 484         = (struct icu_tokenizer *) xmalloc(sizeof(struct icu_tokenizer));
 485
 486     tokenizer->action = action;
 487     tokenizer->bi = 0;
 488     tokenizer->buf16 = 0;
 489     tokenizer->token_count = 0;
 490     tokenizer->token_id = 0;
 491     tokenizer->token_start = 0;
 492     tokenizer->token_end = 0;
 493
 494
 495     switch(tokenizer->action) {
 496     case 'l':
 497     case 'L':
 498         tokenizer->bi = ubrk_open(UBRK_LINE, locale, 0, 0, status);
 499         break;
 500     case 's':
 501     case 'S':
 502         tokenizer->bi = ubrk_open(UBRK_SENTENCE, locale, 0, 0, status);
 503         break;
 504     case 'w':
 505     case 'W':
 506         tokenizer->bi = ubrk_open(UBRK_WORD, locale, 0, 0, status);
 507         break;
 508     case 'c':
 509     case 'C':
 510         tokenizer->bi = ubrk_open(UBRK_CHARACTER, locale, 0, 0, status);
 511         break;
 512     case 't':
 513     case 'T':
 514         tokenizer->bi = ubrk_open(UBRK_TITLE, locale, 0, 0, status);
 515         break;
 516     default:
 517         *status = U_UNSUPPORTED_ERROR;
 518         return 0;
 519         break;
 520     }
 521
 522     /* ICU error stuff is a very  funny business */
 523     if (U_SUCCESS(*status))
 524         return tokenizer;
 525
 526     /* freeing if failed */
 527     icu_tokenizer_destroy(tokenizer);
 528     return 0;
 529 }
 530
 531 void icu_tokenizer_destroy(struct icu_tokenizer * tokenizer)
 532 {
 533     if (tokenizer) {
 534         if (tokenizer->bi)
 535             ubrk_close(tokenizer->bi);
 536         xfree(tokenizer);
 537     }
 538 }
 539
 540 int icu_tokenizer_attach(struct icu_tokenizer * tokenizer,
 541                          struct icu_buf_utf16 * src16,
 542                          UErrorCode *status)
 543 {
 544     if (!tokenizer || !tokenizer->bi || !src16)
 545         return 0;
 546
 547
 548     tokenizer->buf16 = src16;
 549     tokenizer->token_count = 0;
 550     tokenizer->token_id = 0;
 551     tokenizer->token_start = 0;
 552     tokenizer->token_end = 0;
 553
 554     ubrk_setText(tokenizer->bi, src16->utf16, src16->utf16_len, status);
 555
 556
 557     if (U_FAILURE(*status))
 558         return 0;
 559
 560     return 1;
 561 };
 562
 563 int32_t icu_tokenizer_next_token(struct icu_tokenizer * tokenizer,
 564                          struct icu_buf_utf16 * tkn16,
 565                          UErrorCode *status)
 566 {
 567     int32_t tkn_start = 0;
 568     int32_t tkn_end = 0;
 569     int32_t tkn_len = 0;
 570
 571
 572     if (!tokenizer || !tokenizer->bi
 573         || !tokenizer->buf16 || !tokenizer->buf16->utf16_len)
 574         return 0;
 575
 576     /*
 577     never change tokenizer->buf16 and keep always invariant
 578     0 <= tokenizer->token_start
 579        <= tokenizer->token_end
 580        <= tokenizer->buf16->utf16_len
 581     returns length of token
 582     */
 583
 584     if (0 == tokenizer->token_end) /* first call */
 585         tkn_start = ubrk_first(tokenizer->bi);
 586     else /* successive calls */
 587         tkn_start = tokenizer->token_end;
 588
 589     /* get next position */
 590     tkn_end = ubrk_next(tokenizer->bi);
 591
 592     /* repairing invariant at end of ubrk, which is UBRK_DONE = -1 */
 593     if (UBRK_DONE == tkn_end)
 594         tkn_end = tokenizer->buf16->utf16_len;
 595
 596     /* copy out if everything is well */
 597     if(U_FAILURE(*status))
 598         return 0;
 599
 600     /* everything OK, now update internal state */
 601     tkn_len = tkn_end - tkn_start;
 602
 603     if (0 < tkn_len){
 604         tokenizer->token_count++;
 605         tokenizer->token_id++;
 606     } else {
 607         tokenizer->token_id = 0;
 608     }
 609     tokenizer->token_start = tkn_start;
 610     tokenizer->token_end = tkn_end;
 611
 612
 613     /* copying into token buffer if it exists */
 614     if (tkn16){
 615         if (tkn16->utf16_cap < tkn_len)
 616             icu_buf_utf16_resize(tkn16, (size_t) tkn_len * 2);
 617
 618         u_strncpy(tkn16->utf16, &(tokenizer->buf16->utf16)[tkn_start],
 619                   tkn_len);
 620
 621         tkn16->utf16_len = tkn_len;
 622     }
 623
 624     return tkn_len;
 625 }
 626
 627
 628 int32_t icu_tokenizer_token_id(struct icu_tokenizer * tokenizer)
 629 {
 630     return tokenizer->token_id;
 631 }
 632
 633 int32_t icu_tokenizer_token_start(struct icu_tokenizer * tokenizer)
 634 {
 635     return tokenizer->token_start;
 636 }
 637
 638 int32_t icu_tokenizer_token_end(struct icu_tokenizer * tokenizer)
 639 {
 640     return tokenizer->token_end;
 641 }
 642
 643 int32_t icu_tokenizer_token_length(struct icu_tokenizer * tokenizer)
 644 {
 645     return (tokenizer->token_end - tokenizer->token_start);
 646 }
 647
 648 int32_t icu_tokenizer_token_count(struct icu_tokenizer * tokenizer)
 649 {
 650     return tokenizer->token_count;
 651 }
 652
 653
 654
 655 struct icu_normalizer * icu_normalizer_create(const char *rules, char action,
 656                                               UErrorCode *status)
 657 {
 658
 659     struct icu_normalizer * normalizer
 660         = (struct icu_normalizer *) xmalloc(sizeof(struct icu_normalizer));
 661
 662     normalizer->action = action;
 663     normalizer->trans = 0;
 664     normalizer->rules16 =  icu_buf_utf16_create(0);
 665     icu_utf16_from_utf8_cstr(normalizer->rules16, rules, status);
 666
 667     switch(normalizer->action) {
 668     case 'f':
 669     case 'F':
 670         normalizer->trans
 671             = utrans_openU(normalizer->rules16->utf16,
 672                            normalizer->rules16->utf16_len,
 673                            UTRANS_FORWARD,
 674                            0, 0,
 675                            &normalizer->parse_error, status);
 676         break;
 677     case 'r':
 678     case 'R':
 679         normalizer->trans
 680             = utrans_openU(normalizer->rules16->utf16,
 681                            normalizer->rules16->utf16_len,
 682                            UTRANS_REVERSE ,
 683                            0, 0,
 684                            &normalizer->parse_error, status);
 685         break;
 686     default:
 687         *status = U_UNSUPPORTED_ERROR;
 688         return 0;
 689         break;
 690     }
 691
 692     if (U_SUCCESS(*status))
 693         return normalizer;
 694
 695     /* freeing if failed */
 696     icu_normalizer_destroy(normalizer);
 697     return 0;
 698 }
 699
 700
 701 void icu_normalizer_destroy(struct icu_normalizer * normalizer){
 702     if (normalizer) {
 703         if (normalizer->rules16)
 704             icu_buf_utf16_destroy(normalizer->rules16);
 705         if (normalizer->trans)
 706             utrans_close(normalizer->trans);
 707         xfree(normalizer);
 708     }
 709 }
 710
 711
 712
 713 int icu_normalizer_normalize(struct icu_normalizer * normalizer,
 714                              struct icu_buf_utf16 * dest16,
 715                              struct icu_buf_utf16 * src16,
 716                              UErrorCode *status)
 717 {
 718     if (!normalizer || !normalizer->trans
 719         || !src16
 720         || !dest16)
 721         return 0;
 722
 723     if (!src16->utf16_len){           /* guarding for empty source string */
 724         icu_buf_utf16_clear(dest16);
 725         return 0;
 726     }
 727
 728     if (!icu_buf_utf16_copy(dest16, src16))
 729         return 0;
 730
 731
 732     utrans_transUChars (normalizer->trans,
 733                         dest16->utf16, &(dest16->utf16_len),
 734                         dest16->utf16_cap,
 735                         0, &(src16->utf16_len), status);
 736
 737     if (U_FAILURE(*status))
 738         icu_buf_utf16_clear(dest16);
 739
 740     return dest16->utf16_len;
 741 }
 742
 743
 744
 745
 746 struct icu_chain_step * icu_chain_step_create(struct icu_chain * chain,
 747                                               enum icu_chain_step_type type,
 748                                               const uint8_t * rule,
 749                                               struct icu_buf_utf16 * buf16,
 750                                               UErrorCode *status)
 751 {
 752     struct icu_chain_step * step = 0;
 753
 754     if(!chain || !type || !rule)
 755         return 0;
 756
 757     step = (struct icu_chain_step *) xmalloc(sizeof(struct icu_chain_step));
 758
 759     step->type = type;
 760
 761     step->buf16 = buf16;
 762
 763     /* create auxilary objects */
 764     switch(step->type) {
 765     case ICU_chain_step_type_display:
 766         break;
 767     case ICU_chain_step_type_casemap:
 768         step->u.casemap = icu_casemap_create(rule[0], status);
 769         break;
 770     case ICU_chain_step_type_normalize:
 771         step->u.normalizer = icu_normalizer_create((char *) rule, 'f', status);
 772         break;
 773     case ICU_chain_step_type_tokenize:
 774         step->u.tokenizer = icu_tokenizer_create((char *) chain->locale,
 775                                                  (char) rule[0], status);
 776         break;
 777     default:
 778         break;
 779     }
 780
 781     return step;
 782 }
 783
 784
 785 void icu_chain_step_destroy(struct icu_chain_step * step){
 786
 787     if (!step)
 788         return;
 789
 790     icu_chain_step_destroy(step->previous);
 791
 792     switch(step->type) {
 793     case ICU_chain_step_type_display:
 794         break;
 795     case ICU_chain_step_type_casemap:
 796         icu_casemap_destroy(step->u.casemap);
 797         icu_buf_utf16_destroy(step->buf16);
 798         break;
 799     case ICU_chain_step_type_normalize:
 800         icu_normalizer_destroy(step->u.normalizer);
 801         icu_buf_utf16_destroy(step->buf16);
 802         break;
 803     case ICU_chain_step_type_tokenize:
 804         icu_tokenizer_destroy(step->u.tokenizer);
 805         icu_buf_utf16_destroy(step->buf16);
 806         break;
 807     default:
 808         break;
 809     }
 810     xfree(step);
 811 }
 812
 813
 814
 815 struct icu_chain * icu_chain_create(const char *locale,  int sort,
 816                                     UErrorCode * status)
 817 {
 818     struct icu_chain * chain
 819         = (struct icu_chain *) xmalloc(sizeof(struct icu_chain));
 820
 821     *status = U_ZERO_ERROR;
 822
 823     chain->locale = xstrdup(locale);
 824
 825     chain->sort = sort;
 826
 827     chain->coll = ucol_open((const char *) chain->locale, status);
 828
 829     if (U_FAILURE(*status))
 830         return 0;
 831
 832     chain->token_count = 0;
 833
 834     chain->src8cstr = 0;
 835
 836     chain->display8 = icu_buf_utf8_create(0);
 837     chain->norm8 = icu_buf_utf8_create(0);
 838     chain->sort8 = icu_buf_utf8_create(0);
 839
 840     chain->src16 = icu_buf_utf16_create(0);
 841
 842     chain->steps = 0;
 843
 844     return chain;
 845 }
 846
 847
 848 void icu_chain_destroy(struct icu_chain * chain)
 849 {
 850     if (chain)
 851     {
 852         if (chain->coll)
 853             ucol_close(chain->coll);
 854
 855         icu_buf_utf8_destroy(chain->display8);
 856         icu_buf_utf8_destroy(chain->norm8);
 857         icu_buf_utf8_destroy(chain->sort8);
 858
 859         icu_buf_utf16_destroy(chain->src16);
 860
 861         icu_chain_step_destroy(chain->steps);
 862         xfree(chain->locale);
 863         xfree(chain);
 864     }
 865 }
 866
 867
 868
 869 struct icu_chain * icu_chain_xml_config(const xmlNode *xml_node,
 870                                         int sort,
 871                                         UErrorCode * status)
 872 {
 873     xmlNode *node = 0;
 874     struct icu_chain * chain = 0;
 875
 876     *status = U_ZERO_ERROR;
 877
 878     if (!xml_node ||xml_node->type != XML_ELEMENT_NODE)
 879         return 0;
 880
 881     {
 882         xmlChar * xml_locale = xmlGetProp((xmlNode *) xml_node,
 883                                           (xmlChar *) "locale");
 884
 885         if (xml_locale)
 886         {
 887             chain = icu_chain_create((const char *) xml_locale, sort, status);
 888             xmlFree(xml_locale);
 889         }
 890
 891     }
 892     if (!chain)
 893         return 0;
 894
 895     for (node = xml_node->children; node; node = node->next)
 896     {
 897         xmlChar *xml_rule;
 898         struct icu_chain_step * step = 0;
 899
 900         if (node->type != XML_ELEMENT_NODE)
 901             continue;
 902
 903         xml_rule = xmlGetProp(node, (xmlChar *) "rule");
 904
 905         if (!strcmp((const char *) node->name, "casemap"))
 906             step = icu_chain_insert_step(chain, ICU_chain_step_type_casemap,
 907                                          (const uint8_t *) xml_rule, status);
 908         else if (!strcmp((const char *) node->name, "transform"))
 909             step = icu_chain_insert_step(chain, ICU_chain_step_type_normalize,
 910                                          (const uint8_t *) xml_rule, status);
 911         else if (!strcmp((const char *) node->name, "tokenize"))
 912             step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize,
 913                                          (const uint8_t *) xml_rule, status);
 914         else if (!strcmp((const char *) node->name, "display"))
 915             step = icu_chain_insert_step(chain, ICU_chain_step_type_display,
 916                                          (const uint8_t *) "", status);
 917         xmlFree(xml_rule);
 918         if (!step || U_FAILURE(*status))
 919         {
 920             icu_chain_destroy(chain);
 921             return 0;
 922         }
 923
 924
 925     }
 926     return chain;
 927 }
 928
 929
 930
 931 struct icu_chain_step * icu_chain_insert_step(struct icu_chain * chain,
 932                                               enum icu_chain_step_type type,
 933                                               const uint8_t * rule,
 934                                               UErrorCode *status)
 935 {
 936     struct icu_chain_step * step = 0;
 937     struct icu_buf_utf16 * src16 = 0;
 938     struct icu_buf_utf16 * buf16 = 0;
 939
 940     if (!chain || !type || !rule)
 941         return 0;
 942
 943     /* assign utf16 src buffers as needed */
 944     if (chain->steps && chain->steps->buf16)
 945         src16 = chain->steps->buf16;
 946     else if (chain->src16)
 947         src16 = chain->src16;
 948     else
 949         return 0;
 950
 951
 952     /* create utf16 destination buffers as needed, or */
 953     switch(type)
 954     {
 955     case ICU_chain_step_type_display:
 956         buf16 = src16;
 957         break;
 958     case ICU_chain_step_type_casemap:
 959         buf16 = icu_buf_utf16_create(0);
 960         break;
 961     case ICU_chain_step_type_normalize:
 962         buf16 = icu_buf_utf16_create(0);
 963         break;
 964     case ICU_chain_step_type_tokenize:
 965         buf16 = icu_buf_utf16_create(0);
 966         break;
 967     default:
 968         break;
 969     }
 970
 971     /* create actual chain step with this buffer */
 972     step = icu_chain_step_create(chain, type, rule, buf16, status);
 973
 974     step->previous = chain->steps;
 975     chain->steps = step;
 976
 977     return step;
 978 }
 979
 980
 981 int icu_chain_step_next_token(struct icu_chain * chain,
 982                               struct icu_chain_step * step,
 983                               UErrorCode *status)
 984 {
 985     struct icu_buf_utf16 * src16 = 0;
 986     int got_new_token = 0;
 987
 988     if (!chain || !chain->src16 || !step || !step->more_tokens)
 989         return 0;
 990
 991     /* assign utf16 src buffers as neeed, advance in previous steps
 992        tokens until non-zero token met, and setting stop condition */
 993
 994     if (step->previous)
 995     {
 996         src16 = step->previous->buf16;
 997         /* tokens might be killed in previous steps, therefore looping */
 998
 999         while (step->need_new_token
1000                && step->previous->more_tokens
1001                && !got_new_token)
1002             got_new_token
1003                 = icu_chain_step_next_token(chain, step->previous, status);
1004     }
1005     else
1006     { /* first step can only work once on chain->src16 input buffer */
1007         src16 = chain->src16;
1008         step->more_tokens = 0;
1009         got_new_token = 1;
1010     }
1011
1012     if (!src16)
1013         return 0;
1014
1015     /* stop if nothing to process */
1016     if (step->need_new_token && !got_new_token)
1017     {
1018         step->more_tokens = 0;
1019         return 0;
1020     }
1021
1022     /* either an old token not finished yet, or a new token, thus
1023        perform the work, eventually put this steps output in
1024        step->buf16 or the chains UTF8 output buffers  */
1025
1026     switch(step->type)
1027     {
1028     case ICU_chain_step_type_display:
1029         icu_utf16_to_utf8(chain->display8, src16, status);
1030         break;
1031     case ICU_chain_step_type_casemap:
1032         icu_casemap_casemap(step->u.casemap,
1033                             step->buf16, src16, status,
1034                             chain->locale);
1035         break;
1036     case ICU_chain_step_type_normalize:
1037         icu_normalizer_normalize(step->u.normalizer,
1038                                  step->buf16, src16, status);
1039         break;
1040     case ICU_chain_step_type_tokenize:
1041         /* attach to new src16 token only first time during splitting */
1042         if (step->need_new_token)
1043         {
1044             icu_tokenizer_attach(step->u.tokenizer, src16, status);
1045             step->need_new_token = 0;
1046         }
1047
1048         /* splitting one src16 token into multiple buf16 tokens */
1049         step->more_tokens
1050             = icu_tokenizer_next_token(step->u.tokenizer,
1051                                        step->buf16, status);
1052
1053         /* make sure to get new previous token if this one had been used up
1054            by recursive call to _same_ step */
1055
1056         if (!step->more_tokens)
1057         {
1058             step->more_tokens = icu_chain_step_next_token(chain, step, status);
1059             return step->more_tokens;  /* avoid one token count too much! */
1060         }
1061         break;
1062     default:
1063         return 0;
1064         break;
1065     }
1066
1067     if (U_FAILURE(*status))
1068         return 0;
1069
1070     /* if token disappered into thin air, tell caller */
1071     /* if (!step->buf16->utf16_len && !step->more_tokens) */
1072     /*    return 0; */
1073
1074     return 1;
1075 }
1076
1077
1078 int icu_chain_assign_cstr(struct icu_chain * chain,
1079                           const char * src8cstr,
1080                           UErrorCode *status)
1081 {
1082     struct icu_chain_step * stp = 0;
1083
1084     if (!chain || !src8cstr)
1085         return 0;
1086
1087     chain->src8cstr = src8cstr;
1088
1089     stp = chain->steps;
1090
1091     /* clear token count */
1092     chain->token_count = 0;
1093
1094     /* clear all steps stop states */
1095     while (stp)
1096     {
1097         stp->more_tokens = 1;
1098         stp->need_new_token = 1;
1099         stp = stp->previous;
1100     }
1101
1102     /* finally convert UTF8 to UTF16 string if needed */
1103     if (chain->steps || chain->sort)
1104         icu_utf16_from_utf8_cstr(chain->src16, chain->src8cstr, status);
1105
1106     if (U_FAILURE(*status))
1107         return 0;
1108
1109     return 1;
1110 }
1111
1112
1113
1114 int icu_chain_next_token(struct icu_chain * chain,
1115                          UErrorCode *status)
1116 {
1117     int got_token = 0;
1118
1119     *status = U_ZERO_ERROR;
1120
1121     if (!chain)
1122         return 0;
1123
1124     /* special case with no steps - same as index type binary */
1125     if (!chain->steps)
1126     {
1127         if (chain->token_count)
1128             return 0;
1129         else
1130         {
1131             chain->token_count++;
1132
1133             if (chain->sort)
1134                 icu_sortkey8_from_utf16(chain->coll,
1135                                         chain->sort8, chain->steps->buf16,
1136                                         status);
1137             return chain->token_count;
1138         }
1139     }
1140     /* usual case, one or more icu chain steps existing */
1141     else
1142     {
1143         while(!got_token && chain->steps && chain->steps->more_tokens)
1144             got_token = icu_chain_step_next_token(chain, chain->steps, status);
1145
1146         if (got_token)
1147         {
1148             chain->token_count++;
1149
1150             icu_utf16_to_utf8(chain->norm8, chain->steps->buf16, status);
1151
1152             if (chain->sort)
1153                 icu_sortkey8_from_utf16(chain->coll,
1154                                         chain->sort8, chain->steps->buf16,
1155                                         status);
1156
1157             return chain->token_count;
1158         }
1159     }
1160
1161     return 0;
1162 }
1163
1164 int icu_chain_token_number(struct icu_chain * chain)
1165 {
1166     if (!chain)
1167         return 0;
1168
1169     return chain->token_count;
1170 }
1171
1172
1173 const char * icu_chain_token_display(struct icu_chain * chain)
1174 {
1175     if (chain->display8)
1176         return icu_buf_utf8_to_cstr(chain->display8);
1177
1178     return 0;
1179 }
1180
1181 const char * icu_chain_token_norm(struct icu_chain * chain)
1182 {
1183     if (!chain->steps)
1184         return chain->src8cstr;
1185
1186     if (chain->norm8)
1187         return icu_buf_utf8_to_cstr(chain->norm8);
1188
1189     return 0;
1190 }
1191
1192 const char * icu_chain_token_sortkey(struct icu_chain * chain)
1193 {
1194     if (chain->sort8)
1195         return icu_buf_utf8_to_cstr(chain->sort8);
1196
1197     return 0;
1198 }
1199
1200 const UCollator * icu_chain_get_coll(struct icu_chain * chain)
1201 {
1202     return chain->coll;
1203 }
1204
1205 #endif /* YAZ_HAVE_ICU */
1206
1207 /*
1208  * Local variables:
1209  * c-basic-offset: 4
1210  * indent-tabs-mode: nil
1211  * End:
1212  * vim: shiftwidth=4 tabstop=8 expandtab
1213  */