src/icu_I18N.c

   1 /* This file is part of the YAZ toolkit.
   2  * Copyright (C) 1995-2008 Index Data
   3  * See the file LICENSE for details.
   4  */
   5
   6 #if HAVE_CONFIG_H
   7 #include "config.h"
   8 #endif
   9
  10 #define USE_TIMING 0
  11 #if USE_TIMING
  12 #include <yaz/timing.h>
  13 #endif
  14
  15 #if YAZ_HAVE_ICU
  16 #include <yaz/xmalloc.h>
  17
  18 #include <yaz/icu_I18N.h>
  19
  20 #include <yaz/log.h>
  21
  22 #include <string.h>
  23 #include <stdlib.h>
  24 #include <stdio.h>
  25
  26 #include <unicode/ustring.h>  /* some more string fcns*/
  27 #include <unicode/uchar.h>    /* char names           */
  28
  29
  30 #include <unicode/ucol.h>
  31
  32
  33 int icu_check_status (UErrorCode status)
  34 {
  35     if (U_FAILURE(status))
  36     {
  37         yaz_log(YLOG_WARN, "ICU: %d %s\n", status, u_errorName(status));
  38         return 0;
  39     }
  40     return 1;
  41
  42 }
  43
  44
  45
  46 struct icu_buf_utf16 * icu_buf_utf16_create(size_t capacity)
  47 {
  48     struct icu_buf_utf16 * buf16
  49         = (struct icu_buf_utf16 *) xmalloc(sizeof(struct icu_buf_utf16));
  50
  51     buf16->utf16 = 0;
  52     buf16->utf16_len = 0;
  53     buf16->utf16_cap = 0;
  54
  55     if (capacity > 0){
  56         buf16->utf16 = (UChar *) xmalloc(sizeof(UChar) * capacity);
  57         buf16->utf16[0] = (UChar) 0;
  58         buf16->utf16_cap = capacity;
  59     }
  60     return buf16;
  61 }
  62
  63 struct icu_buf_utf16 * icu_buf_utf16_clear(struct icu_buf_utf16 * buf16)
  64 {
  65     if (buf16){
  66         if (buf16->utf16)
  67             buf16->utf16[0] = (UChar) 0;
  68         buf16->utf16_len = 0;
  69     }
  70     return buf16;
  71 }
  72
  73 struct icu_buf_utf16 * icu_buf_utf16_resize(struct icu_buf_utf16 * buf16,
  74                                             size_t capacity)
  75 {
  76     if (!buf16)
  77         return 0;
  78
  79     if (capacity >  0){
  80         if (0 == buf16->utf16)
  81             buf16->utf16 = (UChar *) xmalloc(sizeof(UChar) * capacity);
  82         else
  83             buf16->utf16
  84                 = (UChar *) xrealloc(buf16->utf16, sizeof(UChar) * capacity);
  85
  86         icu_buf_utf16_clear(buf16);
  87         buf16->utf16_cap = capacity;
  88     }
  89     else {
  90         xfree(buf16->utf16);
  91         buf16->utf16 = 0;
  92         buf16->utf16_len = 0;
  93         buf16->utf16_cap = 0;
  94     }
  95
  96     return buf16;
  97 }
  98
  99
 100 struct icu_buf_utf16 * icu_buf_utf16_copy(struct icu_buf_utf16 * dest16,
 101                                           struct icu_buf_utf16 * src16)
 102 {
 103     if(!dest16 || !src16
 104        || dest16 == src16)
 105         return 0;
 106
 107     if (dest16->utf16_cap < src16->utf16_len)
 108         icu_buf_utf16_resize(dest16, src16->utf16_len * 2);
 109
 110     u_strncpy(dest16->utf16, src16->utf16, src16->utf16_len);
 111     dest16->utf16_len = src16->utf16_len;
 112
 113     return dest16;
 114 }
 115
 116
 117 void icu_buf_utf16_destroy(struct icu_buf_utf16 * buf16)
 118 {
 119     if (buf16)
 120         xfree(buf16->utf16);
 121     xfree(buf16);
 122 }
 123
 124
 125
 126 struct icu_buf_utf8 * icu_buf_utf8_create(size_t capacity)
 127 {
 128     struct icu_buf_utf8 * buf8
 129         = (struct icu_buf_utf8 *) xmalloc(sizeof(struct icu_buf_utf8));
 130
 131     buf8->utf8 = 0;
 132     buf8->utf8_len = 0;
 133     buf8->utf8_cap = 0;
 134
 135     if (capacity > 0){
 136         buf8->utf8 = (uint8_t *) xmalloc(sizeof(uint8_t) * capacity);
 137         buf8->utf8[0] = (uint8_t) 0;
 138         buf8->utf8_cap = capacity;
 139     }
 140     return buf8;
 141 }
 142
 143
 144 struct icu_buf_utf8 * icu_buf_utf8_clear(struct icu_buf_utf8 * buf8)
 145 {
 146     if (buf8){
 147         if (buf8->utf8)
 148             buf8->utf8[0] = (uint8_t) 0;
 149         buf8->utf8_len = 0;
 150     }
 151     return buf8;
 152 }
 153
 154
 155 struct icu_buf_utf8 * icu_buf_utf8_resize(struct icu_buf_utf8 * buf8,
 156                                           size_t capacity)
 157 {
 158     if (!buf8)
 159         return 0;
 160
 161     if (capacity >  0){
 162         if (0 == buf8->utf8)
 163             buf8->utf8 = (uint8_t *) xmalloc(sizeof(uint8_t) * capacity);
 164         else
 165             buf8->utf8
 166                 = (uint8_t *) xrealloc(buf8->utf8, sizeof(uint8_t) * capacity);
 167
 168         buf8->utf8_cap = capacity;
 169     }
 170     else {
 171         xfree(buf8->utf8);
 172         buf8->utf8 = 0;
 173         buf8->utf8_len = 0;
 174         buf8->utf8_cap = 0;
 175     }
 176
 177     return buf8;
 178 }
 179
 180
 181 const char *icu_buf_utf8_to_cstr(struct icu_buf_utf8 *src8)
 182 {
 183     if (!src8 || src8->utf8_len == 0)
 184         return "";
 185
 186     if (src8->utf8_len == src8->utf8_cap)
 187         src8 = icu_buf_utf8_resize(src8, src8->utf8_len * 2 + 1);
 188
 189     src8->utf8[src8->utf8_len] = '\0';
 190
 191     return (const char *) src8->utf8;
 192 }
 193
 194
 195 void icu_buf_utf8_destroy(struct icu_buf_utf8 * buf8)
 196 {
 197     if (buf8)
 198         xfree(buf8->utf8);
 199     xfree(buf8);
 200 }
 201
 202
 203
 204 UErrorCode icu_utf16_from_utf8(struct icu_buf_utf16 * dest16,
 205                                struct icu_buf_utf8 * src8,
 206                                UErrorCode * status)
 207 {
 208     int32_t utf16_len = 0;
 209
 210     u_strFromUTF8(dest16->utf16, dest16->utf16_cap,
 211                   &utf16_len,
 212                   (const char *) src8->utf8, src8->utf8_len, status);
 213
 214     /* check for buffer overflow, resize and retry */
 215     if (*status == U_BUFFER_OVERFLOW_ERROR)
 216     {
 217         icu_buf_utf16_resize(dest16, utf16_len * 2);
 218         *status = U_ZERO_ERROR;
 219         u_strFromUTF8(dest16->utf16, dest16->utf16_cap,
 220                       &utf16_len,
 221                       (const char *) src8->utf8, src8->utf8_len, status);
 222     }
 223
 224     if (U_SUCCESS(*status)
 225         && utf16_len <= dest16->utf16_cap)
 226         dest16->utf16_len = utf16_len;
 227     else
 228         icu_buf_utf16_clear(dest16);
 229
 230     return *status;
 231 }
 232
 233
 234
 235 UErrorCode icu_utf16_from_utf8_cstr(struct icu_buf_utf16 * dest16,
 236                                     const char * src8cstr,
 237                                     UErrorCode * status)
 238 {
 239     size_t src8cstr_len = 0;
 240     int32_t utf16_len = 0;
 241
 242     *status = U_ZERO_ERROR;
 243     src8cstr_len = strlen(src8cstr);
 244
 245     u_strFromUTF8(dest16->utf16, dest16->utf16_cap,
 246                   &utf16_len,
 247                   src8cstr, src8cstr_len, status);
 248
 249     /* check for buffer overflow, resize and retry */
 250     if (*status == U_BUFFER_OVERFLOW_ERROR)
 251     {
 252         icu_buf_utf16_resize(dest16, utf16_len * 2);
 253         *status = U_ZERO_ERROR;
 254         u_strFromUTF8(dest16->utf16, dest16->utf16_cap,
 255                       &utf16_len,
 256                       src8cstr, src8cstr_len, status);
 257     }
 258
 259     if (U_SUCCESS(*status)
 260         && utf16_len <= dest16->utf16_cap)
 261         dest16->utf16_len = utf16_len;
 262     else
 263         icu_buf_utf16_clear(dest16);
 264
 265     return *status;
 266 }
 267
 268
 269
 270
 271 UErrorCode icu_utf16_to_utf8(struct icu_buf_utf8 * dest8,
 272                              struct icu_buf_utf16 * src16,
 273                              UErrorCode * status)
 274 {
 275     int32_t utf8_len = 0;
 276
 277     u_strToUTF8((char *) dest8->utf8, dest8->utf8_cap,
 278                 &utf8_len,
 279                 src16->utf16, src16->utf16_len, status);
 280
 281     /* check for buffer overflow, resize and retry */
 282     if (*status == U_BUFFER_OVERFLOW_ERROR)
 283     {
 284         icu_buf_utf8_resize(dest8, utf8_len * 2);
 285         *status = U_ZERO_ERROR;
 286         u_strToUTF8((char *) dest8->utf8, dest8->utf8_cap,
 287                     &utf8_len,
 288                     src16->utf16, src16->utf16_len, status);
 289
 290     }
 291
 292     if (U_SUCCESS(*status)
 293         && utf8_len <= dest8->utf8_cap)
 294         dest8->utf8_len = utf8_len;
 295     else
 296         icu_buf_utf8_clear(dest8);
 297
 298     return *status;
 299 }
 300
 301
 302
 303 struct icu_casemap * icu_casemap_create(char action, UErrorCode *status)
 304 {
 305     struct icu_casemap * casemap
 306         = (struct icu_casemap *) xmalloc(sizeof(struct icu_casemap));
 307     casemap->action = action;
 308
 309     switch(casemap->action) {
 310     case 'l':
 311     case 'L':
 312     case 'u':
 313     case 'U':
 314     case 't':
 315     case 'T':
 316     case 'f':
 317     case 'F':
 318         break;
 319     default:
 320         icu_casemap_destroy(casemap);
 321         return 0;
 322     }
 323
 324     return casemap;
 325 }
 326
 327 void icu_casemap_destroy(struct icu_casemap * casemap)
 328 {
 329     xfree(casemap);
 330 }
 331
 332
 333 int icu_casemap_casemap(struct icu_casemap * casemap,
 334                         struct icu_buf_utf16 * dest16,
 335                         struct icu_buf_utf16 * src16,
 336                         UErrorCode *status,
 337                         const char *locale)
 338 {
 339     if(!casemap)
 340         return 0;
 341
 342     return icu_utf16_casemap(dest16, src16, locale,
 343                              casemap->action, status);
 344 }
 345
 346
 347 int icu_utf16_casemap(struct icu_buf_utf16 * dest16,
 348                       struct icu_buf_utf16 * src16,
 349                       const char *locale, char action,
 350                       UErrorCode *status)
 351 {
 352     int32_t dest16_len = 0;
 353
 354
 355     if (!src16->utf16_len){           /* guarding for empty source string */
 356         if (dest16->utf16)
 357             dest16->utf16[0] = (UChar) 0;
 358         dest16->utf16_len = 0;
 359         return U_ZERO_ERROR;
 360     }
 361
 362
 363     switch(action) {
 364     case 'l':
 365     case 'L':
 366         dest16_len = u_strToLower(dest16->utf16, dest16->utf16_cap,
 367                                   src16->utf16, src16->utf16_len,
 368                                   locale, status);
 369         break;
 370     case 'u':
 371     case 'U':
 372         dest16_len = u_strToUpper(dest16->utf16, dest16->utf16_cap,
 373                                   src16->utf16, src16->utf16_len,
 374                                   locale, status);
 375         break;
 376     case 't':
 377     case 'T':
 378         dest16_len = u_strToTitle(dest16->utf16, dest16->utf16_cap,
 379                                   src16->utf16, src16->utf16_len,
 380                                   0, locale, status);
 381         break;
 382     case 'f':
 383     case 'F':
 384         dest16_len = u_strFoldCase(dest16->utf16, dest16->utf16_cap,
 385                                    src16->utf16, src16->utf16_len,
 386                                    U_FOLD_CASE_DEFAULT, status);
 387         break;
 388
 389     default:
 390         return U_UNSUPPORTED_ERROR;
 391         break;
 392     }
 393
 394     /* check for buffer overflow, resize and retry */
 395     if (*status == U_BUFFER_OVERFLOW_ERROR
 396         && dest16 != src16        /* do not resize if in-place conversion */
 397         ){
 398         icu_buf_utf16_resize(dest16, dest16_len * 2);
 399         *status = U_ZERO_ERROR;
 400
 401
 402         switch(action) {
 403         case 'l':
 404         case 'L':
 405             dest16_len = u_strToLower(dest16->utf16, dest16->utf16_cap,
 406                                       src16->utf16, src16->utf16_len,
 407                                       locale, status);
 408             break;
 409         case 'u':
 410         case 'U':
 411             dest16_len = u_strToUpper(dest16->utf16, dest16->utf16_cap,
 412                                       src16->utf16, src16->utf16_len,
 413                                       locale, status);
 414             break;
 415         case 't':
 416         case 'T':
 417             dest16_len = u_strToTitle(dest16->utf16, dest16->utf16_cap,
 418                                       src16->utf16, src16->utf16_len,
 419                                       0, locale, status);
 420             break;
 421         case 'f':
 422         case 'F':
 423             dest16_len = u_strFoldCase(dest16->utf16, dest16->utf16_cap,
 424                                        src16->utf16, src16->utf16_len,
 425                                        U_FOLD_CASE_DEFAULT, status);
 426             break;
 427
 428         default:
 429             return U_UNSUPPORTED_ERROR;
 430             break;
 431         }
 432     }
 433
 434     if (U_SUCCESS(*status)
 435         && dest16_len <= dest16->utf16_cap)
 436         dest16->utf16_len = dest16_len;
 437     else {
 438         if (dest16->utf16)
 439             dest16->utf16[0] = (UChar) 0;
 440         dest16->utf16_len = 0;
 441     }
 442
 443     return *status;
 444 }
 445
 446
 447
 448 void icu_sortkey8_from_utf16(UCollator *coll,
 449                              struct icu_buf_utf8 * dest8,
 450                              struct icu_buf_utf16 * src16,
 451                              UErrorCode * status)
 452 {
 453
 454     int32_t sortkey_len = 0;
 455
 456     sortkey_len = ucol_getSortKey(coll, src16->utf16, src16->utf16_len,
 457                                   dest8->utf8, dest8->utf8_cap);
 458
 459     /* check for buffer overflow, resize and retry */
 460     if (sortkey_len > dest8->utf8_cap) {
 461         icu_buf_utf8_resize(dest8, sortkey_len * 2);
 462         sortkey_len = ucol_getSortKey(coll, src16->utf16, src16->utf16_len,
 463                                       dest8->utf8, dest8->utf8_cap);
 464     }
 465
 466     if (U_SUCCESS(*status)
 467         && sortkey_len > 0)
 468         dest8->utf8_len = sortkey_len;
 469     else
 470         icu_buf_utf8_clear(dest8);
 471 }
 472
 473
 474
 475 struct icu_tokenizer * icu_tokenizer_create(const char *locale, char action,
 476                                             UErrorCode *status)
 477 {
 478     struct icu_tokenizer * tokenizer
 479         = (struct icu_tokenizer *) xmalloc(sizeof(struct icu_tokenizer));
 480
 481     tokenizer->action = action;
 482     tokenizer->bi = 0;
 483     tokenizer->buf16 = 0;
 484     tokenizer->token_count = 0;
 485     tokenizer->token_id = 0;
 486     tokenizer->token_start = 0;
 487     tokenizer->token_end = 0;
 488
 489
 490     switch(tokenizer->action) {
 491     case 'l':
 492     case 'L':
 493         tokenizer->bi = ubrk_open(UBRK_LINE, locale, 0, 0, status);
 494         break;
 495     case 's':
 496     case 'S':
 497         tokenizer->bi = ubrk_open(UBRK_SENTENCE, locale, 0, 0, status);
 498         break;
 499     case 'w':
 500     case 'W':
 501         tokenizer->bi = ubrk_open(UBRK_WORD, locale, 0, 0, status);
 502         break;
 503     case 'c':
 504     case 'C':
 505         tokenizer->bi = ubrk_open(UBRK_CHARACTER, locale, 0, 0, status);
 506         break;
 507     case 't':
 508     case 'T':
 509         tokenizer->bi = ubrk_open(UBRK_TITLE, locale, 0, 0, status);
 510         break;
 511     default:
 512         *status = U_UNSUPPORTED_ERROR;
 513         return 0;
 514         break;
 515     }
 516
 517     /* ICU error stuff is a very  funny business */
 518     if (U_SUCCESS(*status))
 519         return tokenizer;
 520
 521     /* freeing if failed */
 522     icu_tokenizer_destroy(tokenizer);
 523     return 0;
 524 }
 525
 526 void icu_tokenizer_destroy(struct icu_tokenizer * tokenizer)
 527 {
 528     if (tokenizer) {
 529         if (tokenizer->bi)
 530             ubrk_close(tokenizer->bi);
 531         xfree(tokenizer);
 532     }
 533 }
 534
 535 int icu_tokenizer_attach(struct icu_tokenizer * tokenizer,
 536                          struct icu_buf_utf16 * src16,
 537                          UErrorCode *status)
 538 {
 539     if (!tokenizer || !tokenizer->bi || !src16)
 540         return 0;
 541
 542
 543     tokenizer->buf16 = src16;
 544     tokenizer->token_count = 0;
 545     tokenizer->token_id = 0;
 546     tokenizer->token_start = 0;
 547     tokenizer->token_end = 0;
 548
 549     ubrk_setText(tokenizer->bi, src16->utf16, src16->utf16_len, status);
 550
 551
 552     if (U_FAILURE(*status))
 553         return 0;
 554
 555     return 1;
 556 };
 557
 558 int32_t icu_tokenizer_next_token(struct icu_tokenizer * tokenizer,
 559                          struct icu_buf_utf16 * tkn16,
 560                          UErrorCode *status)
 561 {
 562     int32_t tkn_start = 0;
 563     int32_t tkn_end = 0;
 564     int32_t tkn_len = 0;
 565
 566
 567     if (!tokenizer || !tokenizer->bi
 568         || !tokenizer->buf16 || !tokenizer->buf16->utf16_len)
 569         return 0;
 570
 571     /*
 572     never change tokenizer->buf16 and keep always invariant
 573     0 <= tokenizer->token_start
 574        <= tokenizer->token_end
 575        <= tokenizer->buf16->utf16_len
 576     returns length of token
 577     */
 578
 579     if (0 == tokenizer->token_end) /* first call */
 580         tkn_start = ubrk_first(tokenizer->bi);
 581     else /* successive calls */
 582         tkn_start = tokenizer->token_end;
 583
 584     /* get next position */
 585     tkn_end = ubrk_next(tokenizer->bi);
 586
 587     /* repairing invariant at end of ubrk, which is UBRK_DONE = -1 */
 588     if (UBRK_DONE == tkn_end)
 589         tkn_end = tokenizer->buf16->utf16_len;
 590
 591     /* copy out if everything is well */
 592     if(U_FAILURE(*status))
 593         return 0;
 594
 595     /* everything OK, now update internal state */
 596     tkn_len = tkn_end - tkn_start;
 597
 598     if (0 < tkn_len){
 599         tokenizer->token_count++;
 600         tokenizer->token_id++;
 601     } else {
 602         tokenizer->token_id = 0;
 603     }
 604     tokenizer->token_start = tkn_start;
 605     tokenizer->token_end = tkn_end;
 606
 607
 608     /* copying into token buffer if it exists */
 609     if (tkn16){
 610         if (tkn16->utf16_cap < tkn_len)
 611             icu_buf_utf16_resize(tkn16, (size_t) tkn_len * 2);
 612
 613         u_strncpy(tkn16->utf16, &(tokenizer->buf16->utf16)[tkn_start],
 614                   tkn_len);
 615
 616         tkn16->utf16_len = tkn_len;
 617     }
 618
 619     return tkn_len;
 620 }
 621
 622
 623 int32_t icu_tokenizer_token_id(struct icu_tokenizer * tokenizer)
 624 {
 625     return tokenizer->token_id;
 626 }
 627
 628 int32_t icu_tokenizer_token_start(struct icu_tokenizer * tokenizer)
 629 {
 630     return tokenizer->token_start;
 631 }
 632
 633 int32_t icu_tokenizer_token_end(struct icu_tokenizer * tokenizer)
 634 {
 635     return tokenizer->token_end;
 636 }
 637
 638 int32_t icu_tokenizer_token_length(struct icu_tokenizer * tokenizer)
 639 {
 640     return (tokenizer->token_end - tokenizer->token_start);
 641 }
 642
 643 int32_t icu_tokenizer_token_count(struct icu_tokenizer * tokenizer)
 644 {
 645     return tokenizer->token_count;
 646 }
 647
 648
 649
 650 struct icu_normalizer * icu_normalizer_create(const char *rules, char action,
 651                                               UErrorCode *status)
 652 {
 653
 654     struct icu_normalizer * normalizer
 655         = (struct icu_normalizer *) xmalloc(sizeof(struct icu_normalizer));
 656
 657     normalizer->action = action;
 658     normalizer->trans = 0;
 659     normalizer->rules16 =  icu_buf_utf16_create(0);
 660     icu_utf16_from_utf8_cstr(normalizer->rules16, rules, status);
 661
 662     switch(normalizer->action) {
 663     case 'f':
 664     case 'F':
 665         normalizer->trans
 666             = utrans_openU(normalizer->rules16->utf16,
 667                            normalizer->rules16->utf16_len,
 668                            UTRANS_FORWARD,
 669                            0, 0,
 670                            normalizer->parse_error, status);
 671         break;
 672     case 'r':
 673     case 'R':
 674         normalizer->trans
 675             = utrans_openU(normalizer->rules16->utf16,
 676                            normalizer->rules16->utf16_len,
 677                            UTRANS_REVERSE ,
 678                            0, 0,
 679                            normalizer->parse_error, status);
 680         break;
 681     default:
 682         *status = U_UNSUPPORTED_ERROR;
 683         return 0;
 684         break;
 685     }
 686
 687     if (U_SUCCESS(*status))
 688         return normalizer;
 689
 690     /* freeing if failed */
 691     icu_normalizer_destroy(normalizer);
 692     return 0;
 693 }
 694
 695
 696 void icu_normalizer_destroy(struct icu_normalizer * normalizer){
 697     if (normalizer) {
 698         if (normalizer->rules16)
 699             icu_buf_utf16_destroy(normalizer->rules16);
 700         if (normalizer->trans)
 701             utrans_close(normalizer->trans);
 702         xfree(normalizer);
 703     }
 704 }
 705
 706
 707
 708 int icu_normalizer_normalize(struct icu_normalizer * normalizer,
 709                              struct icu_buf_utf16 * dest16,
 710                              struct icu_buf_utf16 * src16,
 711                              UErrorCode *status)
 712 {
 713     if (!normalizer || !normalizer->trans
 714         || !src16
 715         || !dest16)
 716         return 0;
 717
 718     if (!src16->utf16_len){           /* guarding for empty source string */
 719         icu_buf_utf16_clear(dest16);
 720         return 0;
 721     }
 722
 723     if (!icu_buf_utf16_copy(dest16, src16))
 724         return 0;
 725
 726
 727     utrans_transUChars (normalizer->trans,
 728                         dest16->utf16, &(dest16->utf16_len),
 729                         dest16->utf16_cap,
 730                         0, &(src16->utf16_len), status);
 731
 732     if (U_FAILURE(*status))
 733         icu_buf_utf16_clear(dest16);
 734
 735     return dest16->utf16_len;
 736 }
 737
 738
 739
 740
 741 struct icu_chain_step * icu_chain_step_create(struct icu_chain * chain,
 742                                               enum icu_chain_step_type type,
 743                                               const uint8_t * rule,
 744                                               struct icu_buf_utf16 * buf16,
 745                                               UErrorCode *status)
 746 {
 747     struct icu_chain_step * step = 0;
 748
 749     if(!chain || !type || !rule)
 750         return 0;
 751
 752     step = (struct icu_chain_step *) xmalloc(sizeof(struct icu_chain_step));
 753
 754     step->type = type;
 755
 756     step->buf16 = buf16;
 757
 758     /* create auxilary objects */
 759     switch(step->type) {
 760     case ICU_chain_step_type_display:
 761         break;
 762     case ICU_chain_step_type_casemap:
 763         step->u.casemap = icu_casemap_create(rule[0], status);
 764         break;
 765     case ICU_chain_step_type_normalize:
 766         step->u.normalizer = icu_normalizer_create((char *) rule, 'f', status);
 767         break;
 768     case ICU_chain_step_type_tokenize:
 769         step->u.tokenizer = icu_tokenizer_create((char *) chain->locale,
 770                                                  (char) rule[0], status);
 771         break;
 772     default:
 773         break;
 774     }
 775
 776     return step;
 777 }
 778
 779
 780 void icu_chain_step_destroy(struct icu_chain_step * step){
 781
 782     if (!step)
 783         return;
 784
 785     icu_chain_step_destroy(step->previous);
 786
 787     switch(step->type) {
 788     case ICU_chain_step_type_display:
 789         break;
 790     case ICU_chain_step_type_casemap:
 791         icu_casemap_destroy(step->u.casemap);
 792         icu_buf_utf16_destroy(step->buf16);
 793         break;
 794     case ICU_chain_step_type_normalize:
 795         icu_normalizer_destroy(step->u.normalizer);
 796         icu_buf_utf16_destroy(step->buf16);
 797         break;
 798     case ICU_chain_step_type_tokenize:
 799         icu_tokenizer_destroy(step->u.tokenizer);
 800         icu_buf_utf16_destroy(step->buf16);
 801         break;
 802     default:
 803         break;
 804     }
 805     xfree(step);
 806 }
 807
 808
 809
 810 struct icu_chain * icu_chain_create(const char *locale,  int sort,
 811                                     UErrorCode * status)
 812 {
 813     struct icu_chain * chain
 814         = (struct icu_chain *) xmalloc(sizeof(struct icu_chain));
 815
 816     *status = U_ZERO_ERROR;
 817
 818     chain->locale = xstrdup(locale);
 819
 820     chain->sort = sort;
 821
 822     chain->coll = ucol_open((const char *) chain->locale, status);
 823
 824     if (U_FAILURE(*status))
 825         return 0;
 826
 827     chain->token_count = 0;
 828
 829     chain->src8cstr = 0;
 830
 831     chain->display8 = icu_buf_utf8_create(0);
 832     chain->norm8 = icu_buf_utf8_create(0);
 833     chain->sort8 = icu_buf_utf8_create(0);
 834
 835     chain->src16 = icu_buf_utf16_create(0);
 836
 837     chain->steps = 0;
 838
 839     return chain;
 840 }
 841
 842
 843 void icu_chain_destroy(struct icu_chain * chain)
 844 {
 845     if (chain)
 846     {
 847         if (chain->coll)
 848             ucol_close(chain->coll);
 849
 850         icu_buf_utf8_destroy(chain->display8);
 851         icu_buf_utf8_destroy(chain->norm8);
 852         icu_buf_utf8_destroy(chain->sort8);
 853
 854         icu_buf_utf16_destroy(chain->src16);
 855
 856         icu_chain_step_destroy(chain->steps);
 857         xfree(chain->locale);
 858         xfree(chain);
 859     }
 860 }
 861
 862
 863
 864 struct icu_chain * icu_chain_xml_config(const xmlNode *xml_node,
 865                                         int sort,
 866                                         UErrorCode * status)
 867 {
 868     xmlNode *node = 0;
 869     struct icu_chain * chain = 0;
 870
 871     *status = U_ZERO_ERROR;
 872
 873     if (!xml_node ||xml_node->type != XML_ELEMENT_NODE)
 874         return 0;
 875
 876     {
 877         xmlChar * xml_locale = xmlGetProp((xmlNode *) xml_node,
 878                                           (xmlChar *) "locale");
 879
 880         if (xml_locale)
 881         {
 882             chain = icu_chain_create((const char *) xml_locale, sort, status);
 883             xmlFree(xml_locale);
 884         }
 885
 886     }
 887     if (!chain)
 888         return 0;
 889
 890     for (node = xml_node->children; node; node = node->next)
 891     {
 892         xmlChar *xml_rule;
 893         struct icu_chain_step * step = 0;
 894
 895         if (node->type != XML_ELEMENT_NODE)
 896             continue;
 897
 898         xml_rule = xmlGetProp(node, (xmlChar *) "rule");
 899
 900         if (!strcmp((const char *) node->name, "casemap"))
 901             step = icu_chain_insert_step(chain, ICU_chain_step_type_casemap,
 902                                          (const uint8_t *) xml_rule, status);
 903         else if (!strcmp((const char *) node->name, "transform"))
 904             step = icu_chain_insert_step(chain, ICU_chain_step_type_normalize,
 905                                          (const uint8_t *) xml_rule, status);
 906         else if (!strcmp((const char *) node->name, "tokenize"))
 907             step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize,
 908                                          (const uint8_t *) xml_rule, status);
 909         else if (!strcmp((const char *) node->name, "display"))
 910             step = icu_chain_insert_step(chain, ICU_chain_step_type_display,
 911                                          (const uint8_t *) "", status);
 912         xmlFree(xml_rule);
 913         if (!step || U_FAILURE(*status))
 914         {
 915             icu_chain_destroy(chain);
 916             return 0;
 917         }
 918
 919
 920     }
 921     return chain;
 922 }
 923
 924
 925
 926 struct icu_chain_step * icu_chain_insert_step(struct icu_chain * chain,
 927                                               enum icu_chain_step_type type,
 928                                               const uint8_t * rule,
 929                                               UErrorCode *status)
 930 {
 931     struct icu_chain_step * step = 0;
 932     struct icu_buf_utf16 * src16 = 0;
 933     struct icu_buf_utf16 * buf16 = 0;
 934
 935     if (!chain || !type || !rule)
 936         return 0;
 937
 938     /* assign utf16 src buffers as needed */
 939     if (chain->steps && chain->steps->buf16)
 940         src16 = chain->steps->buf16;
 941     else if (chain->src16)
 942         src16 = chain->src16;
 943     else
 944         return 0;
 945
 946
 947     /* create utf16 destination buffers as needed, or */
 948     switch(type)
 949     {
 950     case ICU_chain_step_type_display:
 951         buf16 = src16;
 952         break;
 953     case ICU_chain_step_type_casemap:
 954         buf16 = icu_buf_utf16_create(0);
 955         break;
 956     case ICU_chain_step_type_normalize:
 957         buf16 = icu_buf_utf16_create(0);
 958         break;
 959     case ICU_chain_step_type_tokenize:
 960         buf16 = icu_buf_utf16_create(0);
 961         break;
 962     default:
 963         break;
 964     }
 965
 966     /* create actual chain step with this buffer */
 967     step = icu_chain_step_create(chain, type, rule, buf16, status);
 968
 969     step->previous = chain->steps;
 970     chain->steps = step;
 971
 972     return step;
 973 }
 974
 975
 976 int icu_chain_step_next_token(struct icu_chain * chain,
 977                               struct icu_chain_step * step,
 978                               UErrorCode *status)
 979 {
 980     struct icu_buf_utf16 * src16 = 0;
 981     int got_new_token = 0;
 982
 983     if (!chain || !chain->src16 || !step || !step->more_tokens)
 984         return 0;
 985
 986     /* assign utf16 src buffers as neeed, advance in previous steps
 987        tokens until non-zero token met, and setting stop condition */
 988
 989     if (step->previous)
 990     {
 991         src16 = step->previous->buf16;
 992         /* tokens might be killed in previous steps, therefore looping */
 993
 994         while (step->need_new_token
 995                && step->previous->more_tokens
 996                && !got_new_token)
 997             got_new_token
 998                 = icu_chain_step_next_token(chain, step->previous, status);
 999     }
1000     else
1001     { /* first step can only work once on chain->src16 input buffer */
1002         src16 = chain->src16;
1003         step->more_tokens = 0;
1004         got_new_token = 1;
1005     }
1006
1007     if (!src16)
1008         return 0;
1009
1010     /* stop if nothing to process */
1011     if (step->need_new_token && !got_new_token)
1012     {
1013         step->more_tokens = 0;
1014         return 0;
1015     }
1016
1017     /* either an old token not finished yet, or a new token, thus
1018        perform the work, eventually put this steps output in
1019        step->buf16 or the chains UTF8 output buffers  */
1020
1021     switch(step->type)
1022     {
1023     case ICU_chain_step_type_display:
1024         icu_utf16_to_utf8(chain->display8, src16, status);
1025         break;
1026     case ICU_chain_step_type_casemap:
1027         icu_casemap_casemap(step->u.casemap,
1028                             step->buf16, src16, status,
1029                             chain->locale);
1030         break;
1031     case ICU_chain_step_type_normalize:
1032         icu_normalizer_normalize(step->u.normalizer,
1033                                  step->buf16, src16, status);
1034         break;
1035     case ICU_chain_step_type_tokenize:
1036         /* attach to new src16 token only first time during splitting */
1037         if (step->need_new_token)
1038         {
1039             icu_tokenizer_attach(step->u.tokenizer, src16, status);
1040             step->need_new_token = 0;
1041         }
1042
1043         /* splitting one src16 token into multiple buf16 tokens */
1044         step->more_tokens
1045             = icu_tokenizer_next_token(step->u.tokenizer,
1046                                        step->buf16, status);
1047
1048         /* make sure to get new previous token if this one had been used up
1049            by recursive call to _same_ step */
1050
1051         if (!step->more_tokens)
1052         {
1053             step->more_tokens = icu_chain_step_next_token(chain, step, status);
1054             return step->more_tokens;  /* avoid one token count too much! */
1055         }
1056         break;
1057     default:
1058         return 0;
1059         break;
1060     }
1061
1062     if (U_FAILURE(*status))
1063         return 0;
1064
1065     /* if token disappered into thin air, tell caller */
1066     /* if (!step->buf16->utf16_len && !step->more_tokens) */
1067     /*    return 0; */
1068
1069     return 1;
1070 }
1071
1072
1073 int icu_chain_assign_cstr(struct icu_chain * chain,
1074                           const char * src8cstr,
1075                           UErrorCode *status)
1076 {
1077     struct icu_chain_step * stp = 0;
1078
1079     if (!chain || !src8cstr)
1080         return 0;
1081
1082     chain->src8cstr = src8cstr;
1083
1084     stp = chain->steps;
1085
1086     /* clear token count */
1087     chain->token_count = 0;
1088
1089     /* clear all steps stop states */
1090     while (stp)
1091     {
1092         stp->more_tokens = 1;
1093         stp->need_new_token = 1;
1094         stp = stp->previous;
1095     }
1096
1097     /* finally convert UTF8 to UTF16 string if needed */
1098     if (chain->steps || chain->sort)
1099         icu_utf16_from_utf8_cstr(chain->src16, chain->src8cstr, status);
1100
1101     if (U_FAILURE(*status))
1102         return 0;
1103
1104     return 1;
1105 }
1106
1107
1108
1109 int icu_chain_next_token(struct icu_chain * chain,
1110                          UErrorCode *status)
1111 {
1112     int got_token = 0;
1113
1114     *status = U_ZERO_ERROR;
1115
1116     if (!chain)
1117         return 0;
1118
1119     /* special case with no steps - same as index type binary */
1120     if (!chain->steps)
1121     {
1122         if (chain->token_count)
1123             return 0;
1124         else
1125         {
1126             chain->token_count++;
1127
1128             if (chain->sort)
1129                 icu_sortkey8_from_utf16(chain->coll,
1130                                         chain->sort8, chain->steps->buf16,
1131                                         status);
1132             return chain->token_count;
1133         }
1134     }
1135     /* usual case, one or more icu chain steps existing */
1136     else
1137     {
1138         while(!got_token && chain->steps && chain->steps->more_tokens)
1139             got_token = icu_chain_step_next_token(chain, chain->steps, status);
1140
1141         if (got_token)
1142         {
1143             chain->token_count++;
1144
1145             icu_utf16_to_utf8(chain->norm8, chain->steps->buf16, status);
1146
1147             if (chain->sort)
1148                 icu_sortkey8_from_utf16(chain->coll,
1149                                         chain->sort8, chain->steps->buf16,
1150                                         status);
1151
1152             return chain->token_count;
1153         }
1154     }
1155
1156     return 0;
1157 }
1158
1159 int icu_chain_token_number(struct icu_chain * chain)
1160 {
1161     if (!chain)
1162         return 0;
1163
1164     return chain->token_count;
1165 }
1166
1167
1168 const char * icu_chain_token_display(struct icu_chain * chain)
1169 {
1170     if (chain->display8)
1171         return icu_buf_utf8_to_cstr(chain->display8);
1172
1173     return 0;
1174 }
1175
1176 const char * icu_chain_token_norm(struct icu_chain * chain)
1177 {
1178     if (!chain->steps)
1179         return chain->src8cstr;
1180
1181     if (chain->norm8)
1182         return icu_buf_utf8_to_cstr(chain->norm8);
1183
1184     return 0;
1185 }
1186
1187 const char * icu_chain_token_sortkey(struct icu_chain * chain)
1188 {
1189     if (chain->sort8)
1190         return icu_buf_utf8_to_cstr(chain->sort8);
1191
1192     return 0;
1193 }
1194
1195 const UCollator * icu_chain_get_coll(struct icu_chain * chain)
1196 {
1197     return chain->coll;
1198 }
1199
1200 #endif /* YAZ_HAVE_ICU */
1201
1202 /*
1203  * Local variables:
1204  * c-basic-offset: 4
1205  * indent-tabs-mode: nil
1206  * End:
1207  * vim: shiftwidth=4 tabstop=8 expandtab
1208  */