src/icu_I18N.c

   1 /* This file is part of the YAZ toolkit.
   2  * Copyright (C) 1995-2009 Index Data
   3  * See the file LICENSE for details.
   4  */
   5
   6 /**
   7  * \file icu_I18N.c
   8  * \brief ICU utilities
   9  */
  10
  11 #if HAVE_CONFIG_H
  12 #include "config.h"
  13 #endif
  14
  15 #define USE_TIMING 0
  16 #if USE_TIMING
  17 #include <yaz/timing.h>
  18 #endif
  19
  20 #if YAZ_HAVE_ICU
  21 #include <yaz/xmalloc.h>
  22
  23 #include <yaz/icu_I18N.h>
  24
  25 #include <yaz/log.h>
  26
  27 #include <string.h>
  28 #include <stdlib.h>
  29 #include <stdio.h>
  30
  31 #include <unicode/ustring.h>  /* some more string fcns*/
  32 #include <unicode/uchar.h>    /* char names           */
  33
  34
  35 #include <unicode/ucol.h>
  36
  37
  38 int icu_check_status (UErrorCode status)
  39 {
  40     if (U_FAILURE(status))
  41     {
  42         yaz_log(YLOG_WARN, "ICU: %d %s\n", status, u_errorName(status));
  43         return 0;
  44     }
  45     return 1;
  46
  47 }
  48
  49
  50
  51 struct icu_buf_utf16 * icu_buf_utf16_create(size_t capacity)
  52 {
  53     struct icu_buf_utf16 * buf16
  54         = (struct icu_buf_utf16 *) xmalloc(sizeof(struct icu_buf_utf16));
  55
  56     buf16->utf16 = 0;
  57     buf16->utf16_len = 0;
  58     buf16->utf16_cap = 0;
  59
  60     if (capacity > 0){
  61         buf16->utf16 = (UChar *) xmalloc(sizeof(UChar) * capacity);
  62         buf16->utf16[0] = (UChar) 0;
  63         buf16->utf16_cap = capacity;
  64     }
  65     return buf16;
  66 }
  67
  68 struct icu_buf_utf16 * icu_buf_utf16_clear(struct icu_buf_utf16 * buf16)
  69 {
  70     if (buf16){
  71         if (buf16->utf16)
  72             buf16->utf16[0] = (UChar) 0;
  73         buf16->utf16_len = 0;
  74     }
  75     return buf16;
  76 }
  77
  78 struct icu_buf_utf16 * icu_buf_utf16_resize(struct icu_buf_utf16 * buf16,
  79                                             size_t capacity)
  80 {
  81     if (!buf16)
  82         return 0;
  83
  84     if (capacity >  0){
  85         if (0 == buf16->utf16)
  86             buf16->utf16 = (UChar *) xmalloc(sizeof(UChar) * capacity);
  87         else
  88             buf16->utf16
  89                 = (UChar *) xrealloc(buf16->utf16, sizeof(UChar) * capacity);
  90
  91         icu_buf_utf16_clear(buf16);
  92         buf16->utf16_cap = capacity;
  93     }
  94     else {
  95         xfree(buf16->utf16);
  96         buf16->utf16 = 0;
  97         buf16->utf16_len = 0;
  98         buf16->utf16_cap = 0;
  99     }
 100
 101     return buf16;
 102 }
 103
 104
 105 struct icu_buf_utf16 * icu_buf_utf16_copy(struct icu_buf_utf16 * dest16,
 106                                           struct icu_buf_utf16 * src16)
 107 {
 108     if(!dest16 || !src16
 109        || dest16 == src16)
 110         return 0;
 111
 112     if (dest16->utf16_cap < src16->utf16_len)
 113         icu_buf_utf16_resize(dest16, src16->utf16_len * 2);
 114
 115     u_strncpy(dest16->utf16, src16->utf16, src16->utf16_len);
 116     dest16->utf16_len = src16->utf16_len;
 117
 118     return dest16;
 119 }
 120
 121
 122 void icu_buf_utf16_destroy(struct icu_buf_utf16 * buf16)
 123 {
 124     if (buf16)
 125         xfree(buf16->utf16);
 126     xfree(buf16);
 127 }
 128
 129
 130
 131 struct icu_buf_utf8 * icu_buf_utf8_create(size_t capacity)
 132 {
 133     struct icu_buf_utf8 * buf8
 134         = (struct icu_buf_utf8 *) xmalloc(sizeof(struct icu_buf_utf8));
 135
 136     buf8->utf8 = 0;
 137     buf8->utf8_len = 0;
 138     buf8->utf8_cap = 0;
 139
 140     if (capacity > 0){
 141         buf8->utf8 = (uint8_t *) xmalloc(sizeof(uint8_t) * capacity);
 142         buf8->utf8[0] = (uint8_t) 0;
 143         buf8->utf8_cap = capacity;
 144     }
 145     return buf8;
 146 }
 147
 148
 149 struct icu_buf_utf8 * icu_buf_utf8_clear(struct icu_buf_utf8 * buf8)
 150 {
 151     if (buf8){
 152         if (buf8->utf8)
 153             buf8->utf8[0] = (uint8_t) 0;
 154         buf8->utf8_len = 0;
 155     }
 156     return buf8;
 157 }
 158
 159
 160 struct icu_buf_utf8 * icu_buf_utf8_resize(struct icu_buf_utf8 * buf8,
 161                                           size_t capacity)
 162 {
 163     if (!buf8)
 164         return 0;
 165
 166     if (capacity >  0){
 167         if (0 == buf8->utf8)
 168             buf8->utf8 = (uint8_t *) xmalloc(sizeof(uint8_t) * capacity);
 169         else
 170             buf8->utf8
 171                 = (uint8_t *) xrealloc(buf8->utf8, sizeof(uint8_t) * capacity);
 172
 173         buf8->utf8_cap = capacity;
 174     }
 175     else {
 176         xfree(buf8->utf8);
 177         buf8->utf8 = 0;
 178         buf8->utf8_len = 0;
 179         buf8->utf8_cap = 0;
 180     }
 181
 182     return buf8;
 183 }
 184
 185
 186 const char *icu_buf_utf8_to_cstr(struct icu_buf_utf8 *src8)
 187 {
 188     if (!src8 || src8->utf8_len == 0)
 189         return "";
 190
 191     if (src8->utf8_len == src8->utf8_cap)
 192         src8 = icu_buf_utf8_resize(src8, src8->utf8_len * 2 + 1);
 193
 194     src8->utf8[src8->utf8_len] = '\0';
 195
 196     return (const char *) src8->utf8;
 197 }
 198
 199
 200 void icu_buf_utf8_destroy(struct icu_buf_utf8 * buf8)
 201 {
 202     if (buf8)
 203         xfree(buf8->utf8);
 204     xfree(buf8);
 205 }
 206
 207
 208
 209 UErrorCode icu_utf16_from_utf8(struct icu_buf_utf16 * dest16,
 210                                struct icu_buf_utf8 * src8,
 211                                UErrorCode * status)
 212 {
 213     int32_t utf16_len = 0;
 214
 215     u_strFromUTF8(dest16->utf16, dest16->utf16_cap,
 216                   &utf16_len,
 217                   (const char *) src8->utf8, src8->utf8_len, status);
 218
 219     /* check for buffer overflow, resize and retry */
 220     if (*status == U_BUFFER_OVERFLOW_ERROR)
 221     {
 222         icu_buf_utf16_resize(dest16, utf16_len * 2);
 223         *status = U_ZERO_ERROR;
 224         u_strFromUTF8(dest16->utf16, dest16->utf16_cap,
 225                       &utf16_len,
 226                       (const char *) src8->utf8, src8->utf8_len, status);
 227     }
 228
 229     if (U_SUCCESS(*status)
 230         && utf16_len <= dest16->utf16_cap)
 231         dest16->utf16_len = utf16_len;
 232     else
 233         icu_buf_utf16_clear(dest16);
 234
 235     return *status;
 236 }
 237
 238
 239
 240 UErrorCode icu_utf16_from_utf8_cstr(struct icu_buf_utf16 * dest16,
 241                                     const char * src8cstr,
 242                                     UErrorCode * status)
 243 {
 244     size_t src8cstr_len = 0;
 245     int32_t utf16_len = 0;
 246
 247     *status = U_ZERO_ERROR;
 248     src8cstr_len = strlen(src8cstr);
 249
 250     u_strFromUTF8(dest16->utf16, dest16->utf16_cap,
 251                   &utf16_len,
 252                   src8cstr, src8cstr_len, status);
 253
 254     /* check for buffer overflow, resize and retry */
 255     if (*status == U_BUFFER_OVERFLOW_ERROR)
 256     {
 257         icu_buf_utf16_resize(dest16, utf16_len * 2);
 258         *status = U_ZERO_ERROR;
 259         u_strFromUTF8(dest16->utf16, dest16->utf16_cap,
 260                       &utf16_len,
 261                       src8cstr, src8cstr_len, status);
 262     }
 263
 264     if (U_SUCCESS(*status)
 265         && utf16_len <= dest16->utf16_cap)
 266         dest16->utf16_len = utf16_len;
 267     else
 268         icu_buf_utf16_clear(dest16);
 269
 270     return *status;
 271 }
 272
 273
 274
 275
 276 UErrorCode icu_utf16_to_utf8(struct icu_buf_utf8 * dest8,
 277                              struct icu_buf_utf16 * src16,
 278                              UErrorCode * status)
 279 {
 280     int32_t utf8_len = 0;
 281
 282     u_strToUTF8((char *) dest8->utf8, dest8->utf8_cap,
 283                 &utf8_len,
 284                 src16->utf16, src16->utf16_len, status);
 285
 286     /* check for buffer overflow, resize and retry */
 287     if (*status == U_BUFFER_OVERFLOW_ERROR)
 288     {
 289         icu_buf_utf8_resize(dest8, utf8_len * 2);
 290         *status = U_ZERO_ERROR;
 291         u_strToUTF8((char *) dest8->utf8, dest8->utf8_cap,
 292                     &utf8_len,
 293                     src16->utf16, src16->utf16_len, status);
 294
 295     }
 296
 297     if (U_SUCCESS(*status)
 298         && utf8_len <= dest8->utf8_cap)
 299         dest8->utf8_len = utf8_len;
 300     else
 301         icu_buf_utf8_clear(dest8);
 302
 303     return *status;
 304 }
 305
 306
 307
 308 struct icu_casemap * icu_casemap_create(char action, UErrorCode *status)
 309 {
 310     struct icu_casemap * casemap
 311         = (struct icu_casemap *) xmalloc(sizeof(struct icu_casemap));
 312     casemap->action = action;
 313
 314     switch(casemap->action) {
 315     case 'l':
 316     case 'L':
 317     case 'u':
 318     case 'U':
 319     case 't':
 320     case 'T':
 321     case 'f':
 322     case 'F':
 323         break;
 324     default:
 325         icu_casemap_destroy(casemap);
 326         return 0;
 327     }
 328
 329     return casemap;
 330 }
 331
 332 void icu_casemap_destroy(struct icu_casemap * casemap)
 333 {
 334     xfree(casemap);
 335 }
 336
 337
 338 int icu_casemap_casemap(struct icu_casemap * casemap,
 339                         struct icu_buf_utf16 * dest16,
 340                         struct icu_buf_utf16 * src16,
 341                         UErrorCode *status,
 342                         const char *locale)
 343 {
 344     if(!casemap)
 345         return 0;
 346
 347     return icu_utf16_casemap(dest16, src16, locale,
 348                              casemap->action, status);
 349 }
 350
 351
 352 int icu_utf16_casemap(struct icu_buf_utf16 * dest16,
 353                       struct icu_buf_utf16 * src16,
 354                       const char *locale, char action,
 355                       UErrorCode *status)
 356 {
 357     int32_t dest16_len = 0;
 358
 359
 360     if (!src16->utf16_len){           /* guarding for empty source string */
 361         if (dest16->utf16)
 362             dest16->utf16[0] = (UChar) 0;
 363         dest16->utf16_len = 0;
 364         return U_ZERO_ERROR;
 365     }
 366
 367
 368     switch(action) {
 369     case 'l':
 370     case 'L':
 371         dest16_len = u_strToLower(dest16->utf16, dest16->utf16_cap,
 372                                   src16->utf16, src16->utf16_len,
 373                                   locale, status);
 374         break;
 375     case 'u':
 376     case 'U':
 377         dest16_len = u_strToUpper(dest16->utf16, dest16->utf16_cap,
 378                                   src16->utf16, src16->utf16_len,
 379                                   locale, status);
 380         break;
 381     case 't':
 382     case 'T':
 383         dest16_len = u_strToTitle(dest16->utf16, dest16->utf16_cap,
 384                                   src16->utf16, src16->utf16_len,
 385                                   0, locale, status);
 386         break;
 387     case 'f':
 388     case 'F':
 389         dest16_len = u_strFoldCase(dest16->utf16, dest16->utf16_cap,
 390                                    src16->utf16, src16->utf16_len,
 391                                    U_FOLD_CASE_DEFAULT, status);
 392         break;
 393
 394     default:
 395         return U_UNSUPPORTED_ERROR;
 396         break;
 397     }
 398
 399     /* check for buffer overflow, resize and retry */
 400     if (*status == U_BUFFER_OVERFLOW_ERROR
 401         && dest16 != src16        /* do not resize if in-place conversion */
 402         ){
 403         icu_buf_utf16_resize(dest16, dest16_len * 2);
 404         *status = U_ZERO_ERROR;
 405
 406
 407         switch(action) {
 408         case 'l':
 409         case 'L':
 410             dest16_len = u_strToLower(dest16->utf16, dest16->utf16_cap,
 411                                       src16->utf16, src16->utf16_len,
 412                                       locale, status);
 413             break;
 414         case 'u':
 415         case 'U':
 416             dest16_len = u_strToUpper(dest16->utf16, dest16->utf16_cap,
 417                                       src16->utf16, src16->utf16_len,
 418                                       locale, status);
 419             break;
 420         case 't':
 421         case 'T':
 422             dest16_len = u_strToTitle(dest16->utf16, dest16->utf16_cap,
 423                                       src16->utf16, src16->utf16_len,
 424                                       0, locale, status);
 425             break;
 426         case 'f':
 427         case 'F':
 428             dest16_len = u_strFoldCase(dest16->utf16, dest16->utf16_cap,
 429                                        src16->utf16, src16->utf16_len,
 430                                        U_FOLD_CASE_DEFAULT, status);
 431             break;
 432
 433         default:
 434             return U_UNSUPPORTED_ERROR;
 435             break;
 436         }
 437     }
 438
 439     if (U_SUCCESS(*status)
 440         && dest16_len <= dest16->utf16_cap)
 441         dest16->utf16_len = dest16_len;
 442     else {
 443         if (dest16->utf16)
 444             dest16->utf16[0] = (UChar) 0;
 445         dest16->utf16_len = 0;
 446     }
 447
 448     return *status;
 449 }
 450
 451
 452
 453 void icu_sortkey8_from_utf16(UCollator *coll,
 454                              struct icu_buf_utf8 * dest8,
 455                              struct icu_buf_utf16 * src16,
 456                              UErrorCode * status)
 457 {
 458
 459     int32_t sortkey_len = 0;
 460
 461     sortkey_len = ucol_getSortKey(coll, src16->utf16, src16->utf16_len,
 462                                   dest8->utf8, dest8->utf8_cap);
 463
 464     /* check for buffer overflow, resize and retry */
 465     if (sortkey_len > dest8->utf8_cap) {
 466         icu_buf_utf8_resize(dest8, sortkey_len * 2);
 467         sortkey_len = ucol_getSortKey(coll, src16->utf16, src16->utf16_len,
 468                                       dest8->utf8, dest8->utf8_cap);
 469     }
 470
 471     if (U_SUCCESS(*status)
 472         && sortkey_len > 0)
 473         dest8->utf8_len = sortkey_len;
 474     else
 475         icu_buf_utf8_clear(dest8);
 476 }
 477
 478
 479
 480 struct icu_tokenizer * icu_tokenizer_create(const char *locale, char action,
 481                                             UErrorCode *status)
 482 {
 483     struct icu_tokenizer * tokenizer
 484         = (struct icu_tokenizer *) xmalloc(sizeof(struct icu_tokenizer));
 485
 486     tokenizer->action = action;
 487     tokenizer->bi = 0;
 488     tokenizer->buf16 = 0;
 489     tokenizer->token_count = 0;
 490     tokenizer->token_id = 0;
 491     tokenizer->token_start = 0;
 492     tokenizer->token_end = 0;
 493
 494
 495     switch(tokenizer->action) {
 496     case 'l':
 497     case 'L':
 498         tokenizer->bi = ubrk_open(UBRK_LINE, locale, 0, 0, status);
 499         break;
 500     case 's':
 501     case 'S':
 502         tokenizer->bi = ubrk_open(UBRK_SENTENCE, locale, 0, 0, status);
 503         break;
 504     case 'w':
 505     case 'W':
 506         tokenizer->bi = ubrk_open(UBRK_WORD, locale, 0, 0, status);
 507         break;
 508     case 'c':
 509     case 'C':
 510         tokenizer->bi = ubrk_open(UBRK_CHARACTER, locale, 0, 0, status);
 511         break;
 512     case 't':
 513     case 'T':
 514         tokenizer->bi = ubrk_open(UBRK_TITLE, locale, 0, 0, status);
 515         break;
 516     default:
 517         *status = U_UNSUPPORTED_ERROR;
 518         return 0;
 519         break;
 520     }
 521
 522     /* ICU error stuff is a very  funny business */
 523     if (U_SUCCESS(*status))
 524         return tokenizer;
 525
 526     /* freeing if failed */
 527     icu_tokenizer_destroy(tokenizer);
 528     return 0;
 529 }
 530
 531 void icu_tokenizer_destroy(struct icu_tokenizer * tokenizer)
 532 {
 533     if (tokenizer) {
 534         if (tokenizer->bi)
 535             ubrk_close(tokenizer->bi);
 536         xfree(tokenizer);
 537     }
 538 }
 539
 540 int icu_tokenizer_attach(struct icu_tokenizer * tokenizer,
 541                          struct icu_buf_utf16 * src16,
 542                          UErrorCode *status)
 543 {
 544     if (!tokenizer || !tokenizer->bi || !src16)
 545         return 0;
 546
 547
 548     tokenizer->buf16 = src16;
 549     tokenizer->token_count = 0;
 550     tokenizer->token_id = 0;
 551     tokenizer->token_start = 0;
 552     tokenizer->token_end = 0;
 553
 554     ubrk_setText(tokenizer->bi, src16->utf16, src16->utf16_len, status);
 555
 556
 557     if (U_FAILURE(*status))
 558         return 0;
 559
 560     return 1;
 561 };
 562
 563 int32_t icu_tokenizer_next_token(struct icu_tokenizer * tokenizer,
 564                          struct icu_buf_utf16 * tkn16,
 565                          UErrorCode *status)
 566 {
 567     int32_t tkn_start = 0;
 568     int32_t tkn_end = 0;
 569     int32_t tkn_len = 0;
 570
 571
 572     if (!tokenizer || !tokenizer->bi
 573         || !tokenizer->buf16 || !tokenizer->buf16->utf16_len)
 574         return 0;
 575
 576     /*
 577     never change tokenizer->buf16 and keep always invariant
 578     0 <= tokenizer->token_start
 579        <= tokenizer->token_end
 580        <= tokenizer->buf16->utf16_len
 581     returns length of token
 582     */
 583
 584     if (0 == tokenizer->token_end) /* first call */
 585         tkn_start = ubrk_first(tokenizer->bi);
 586     else /* successive calls */
 587         tkn_start = tokenizer->token_end;
 588
 589     /* get next position */
 590     tkn_end = ubrk_next(tokenizer->bi);
 591
 592     /* repairing invariant at end of ubrk, which is UBRK_DONE = -1 */
 593     if (UBRK_DONE == tkn_end)
 594         tkn_end = tokenizer->buf16->utf16_len;
 595
 596     /* copy out if everything is well */
 597     if(U_FAILURE(*status))
 598         return 0;
 599
 600     /* everything OK, now update internal state */
 601     tkn_len = tkn_end - tkn_start;
 602
 603     if (0 < tkn_len){
 604         tokenizer->token_count++;
 605         tokenizer->token_id++;
 606     } else {
 607         tokenizer->token_id = 0;
 608     }
 609     tokenizer->token_start = tkn_start;
 610     tokenizer->token_end = tkn_end;
 611
 612
 613     /* copying into token buffer if it exists */
 614     if (tkn16){
 615         if (tkn16->utf16_cap < tkn_len)
 616             icu_buf_utf16_resize(tkn16, (size_t) tkn_len * 2);
 617
 618         u_strncpy(tkn16->utf16, &(tokenizer->buf16->utf16)[tkn_start],
 619                   tkn_len);
 620
 621         tkn16->utf16_len = tkn_len;
 622     }
 623
 624     return tkn_len;
 625 }
 626
 627
 628 int32_t icu_tokenizer_token_id(struct icu_tokenizer * tokenizer)
 629 {
 630     return tokenizer->token_id;
 631 }
 632
 633 int32_t icu_tokenizer_token_start(struct icu_tokenizer * tokenizer)
 634 {
 635     return tokenizer->token_start;
 636 }
 637
 638 int32_t icu_tokenizer_token_end(struct icu_tokenizer * tokenizer)
 639 {
 640     return tokenizer->token_end;
 641 }
 642
 643 int32_t icu_tokenizer_token_length(struct icu_tokenizer * tokenizer)
 644 {
 645     return (tokenizer->token_end - tokenizer->token_start);
 646 }
 647
 648 int32_t icu_tokenizer_token_count(struct icu_tokenizer * tokenizer)
 649 {
 650     return tokenizer->token_count;
 651 }
 652
 653
 654
 655 struct icu_transform * icu_transform_create(const char *rules, char action,
 656                                             UErrorCode *status)
 657 {
 658
 659     struct icu_transform * transform
 660         = (struct icu_transform *) xmalloc(sizeof(struct icu_transform));
 661
 662     transform->action = action;
 663     transform->trans = 0;
 664     transform->rules16 =  icu_buf_utf16_create(0);
 665     icu_utf16_from_utf8_cstr(transform->rules16, rules, status);
 666
 667     switch(transform->action)
 668     {
 669     case 'f':
 670     case 'F':
 671         transform->trans
 672             = utrans_openU(transform->rules16->utf16,
 673                            transform->rules16->utf16_len,
 674                            UTRANS_FORWARD,
 675                            0, 0,
 676                            &transform->parse_error, status);
 677         break;
 678     case 'r':
 679     case 'R':
 680         transform->trans
 681             = utrans_openU(transform->rules16->utf16,
 682                            transform->rules16->utf16_len,
 683                            UTRANS_REVERSE ,
 684                            0, 0,
 685                            &transform->parse_error, status);
 686         break;
 687     default:
 688         *status = U_UNSUPPORTED_ERROR;
 689         return 0;
 690         break;
 691     }
 692
 693     if (U_SUCCESS(*status))
 694         return transform;
 695
 696     /* freeing if failed */
 697     icu_transform_destroy(transform);
 698     return 0;
 699 }
 700
 701
 702 void icu_transform_destroy(struct icu_transform * transform){
 703     if (transform) {
 704         if (transform->rules16)
 705             icu_buf_utf16_destroy(transform->rules16);
 706         if (transform->trans)
 707             utrans_close(transform->trans);
 708         xfree(transform);
 709     }
 710 }
 711
 712
 713
 714 int icu_transform_trans(struct icu_transform * transform,
 715                         struct icu_buf_utf16 * dest16,
 716                         struct icu_buf_utf16 * src16,
 717                         UErrorCode *status)
 718 {
 719     if (!transform || !transform->trans
 720         || !src16
 721         || !dest16)
 722         return 0;
 723
 724     if (!src16->utf16_len){           /* guarding for empty source string */
 725         icu_buf_utf16_clear(dest16);
 726         return 0;
 727     }
 728
 729     if (!icu_buf_utf16_copy(dest16, src16))
 730         return 0;
 731
 732
 733     utrans_transUChars (transform->trans,
 734                         dest16->utf16, &(dest16->utf16_len),
 735                         dest16->utf16_cap,
 736                         0, &(src16->utf16_len), status);
 737
 738     if (U_FAILURE(*status))
 739         icu_buf_utf16_clear(dest16);
 740
 741     return dest16->utf16_len;
 742 }
 743
 744
 745
 746
 747 struct icu_chain_step * icu_chain_step_create(struct icu_chain * chain,
 748                                               enum icu_chain_step_type type,
 749                                               const uint8_t * rule,
 750                                               struct icu_buf_utf16 * buf16,
 751                                               UErrorCode *status)
 752 {
 753     struct icu_chain_step * step = 0;
 754
 755     if(!chain || !type || !rule)
 756         return 0;
 757
 758     step = (struct icu_chain_step *) xmalloc(sizeof(struct icu_chain_step));
 759
 760     step->type = type;
 761
 762     step->buf16 = buf16;
 763
 764     /* create auxilary objects */
 765     switch(step->type) {
 766     case ICU_chain_step_type_display:
 767         break;
 768     case ICU_chain_step_type_casemap:
 769         step->u.casemap = icu_casemap_create(rule[0], status);
 770         break;
 771     case ICU_chain_step_type_transform:
 772         step->u.transform = icu_transform_create((char *) rule, 'f', status);
 773         break;
 774     case ICU_chain_step_type_tokenize:
 775         step->u.tokenizer = icu_tokenizer_create((char *) chain->locale,
 776                                                  (char) rule[0], status);
 777         break;
 778     default:
 779         break;
 780     }
 781
 782     return step;
 783 }
 784
 785
 786 void icu_chain_step_destroy(struct icu_chain_step * step){
 787
 788     if (!step)
 789         return;
 790
 791     icu_chain_step_destroy(step->previous);
 792
 793     switch(step->type) {
 794     case ICU_chain_step_type_display:
 795         break;
 796     case ICU_chain_step_type_casemap:
 797         icu_casemap_destroy(step->u.casemap);
 798         icu_buf_utf16_destroy(step->buf16);
 799         break;
 800     case ICU_chain_step_type_transform:
 801         icu_transform_destroy(step->u.transform);
 802         icu_buf_utf16_destroy(step->buf16);
 803         break;
 804     case ICU_chain_step_type_tokenize:
 805         icu_tokenizer_destroy(step->u.tokenizer);
 806         icu_buf_utf16_destroy(step->buf16);
 807         break;
 808     default:
 809         break;
 810     }
 811     xfree(step);
 812 }
 813
 814
 815
 816 struct icu_chain * icu_chain_create(const char *locale,  int sort,
 817                                     UErrorCode * status)
 818 {
 819     struct icu_chain * chain
 820         = (struct icu_chain *) xmalloc(sizeof(struct icu_chain));
 821
 822     *status = U_ZERO_ERROR;
 823
 824     chain->locale = xstrdup(locale);
 825
 826     chain->sort = sort;
 827
 828     chain->coll = ucol_open((const char *) chain->locale, status);
 829
 830     if (U_FAILURE(*status))
 831         return 0;
 832
 833     chain->token_count = 0;
 834
 835     chain->src8cstr = 0;
 836
 837     chain->display8 = icu_buf_utf8_create(0);
 838     chain->norm8 = icu_buf_utf8_create(0);
 839     chain->sort8 = icu_buf_utf8_create(0);
 840
 841     chain->src16 = icu_buf_utf16_create(0);
 842
 843     chain->steps = 0;
 844
 845     return chain;
 846 }
 847
 848
 849 void icu_chain_destroy(struct icu_chain * chain)
 850 {
 851     if (chain)
 852     {
 853         if (chain->coll)
 854             ucol_close(chain->coll);
 855
 856         icu_buf_utf8_destroy(chain->display8);
 857         icu_buf_utf8_destroy(chain->norm8);
 858         icu_buf_utf8_destroy(chain->sort8);
 859
 860         icu_buf_utf16_destroy(chain->src16);
 861
 862         icu_chain_step_destroy(chain->steps);
 863         xfree(chain->locale);
 864         xfree(chain);
 865     }
 866 }
 867
 868
 869
 870 struct icu_chain * icu_chain_xml_config(const xmlNode *xml_node,
 871                                         int sort,
 872                                         UErrorCode * status)
 873 {
 874     xmlNode *node = 0;
 875     struct icu_chain * chain = 0;
 876
 877     *status = U_ZERO_ERROR;
 878
 879     if (!xml_node ||xml_node->type != XML_ELEMENT_NODE)
 880         return 0;
 881
 882     {
 883         xmlChar * xml_locale = xmlGetProp((xmlNode *) xml_node,
 884                                           (xmlChar *) "locale");
 885
 886         if (xml_locale)
 887         {
 888             chain = icu_chain_create((const char *) xml_locale, sort, status);
 889             xmlFree(xml_locale);
 890         }
 891
 892     }
 893     if (!chain)
 894         return 0;
 895
 896     for (node = xml_node->children; node; node = node->next)
 897     {
 898         xmlChar *xml_rule;
 899         struct icu_chain_step * step = 0;
 900
 901         if (node->type != XML_ELEMENT_NODE)
 902             continue;
 903
 904         xml_rule = xmlGetProp(node, (xmlChar *) "rule");
 905
 906         if (!strcmp((const char *) node->name, "casemap"))
 907             step = icu_chain_insert_step(chain, ICU_chain_step_type_casemap,
 908                                          (const uint8_t *) xml_rule, status);
 909         else if (!strcmp((const char *) node->name, "transform"))
 910             step = icu_chain_insert_step(chain, ICU_chain_step_type_transform,
 911                                          (const uint8_t *) xml_rule, status);
 912         else if (!strcmp((const char *) node->name, "tokenize"))
 913             step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize,
 914                                          (const uint8_t *) xml_rule, status);
 915         else if (!strcmp((const char *) node->name, "display"))
 916             step = icu_chain_insert_step(chain, ICU_chain_step_type_display,
 917                                          (const uint8_t *) "", status);
 918         else if (!strcmp((const char *) node->name, "normalize"))
 919         {
 920             yaz_log(YLOG_WARN, "Element %s is deprecated. "
 921                     "Use transform instead", node->name);
 922             step = icu_chain_insert_step(chain, ICU_chain_step_type_transform,
 923                                          (const uint8_t *) xml_rule, status);
 924         }
 925         else if (!strcmp((const char *) node->name, "index")
 926                  || !strcmp((const char *) node->name, "sortkey"))
 927         {
 928             yaz_log(YLOG_WARN, "Element %s is no longer needed. "
 929                     "Remove it from the configuration", node->name);
 930         }
 931         else
 932         {
 933             yaz_log(YLOG_WARN, "Unknown element %s", node->name);
 934             icu_chain_destroy(chain);
 935             return 0;
 936         }
 937         xmlFree(xml_rule);
 938         if (step && U_FAILURE(*status))
 939         {
 940             icu_chain_destroy(chain);
 941             return 0;
 942         }
 943     }
 944     return chain;
 945 }
 946
 947
 948
 949 struct icu_chain_step * icu_chain_insert_step(struct icu_chain * chain,
 950                                               enum icu_chain_step_type type,
 951                                               const uint8_t * rule,
 952                                               UErrorCode *status)
 953 {
 954     struct icu_chain_step * step = 0;
 955     struct icu_buf_utf16 * src16 = 0;
 956     struct icu_buf_utf16 * buf16 = 0;
 957
 958     if (!chain || !type || !rule)
 959         return 0;
 960
 961     /* assign utf16 src buffers as needed */
 962     if (chain->steps && chain->steps->buf16)
 963         src16 = chain->steps->buf16;
 964     else if (chain->src16)
 965         src16 = chain->src16;
 966     else
 967         return 0;
 968
 969
 970     /* create utf16 destination buffers as needed, or */
 971     switch(type)
 972     {
 973     case ICU_chain_step_type_display:
 974         buf16 = src16;
 975         break;
 976     case ICU_chain_step_type_casemap:
 977         buf16 = icu_buf_utf16_create(0);
 978         break;
 979     case ICU_chain_step_type_transform:
 980         buf16 = icu_buf_utf16_create(0);
 981         break;
 982     case ICU_chain_step_type_tokenize:
 983         buf16 = icu_buf_utf16_create(0);
 984         break;
 985     default:
 986         break;
 987     }
 988
 989     /* create actual chain step with this buffer */
 990     step = icu_chain_step_create(chain, type, rule, buf16, status);
 991
 992     step->previous = chain->steps;
 993     chain->steps = step;
 994
 995     return step;
 996 }
 997
 998
 999 int icu_chain_step_next_token(struct icu_chain * chain,
1000                               struct icu_chain_step * step,
1001                               UErrorCode *status)
1002 {
1003     struct icu_buf_utf16 * src16 = 0;
1004     int got_new_token = 0;
1005
1006     if (!chain || !chain->src16 || !step || !step->more_tokens)
1007         return 0;
1008
1009     /* assign utf16 src buffers as neeed, advance in previous steps
1010        tokens until non-zero token met, and setting stop condition */
1011
1012     if (step->previous)
1013     {
1014         src16 = step->previous->buf16;
1015         /* tokens might be killed in previous steps, therefore looping */
1016
1017         while (step->need_new_token
1018                && step->previous->more_tokens
1019                && !got_new_token)
1020             got_new_token
1021                 = icu_chain_step_next_token(chain, step->previous, status);
1022     }
1023     else
1024     { /* first step can only work once on chain->src16 input buffer */
1025         src16 = chain->src16;
1026         step->more_tokens = 0;
1027         got_new_token = 1;
1028     }
1029
1030     if (!src16)
1031         return 0;
1032
1033     /* stop if nothing to process */
1034     if (step->need_new_token && !got_new_token)
1035     {
1036         step->more_tokens = 0;
1037         return 0;
1038     }
1039
1040     /* either an old token not finished yet, or a new token, thus
1041        perform the work, eventually put this steps output in
1042        step->buf16 or the chains UTF8 output buffers  */
1043
1044     switch(step->type)
1045     {
1046     case ICU_chain_step_type_display:
1047         icu_utf16_to_utf8(chain->display8, src16, status);
1048         break;
1049     case ICU_chain_step_type_casemap:
1050         icu_casemap_casemap(step->u.casemap,
1051                             step->buf16, src16, status,
1052                             chain->locale);
1053         break;
1054     case ICU_chain_step_type_transform:
1055         icu_transform_trans(step->u.transform,
1056                             step->buf16, src16, status);
1057         break;
1058     case ICU_chain_step_type_tokenize:
1059         /* attach to new src16 token only first time during splitting */
1060         if (step->need_new_token)
1061         {
1062             icu_tokenizer_attach(step->u.tokenizer, src16, status);
1063             step->need_new_token = 0;
1064         }
1065
1066         /* splitting one src16 token into multiple buf16 tokens */
1067         step->more_tokens
1068             = icu_tokenizer_next_token(step->u.tokenizer,
1069                                        step->buf16, status);
1070
1071         /* make sure to get new previous token if this one had been used up
1072            by recursive call to _same_ step */
1073
1074         if (!step->more_tokens)
1075         {
1076             step->more_tokens = icu_chain_step_next_token(chain, step, status);
1077             return step->more_tokens;  /* avoid one token count too much! */
1078         }
1079         break;
1080     default:
1081         return 0;
1082         break;
1083     }
1084
1085     if (U_FAILURE(*status))
1086         return 0;
1087
1088     /* if token disappered into thin air, tell caller */
1089     /* if (!step->buf16->utf16_len && !step->more_tokens) */
1090     /*    return 0; */
1091
1092     return 1;
1093 }
1094
1095
1096 int icu_chain_assign_cstr(struct icu_chain * chain,
1097                           const char * src8cstr,
1098                           UErrorCode *status)
1099 {
1100     struct icu_chain_step * stp = 0;
1101
1102     if (!chain || !src8cstr)
1103         return 0;
1104
1105     chain->src8cstr = src8cstr;
1106
1107     stp = chain->steps;
1108
1109     /* clear token count */
1110     chain->token_count = 0;
1111
1112     /* clear all steps stop states */
1113     while (stp)
1114     {
1115         stp->more_tokens = 1;
1116         stp->need_new_token = 1;
1117         stp = stp->previous;
1118     }
1119
1120     /* finally convert UTF8 to UTF16 string if needed */
1121     if (chain->steps || chain->sort)
1122         icu_utf16_from_utf8_cstr(chain->src16, chain->src8cstr, status);
1123
1124     if (U_FAILURE(*status))
1125         return 0;
1126
1127     return 1;
1128 }
1129
1130
1131
1132 int icu_chain_next_token(struct icu_chain * chain,
1133                          UErrorCode *status)
1134 {
1135     int got_token = 0;
1136
1137     *status = U_ZERO_ERROR;
1138
1139     if (!chain)
1140         return 0;
1141
1142     /* special case with no steps - same as index type binary */
1143     if (!chain->steps)
1144     {
1145         if (chain->token_count)
1146             return 0;
1147         else
1148         {
1149             chain->token_count++;
1150
1151             if (chain->sort)
1152                 icu_sortkey8_from_utf16(chain->coll,
1153                                         chain->sort8, chain->steps->buf16,
1154                                         status);
1155             return chain->token_count;
1156         }
1157     }
1158     /* usual case, one or more icu chain steps existing */
1159     else
1160     {
1161         while(!got_token && chain->steps && chain->steps->more_tokens)
1162             got_token = icu_chain_step_next_token(chain, chain->steps, status);
1163
1164         if (got_token)
1165         {
1166             chain->token_count++;
1167
1168             icu_utf16_to_utf8(chain->norm8, chain->steps->buf16, status);
1169
1170             if (chain->sort)
1171                 icu_sortkey8_from_utf16(chain->coll,
1172                                         chain->sort8, chain->steps->buf16,
1173                                         status);
1174
1175             return chain->token_count;
1176         }
1177     }
1178
1179     return 0;
1180 }
1181
1182 int icu_chain_token_number(struct icu_chain * chain)
1183 {
1184     if (!chain)
1185         return 0;
1186
1187     return chain->token_count;
1188 }
1189
1190
1191 const char * icu_chain_token_display(struct icu_chain * chain)
1192 {
1193     if (chain->display8)
1194         return icu_buf_utf8_to_cstr(chain->display8);
1195
1196     return 0;
1197 }
1198
1199 const char * icu_chain_token_norm(struct icu_chain * chain)
1200 {
1201     if (!chain->steps)
1202         return chain->src8cstr;
1203
1204     if (chain->norm8)
1205         return icu_buf_utf8_to_cstr(chain->norm8);
1206
1207     return 0;
1208 }
1209
1210 const char * icu_chain_token_sortkey(struct icu_chain * chain)
1211 {
1212     if (chain->sort8)
1213         return icu_buf_utf8_to_cstr(chain->sort8);
1214
1215     return 0;
1216 }
1217
1218 const UCollator * icu_chain_get_coll(struct icu_chain * chain)
1219 {
1220     return chain->coll;
1221 }
1222
1223 #endif /* YAZ_HAVE_ICU */
1224
1225 /*
1226  * Local variables:
1227  * c-basic-offset: 4
1228  * c-file-style: "Stroustrup"
1229  * indent-tabs-mode: nil
1230  * End:
1231  * vim: shiftwidth=4 tabstop=8 expandtab
1232  */
1233