src/icu_I18N.c

   1 /*
   2  * Copyright (C) 1995-2007, Index Data ApS
   3  * See the file LICENSE for details.
   4  *
   5  * $Id: icu_I18N.c,v 1.22 2007-12-17 11:23:16 adam Exp $
   6  */
   7
   8 #if HAVE_CONFIG_H
   9 #include "config.h"
  10 #endif
  11
  12 #define USE_TIMING 0
  13 #if USE_TIMING
  14 #include <yaz/timing.h>
  15 #endif
  16
  17 #if YAZ_HAVE_ICU
  18 #include <yaz/xmalloc.h>
  19
  20 #include <yaz/icu_I18N.h>
  21
  22 #include <yaz/log.h>
  23
  24 #include <string.h>
  25 #include <stdlib.h>
  26 #include <stdio.h>
  27
  28 #include <unicode/ustring.h>  /* some more string fcns*/
  29 #include <unicode/uchar.h>    /* char names           */
  30
  31
  32 #include <unicode/ucol.h>
  33
  34
  35 int icu_check_status (UErrorCode status)
  36 {
  37     if (U_FAILURE(status))
  38     {
  39         yaz_log(YLOG_WARN, "ICU: %d %s\n", status, u_errorName(status));
  40         return 0;
  41     }
  42     return 1;
  43
  44 }
  45
  46
  47
  48 struct icu_buf_utf16 * icu_buf_utf16_create(size_t capacity)
  49 {
  50     struct icu_buf_utf16 * buf16
  51         = (struct icu_buf_utf16 *) xmalloc(sizeof(struct icu_buf_utf16));
  52
  53     buf16->utf16 = 0;
  54     buf16->utf16_len = 0;
  55     buf16->utf16_cap = 0;
  56
  57     if (capacity > 0){
  58         buf16->utf16 = (UChar *) xmalloc(sizeof(UChar) * capacity);
  59         buf16->utf16[0] = (UChar) 0;
  60         buf16->utf16_cap = capacity;
  61     }
  62     return buf16;
  63 }
  64
  65 struct icu_buf_utf16 * icu_buf_utf16_clear(struct icu_buf_utf16 * buf16)
  66 {
  67     if (buf16){
  68         if (buf16->utf16)
  69             buf16->utf16[0] = (UChar) 0;
  70         buf16->utf16_len = 0;
  71     }
  72     return buf16;
  73 }
  74
  75 struct icu_buf_utf16 * icu_buf_utf16_resize(struct icu_buf_utf16 * buf16,
  76                                             size_t capacity)
  77 {
  78     if (!buf16)
  79         return 0;
  80
  81     if (capacity >  0){
  82         if (0 == buf16->utf16)
  83             buf16->utf16 = (UChar *) xmalloc(sizeof(UChar) * capacity);
  84         else
  85             buf16->utf16
  86                 = (UChar *) xrealloc(buf16->utf16, sizeof(UChar) * capacity);
  87
  88         icu_buf_utf16_clear(buf16);
  89         buf16->utf16_cap = capacity;
  90     }
  91     else {
  92         xfree(buf16->utf16);
  93         buf16->utf16 = 0;
  94         buf16->utf16_len = 0;
  95         buf16->utf16_cap = 0;
  96     }
  97
  98     return buf16;
  99 }
 100
 101
 102 struct icu_buf_utf16 * icu_buf_utf16_copy(struct icu_buf_utf16 * dest16,
 103                                           struct icu_buf_utf16 * src16)
 104 {
 105     if(!dest16 || !src16
 106        || dest16 == src16)
 107         return 0;
 108
 109     if (dest16->utf16_cap < src16->utf16_len)
 110         icu_buf_utf16_resize(dest16, src16->utf16_len * 2);
 111
 112     u_strncpy(dest16->utf16, src16->utf16, src16->utf16_len);
 113     dest16->utf16_len = src16->utf16_len;
 114
 115     return dest16;
 116 }
 117
 118
 119 void icu_buf_utf16_destroy(struct icu_buf_utf16 * buf16)
 120 {
 121     if (buf16)
 122         xfree(buf16->utf16);
 123     xfree(buf16);
 124 }
 125
 126
 127
 128 struct icu_buf_utf8 * icu_buf_utf8_create(size_t capacity)
 129 {
 130     struct icu_buf_utf8 * buf8
 131         = (struct icu_buf_utf8 *) xmalloc(sizeof(struct icu_buf_utf8));
 132
 133     buf8->utf8 = 0;
 134     buf8->utf8_len = 0;
 135     buf8->utf8_cap = 0;
 136
 137     if (capacity > 0){
 138         buf8->utf8 = (uint8_t *) xmalloc(sizeof(uint8_t) * capacity);
 139         buf8->utf8[0] = (uint8_t) 0;
 140         buf8->utf8_cap = capacity;
 141     }
 142     return buf8;
 143 }
 144
 145
 146 struct icu_buf_utf8 * icu_buf_utf8_clear(struct icu_buf_utf8 * buf8)
 147 {
 148     if (buf8){
 149         if (buf8->utf8)
 150             buf8->utf8[0] = (uint8_t) 0;
 151         buf8->utf8_len = 0;
 152     }
 153     return buf8;
 154 }
 155
 156
 157 struct icu_buf_utf8 * icu_buf_utf8_resize(struct icu_buf_utf8 * buf8,
 158                                           size_t capacity)
 159 {
 160     if (!buf8)
 161         return 0;
 162
 163     if (capacity >  0){
 164         if (0 == buf8->utf8)
 165             buf8->utf8 = (uint8_t *) xmalloc(sizeof(uint8_t) * capacity);
 166         else
 167             buf8->utf8
 168                 = (uint8_t *) xrealloc(buf8->utf8, sizeof(uint8_t) * capacity);
 169
 170         buf8->utf8_cap = capacity;
 171     }
 172     else {
 173         xfree(buf8->utf8);
 174         buf8->utf8 = 0;
 175         buf8->utf8_len = 0;
 176         buf8->utf8_cap = 0;
 177     }
 178
 179     return buf8;
 180 }
 181
 182
 183 const char *icu_buf_utf8_to_cstr(struct icu_buf_utf8 *src8)
 184 {
 185     if (!src8 || src8->utf8_len == 0)
 186         return "";
 187
 188     if (src8->utf8_len == src8->utf8_cap)
 189         src8 = icu_buf_utf8_resize(src8, src8->utf8_len * 2 + 1);
 190
 191     src8->utf8[src8->utf8_len] = '\0';
 192
 193     return (const char *) src8->utf8;
 194 }
 195
 196
 197 void icu_buf_utf8_destroy(struct icu_buf_utf8 * buf8)
 198 {
 199     if (buf8)
 200         xfree(buf8->utf8);
 201     xfree(buf8);
 202 }
 203
 204
 205
 206 UErrorCode icu_utf16_from_utf8(struct icu_buf_utf16 * dest16,
 207                                struct icu_buf_utf8 * src8,
 208                                UErrorCode * status)
 209 {
 210     int32_t utf16_len = 0;
 211
 212     u_strFromUTF8(dest16->utf16, dest16->utf16_cap,
 213                   &utf16_len,
 214                   (const char *) src8->utf8, src8->utf8_len, status);
 215
 216     /* check for buffer overflow, resize and retry */
 217     if (*status == U_BUFFER_OVERFLOW_ERROR)
 218     {
 219         icu_buf_utf16_resize(dest16, utf16_len * 2);
 220         *status = U_ZERO_ERROR;
 221         u_strFromUTF8(dest16->utf16, dest16->utf16_cap,
 222                       &utf16_len,
 223                       (const char *) src8->utf8, src8->utf8_len, status);
 224     }
 225
 226     if (U_SUCCESS(*status)
 227         && utf16_len <= dest16->utf16_cap)
 228         dest16->utf16_len = utf16_len;
 229     else
 230         icu_buf_utf16_clear(dest16);
 231
 232     return *status;
 233 }
 234
 235
 236
 237 UErrorCode icu_utf16_from_utf8_cstr(struct icu_buf_utf16 * dest16,
 238                                     const char * src8cstr,
 239                                     UErrorCode * status)
 240 {
 241     size_t src8cstr_len = 0;
 242     int32_t utf16_len = 0;
 243
 244     *status = U_ZERO_ERROR;
 245     src8cstr_len = strlen(src8cstr);
 246
 247     u_strFromUTF8(dest16->utf16, dest16->utf16_cap,
 248                   &utf16_len,
 249                   src8cstr, src8cstr_len, status);
 250
 251     /* check for buffer overflow, resize and retry */
 252     if (*status == U_BUFFER_OVERFLOW_ERROR)
 253     {
 254         icu_buf_utf16_resize(dest16, utf16_len * 2);
 255         *status = U_ZERO_ERROR;
 256         u_strFromUTF8(dest16->utf16, dest16->utf16_cap,
 257                       &utf16_len,
 258                       src8cstr, src8cstr_len, status);
 259     }
 260
 261     if (U_SUCCESS(*status)
 262         && utf16_len <= dest16->utf16_cap)
 263         dest16->utf16_len = utf16_len;
 264     else
 265         icu_buf_utf16_clear(dest16);
 266
 267     return *status;
 268 }
 269
 270
 271
 272
 273 UErrorCode icu_utf16_to_utf8(struct icu_buf_utf8 * dest8,
 274                              struct icu_buf_utf16 * src16,
 275                              UErrorCode * status)
 276 {
 277     int32_t utf8_len = 0;
 278
 279     u_strToUTF8((char *) dest8->utf8, dest8->utf8_cap,
 280                 &utf8_len,
 281                 src16->utf16, src16->utf16_len, status);
 282
 283     /* check for buffer overflow, resize and retry */
 284     if (*status == U_BUFFER_OVERFLOW_ERROR)
 285     {
 286         icu_buf_utf8_resize(dest8, utf8_len * 2);
 287         *status = U_ZERO_ERROR;
 288         u_strToUTF8((char *) dest8->utf8, dest8->utf8_cap,
 289                     &utf8_len,
 290                     src16->utf16, src16->utf16_len, status);
 291
 292     }
 293
 294     if (U_SUCCESS(*status)
 295         && utf8_len <= dest8->utf8_cap)
 296         dest8->utf8_len = utf8_len;
 297     else
 298         icu_buf_utf8_clear(dest8);
 299
 300     return *status;
 301 }
 302
 303
 304
 305 struct icu_casemap * icu_casemap_create(char action, UErrorCode *status)
 306 {
 307     struct icu_casemap * casemap
 308         = (struct icu_casemap *) xmalloc(sizeof(struct icu_casemap));
 309     casemap->action = action;
 310
 311     switch(casemap->action) {
 312     case 'l':
 313     case 'L':
 314     case 'u':
 315     case 'U':
 316     case 't':
 317     case 'T':
 318     case 'f':
 319     case 'F':
 320         break;
 321     default:
 322         icu_casemap_destroy(casemap);
 323         return 0;
 324     }
 325
 326     return casemap;
 327 }
 328
 329 void icu_casemap_destroy(struct icu_casemap * casemap)
 330 {
 331     xfree(casemap);
 332 }
 333
 334
 335 int icu_casemap_casemap(struct icu_casemap * casemap,
 336                         struct icu_buf_utf16 * dest16,
 337                         struct icu_buf_utf16 * src16,
 338                         UErrorCode *status,
 339                         const char *locale)
 340 {
 341     if(!casemap)
 342         return 0;
 343
 344     return icu_utf16_casemap(dest16, src16, locale,
 345                              casemap->action, status);
 346 }
 347
 348
 349 int icu_utf16_casemap(struct icu_buf_utf16 * dest16,
 350                       struct icu_buf_utf16 * src16,
 351                       const char *locale, char action,
 352                       UErrorCode *status)
 353 {
 354     int32_t dest16_len = 0;
 355
 356
 357     if (!src16->utf16_len){           /* guarding for empty source string */
 358         if (dest16->utf16)
 359             dest16->utf16[0] = (UChar) 0;
 360         dest16->utf16_len = 0;
 361         return U_ZERO_ERROR;
 362     }
 363
 364
 365     switch(action) {
 366     case 'l':
 367     case 'L':
 368         dest16_len = u_strToLower(dest16->utf16, dest16->utf16_cap,
 369                                   src16->utf16, src16->utf16_len,
 370                                   locale, status);
 371         break;
 372     case 'u':
 373     case 'U':
 374         dest16_len = u_strToUpper(dest16->utf16, dest16->utf16_cap,
 375                                   src16->utf16, src16->utf16_len,
 376                                   locale, status);
 377         break;
 378     case 't':
 379     case 'T':
 380         dest16_len = u_strToTitle(dest16->utf16, dest16->utf16_cap,
 381                                   src16->utf16, src16->utf16_len,
 382                                   0, locale, status);
 383         break;
 384     case 'f':
 385     case 'F':
 386         dest16_len = u_strFoldCase(dest16->utf16, dest16->utf16_cap,
 387                                    src16->utf16, src16->utf16_len,
 388                                    U_FOLD_CASE_DEFAULT, status);
 389         break;
 390
 391     default:
 392         return U_UNSUPPORTED_ERROR;
 393         break;
 394     }
 395
 396     /* check for buffer overflow, resize and retry */
 397     if (*status == U_BUFFER_OVERFLOW_ERROR
 398         && dest16 != src16        /* do not resize if in-place conversion */
 399         ){
 400         icu_buf_utf16_resize(dest16, dest16_len * 2);
 401         *status = U_ZERO_ERROR;
 402
 403
 404         switch(action) {
 405         case 'l':
 406         case 'L':
 407             dest16_len = u_strToLower(dest16->utf16, dest16->utf16_cap,
 408                                       src16->utf16, src16->utf16_len,
 409                                       locale, status);
 410             break;
 411         case 'u':
 412         case 'U':
 413             dest16_len = u_strToUpper(dest16->utf16, dest16->utf16_cap,
 414                                       src16->utf16, src16->utf16_len,
 415                                       locale, status);
 416             break;
 417         case 't':
 418         case 'T':
 419             dest16_len = u_strToTitle(dest16->utf16, dest16->utf16_cap,
 420                                       src16->utf16, src16->utf16_len,
 421                                       0, locale, status);
 422             break;
 423         case 'f':
 424         case 'F':
 425             dest16_len = u_strFoldCase(dest16->utf16, dest16->utf16_cap,
 426                                        src16->utf16, src16->utf16_len,
 427                                        U_FOLD_CASE_DEFAULT, status);
 428             break;
 429
 430         default:
 431             return U_UNSUPPORTED_ERROR;
 432             break;
 433         }
 434     }
 435
 436     if (U_SUCCESS(*status)
 437         && dest16_len <= dest16->utf16_cap)
 438         dest16->utf16_len = dest16_len;
 439     else {
 440         if (dest16->utf16)
 441             dest16->utf16[0] = (UChar) 0;
 442         dest16->utf16_len = 0;
 443     }
 444
 445     return *status;
 446 }
 447
 448
 449
 450 void icu_sortkey8_from_utf16(UCollator *coll,
 451                              struct icu_buf_utf8 * dest8,
 452                              struct icu_buf_utf16 * src16,
 453                              UErrorCode * status)
 454 {
 455
 456     int32_t sortkey_len = 0;
 457
 458     sortkey_len = ucol_getSortKey(coll, src16->utf16, src16->utf16_len,
 459                                   dest8->utf8, dest8->utf8_cap);
 460
 461     /* check for buffer overflow, resize and retry */
 462     if (sortkey_len > dest8->utf8_cap) {
 463         icu_buf_utf8_resize(dest8, sortkey_len * 2);
 464         sortkey_len = ucol_getSortKey(coll, src16->utf16, src16->utf16_len,
 465                                       dest8->utf8, dest8->utf8_cap);
 466     }
 467
 468     if (U_SUCCESS(*status)
 469         && sortkey_len > 0)
 470         dest8->utf8_len = sortkey_len;
 471     else
 472         icu_buf_utf8_clear(dest8);
 473 }
 474
 475
 476
 477 struct icu_tokenizer * icu_tokenizer_create(const char *locale, char action,
 478                                             UErrorCode *status)
 479 {
 480     struct icu_tokenizer * tokenizer
 481         = (struct icu_tokenizer *) xmalloc(sizeof(struct icu_tokenizer));
 482
 483     tokenizer->action = action;
 484     tokenizer->bi = 0;
 485     tokenizer->buf16 = 0;
 486     tokenizer->token_count = 0;
 487     tokenizer->token_id = 0;
 488     tokenizer->token_start = 0;
 489     tokenizer->token_end = 0;
 490
 491
 492     switch(tokenizer->action) {
 493     case 'l':
 494     case 'L':
 495         tokenizer->bi = ubrk_open(UBRK_LINE, locale, 0, 0, status);
 496         break;
 497     case 's':
 498     case 'S':
 499         tokenizer->bi = ubrk_open(UBRK_SENTENCE, locale, 0, 0, status);
 500         break;
 501     case 'w':
 502     case 'W':
 503         tokenizer->bi = ubrk_open(UBRK_WORD, locale, 0, 0, status);
 504         break;
 505     case 'c':
 506     case 'C':
 507         tokenizer->bi = ubrk_open(UBRK_CHARACTER, locale, 0, 0, status);
 508         break;
 509     case 't':
 510     case 'T':
 511         tokenizer->bi = ubrk_open(UBRK_TITLE, locale, 0, 0, status);
 512         break;
 513     default:
 514         *status = U_UNSUPPORTED_ERROR;
 515         return 0;
 516         break;
 517     }
 518
 519     /* ICU error stuff is a very  funny business */
 520     if (U_SUCCESS(*status))
 521         return tokenizer;
 522
 523     /* freeing if failed */
 524     icu_tokenizer_destroy(tokenizer);
 525     return 0;
 526 }
 527
 528 void icu_tokenizer_destroy(struct icu_tokenizer * tokenizer)
 529 {
 530     if (tokenizer) {
 531         if (tokenizer->bi)
 532             ubrk_close(tokenizer->bi);
 533         xfree(tokenizer);
 534     }
 535 }
 536
 537 int icu_tokenizer_attach(struct icu_tokenizer * tokenizer,
 538                          struct icu_buf_utf16 * src16,
 539                          UErrorCode *status)
 540 {
 541     if (!tokenizer || !tokenizer->bi || !src16)
 542         return 0;
 543
 544
 545     tokenizer->buf16 = src16;
 546     tokenizer->token_count = 0;
 547     tokenizer->token_id = 0;
 548     tokenizer->token_start = 0;
 549     tokenizer->token_end = 0;
 550
 551     ubrk_setText(tokenizer->bi, src16->utf16, src16->utf16_len, status);
 552
 553
 554     if (U_FAILURE(*status))
 555         return 0;
 556
 557     return 1;
 558 };
 559
 560 int32_t icu_tokenizer_next_token(struct icu_tokenizer * tokenizer,
 561                          struct icu_buf_utf16 * tkn16,
 562                          UErrorCode *status)
 563 {
 564     int32_t tkn_start = 0;
 565     int32_t tkn_end = 0;
 566     int32_t tkn_len = 0;
 567
 568
 569     if (!tokenizer || !tokenizer->bi
 570         || !tokenizer->buf16 || !tokenizer->buf16->utf16_len)
 571         return 0;
 572
 573     /*
 574     never change tokenizer->buf16 and keep always invariant
 575     0 <= tokenizer->token_start
 576        <= tokenizer->token_end
 577        <= tokenizer->buf16->utf16_len
 578     returns length of token
 579     */
 580
 581     if (0 == tokenizer->token_end) /* first call */
 582         tkn_start = ubrk_first(tokenizer->bi);
 583     else /* successive calls */
 584         tkn_start = tokenizer->token_end;
 585
 586     /* get next position */
 587     tkn_end = ubrk_next(tokenizer->bi);
 588
 589     /* repairing invariant at end of ubrk, which is UBRK_DONE = -1 */
 590     if (UBRK_DONE == tkn_end)
 591         tkn_end = tokenizer->buf16->utf16_len;
 592
 593     /* copy out if everything is well */
 594     if(U_FAILURE(*status))
 595         return 0;
 596
 597     /* everything OK, now update internal state */
 598     tkn_len = tkn_end - tkn_start;
 599
 600     if (0 < tkn_len){
 601         tokenizer->token_count++;
 602         tokenizer->token_id++;
 603     } else {
 604         tokenizer->token_id = 0;
 605     }
 606     tokenizer->token_start = tkn_start;
 607     tokenizer->token_end = tkn_end;
 608
 609
 610     /* copying into token buffer if it exists */
 611     if (tkn16){
 612         if (tkn16->utf16_cap < tkn_len)
 613             icu_buf_utf16_resize(tkn16, (size_t) tkn_len * 2);
 614
 615         u_strncpy(tkn16->utf16, &(tokenizer->buf16->utf16)[tkn_start],
 616                   tkn_len);
 617
 618         tkn16->utf16_len = tkn_len;
 619     }
 620
 621     return tkn_len;
 622 }
 623
 624
 625 int32_t icu_tokenizer_token_id(struct icu_tokenizer * tokenizer)
 626 {
 627     return tokenizer->token_id;
 628 }
 629
 630 int32_t icu_tokenizer_token_start(struct icu_tokenizer * tokenizer)
 631 {
 632     return tokenizer->token_start;
 633 }
 634
 635 int32_t icu_tokenizer_token_end(struct icu_tokenizer * tokenizer)
 636 {
 637     return tokenizer->token_end;
 638 }
 639
 640 int32_t icu_tokenizer_token_length(struct icu_tokenizer * tokenizer)
 641 {
 642     return (tokenizer->token_end - tokenizer->token_start);
 643 }
 644
 645 int32_t icu_tokenizer_token_count(struct icu_tokenizer * tokenizer)
 646 {
 647     return tokenizer->token_count;
 648 }
 649
 650
 651
 652 struct icu_normalizer * icu_normalizer_create(const char *rules, char action,
 653                                               UErrorCode *status)
 654 {
 655
 656     struct icu_normalizer * normalizer
 657         = (struct icu_normalizer *) xmalloc(sizeof(struct icu_normalizer));
 658
 659     normalizer->action = action;
 660     normalizer->trans = 0;
 661     normalizer->rules16 =  icu_buf_utf16_create(0);
 662     icu_utf16_from_utf8_cstr(normalizer->rules16, rules, status);
 663
 664     switch(normalizer->action) {
 665     case 'f':
 666     case 'F':
 667         normalizer->trans
 668             = utrans_openU(normalizer->rules16->utf16,
 669                            normalizer->rules16->utf16_len,
 670                            UTRANS_FORWARD,
 671                            0, 0,
 672                            normalizer->parse_error, status);
 673         break;
 674     case 'r':
 675     case 'R':
 676         normalizer->trans
 677             = utrans_openU(normalizer->rules16->utf16,
 678                            normalizer->rules16->utf16_len,
 679                            UTRANS_REVERSE ,
 680                            0, 0,
 681                            normalizer->parse_error, status);
 682         break;
 683     default:
 684         *status = U_UNSUPPORTED_ERROR;
 685         return 0;
 686         break;
 687     }
 688
 689     if (U_SUCCESS(*status))
 690         return normalizer;
 691
 692     /* freeing if failed */
 693     icu_normalizer_destroy(normalizer);
 694     return 0;
 695 }
 696
 697
 698 void icu_normalizer_destroy(struct icu_normalizer * normalizer){
 699     if (normalizer) {
 700         if (normalizer->rules16)
 701             icu_buf_utf16_destroy(normalizer->rules16);
 702         if (normalizer->trans)
 703             utrans_close(normalizer->trans);
 704         xfree(normalizer);
 705     }
 706 }
 707
 708
 709
 710 int icu_normalizer_normalize(struct icu_normalizer * normalizer,
 711                              struct icu_buf_utf16 * dest16,
 712                              struct icu_buf_utf16 * src16,
 713                              UErrorCode *status)
 714 {
 715     if (!normalizer || !normalizer->trans
 716         || !src16
 717         || !dest16)
 718         return 0;
 719
 720     if (!src16->utf16_len){           /* guarding for empty source string */
 721         icu_buf_utf16_clear(dest16);
 722         return 0;
 723     }
 724
 725     if (!icu_buf_utf16_copy(dest16, src16))
 726         return 0;
 727
 728
 729     utrans_transUChars (normalizer->trans,
 730                         dest16->utf16, &(dest16->utf16_len),
 731                         dest16->utf16_cap,
 732                         0, &(src16->utf16_len), status);
 733
 734     if (U_FAILURE(*status))
 735         icu_buf_utf16_clear(dest16);
 736
 737     return dest16->utf16_len;
 738 }
 739
 740
 741
 742
 743 struct icu_chain_step * icu_chain_step_create(struct icu_chain * chain,
 744                                               enum icu_chain_step_type type,
 745                                               const uint8_t * rule,
 746                                               struct icu_buf_utf16 * buf16,
 747                                               UErrorCode *status)
 748 {
 749     struct icu_chain_step * step = 0;
 750
 751     if(!chain || !type || !rule)
 752         return 0;
 753
 754     step = (struct icu_chain_step *) xmalloc(sizeof(struct icu_chain_step));
 755
 756     step->type = type;
 757
 758     step->buf16 = buf16;
 759
 760     /* create auxilary objects */
 761     switch(step->type) {
 762     case ICU_chain_step_type_display:
 763         break;
 764     case ICU_chain_step_type_casemap:
 765         step->u.casemap = icu_casemap_create(rule[0], status);
 766         break;
 767     case ICU_chain_step_type_normalize:
 768         step->u.normalizer = icu_normalizer_create((char *) rule, 'f', status);
 769         break;
 770     case ICU_chain_step_type_tokenize:
 771         step->u.tokenizer = icu_tokenizer_create((char *) chain->locale,
 772                                                  (char) rule[0], status);
 773         break;
 774     default:
 775         break;
 776     }
 777
 778     return step;
 779 }
 780
 781
 782 void icu_chain_step_destroy(struct icu_chain_step * step){
 783
 784     if (!step)
 785         return;
 786
 787     icu_chain_step_destroy(step->previous);
 788
 789     switch(step->type) {
 790     case ICU_chain_step_type_display:
 791         break;
 792     case ICU_chain_step_type_casemap:
 793         icu_casemap_destroy(step->u.casemap);
 794         icu_buf_utf16_destroy(step->buf16);
 795         break;
 796     case ICU_chain_step_type_normalize:
 797         icu_normalizer_destroy(step->u.normalizer);
 798         icu_buf_utf16_destroy(step->buf16);
 799         break;
 800     case ICU_chain_step_type_tokenize:
 801         icu_tokenizer_destroy(step->u.tokenizer);
 802         icu_buf_utf16_destroy(step->buf16);
 803         break;
 804     default:
 805         break;
 806     }
 807     xfree(step);
 808 }
 809
 810
 811
 812 struct icu_chain * icu_chain_create(const char *locale,  int sort,
 813                                     UErrorCode * status)
 814 {
 815     struct icu_chain * chain
 816         = (struct icu_chain *) xmalloc(sizeof(struct icu_chain));
 817
 818     *status = U_ZERO_ERROR;
 819
 820     chain->locale = xstrdup(locale);
 821
 822     chain->sort = sort;
 823
 824     chain->coll = ucol_open((const char *) chain->locale, status);
 825
 826     if (U_FAILURE(*status))
 827         return 0;
 828
 829     chain->token_count = 0;
 830
 831     chain->src8cstr = 0;
 832
 833     chain->display8 = icu_buf_utf8_create(0);
 834     chain->norm8 = icu_buf_utf8_create(0);
 835     chain->sort8 = icu_buf_utf8_create(0);
 836
 837     chain->src16 = icu_buf_utf16_create(0);
 838
 839     chain->steps = 0;
 840
 841     return chain;
 842 }
 843
 844
 845 void icu_chain_destroy(struct icu_chain * chain)
 846 {
 847     if (chain)
 848     {
 849         if (chain->coll)
 850             ucol_close(chain->coll);
 851
 852         icu_buf_utf8_destroy(chain->display8);
 853         icu_buf_utf8_destroy(chain->norm8);
 854         icu_buf_utf8_destroy(chain->sort8);
 855
 856         icu_buf_utf16_destroy(chain->src16);
 857
 858         icu_chain_step_destroy(chain->steps);
 859         xfree(chain->locale);
 860         xfree(chain);
 861     }
 862 }
 863
 864
 865
 866 struct icu_chain * icu_chain_xml_config(const xmlNode *xml_node,
 867                                         int sort,
 868                                         UErrorCode * status)
 869 {
 870     xmlNode *node = 0;
 871     struct icu_chain * chain = 0;
 872
 873     *status = U_ZERO_ERROR;
 874
 875     if (!xml_node ||xml_node->type != XML_ELEMENT_NODE)
 876         return 0;
 877
 878     {
 879         xmlChar * xml_locale = xmlGetProp((xmlNode *) xml_node,
 880                                           (xmlChar *) "locale");
 881
 882         if (xml_locale)
 883         {
 884             chain = icu_chain_create((const char *) xml_locale, sort, status);
 885             xmlFree(xml_locale);
 886         }
 887
 888     }
 889     if (!chain)
 890         return 0;
 891
 892     for (node = xml_node->children; node; node = node->next)
 893     {
 894         xmlChar *xml_rule;
 895         struct icu_chain_step * step = 0;
 896
 897         if (node->type != XML_ELEMENT_NODE)
 898             continue;
 899
 900         xml_rule = xmlGetProp(node, (xmlChar *) "rule");
 901
 902         if (!strcmp((const char *) node->name, "casemap"))
 903             step = icu_chain_insert_step(chain, ICU_chain_step_type_casemap,
 904                                          (const uint8_t *) xml_rule, status);
 905         else if (!strcmp((const char *) node->name, "transform"))
 906             step = icu_chain_insert_step(chain, ICU_chain_step_type_normalize,
 907                                          (const uint8_t *) xml_rule, status);
 908         else if (!strcmp((const char *) node->name, "tokenize"))
 909             step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize,
 910                                          (const uint8_t *) xml_rule, status);
 911         else if (!strcmp((const char *) node->name, "display"))
 912             step = icu_chain_insert_step(chain, ICU_chain_step_type_display,
 913                                          (const uint8_t *) "", status);
 914         xmlFree(xml_rule);
 915         if (!step || U_FAILURE(*status))
 916         {
 917             icu_chain_destroy(chain);
 918             return 0;
 919         }
 920
 921
 922     }
 923     return chain;
 924 }
 925
 926
 927
 928 struct icu_chain_step * icu_chain_insert_step(struct icu_chain * chain,
 929                                               enum icu_chain_step_type type,
 930                                               const uint8_t * rule,
 931                                               UErrorCode *status)
 932 {
 933     struct icu_chain_step * step = 0;
 934     struct icu_buf_utf16 * src16 = 0;
 935     struct icu_buf_utf16 * buf16 = 0;
 936
 937     if (!chain || !type || !rule)
 938         return 0;
 939
 940     /* assign utf16 src buffers as needed */
 941     if (chain->steps && chain->steps->buf16)
 942         src16 = chain->steps->buf16;
 943     else if (chain->src16)
 944         src16 = chain->src16;
 945     else
 946         return 0;
 947
 948
 949     /* create utf16 destination buffers as needed, or */
 950     switch(type)
 951     {
 952     case ICU_chain_step_type_display:
 953         buf16 = src16;
 954         break;
 955     case ICU_chain_step_type_casemap:
 956         buf16 = icu_buf_utf16_create(0);
 957         break;
 958     case ICU_chain_step_type_normalize:
 959         buf16 = icu_buf_utf16_create(0);
 960         break;
 961     case ICU_chain_step_type_tokenize:
 962         buf16 = icu_buf_utf16_create(0);
 963         break;
 964     default:
 965         break;
 966     }
 967
 968     /* create actual chain step with this buffer */
 969     step = icu_chain_step_create(chain, type, rule, buf16, status);
 970
 971     step->previous = chain->steps;
 972     chain->steps = step;
 973
 974     return step;
 975 }
 976
 977
 978 int icu_chain_step_next_token(struct icu_chain * chain,
 979                               struct icu_chain_step * step,
 980                               UErrorCode *status)
 981 {
 982     struct icu_buf_utf16 * src16 = 0;
 983     int got_new_token = 0;
 984
 985     if (!chain || !chain->src16 || !step || !step->more_tokens)
 986         return 0;
 987
 988     /* assign utf16 src buffers as neeed, advance in previous steps
 989        tokens until non-zero token met, and setting stop condition */
 990
 991     if (step->previous)
 992     {
 993         src16 = step->previous->buf16;
 994         /* tokens might be killed in previous steps, therefore looping */
 995
 996         while (step->need_new_token
 997                && step->previous->more_tokens
 998                && !got_new_token)
 999             got_new_token
1000                 = icu_chain_step_next_token(chain, step->previous, status);
1001     }
1002     else
1003     { /* first step can only work once on chain->src16 input buffer */
1004         src16 = chain->src16;
1005         step->more_tokens = 0;
1006         got_new_token = 1;
1007     }
1008
1009     if (!src16)
1010         return 0;
1011
1012     /* stop if nothing to process */
1013     if (step->need_new_token && !got_new_token)
1014     {
1015         step->more_tokens = 0;
1016         return 0;
1017     }
1018
1019     /* either an old token not finished yet, or a new token, thus
1020        perform the work, eventually put this steps output in
1021        step->buf16 or the chains UTF8 output buffers  */
1022
1023     switch(step->type)
1024     {
1025     case ICU_chain_step_type_display:
1026         icu_utf16_to_utf8(chain->display8, src16, status);
1027         break;
1028     case ICU_chain_step_type_casemap:
1029         icu_casemap_casemap(step->u.casemap,
1030                             step->buf16, src16, status,
1031                             chain->locale);
1032         break;
1033     case ICU_chain_step_type_normalize:
1034         icu_normalizer_normalize(step->u.normalizer,
1035                                  step->buf16, src16, status);
1036         break;
1037     case ICU_chain_step_type_tokenize:
1038         /* attach to new src16 token only first time during splitting */
1039         if (step->need_new_token)
1040         {
1041             icu_tokenizer_attach(step->u.tokenizer, src16, status);
1042             step->need_new_token = 0;
1043         }
1044
1045         /* splitting one src16 token into multiple buf16 tokens */
1046         step->more_tokens
1047             = icu_tokenizer_next_token(step->u.tokenizer,
1048                                        step->buf16, status);
1049
1050         /* make sure to get new previous token if this one had been used up
1051            by recursive call to _same_ step */
1052
1053         if (!step->more_tokens)
1054         {
1055             step->more_tokens = icu_chain_step_next_token(chain, step, status);
1056             return step->more_tokens;  /* avoid one token count too much! */
1057         }
1058         break;
1059     default:
1060         return 0;
1061         break;
1062     }
1063
1064     if (U_FAILURE(*status))
1065         return 0;
1066
1067     /* if token disappered into thin air, tell caller */
1068     /* if (!step->buf16->utf16_len && !step->more_tokens) */
1069     /*    return 0; */
1070
1071     return 1;
1072 }
1073
1074
1075 int icu_chain_assign_cstr(struct icu_chain * chain,
1076                           const char * src8cstr,
1077                           UErrorCode *status)
1078 {
1079     struct icu_chain_step * stp = 0;
1080
1081     if (!chain || !src8cstr)
1082         return 0;
1083
1084     chain->src8cstr = src8cstr;
1085
1086     stp = chain->steps;
1087
1088     /* clear token count */
1089     chain->token_count = 0;
1090
1091     /* clear all steps stop states */
1092     while (stp)
1093     {
1094         stp->more_tokens = 1;
1095         stp->need_new_token = 1;
1096         stp = stp->previous;
1097     }
1098
1099     /* finally convert UTF8 to UTF16 string if needed */
1100     if (chain->steps || chain->sort)
1101         icu_utf16_from_utf8_cstr(chain->src16, chain->src8cstr, status);
1102
1103     if (U_FAILURE(*status))
1104         return 0;
1105
1106     return 1;
1107 }
1108
1109
1110
1111 int icu_chain_next_token(struct icu_chain * chain,
1112                          UErrorCode *status)
1113 {
1114     int got_token = 0;
1115
1116     *status = U_ZERO_ERROR;
1117
1118     if (!chain)
1119         return 0;
1120
1121     /* special case with no steps - same as index type binary */
1122     if (!chain->steps)
1123     {
1124         if (chain->token_count)
1125             return 0;
1126         else
1127         {
1128             chain->token_count++;
1129
1130             if (chain->sort)
1131                 icu_sortkey8_from_utf16(chain->coll,
1132                                         chain->sort8, chain->steps->buf16,
1133                                         status);
1134             return chain->token_count;
1135         }
1136     }
1137     /* usual case, one or more icu chain steps existing */
1138     else
1139     {
1140         while(!got_token && chain->steps && chain->steps->more_tokens)
1141             got_token = icu_chain_step_next_token(chain, chain->steps, status);
1142
1143         if (got_token)
1144         {
1145             chain->token_count++;
1146
1147             icu_utf16_to_utf8(chain->norm8, chain->steps->buf16, status);
1148
1149             if (chain->sort)
1150                 icu_sortkey8_from_utf16(chain->coll,
1151                                         chain->sort8, chain->steps->buf16,
1152                                         status);
1153
1154             return chain->token_count;
1155         }
1156     }
1157
1158     return 0;
1159 }
1160
1161 int icu_chain_token_number(struct icu_chain * chain)
1162 {
1163     if (!chain)
1164         return 0;
1165
1166     return chain->token_count;
1167 }
1168
1169
1170 const char * icu_chain_token_display(struct icu_chain * chain)
1171 {
1172     if (chain->display8)
1173         return icu_buf_utf8_to_cstr(chain->display8);
1174
1175     return 0;
1176 }
1177
1178 const char * icu_chain_token_norm(struct icu_chain * chain)
1179 {
1180     if (!chain->steps)
1181         return chain->src8cstr;
1182
1183     if (chain->norm8)
1184         return icu_buf_utf8_to_cstr(chain->norm8);
1185
1186     return 0;
1187 }
1188
1189 const char * icu_chain_token_sortkey(struct icu_chain * chain)
1190 {
1191     if (chain->sort8)
1192         return icu_buf_utf8_to_cstr(chain->sort8);
1193
1194     return 0;
1195 }
1196
1197 const UCollator * icu_chain_get_coll(struct icu_chain * chain)
1198 {
1199     return chain->coll;
1200 }
1201
1202 #endif /* YAZ_HAVE_ICU */
1203
1204 /*
1205  * Local variables:
1206  * c-basic-offset: 4
1207  * indent-tabs-mode: nil
1208  * End:
1209  * vim: shiftwidth=4 tabstop=8 expandtab
1210  */