src/icu_I18N.c

   1 /* This file is part of the YAZ toolkit.
   2  * Copyright (C) 1995-2009 Index Data
   3  * See the file LICENSE for details.
   4  */
   5
   6 /**
   7  * \file icu_I18N.c
   8  * \brief ICU utilities
   9  */
  10
  11 #if HAVE_CONFIG_H
  12 #include "config.h"
  13 #endif
  14
  15 #define USE_TIMING 0
  16 #if USE_TIMING
  17 #include <yaz/timing.h>
  18 #endif
  19
  20 #if YAZ_HAVE_ICU
  21 #include <yaz/xmalloc.h>
  22
  23 #include <yaz/icu_I18N.h>
  24
  25 #include <yaz/log.h>
  26
  27 #include <string.h>
  28 #include <stdlib.h>
  29 #include <stdio.h>
  30
  31 #include <unicode/ustring.h>  /* some more string fcns*/
  32 #include <unicode/uchar.h>    /* char names           */
  33
  34
  35 #include <unicode/ucol.h>
  36
  37
  38 int icu_check_status (UErrorCode status)
  39 {
  40     if (U_FAILURE(status))
  41     {
  42         yaz_log(YLOG_WARN, "ICU: %d %s\n", status, u_errorName(status));
  43         return 0;
  44     }
  45     return 1;
  46
  47 }
  48
  49
  50
  51 struct icu_buf_utf16 * icu_buf_utf16_create(size_t capacity)
  52 {
  53     struct icu_buf_utf16 * buf16
  54         = (struct icu_buf_utf16 *) xmalloc(sizeof(struct icu_buf_utf16));
  55
  56     buf16->utf16 = 0;
  57     buf16->utf16_len = 0;
  58     buf16->utf16_cap = 0;
  59
  60     if (capacity > 0){
  61         buf16->utf16 = (UChar *) xmalloc(sizeof(UChar) * capacity);
  62         buf16->utf16[0] = (UChar) 0;
  63         buf16->utf16_cap = capacity;
  64     }
  65     return buf16;
  66 }
  67
  68 struct icu_buf_utf16 * icu_buf_utf16_clear(struct icu_buf_utf16 * buf16)
  69 {
  70     if (buf16){
  71         if (buf16->utf16)
  72             buf16->utf16[0] = (UChar) 0;
  73         buf16->utf16_len = 0;
  74     }
  75     return buf16;
  76 }
  77
  78 struct icu_buf_utf16 * icu_buf_utf16_resize(struct icu_buf_utf16 * buf16,
  79                                             size_t capacity)
  80 {
  81     if (!buf16)
  82         return 0;
  83
  84     if (capacity >  0){
  85         if (0 == buf16->utf16)
  86             buf16->utf16 = (UChar *) xmalloc(sizeof(UChar) * capacity);
  87         else
  88             buf16->utf16
  89                 = (UChar *) xrealloc(buf16->utf16, sizeof(UChar) * capacity);
  90
  91         icu_buf_utf16_clear(buf16);
  92         buf16->utf16_cap = capacity;
  93     }
  94     else {
  95         xfree(buf16->utf16);
  96         buf16->utf16 = 0;
  97         buf16->utf16_len = 0;
  98         buf16->utf16_cap = 0;
  99     }
 100
 101     return buf16;
 102 }
 103
 104
 105 struct icu_buf_utf16 * icu_buf_utf16_copy(struct icu_buf_utf16 * dest16,
 106                                           struct icu_buf_utf16 * src16)
 107 {
 108     if(!dest16 || !src16
 109        || dest16 == src16)
 110         return 0;
 111
 112     if (dest16->utf16_cap < src16->utf16_len)
 113         icu_buf_utf16_resize(dest16, src16->utf16_len * 2);
 114
 115     u_strncpy(dest16->utf16, src16->utf16, src16->utf16_len);
 116     dest16->utf16_len = src16->utf16_len;
 117
 118     return dest16;
 119 }
 120
 121
 122 void icu_buf_utf16_destroy(struct icu_buf_utf16 * buf16)
 123 {
 124     if (buf16)
 125         xfree(buf16->utf16);
 126     xfree(buf16);
 127 }
 128
 129
 130
 131 struct icu_buf_utf8 * icu_buf_utf8_create(size_t capacity)
 132 {
 133     struct icu_buf_utf8 * buf8
 134         = (struct icu_buf_utf8 *) xmalloc(sizeof(struct icu_buf_utf8));
 135
 136     buf8->utf8 = 0;
 137     buf8->utf8_len = 0;
 138     buf8->utf8_cap = 0;
 139
 140     if (capacity > 0){
 141         buf8->utf8 = (uint8_t *) xmalloc(sizeof(uint8_t) * capacity);
 142         buf8->utf8[0] = (uint8_t) 0;
 143         buf8->utf8_cap = capacity;
 144     }
 145     return buf8;
 146 }
 147
 148
 149 struct icu_buf_utf8 * icu_buf_utf8_clear(struct icu_buf_utf8 * buf8)
 150 {
 151     if (buf8){
 152         if (buf8->utf8)
 153             buf8->utf8[0] = (uint8_t) 0;
 154         buf8->utf8_len = 0;
 155     }
 156     return buf8;
 157 }
 158
 159
 160 struct icu_buf_utf8 * icu_buf_utf8_resize(struct icu_buf_utf8 * buf8,
 161                                           size_t capacity)
 162 {
 163     if (!buf8)
 164         return 0;
 165
 166     if (capacity >  0){
 167         if (0 == buf8->utf8)
 168             buf8->utf8 = (uint8_t *) xmalloc(sizeof(uint8_t) * capacity);
 169         else
 170             buf8->utf8
 171                 = (uint8_t *) xrealloc(buf8->utf8, sizeof(uint8_t) * capacity);
 172
 173         buf8->utf8_cap = capacity;
 174     }
 175     else {
 176         xfree(buf8->utf8);
 177         buf8->utf8 = 0;
 178         buf8->utf8_len = 0;
 179         buf8->utf8_cap = 0;
 180     }
 181
 182     return buf8;
 183 }
 184
 185
 186 const char *icu_buf_utf8_to_cstr(struct icu_buf_utf8 *src8)
 187 {
 188     if (!src8 || src8->utf8_len == 0)
 189         return "";
 190
 191     if (src8->utf8_len == src8->utf8_cap)
 192         src8 = icu_buf_utf8_resize(src8, src8->utf8_len * 2 + 1);
 193
 194     src8->utf8[src8->utf8_len] = '\0';
 195
 196     return (const char *) src8->utf8;
 197 }
 198
 199
 200 void icu_buf_utf8_destroy(struct icu_buf_utf8 * buf8)
 201 {
 202     if (buf8)
 203         xfree(buf8->utf8);
 204     xfree(buf8);
 205 }
 206
 207
 208 UErrorCode icu_utf16_from_utf8_cstr(struct icu_buf_utf16 * dest16,
 209                                     const char * src8cstr,
 210                                     UErrorCode * status)
 211 {
 212     size_t src8cstr_len = 0;
 213     int32_t utf16_len = 0;
 214
 215     *status = U_ZERO_ERROR;
 216     src8cstr_len = strlen(src8cstr);
 217
 218     u_strFromUTF8(dest16->utf16, dest16->utf16_cap,
 219                   &utf16_len,
 220                   src8cstr, src8cstr_len, status);
 221
 222     /* check for buffer overflow, resize and retry */
 223     if (*status == U_BUFFER_OVERFLOW_ERROR)
 224     {
 225         icu_buf_utf16_resize(dest16, utf16_len * 2);
 226         *status = U_ZERO_ERROR;
 227         u_strFromUTF8(dest16->utf16, dest16->utf16_cap,
 228                       &utf16_len,
 229                       src8cstr, src8cstr_len, status);
 230     }
 231
 232     if (U_SUCCESS(*status)
 233         && utf16_len <= dest16->utf16_cap)
 234         dest16->utf16_len = utf16_len;
 235     else
 236         icu_buf_utf16_clear(dest16);
 237
 238     return *status;
 239 }
 240
 241
 242
 243
 244 UErrorCode icu_utf16_to_utf8(struct icu_buf_utf8 * dest8,
 245                              struct icu_buf_utf16 * src16,
 246                              UErrorCode * status)
 247 {
 248     int32_t utf8_len = 0;
 249
 250     u_strToUTF8((char *) dest8->utf8, dest8->utf8_cap,
 251                 &utf8_len,
 252                 src16->utf16, src16->utf16_len, status);
 253
 254     /* check for buffer overflow, resize and retry */
 255     if (*status == U_BUFFER_OVERFLOW_ERROR)
 256     {
 257         icu_buf_utf8_resize(dest8, utf8_len * 2);
 258         *status = U_ZERO_ERROR;
 259         u_strToUTF8((char *) dest8->utf8, dest8->utf8_cap,
 260                     &utf8_len,
 261                     src16->utf16, src16->utf16_len, status);
 262
 263     }
 264
 265     if (U_SUCCESS(*status)
 266         && utf8_len <= dest8->utf8_cap)
 267         dest8->utf8_len = utf8_len;
 268     else
 269         icu_buf_utf8_clear(dest8);
 270
 271     return *status;
 272 }
 273
 274
 275
 276 struct icu_casemap * icu_casemap_create(char action, UErrorCode *status)
 277 {
 278     struct icu_casemap * casemap
 279         = (struct icu_casemap *) xmalloc(sizeof(struct icu_casemap));
 280     casemap->action = action;
 281
 282     switch(casemap->action) {
 283     case 'l':
 284     case 'L':
 285     case 'u':
 286     case 'U':
 287     case 't':
 288     case 'T':
 289     case 'f':
 290     case 'F':
 291         break;
 292     default:
 293         icu_casemap_destroy(casemap);
 294         return 0;
 295     }
 296
 297     return casemap;
 298 }
 299
 300 void icu_casemap_destroy(struct icu_casemap * casemap)
 301 {
 302     xfree(casemap);
 303 }
 304
 305
 306 int icu_casemap_casemap(struct icu_casemap * casemap,
 307                         struct icu_buf_utf16 * dest16,
 308                         struct icu_buf_utf16 * src16,
 309                         UErrorCode *status,
 310                         const char *locale)
 311 {
 312     if(!casemap)
 313         return 0;
 314
 315     return icu_utf16_casemap(dest16, src16, locale,
 316                              casemap->action, status);
 317 }
 318
 319
 320 int icu_utf16_casemap(struct icu_buf_utf16 * dest16,
 321                       struct icu_buf_utf16 * src16,
 322                       const char *locale, char action,
 323                       UErrorCode *status)
 324 {
 325     int32_t dest16_len = 0;
 326
 327
 328     if (!src16->utf16_len){           /* guarding for empty source string */
 329         if (dest16->utf16)
 330             dest16->utf16[0] = (UChar) 0;
 331         dest16->utf16_len = 0;
 332         return U_ZERO_ERROR;
 333     }
 334
 335
 336     switch(action) {
 337     case 'l':
 338     case 'L':
 339         dest16_len = u_strToLower(dest16->utf16, dest16->utf16_cap,
 340                                   src16->utf16, src16->utf16_len,
 341                                   locale, status);
 342         break;
 343     case 'u':
 344     case 'U':
 345         dest16_len = u_strToUpper(dest16->utf16, dest16->utf16_cap,
 346                                   src16->utf16, src16->utf16_len,
 347                                   locale, status);
 348         break;
 349     case 't':
 350     case 'T':
 351         dest16_len = u_strToTitle(dest16->utf16, dest16->utf16_cap,
 352                                   src16->utf16, src16->utf16_len,
 353                                   0, locale, status);
 354         break;
 355     case 'f':
 356     case 'F':
 357         dest16_len = u_strFoldCase(dest16->utf16, dest16->utf16_cap,
 358                                    src16->utf16, src16->utf16_len,
 359                                    U_FOLD_CASE_DEFAULT, status);
 360         break;
 361
 362     default:
 363         return U_UNSUPPORTED_ERROR;
 364         break;
 365     }
 366
 367     /* check for buffer overflow, resize and retry */
 368     if (*status == U_BUFFER_OVERFLOW_ERROR
 369         && dest16 != src16        /* do not resize if in-place conversion */
 370         ){
 371         icu_buf_utf16_resize(dest16, dest16_len * 2);
 372         *status = U_ZERO_ERROR;
 373
 374
 375         switch(action) {
 376         case 'l':
 377         case 'L':
 378             dest16_len = u_strToLower(dest16->utf16, dest16->utf16_cap,
 379                                       src16->utf16, src16->utf16_len,
 380                                       locale, status);
 381             break;
 382         case 'u':
 383         case 'U':
 384             dest16_len = u_strToUpper(dest16->utf16, dest16->utf16_cap,
 385                                       src16->utf16, src16->utf16_len,
 386                                       locale, status);
 387             break;
 388         case 't':
 389         case 'T':
 390             dest16_len = u_strToTitle(dest16->utf16, dest16->utf16_cap,
 391                                       src16->utf16, src16->utf16_len,
 392                                       0, locale, status);
 393             break;
 394         case 'f':
 395         case 'F':
 396             dest16_len = u_strFoldCase(dest16->utf16, dest16->utf16_cap,
 397                                        src16->utf16, src16->utf16_len,
 398                                        U_FOLD_CASE_DEFAULT, status);
 399             break;
 400
 401         default:
 402             return U_UNSUPPORTED_ERROR;
 403             break;
 404         }
 405     }
 406
 407     if (U_SUCCESS(*status)
 408         && dest16_len <= dest16->utf16_cap)
 409         dest16->utf16_len = dest16_len;
 410     else {
 411         if (dest16->utf16)
 412             dest16->utf16[0] = (UChar) 0;
 413         dest16->utf16_len = 0;
 414     }
 415
 416     return *status;
 417 }
 418
 419
 420
 421 void icu_sortkey8_from_utf16(UCollator *coll,
 422                              struct icu_buf_utf8 * dest8,
 423                              struct icu_buf_utf16 * src16,
 424                              UErrorCode * status)
 425 {
 426
 427     int32_t sortkey_len = 0;
 428
 429     sortkey_len = ucol_getSortKey(coll, src16->utf16, src16->utf16_len,
 430                                   dest8->utf8, dest8->utf8_cap);
 431
 432     /* check for buffer overflow, resize and retry */
 433     if (sortkey_len > dest8->utf8_cap) {
 434         icu_buf_utf8_resize(dest8, sortkey_len * 2);
 435         sortkey_len = ucol_getSortKey(coll, src16->utf16, src16->utf16_len,
 436                                       dest8->utf8, dest8->utf8_cap);
 437     }
 438
 439     if (U_SUCCESS(*status)
 440         && sortkey_len > 0)
 441         dest8->utf8_len = sortkey_len;
 442     else
 443         icu_buf_utf8_clear(dest8);
 444 }
 445
 446
 447
 448 struct icu_tokenizer * icu_tokenizer_create(const char *locale, char action,
 449                                             UErrorCode *status)
 450 {
 451     struct icu_tokenizer * tokenizer
 452         = (struct icu_tokenizer *) xmalloc(sizeof(struct icu_tokenizer));
 453
 454     tokenizer->action = action;
 455     tokenizer->bi = 0;
 456     tokenizer->buf16 = 0;
 457     tokenizer->token_count = 0;
 458     tokenizer->token_id = 0;
 459     tokenizer->token_start = 0;
 460     tokenizer->token_end = 0;
 461
 462
 463     switch(tokenizer->action) {
 464     case 'l':
 465     case 'L':
 466         tokenizer->bi = ubrk_open(UBRK_LINE, locale, 0, 0, status);
 467         break;
 468     case 's':
 469     case 'S':
 470         tokenizer->bi = ubrk_open(UBRK_SENTENCE, locale, 0, 0, status);
 471         break;
 472     case 'w':
 473     case 'W':
 474         tokenizer->bi = ubrk_open(UBRK_WORD, locale, 0, 0, status);
 475         break;
 476     case 'c':
 477     case 'C':
 478         tokenizer->bi = ubrk_open(UBRK_CHARACTER, locale, 0, 0, status);
 479         break;
 480     case 't':
 481     case 'T':
 482         tokenizer->bi = ubrk_open(UBRK_TITLE, locale, 0, 0, status);
 483         break;
 484     default:
 485         *status = U_UNSUPPORTED_ERROR;
 486         return 0;
 487         break;
 488     }
 489
 490     /* ICU error stuff is a very  funny business */
 491     if (U_SUCCESS(*status))
 492         return tokenizer;
 493
 494     /* freeing if failed */
 495     icu_tokenizer_destroy(tokenizer);
 496     return 0;
 497 }
 498
 499 void icu_tokenizer_destroy(struct icu_tokenizer * tokenizer)
 500 {
 501     if (tokenizer) {
 502         if (tokenizer->bi)
 503             ubrk_close(tokenizer->bi);
 504         xfree(tokenizer);
 505     }
 506 }
 507
 508 int icu_tokenizer_attach(struct icu_tokenizer * tokenizer,
 509                          struct icu_buf_utf16 * src16,
 510                          UErrorCode *status)
 511 {
 512     if (!tokenizer || !tokenizer->bi || !src16)
 513         return 0;
 514
 515
 516     tokenizer->buf16 = src16;
 517     tokenizer->token_count = 0;
 518     tokenizer->token_id = 0;
 519     tokenizer->token_start = 0;
 520     tokenizer->token_end = 0;
 521
 522     ubrk_setText(tokenizer->bi, src16->utf16, src16->utf16_len, status);
 523
 524
 525     if (U_FAILURE(*status))
 526         return 0;
 527
 528     return 1;
 529 };
 530
 531 int32_t icu_tokenizer_next_token(struct icu_tokenizer * tokenizer,
 532                          struct icu_buf_utf16 * tkn16,
 533                          UErrorCode *status)
 534 {
 535     int32_t tkn_start = 0;
 536     int32_t tkn_end = 0;
 537     int32_t tkn_len = 0;
 538
 539
 540     if (!tokenizer || !tokenizer->bi
 541         || !tokenizer->buf16 || !tokenizer->buf16->utf16_len)
 542         return 0;
 543
 544     /*
 545     never change tokenizer->buf16 and keep always invariant
 546     0 <= tokenizer->token_start
 547        <= tokenizer->token_end
 548        <= tokenizer->buf16->utf16_len
 549     returns length of token
 550     */
 551
 552     if (0 == tokenizer->token_end) /* first call */
 553         tkn_start = ubrk_first(tokenizer->bi);
 554     else /* successive calls */
 555         tkn_start = tokenizer->token_end;
 556
 557     /* get next position */
 558     tkn_end = ubrk_next(tokenizer->bi);
 559
 560     /* repairing invariant at end of ubrk, which is UBRK_DONE = -1 */
 561     if (UBRK_DONE == tkn_end)
 562         tkn_end = tokenizer->buf16->utf16_len;
 563
 564     /* copy out if everything is well */
 565     if(U_FAILURE(*status))
 566         return 0;
 567
 568     /* everything OK, now update internal state */
 569     tkn_len = tkn_end - tkn_start;
 570
 571     if (0 < tkn_len){
 572         tokenizer->token_count++;
 573         tokenizer->token_id++;
 574     } else {
 575         tokenizer->token_id = 0;
 576     }
 577     tokenizer->token_start = tkn_start;
 578     tokenizer->token_end = tkn_end;
 579
 580
 581     /* copying into token buffer if it exists */
 582     if (tkn16){
 583         if (tkn16->utf16_cap < tkn_len)
 584             icu_buf_utf16_resize(tkn16, (size_t) tkn_len * 2);
 585
 586         u_strncpy(tkn16->utf16, &(tokenizer->buf16->utf16)[tkn_start],
 587                   tkn_len);
 588
 589         tkn16->utf16_len = tkn_len;
 590     }
 591
 592     return tkn_len;
 593 }
 594
 595
 596 int32_t icu_tokenizer_token_id(struct icu_tokenizer * tokenizer)
 597 {
 598     return tokenizer->token_id;
 599 }
 600
 601 int32_t icu_tokenizer_token_start(struct icu_tokenizer * tokenizer)
 602 {
 603     return tokenizer->token_start;
 604 }
 605
 606 int32_t icu_tokenizer_token_end(struct icu_tokenizer * tokenizer)
 607 {
 608     return tokenizer->token_end;
 609 }
 610
 611 int32_t icu_tokenizer_token_length(struct icu_tokenizer * tokenizer)
 612 {
 613     return (tokenizer->token_end - tokenizer->token_start);
 614 }
 615
 616 int32_t icu_tokenizer_token_count(struct icu_tokenizer * tokenizer)
 617 {
 618     return tokenizer->token_count;
 619 }
 620
 621
 622
 623 struct icu_transform * icu_transform_create(const char *id, char action,
 624                                             const char *rules,
 625                                             UErrorCode *status)
 626 {
 627     struct icu_buf_utf16 *id16 = icu_buf_utf16_create(0);
 628     struct icu_buf_utf16 *rules16 = icu_buf_utf16_create(0);
 629
 630     struct icu_transform * transform
 631         = (struct icu_transform *) xmalloc(sizeof(struct icu_transform));
 632
 633     transform->action = action;
 634     transform->trans = 0;
 635
 636     if (id)
 637         icu_utf16_from_utf8_cstr(id16, id, status);
 638     if (rules)
 639         icu_utf16_from_utf8_cstr(rules16, rules, status);
 640
 641     switch(transform->action)
 642     {
 643     case 'f':
 644     case 'F':
 645         transform->trans
 646             = utrans_openU(id16->utf16,
 647                            id16->utf16_len,
 648                            UTRANS_FORWARD,
 649                            rules16->utf16,
 650                            rules16->utf16_len,
 651                            &transform->parse_error, status);
 652         break;
 653     case 'r':
 654     case 'R':
 655         transform->trans
 656             = utrans_openU(id16->utf16,
 657                            id16->utf16_len,
 658                            UTRANS_REVERSE ,
 659                            rules16->utf16,
 660                            rules16->utf16_len,
 661                            &transform->parse_error, status);
 662         break;
 663     default:
 664         *status = U_UNSUPPORTED_ERROR;
 665         break;
 666     }
 667     icu_buf_utf16_destroy(rules16);
 668     icu_buf_utf16_destroy(id16);
 669
 670     if (U_SUCCESS(*status))
 671         return transform;
 672
 673     /* freeing if failed */
 674     icu_transform_destroy(transform);
 675     return 0;
 676 }
 677
 678
 679 void icu_transform_destroy(struct icu_transform * transform){
 680     if (transform) {
 681         if (transform->trans)
 682             utrans_close(transform->trans);
 683         xfree(transform);
 684     }
 685 }
 686
 687
 688
 689 int icu_transform_trans(struct icu_transform * transform,
 690                         struct icu_buf_utf16 * dest16,
 691                         struct icu_buf_utf16 * src16,
 692                         UErrorCode *status)
 693 {
 694     if (!transform || !transform->trans
 695         || !src16
 696         || !dest16)
 697         return 0;
 698
 699     if (!src16->utf16_len){           /* guarding for empty source string */
 700         icu_buf_utf16_clear(dest16);
 701         return 0;
 702     }
 703
 704     if (!icu_buf_utf16_copy(dest16, src16))
 705         return 0;
 706
 707
 708     utrans_transUChars (transform->trans,
 709                         dest16->utf16, &(dest16->utf16_len),
 710                         dest16->utf16_cap,
 711                         0, &(src16->utf16_len), status);
 712
 713     if (U_FAILURE(*status))
 714         icu_buf_utf16_clear(dest16);
 715
 716     return dest16->utf16_len;
 717 }
 718
 719
 720
 721
 722 struct icu_chain_step * icu_chain_step_create(struct icu_chain * chain,
 723                                               enum icu_chain_step_type type,
 724                                               const uint8_t * rule,
 725                                               struct icu_buf_utf16 * buf16,
 726                                               UErrorCode *status)
 727 {
 728     struct icu_chain_step * step = 0;
 729
 730     if(!chain || !type || !rule)
 731         return 0;
 732
 733     step = (struct icu_chain_step *) xmalloc(sizeof(struct icu_chain_step));
 734
 735     step->type = type;
 736
 737     step->buf16 = buf16;
 738
 739     /* create auxilary objects */
 740     switch(step->type) {
 741     case ICU_chain_step_type_display:
 742         break;
 743     case ICU_chain_step_type_casemap:
 744         step->u.casemap = icu_casemap_create(rule[0], status);
 745         break;
 746     case ICU_chain_step_type_transform:
 747         /* rule omitted. Only ID used */
 748         step->u.transform = icu_transform_create((const char *) rule, 'f',
 749                                                  0, status);
 750         break;
 751     case ICU_chain_step_type_tokenize:
 752         step->u.tokenizer = icu_tokenizer_create((char *) chain->locale,
 753                                                  (char) rule[0], status);
 754         break;
 755     case ICU_chain_step_type_transliterate:
 756         /* we pass a dummy ID to utrans_openU.. */
 757         step->u.transform = icu_transform_create("custom", 'f',
 758                                                  (const char *) rule, status);
 759         break;
 760     default:
 761         break;
 762     }
 763
 764     return step;
 765 }
 766
 767
 768 void icu_chain_step_destroy(struct icu_chain_step * step){
 769
 770     if (!step)
 771         return;
 772
 773     icu_chain_step_destroy(step->previous);
 774
 775     switch(step->type) {
 776     case ICU_chain_step_type_display:
 777         break;
 778     case ICU_chain_step_type_casemap:
 779         icu_casemap_destroy(step->u.casemap);
 780         icu_buf_utf16_destroy(step->buf16);
 781         break;
 782     case ICU_chain_step_type_transform:
 783     case ICU_chain_step_type_transliterate:
 784         icu_transform_destroy(step->u.transform);
 785         icu_buf_utf16_destroy(step->buf16);
 786         break;
 787     case ICU_chain_step_type_tokenize:
 788         icu_tokenizer_destroy(step->u.tokenizer);
 789         icu_buf_utf16_destroy(step->buf16);
 790         break;
 791     default:
 792         break;
 793     }
 794     xfree(step);
 795 }
 796
 797
 798
 799 struct icu_chain * icu_chain_create(const char *locale,  int sort,
 800                                     UErrorCode * status)
 801 {
 802     struct icu_chain * chain
 803         = (struct icu_chain *) xmalloc(sizeof(struct icu_chain));
 804
 805     *status = U_ZERO_ERROR;
 806
 807     chain->locale = xstrdup(locale);
 808
 809     chain->sort = sort;
 810
 811     chain->coll = ucol_open((const char *) chain->locale, status);
 812
 813     if (U_FAILURE(*status))
 814         return 0;
 815
 816     chain->token_count = 0;
 817
 818     chain->src8cstr = 0;
 819
 820     chain->display8 = icu_buf_utf8_create(0);
 821     chain->norm8 = icu_buf_utf8_create(0);
 822     chain->sort8 = icu_buf_utf8_create(0);
 823
 824     chain->src16 = icu_buf_utf16_create(0);
 825
 826     chain->steps = 0;
 827
 828     return chain;
 829 }
 830
 831
 832 void icu_chain_destroy(struct icu_chain * chain)
 833 {
 834     if (chain)
 835     {
 836         if (chain->coll)
 837             ucol_close(chain->coll);
 838
 839         icu_buf_utf8_destroy(chain->display8);
 840         icu_buf_utf8_destroy(chain->norm8);
 841         icu_buf_utf8_destroy(chain->sort8);
 842
 843         icu_buf_utf16_destroy(chain->src16);
 844
 845         icu_chain_step_destroy(chain->steps);
 846         xfree(chain->locale);
 847         xfree(chain);
 848     }
 849 }
 850
 851
 852
 853 struct icu_chain * icu_chain_xml_config(const xmlNode *xml_node,
 854                                         int sort,
 855                                         UErrorCode * status)
 856 {
 857     xmlNode *node = 0;
 858     struct icu_chain * chain = 0;
 859
 860     *status = U_ZERO_ERROR;
 861
 862     if (!xml_node ||xml_node->type != XML_ELEMENT_NODE)
 863         return 0;
 864
 865     {
 866         xmlChar * xml_locale = xmlGetProp((xmlNode *) xml_node,
 867                                           (xmlChar *) "locale");
 868
 869         if (xml_locale)
 870         {
 871             chain = icu_chain_create((const char *) xml_locale, sort, status);
 872             xmlFree(xml_locale);
 873         }
 874
 875     }
 876     if (!chain)
 877         return 0;
 878
 879     for (node = xml_node->children; node; node = node->next)
 880     {
 881         xmlChar *xml_rule;
 882         struct icu_chain_step * step = 0;
 883
 884         if (node->type != XML_ELEMENT_NODE)
 885             continue;
 886
 887         xml_rule = xmlGetProp(node, (xmlChar *) "rule");
 888
 889         if (!strcmp((const char *) node->name, "casemap"))
 890             step = icu_chain_insert_step(chain, ICU_chain_step_type_casemap,
 891                                          (const uint8_t *) xml_rule, status);
 892         else if (!strcmp((const char *) node->name, "transform"))
 893             step = icu_chain_insert_step(chain, ICU_chain_step_type_transform,
 894                                          (const uint8_t *) xml_rule, status);
 895         else if (!strcmp((const char *) node->name, "transliterate"))
 896             step = icu_chain_insert_step(chain, ICU_chain_step_type_transliterate,
 897                                          (const uint8_t *) xml_rule, status);
 898         else if (!strcmp((const char *) node->name, "tokenize"))
 899             step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize,
 900                                          (const uint8_t *) xml_rule, status);
 901         else if (!strcmp((const char *) node->name, "display"))
 902             step = icu_chain_insert_step(chain, ICU_chain_step_type_display,
 903                                          (const uint8_t *) "", status);
 904         else if (!strcmp((const char *) node->name, "normalize"))
 905         {
 906             yaz_log(YLOG_WARN, "Element %s is deprecated. "
 907                     "Use transform instead", node->name);
 908             step = icu_chain_insert_step(chain, ICU_chain_step_type_transform,
 909                                          (const uint8_t *) xml_rule, status);
 910         }
 911         else if (!strcmp((const char *) node->name, "index")
 912                  || !strcmp((const char *) node->name, "sortkey"))
 913         {
 914             yaz_log(YLOG_WARN, "Element %s is no longer needed. "
 915                     "Remove it from the configuration", node->name);
 916         }
 917         else
 918         {
 919             yaz_log(YLOG_WARN, "Unknown element %s", node->name);
 920             icu_chain_destroy(chain);
 921             return 0;
 922         }
 923         xmlFree(xml_rule);
 924         if (step && U_FAILURE(*status))
 925         {
 926             icu_chain_destroy(chain);
 927             return 0;
 928         }
 929     }
 930     return chain;
 931 }
 932
 933
 934
 935 struct icu_chain_step * icu_chain_insert_step(struct icu_chain * chain,
 936                                               enum icu_chain_step_type type,
 937                                               const uint8_t * rule,
 938                                               UErrorCode *status)
 939 {
 940     struct icu_chain_step * step = 0;
 941     struct icu_buf_utf16 * src16 = 0;
 942     struct icu_buf_utf16 * buf16 = 0;
 943
 944     if (!chain || !type || !rule)
 945         return 0;
 946
 947     /* assign utf16 src buffers as needed */
 948     if (chain->steps && chain->steps->buf16)
 949         src16 = chain->steps->buf16;
 950     else if (chain->src16)
 951         src16 = chain->src16;
 952     else
 953         return 0;
 954
 955
 956     /* create utf16 destination buffers as needed, or */
 957     switch(type)
 958     {
 959     case ICU_chain_step_type_display:
 960         buf16 = src16;
 961         break;
 962     case ICU_chain_step_type_casemap:
 963         buf16 = icu_buf_utf16_create(0);
 964         break;
 965     case ICU_chain_step_type_transform:
 966     case ICU_chain_step_type_transliterate:
 967         buf16 = icu_buf_utf16_create(0);
 968         break;
 969     case ICU_chain_step_type_tokenize:
 970         buf16 = icu_buf_utf16_create(0);
 971         break;
 972         break;
 973     default:
 974         break;
 975     }
 976
 977     /* create actual chain step with this buffer */
 978     step = icu_chain_step_create(chain, type, rule, buf16, status);
 979
 980     step->previous = chain->steps;
 981     chain->steps = step;
 982
 983     return step;
 984 }
 985
 986
 987 int icu_chain_step_next_token(struct icu_chain * chain,
 988                               struct icu_chain_step * step,
 989                               UErrorCode *status)
 990 {
 991     struct icu_buf_utf16 * src16 = 0;
 992     int got_new_token = 0;
 993
 994     if (!chain || !chain->src16 || !step || !step->more_tokens)
 995         return 0;
 996
 997     /* assign utf16 src buffers as neeed, advance in previous steps
 998        tokens until non-zero token met, and setting stop condition */
 999
1000     if (step->previous)
1001     {
1002         src16 = step->previous->buf16;
1003         /* tokens might be killed in previous steps, therefore looping */
1004
1005         while (step->need_new_token
1006                && step->previous->more_tokens
1007                && !got_new_token)
1008             got_new_token
1009                 = icu_chain_step_next_token(chain, step->previous, status);
1010     }
1011     else
1012     { /* first step can only work once on chain->src16 input buffer */
1013         src16 = chain->src16;
1014         step->more_tokens = 0;
1015         got_new_token = 1;
1016     }
1017
1018     if (!src16)
1019         return 0;
1020
1021     /* stop if nothing to process */
1022     if (step->need_new_token && !got_new_token)
1023     {
1024         step->more_tokens = 0;
1025         return 0;
1026     }
1027
1028     /* either an old token not finished yet, or a new token, thus
1029        perform the work, eventually put this steps output in
1030        step->buf16 or the chains UTF8 output buffers  */
1031
1032     switch(step->type)
1033     {
1034     case ICU_chain_step_type_display:
1035         icu_utf16_to_utf8(chain->display8, src16, status);
1036         break;
1037     case ICU_chain_step_type_casemap:
1038         icu_casemap_casemap(step->u.casemap,
1039                             step->buf16, src16, status,
1040                             chain->locale);
1041         break;
1042     case ICU_chain_step_type_transform:
1043     case ICU_chain_step_type_transliterate:
1044         icu_transform_trans(step->u.transform,
1045                             step->buf16, src16, status);
1046         break;
1047     case ICU_chain_step_type_tokenize:
1048         /* attach to new src16 token only first time during splitting */
1049         if (step->need_new_token)
1050         {
1051             icu_tokenizer_attach(step->u.tokenizer, src16, status);
1052             step->need_new_token = 0;
1053         }
1054
1055         /* splitting one src16 token into multiple buf16 tokens */
1056         step->more_tokens
1057             = icu_tokenizer_next_token(step->u.tokenizer,
1058                                        step->buf16, status);
1059
1060         /* make sure to get new previous token if this one had been used up
1061            by recursive call to _same_ step */
1062
1063         if (!step->more_tokens)
1064         {
1065             step->more_tokens = icu_chain_step_next_token(chain, step, status);
1066             return step->more_tokens;  /* avoid one token count too much! */
1067         }
1068         break;
1069     default:
1070         return 0;
1071         break;
1072     }
1073
1074     if (U_FAILURE(*status))
1075         return 0;
1076
1077     /* if token disappered into thin air, tell caller */
1078     /* if (!step->buf16->utf16_len && !step->more_tokens) */
1079     /*    return 0; */
1080
1081     return 1;
1082 }
1083
1084
1085 int icu_chain_assign_cstr(struct icu_chain * chain,
1086                           const char * src8cstr,
1087                           UErrorCode *status)
1088 {
1089     struct icu_chain_step * stp = 0;
1090
1091     if (!chain || !src8cstr)
1092         return 0;
1093
1094     chain->src8cstr = src8cstr;
1095
1096     stp = chain->steps;
1097
1098     /* clear token count */
1099     chain->token_count = 0;
1100
1101     /* clear all steps stop states */
1102     while (stp)
1103     {
1104         stp->more_tokens = 1;
1105         stp->need_new_token = 1;
1106         stp = stp->previous;
1107     }
1108
1109     /* finally convert UTF8 to UTF16 string if needed */
1110     if (chain->steps || chain->sort)
1111         icu_utf16_from_utf8_cstr(chain->src16, chain->src8cstr, status);
1112
1113     if (U_FAILURE(*status))
1114         return 0;
1115
1116     return 1;
1117 }
1118
1119
1120
1121 int icu_chain_next_token(struct icu_chain * chain,
1122                          UErrorCode *status)
1123 {
1124     int got_token = 0;
1125
1126     *status = U_ZERO_ERROR;
1127
1128     if (!chain)
1129         return 0;
1130
1131     /* special case with no steps - same as index type binary */
1132     if (!chain->steps)
1133     {
1134         if (chain->token_count)
1135             return 0;
1136         else
1137         {
1138             chain->token_count++;
1139
1140             if (chain->sort)
1141                 icu_sortkey8_from_utf16(chain->coll,
1142                                         chain->sort8, chain->steps->buf16,
1143                                         status);
1144             return chain->token_count;
1145         }
1146     }
1147     /* usual case, one or more icu chain steps existing */
1148     else
1149     {
1150         while(!got_token && chain->steps && chain->steps->more_tokens)
1151             got_token = icu_chain_step_next_token(chain, chain->steps, status);
1152
1153         if (got_token)
1154         {
1155             chain->token_count++;
1156
1157             icu_utf16_to_utf8(chain->norm8, chain->steps->buf16, status);
1158
1159             if (chain->sort)
1160                 icu_sortkey8_from_utf16(chain->coll,
1161                                         chain->sort8, chain->steps->buf16,
1162                                         status);
1163
1164             return chain->token_count;
1165         }
1166     }
1167
1168     return 0;
1169 }
1170
1171 int icu_chain_token_number(struct icu_chain * chain)
1172 {
1173     if (!chain)
1174         return 0;
1175
1176     return chain->token_count;
1177 }
1178
1179
1180 const char * icu_chain_token_display(struct icu_chain * chain)
1181 {
1182     if (chain->display8)
1183         return icu_buf_utf8_to_cstr(chain->display8);
1184
1185     return 0;
1186 }
1187
1188 const char * icu_chain_token_norm(struct icu_chain * chain)
1189 {
1190     if (!chain->steps)
1191         return chain->src8cstr;
1192
1193     if (chain->norm8)
1194         return icu_buf_utf8_to_cstr(chain->norm8);
1195
1196     return 0;
1197 }
1198
1199 const char * icu_chain_token_sortkey(struct icu_chain * chain)
1200 {
1201     if (chain->sort8)
1202         return icu_buf_utf8_to_cstr(chain->sort8);
1203
1204     return 0;
1205 }
1206
1207 const UCollator * icu_chain_get_coll(struct icu_chain * chain)
1208 {
1209     return chain->coll;
1210 }
1211
1212 #endif /* YAZ_HAVE_ICU */
1213
1214 /*
1215  * Local variables:
1216  * c-basic-offset: 4
1217  * c-file-style: "Stroustrup"
1218  * indent-tabs-mode: nil
1219  * End:
1220  * vim: shiftwidth=4 tabstop=8 expandtab
1221  */
1222