src/icu_I18N.h

   1 /* $Id: icu_I18N.h,v 1.12 2007-05-14 13:51:24 marc Exp $
   2    Copyright (c) 2006-2007, Index Data.
   3
   4    This file is part of Pazpar2.
   5
   6    Pazpar2 is free software; you can redistribute it and/or modify it under
   7    the terms of the GNU General Public License as published by the Free
   8    Software Foundation; either version 2, or (at your option) any later
   9    version.
  10
  11    Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY
  12    WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14    for more details.
  15
  16    You should have received a copy of the GNU General Public License
  17    along with Pazpar2; see the file LICENSE.  If not, write to the
  18    Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
  19    02111-1307, USA.
  20 */
  21
  22 #ifndef ICU_I18NL_H
  23 #define ICU_I18NL_H
  24
  25 #ifdef HAVE_ICU
  26
  27 #include <yaz/nmem.h>
  28
  29
  30 #include <unicode/utypes.h>   /* Basic ICU data types */
  31 #include <unicode/uchar.h>    /* char names           */
  32
  33 //#include <unicode/ustdio.h>
  34 #include <unicode/ucol.h>
  35 //#include <unicode/ucnv.h>     /* C   Converter API    */
  36 //#include <unicode/ustring.h>  /* some more string fcns*/
  37 //#include <unicode/uloc.h>
  38 #include <unicode/ubrk.h>
  39 //#include <unicode/unistr.h>
  40 #include <unicode/utrans.h>
  41
  42
  43
  44 // declared structs and functions
  45
  46 int icu_check_status (UErrorCode status);
  47
  48 struct icu_buf_utf16
  49 {
  50   UChar * utf16;
  51   int32_t utf16_len;
  52   int32_t utf16_cap;
  53 };
  54
  55 struct icu_buf_utf16 * icu_buf_utf16_create(size_t capacity);
  56 struct icu_buf_utf16 * icu_buf_utf16_resize(struct icu_buf_utf16 * buf16,
  57                                             size_t capacity);
  58 struct icu_buf_utf16 * icu_buf_utf16_copy(struct icu_buf_utf16 * dest16,
  59                                           struct icu_buf_utf16 * src16);
  60 void icu_buf_utf16_destroy(struct icu_buf_utf16 * buf16);
  61
  62
  63
  64 struct icu_buf_utf8
  65 {
  66   uint8_t * utf8;
  67   int32_t utf8_len;
  68   int32_t utf8_cap;
  69 };
  70
  71 struct icu_buf_utf8 * icu_buf_utf8_create(size_t capacity);
  72 struct icu_buf_utf8 * icu_buf_utf8_resize(struct icu_buf_utf8 * buf8,
  73                                           size_t capacity);
  74 void icu_buf_utf8_destroy(struct icu_buf_utf8 * buf8);
  75
  76
  77 UErrorCode icu_utf16_from_utf8(struct icu_buf_utf16 * dest16,
  78                                struct icu_buf_utf8 * src8,
  79                                UErrorCode * status);
  80
  81 UErrorCode icu_utf16_from_utf8_cstr(struct icu_buf_utf16 * dest16,
  82                                     const char * src8cstr,
  83                                     UErrorCode * status);
  84
  85
  86 UErrorCode icu_utf16_to_utf8(struct icu_buf_utf8 * dest8,
  87                              struct icu_buf_utf16 * src16,
  88                              UErrorCode * status);
  89
  90 int icu_utf16_casemap(struct icu_buf_utf16 * dest16,
  91                       struct icu_buf_utf16 * src16,
  92                       const char *locale, char action,
  93                       UErrorCode *status);
  94
  95 UErrorCode icu_sortkey8_from_utf16(UCollator *coll,
  96                                    struct icu_buf_utf8 * dest8,
  97                                    struct icu_buf_utf16 * src16,
  98                                    UErrorCode * status);
  99
 100 struct icu_tokenizer
 101 {
 102   char locale[16];
 103   char action;
 104   UBreakIterator* bi;
 105   struct icu_buf_utf16 * buf16;
 106   int32_t token_count;
 107   int32_t token_id;
 108   int32_t token_start;
 109   int32_t token_end;
 110   // keep always invariant
 111   // 0 <= token_start
 112   //   <= token_end
 113   //   <= buf16->utf16_len
 114   // and invariant
 115   // 0 <= token_id <= token_count
 116 };
 117
 118 struct icu_tokenizer * icu_tokenizer_create(const char *locale, char action,
 119                                             UErrorCode *status);
 120
 121 void icu_tokenizer_destroy(struct icu_tokenizer * tokenizer);
 122
 123 int icu_tokenizer_attach(struct icu_tokenizer * tokenizer,
 124                          struct icu_buf_utf16 * src16, UErrorCode *status);
 125
 126 int32_t icu_tokenizer_next_token(struct icu_tokenizer * tokenizer,
 127                                  struct icu_buf_utf16 * tkn16,
 128                                  UErrorCode *status);
 129
 130 int32_t icu_tokenizer_token_id(struct icu_tokenizer * tokenizer);
 131 int32_t icu_tokenizer_token_start(struct icu_tokenizer * tokenizer);
 132 int32_t icu_tokenizer_token_end(struct icu_tokenizer * tokenizer);
 133 int32_t icu_tokenizer_token_length(struct icu_tokenizer * tokenizer);
 134 int32_t icu_tokenizer_token_count(struct icu_tokenizer * tokenizer);
 135
 136
 137
 138 struct icu_normalizer
 139 {
 140   char action;
 141   struct icu_buf_utf16 * rules16;
 142   UParseError parse_error[256];
 143   UTransliterator * trans;
 144 };
 145
 146 struct icu_normalizer * icu_normalizer_create(const char *rules, char action,
 147                                               UErrorCode *status);
 148
 149
 150 void icu_normalizer_destroy(struct icu_normalizer * normalizer);
 151
 152 int icu_normalizer_normalize(struct icu_normalizer * normalizer,
 153                              struct icu_buf_utf16 * dest16,
 154                              struct icu_buf_utf16 * src16,
 155                              UErrorCode *status);
 156
 157
 158 #if 0
 159 struct icu_token
 160 {
 161   int32_t token_id;
 162   uint8_t * display8;
 163   uint8_t * norm8;
 164   uint8_t * sort8;
 165 }
 166 #endif
 167
 168 enum icu_chain_step_type {
 169     ICU_chain_step_type_none,      //
 170     ICU_chain_step_type_display,   // convert to utf8 display format
 171     ICU_chain_step_type_norm,      // convert to utf8 norm format
 172     ICU_chain_step_type_sort,      // convert to utf8 sort format
 173     ICU_chain_step_type_charmap,   // apply utf16 charmap
 174     ICU_chain_step_type_normalize, // apply utf16 normalization
 175     ICU_chain_step_type_tokenize   // apply utf16 tokenization
 176 };
 177
 178
 179
 180 struct icu_chain_step
 181 {
 182   // type and action object
 183   enum icu_chain_step_type type;
 184   union {
 185     struct icu_normalizer * normalizer;
 186     struct icu_tokenizer * tokenizer;
 187   } u;
 188   // temprary post-action utf16 buffer
 189   struct icu_buf_utf16 * buf16;
 190   struct icu_chain_step * next;
 191 };
 192
 193
 194 struct icu_chain
 195 {
 196   uint8_t identifier[128];
 197   uint8_t locale[16];
 198
 199   // number of tokens returned so far
 200   int32_t token_count;
 201
 202   // utf8 output buffers
 203   struct icu_buf_utf8 * display8;
 204   struct icu_buf_utf8 * norm8;
 205   struct icu_buf_utf8 * sort8;
 206
 207   // utf16 source buffer
 208   struct icu_buf_utf16 * src16;
 209
 210   // linked list of chain steps
 211   struct icu_chain_step * steps;
 212 };
 213
 214 struct icu_chain * icu_chain_create(const uint8_t * identifier,
 215                                     const uint8_t * locale);
 216
 217 void icu_chain_destroy(struct icu_chain * chain);
 218
 219 struct icu_chain_step * icu_chain_append_step(struct icu_chain * chain,
 220                                               enum icu_chain_step_type type,
 221                                               const uint8_t * rule);
 222
 223 void icu_chain_step_destroy(struct icu_chain_step * step);
 224
 225
 226
 227 #endif // HAVE_ICU
 228 #endif // ICU_I18NL_H