util/yaz-icu.c

   1 /*
   2  * Copyright (C) 1995-2007, Index Data ApS
   3  * See the file LICENSE for details.
   4  *
   5  * $Id: yaz-icu.c,v 1.11 2007-11-08 13:35:14 adam Exp $
   6  */
   7
   8 #if HAVE_CONFIG_H
   9 #include "config.h"
  10 #endif
  11
  12 #include <string.h>
  13
  14 #include <stdio.h>
  15 #include <stdlib.h>
  16
  17 #include <yaz/options.h>
  18
  19
  20 #if YAZ_HAVE_ICU
  21
  22 #include <unicode/ucnv.h>
  23 #include <unicode/ustring.h>
  24 #include <unicode/ucol.h>
  25 #include <unicode/ubrk.h>
  26 #include <unicode/utrans.h>
  27
  28 #include <yaz/icu.h>
  29
  30 /* commando line and config parameters */
  31 static struct config_t {
  32     char conffile[1024];
  33     char print[1024];
  34     int xmloutput;
  35     yaz_icu_chain_t chain;
  36     FILE * infile;
  37     FILE * outfile;
  38 } config;
  39
  40
  41
  42 void print_option_error(const struct config_t *p_config)
  43 {
  44     fprintf(stderr, "Calling error, valid options are :\n");
  45     fprintf(stderr, "yaz-icu\n"
  46             "   [-c (path/to/config/file.xml)]\n"
  47             "   [-p (a|c|l|t)] print ICU info \n"
  48             "   [-x] XML output\n"
  49             "\n"
  50             "Examples:\n"
  51             "cat hugetextfile.txt | ./yaz-icu -c config.xml \n"
  52             "./yaz-icu -p c\n"
  53             "./yaz-icu -p l -x\n"
  54             "./yaz-icu -p t -x\n"
  55             "\n"
  56             "Example ICU chain XML configuration file:\n"
  57             "<icu_chain id=\"en:word\" locale=\"en\">\n"
  58             "  <normalize rule=\"[:Control:] Any-Remove\"/>\n"
  59             "  <tokenize rule=\"l\"/>\n"
  60             "  <normalize rule=\"[[:WhiteSpace:][:Punctuation:]] Remove\"/>\n"
  61             "  <casemap rule=\"l\"/>\n"
  62             "</icu_chain>\n"
  63           );
  64     exit(1);
  65 }
  66
  67 void read_params(int argc, char **argv, struct config_t *p_config)
  68 {
  69     char *arg;
  70     int ret;
  71
  72     /* set default parameters */
  73     p_config->conffile[0] = 0;
  74     p_config->print[0] = 0;
  75     p_config->xmloutput = 0;
  76     p_config->chain = 0;
  77     p_config->infile = stdin;
  78     p_config->outfile = stdout;
  79
  80     /* set up command line parameters */
  81
  82     while ((ret = options("c:p:x", argv, argc, &arg)) != -2)
  83     {
  84         switch (ret)
  85         {
  86         case 'c':
  87             strcpy(p_config->conffile, arg);
  88             break;
  89         case 'p':
  90             strcpy(p_config->print, arg);
  91             break;
  92         case 'x':
  93             p_config->xmloutput = 1;
  94             break;
  95         default:
  96             print_option_error(p_config);
  97         }
  98     }
  99
 100     if ((!strlen(p_config->conffile)
 101          && !strlen(p_config->print))
 102         || !config.infile
 103         || !config.outfile)
 104
 105         print_option_error(p_config);
 106 }
 107
 108
 109 /*     UConverter *conv; */
 110 /*     conv = ucnv_open("utf-8", &status); */
 111 /*     assert(U_SUCCESS(status)); */
 112
 113 /*     *ustr16_len  */
 114 /*       = ucnv_toUChars(conv, ustr16, 1024,  */
 115 /*                       (const char *) *xstr8, strlen((const char *) *xstr8), */
 116 /*                       &status); */
 117
 118
 119
 120 /*      ucnv_fromUChars(conv, */
 121 /*                      (char *) *xstr8, strlen((const char *) *xstr8), */
 122 /*                      ustr16, *ustr16_len, */
 123 /*                      &status); */
 124 /*      ucnv_close(conv); */
 125
 126
 127 static void print_icu_converters(const struct config_t *p_config)
 128 {
 129     int32_t count;
 130     int32_t i;
 131
 132     count = ucnv_countAvailable();
 133     if (p_config->xmloutput)
 134         fprintf(config.outfile, "<converters count=\"%d\" default=\"%s\">\n",
 135                 count, ucnv_getDefaultName());
 136     else {
 137         fprintf(config.outfile, "Available ICU converters: %d\n", count);
 138         fprintf(config.outfile, "Default ICU Converter is: '%s'\n",
 139                 ucnv_getDefaultName());
 140     }
 141
 142     for(i=0;i<count;i++){
 143         if (p_config->xmloutput)
 144             fprintf(config.outfile, "<converter id=\"%s\"/>\n",
 145                     ucnv_getAvailableName(i));
 146         else
 147             fprintf(config.outfile, "%s ", ucnv_getAvailableName(i));
 148     }
 149
 150     if (p_config->xmloutput)
 151         fprintf(config.outfile, "</converters>\n");
 152     else
 153         fprintf(config.outfile, "\n");
 154 }
 155
 156 static void print_icu_transliterators(const struct config_t *p_config)
 157 {
 158     int32_t buf_cap = 128;
 159     char buf[buf_cap];
 160     int32_t i;
 161     int32_t count = utrans_countAvailableIDs();
 162
 163     if (p_config->xmloutput)
 164         fprintf(config.outfile, "<transliterators count=\"%d\">\n",  count);
 165     else
 166         fprintf(config.outfile, "Available ICU transliterators: %d\n", count);
 167
 168     for(i = 0; i <count; i++)
 169     {
 170         utrans_getAvailableID(i, buf, buf_cap);
 171         if (p_config->xmloutput)
 172             fprintf(config.outfile, "<transliterator id=\"%s\"/>\n", buf);
 173         else
 174             fprintf(config.outfile, " %s", buf);
 175     }
 176
 177     if (p_config->xmloutput){
 178         fprintf(config.outfile, "</transliterators>\n");
 179     }
 180     else
 181     {
 182         fprintf(config.outfile, "\n\nUnicode Set Patterns:\n"
 183                 "   Pattern         Description\n"
 184                 "   Ranges          [a-z]       The lower case letters a through z\n"
 185                 "   Named Chars     [abc123] The six characters a,b,c,1,2 and 3\n"
 186                 "   String          [abc{def}] chars a, b and c, and string 'def'\n"
 187                 "   Categories      [\\p{Letter}] Perl General Category 'Letter'.\n"
 188                 "   Categories      [:Letter:] Posix General Category 'Letter'.\n"
 189                 "\n"
 190                 "   Combination     Example\n"
 191                 "   Union           [[:Greek:] [:letter:]]\n"
 192                 "   Intersection    [[:Greek:] & [:letter:]]\n"
 193                 "   Set Complement  [[:Greek:] - [:letter:]]\n"
 194                 "   Complement      [^[:Greek:] [:letter:]]\n"
 195                 "\n"
 196              "see: http://icu.sourceforge.net/userguide/unicodeSet.html\n"
 197                 "\n"
 198                 "Examples:\n"
 199                 "   [:Punctuation:] Any-Remove\n"
 200                 "   [:Cased-Letter:] Any-Upper\n"
 201                 "   [:Control:] Any-Remove\n"
 202                 "   [:Decimal_Number:] Any-Remove\n"
 203                 "   [:Final_Punctuation:] Any-Remove\n"
 204                 "   [:Georgian:] Any-Upper\n"
 205                 "   [:Katakana:] Any-Remove\n"
 206                 "   [:Arabic:] Any-Remove\n"
 207                 "   [:Punctuation:] Remove\n"
 208                 "   [[:Punctuation:]-[.,]] Remove\n"
 209                 "   [:Line_Separator:] Any-Remove\n"
 210                 "   [:Math_Symbol:] Any-Remove\n"
 211                 "   Lower; [:^Letter:] Remove (word tokenization)\n"
 212                 "   [:^Number:] Remove (numeric tokenization)\n"
 213                 "   [:^Katagana:] Remove (remove everything except Katagana)\n"
 214                 "   Lower;[[:WhiteSpace:][:Punctuation:]] Remove (word tokenization)\n"
 215                 "   NFD; [:Nonspacing Mark:] Remove; NFC   (removes accents from characters)\n"
 216                 "   [A-Za-z]; Lower(); Latin-Katakana; Katakana-Hiragana (transforms latin and katagana to hiragana)\n"
 217                 "   [[:separator:][:start punctuation:][:initial punctuation:]] Remove \n"
 218                 "\n"
 219                 "see http://icu.sourceforge.net/userguide/Transform.html\n"
 220                 "    http://www.unicode.org/Public/UNIDATA/UCD.html\n"
 221                 "    http://icu.sourceforge.net/userguide/Transform.html\n"
 222                 "    http://icu.sourceforge.net/userguide/TransformRule.html\n"
 223             );
 224
 225
 226         fprintf(config.outfile, "\n\n");
 227
 228     }
 229 }
 230
 231 static void print_icu_xml_locales(const struct config_t *p_config)
 232 {
 233     int32_t count;
 234     int32_t i;
 235     UErrorCode status = U_ZERO_ERROR;
 236
 237     UChar keyword[64];
 238     int32_t keyword_len = 0;
 239     char keyword_str[128];
 240     int32_t keyword_str_len = 0;
 241
 242     UChar language[64];
 243     int32_t language_len = 0;
 244     char lang_str[128];
 245     int32_t lang_str_len = 0;
 246
 247     UChar script[64];
 248     int32_t script_len = 0;
 249     char script_str[128];
 250     int32_t script_str_len = 0;
 251
 252     UChar location[64];
 253     int32_t location_len = 0;
 254     char location_str[128];
 255     int32_t location_str_len = 0;
 256
 257     UChar variant[64];
 258     int32_t variant_len = 0;
 259     char variant_str[128];
 260     int32_t variant_str_len = 0;
 261
 262     UChar name[64];
 263     int32_t name_len = 0;
 264     char name_str[128];
 265     int32_t name_str_len = 0;
 266
 267     UChar localname[64];
 268     int32_t localname_len = 0;
 269     char localname_str[128];
 270     int32_t localname_str_len = 0;
 271
 272     count = uloc_countAvailable() ;
 273
 274     if (p_config->xmloutput){
 275
 276         fprintf(config.outfile, "<locales count=\"%d\" default=\"%s\" collations=\"%d\">\n",
 277                 count, uloc_getDefault(), ucol_countAvailable());
 278     }
 279
 280     for(i=0;i<count;i++)
 281     {
 282
 283         keyword_len
 284             = uloc_getDisplayKeyword(uloc_getAvailable(i), "en",
 285                                      keyword, 64,
 286                                      &status);
 287
 288         u_strToUTF8(keyword_str, 128, &keyword_str_len,
 289                     keyword, keyword_len,
 290                     &status);
 291
 292
 293         language_len
 294             = uloc_getDisplayLanguage(uloc_getAvailable(i), "en",
 295                                       language, 64,
 296                                       &status);
 297
 298         u_strToUTF8(lang_str, 128, &lang_str_len,
 299                     language, language_len,
 300                     &status);
 301
 302
 303         script_len
 304             = uloc_getDisplayScript(uloc_getAvailable(i), "en",
 305                                     script, 64,
 306                                     &status);
 307
 308         u_strToUTF8(script_str, 128, &script_str_len,
 309                     script, script_len,
 310                     &status);
 311
 312         location_len
 313             = uloc_getDisplayCountry(uloc_getAvailable(i), "en",
 314                                      location, 64,
 315                                      &status);
 316
 317         u_strToUTF8(location_str, 128, &location_str_len,
 318                     location, location_len,
 319                     &status);
 320
 321         variant_len
 322             = uloc_getDisplayVariant(uloc_getAvailable(i), "en",
 323                                      variant, 64,
 324                                      &status);
 325
 326         u_strToUTF8(variant_str, 128, &variant_str_len,
 327                     variant, variant_len,
 328                     &status);
 329
 330         name_len
 331             = uloc_getDisplayName(uloc_getAvailable(i), "en",
 332                                   name, 64,
 333                                   &status);
 334
 335         u_strToUTF8(name_str, 128, &name_str_len,
 336                     name, name_len,
 337                     &status);
 338
 339         localname_len
 340             = uloc_getDisplayName(uloc_getAvailable(i), uloc_getAvailable(i),
 341                                   localname, 64,
 342                                   &status);
 343
 344         u_strToUTF8(localname_str, 128, &localname_str_len,
 345                     localname, localname_len,
 346                     &status);
 347
 348
 349         if (p_config->xmloutput){
 350             fprintf(config.outfile, "<locale id=\"%s\"", uloc_getAvailable(i));
 351             /* fprintf(config.outfile, " locale=\"%s\"", uloc_getAvailable(i)); */
 352             /* if (strlen(keyword_str)) */
 353             /*   fprintf(config.outfile, " keyword=\"%s\"", keyword_str); */
 354             /* if (ucol_getAvailable(i)) */
 355             /*   fprintf(config.outfile, " collation=\"1\""); */
 356             if (strlen(lang_str))
 357                 fprintf(config.outfile, " language=\"%s\"", lang_str);
 358             if (strlen(script_str))
 359                 fprintf(config.outfile, " script=\"%s\"", script_str);
 360             if (strlen(location_str))
 361                 fprintf(config.outfile, " location=\"%s\"", location_str);
 362             if (strlen(variant_str))
 363                 fprintf(config.outfile, " variant=\"%s\"", variant_str);
 364             if (strlen(name_str))
 365                 fprintf(config.outfile, " name=\"%s\"", name_str);
 366             if (strlen(localname_str))
 367                 fprintf(config.outfile, " localname=\"%s\"", localname_str);
 368             fprintf(config.outfile, ">");
 369             if (strlen(localname_str))
 370                 fprintf(config.outfile, "%s", localname_str);
 371             fprintf(config.outfile, "</locale>\n");
 372         }
 373         else if (1 == p_config->xmloutput){
 374             fprintf(config.outfile, "%s", uloc_getAvailable(i));
 375             fprintf(config.outfile, " | ");
 376             if (strlen(name_str))
 377                 fprintf(config.outfile, "%s", name_str);
 378             fprintf(config.outfile, " | ");
 379             if (strlen(localname_str))
 380                 fprintf(config.outfile, "%s", localname_str);
 381             fprintf(config.outfile, "\n");
 382         }
 383         else
 384             fprintf(config.outfile, "%s ", uloc_getAvailable(i));
 385     }
 386     if (p_config->xmloutput)
 387         fprintf(config.outfile, "</locales>\n");
 388     else
 389         fprintf(config.outfile, "\n");
 390
 391     if(U_FAILURE(status)) {
 392         fprintf(stderr, "ICU Error: %d %s\n", status, u_errorName(status));
 393         exit(status);
 394     }
 395 }
 396
 397
 398 static void print_info(const struct config_t *p_config)
 399 {
 400     if (p_config->xmloutput)
 401         fprintf(config.outfile, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
 402                 "<icu>\n");
 403
 404     if ('c' == config.print[0])
 405         print_icu_converters(&config);
 406     else if ('l' == config.print[0])
 407         print_icu_xml_locales(&config);
 408     else if ('t' == config.print[0])
 409         print_icu_transliterators(&config);
 410     else {
 411         print_icu_converters(&config);
 412         print_icu_xml_locales(&config);
 413         print_icu_transliterators(&config);
 414     }
 415
 416     if (p_config->xmloutput)
 417         fprintf(config.outfile, "</icu>\n");
 418
 419     exit(0);
 420 }
 421
 422
 423
 424 static void process_text_file(const struct config_t *p_config)
 425 {
 426     char *line = 0;
 427     char linebuf[1024];
 428
 429     xmlDoc *doc = xmlParseFile(config.conffile);
 430     xmlNode *xml_node = xmlDocGetRootElement(doc);
 431
 432     long unsigned int token_count = 0;
 433     long unsigned int line_count = 0;
 434
 435     UErrorCode status = U_ZERO_ERROR;
 436     int success = 0;
 437
 438     if (! xml_node) {
 439         printf("Could not parse XML config file '%s' \n",
 440                 config.conffile);
 441         exit (1);
 442     }
 443
 444     config.chain = icu_chain_xml_config(xml_node, 0, &status);
 445
 446     if (config.chain && U_SUCCESS(status))
 447         success = 1;
 448     else {
 449         printf("Could not set up ICU chain from config file '%s' \n",
 450                 config.conffile);
 451         exit (1);
 452     }
 453
 454
 455
 456     if (p_config->xmloutput)
 457         fprintf(config.outfile,
 458                 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
 459                 "<icu>\n"
 460                 "<tokens>\n");
 461
 462     /* read input lines for processing */
 463     while ((line=fgets(linebuf, sizeof(linebuf)-1, config.infile)))
 464     {
 465         success = icu_chain_assign_cstr(config.chain, line, &status);
 466         line_count++;
 467
 468         while (success && icu_chain_next_token(config.chain, &status)){
 469             if (U_FAILURE(status))
 470                 success = 0;
 471             else {
 472                 token_count++;
 473                 if (p_config->xmloutput)
 474                     fprintf(config.outfile,
 475                             "<token id=\%lu\" line=\"%lu\""
 476                             " norm=\"%s\" display=\"%s\"/>\n",
 477                             token_count,
 478                             line_count,
 479                             icu_chain_token_norm(config.chain),
 480                             icu_chain_token_display(config.chain));
 481                 else
 482                     fprintf(config.outfile, "%lu %lu '%s' '%s'\n",
 483                             token_count,
 484                             line_count,
 485                             icu_chain_token_norm(config.chain),
 486                             icu_chain_token_display(config.chain));
 487             }
 488         }
 489
 490     }
 491
 492     if (p_config->xmloutput)
 493         fprintf(config.outfile,
 494                 "</tokens>\n"
 495                 "</icu>\n");
 496
 497     icu_chain_destroy(config.chain);
 498     xmlFreeDoc(doc);
 499     if (line)
 500         free(line);
 501 }
 502
 503 #endif /* YAZ_HAVE_ICU */
 504
 505
 506 int main(int argc, char **argv)
 507 {
 508
 509 #if YAZ_HAVE_ICU
 510
 511     read_params(argc, argv, &config);
 512
 513     if (config.conffile && strlen(config.conffile))
 514         process_text_file(&config);
 515
 516     if (config.print && strlen(config.print))
 517         print_info(&config);
 518
 519 #else /* YAZ_HAVE_ICU */
 520
 521     printf("ICU not available on your system.\n"
 522            "Please install libicu36-dev and icu-doc or similar, "
 523            "re-configure and re-compile\n");
 524
 525
 526 #endif /* YAZ_HAVE_ICU */
 527
 528     return(0);
 529 }
 530
 531
 532 /*
 533  * Local variables:
 534  * c-basic-offset: 4
 535  * indent-tabs-mode: nil
 536  * End:
 537  * vim: shiftwidth=4 tabstop=8 expandtab
 538  */
 539