src/icu_chain_test.c

   1 /**
   2  gcc -I/usr/include/libxml2 -lxml2 -o icu-xml-convert icu-xml-convert.c
   3  */
   4
   5 #if HAVE_CONFIG_H
   6 #include "cconfig.h"
   7 #endif
   8
   9 #define _GNU_SOURCE
  10 #include <string.h>
  11
  12 #include <stdio.h>
  13 #include <stdlib.h>
  14
  15 //#include <yaz/xmalloc.h>
  16 #include <yaz/options.h>
  17
  18
  19 #ifdef HAVE_ICU
  20
  21 #include <unicode/ucnv.h>
  22 #include <unicode/ustring.h>
  23
  24 #include "icu_I18N.h"
  25
  26 /* commando line and config parameters */
  27 static struct config_t {
  28     char conffile[1024];
  29     char print[1024];
  30     int xmloutput;
  31     struct icu_chain * chain;
  32     FILE * infile;
  33     FILE * outfile;
  34 } config;
  35
  36
  37
  38 void print_option_error(const struct config_t *p_config)
  39 {
  40   fprintf(stderr, "Calling error, valid options are :\n");
  41   fprintf(stderr, "icu_chain_test\n"
  42           "   [-c (path/to/config/file.xml)]\n"
  43           "   [-p (a|c|l|t)] print ICU info \n"
  44           "   [-x] XML output\n"
  45           "\n"
  46           "Examples:\n"
  47           "cat hugetextfile.txt | ./icu_chain_test -c config.xml \n"
  48           "./icu_chain_test -p c\n"
  49           "./icu_chain_test -p l -x\n"
  50           "./icu_chain_test -p t -x\n"
  51           );
  52   exit(1);
  53 }
  54
  55 void read_params(int argc, char **argv, struct config_t *p_config){
  56   char *arg;
  57   int ret;
  58
  59   /* set default parameters */
  60   p_config->conffile[0] = 0;
  61   p_config->print[0] = 0;
  62   p_config->xmloutput = 0;
  63   p_config->chain = 0;
  64   p_config->infile = stdin;
  65   p_config->outfile = stdout;
  66
  67   /* set up command line parameters */
  68
  69   while ((ret = options("c:p:x", argv, argc, &arg)) != -2)
  70     {
  71       switch (ret)
  72         {
  73         case 'c':
  74           strcpy(p_config->conffile, arg);
  75           break;
  76         case 'p':
  77           strcpy(p_config->print, arg);
  78           break;
  79         case 'x':
  80             p_config->xmloutput = 1;
  81           break;
  82         default:
  83           print_option_error(p_config);
  84         }
  85     }
  86
  87     //p_config->infile = fopen("/etc/passwd", "r");
  88
  89
  90
  91   if ((!strlen(p_config->conffile)
  92       && !strlen(p_config->print))
  93       || !config.infile
  94       || !config.outfile)
  95
  96     print_option_error(p_config);
  97 };
  98
  99
 100 /*     UConverter *conv; */
 101 /*     conv = ucnv_open("utf-8", &status); */
 102 /*     assert(U_SUCCESS(status)); */
 103
 104 /*     *ustr16_len  */
 105 /*       = ucnv_toUChars(conv, ustr16, 1024,  */
 106 /*                       (const char *) *xstr8, strlen((const char *) *xstr8), */
 107 /*                       &status); */
 108
 109
 110
 111 /*      ucnv_fromUChars(conv, */
 112 /*                      (char *) *xstr8, strlen((const char *) *xstr8), */
 113 /*                      ustr16, *ustr16_len, */
 114 /*                      &status); */
 115 /*      ucnv_close(conv); */
 116
 117
 118 static void print_icu_converters(const struct config_t *p_config)
 119 {
 120     int32_t count;
 121     int32_t i;
 122
 123     count = ucnv_countAvailable();
 124     if (p_config->xmloutput)
 125         fprintf(config.outfile, "<converters count=\"%d\" default=\"%s\">\n",
 126                count, ucnv_getDefaultName());
 127     else {
 128         fprintf(config.outfile, "Available ICU converters: %d\n", count);
 129         fprintf(config.outfile, "Default ICU Converter is: '%s'\n", ucnv_getDefaultName());
 130     }
 131
 132     for(i=0;i<count;i++){
 133         if (p_config->xmloutput)
 134             fprintf(config.outfile, "<converter id=\"%s\"/>\n", ucnv_getAvailableName(i));
 135         else
 136             fprintf(config.outfile, "%s ", ucnv_getAvailableName(i));
 137     }
 138
 139     if (p_config->xmloutput)
 140         fprintf(config.outfile, "</converters>\n");
 141     else
 142         fprintf(config.outfile, "\n");
 143 }
 144
 145 static void print_icu_transliterators(const struct config_t *p_config)
 146 {
 147   int32_t count;
 148   int32_t i;
 149
 150   count = utrans_countAvailableIDs();
 151
 152   int32_t buf_cap = 128;
 153   char buf[buf_cap];
 154
 155   if (p_config->xmloutput)
 156     fprintf(config.outfile, "<transliterators count=\"%d\">\n",  count);
 157    else
 158     fprintf(config.outfile, "Available ICU transliterators: %d\n", count);
 159
 160   for(i=0;i<count;i++)
 161     {
 162       utrans_getAvailableID(i, buf, buf_cap);
 163        if (p_config->xmloutput)
 164          fprintf(config.outfile, "<transliterator id=\"%s\"/>\n", buf);
 165        else
 166          fprintf(config.outfile, " %s", buf);
 167     }
 168
 169   if (p_config->xmloutput){
 170     fprintf(config.outfile, "</transliterators>\n");
 171   }
 172   else
 173     {
 174       fprintf(config.outfile, "\n\nUnicode Set Patterns:\n"
 175              "   Pattern         Description\n"
 176              "   Ranges          [a-z]  The lower case letters a through z\n"
 177              "   Named Chars     [abc123] The six characters a,b,c,1,2 and 3\n"
 178              "   String          [abc{def}] chars a, b and c, and string 'def'\n"
 179              "   Categories      [\\p{Letter}] Perl General Category 'Letter'.\n"
 180              "   Categories      [:Letter:] Posix General Category 'Letter'.\n"
 181              "\n"
 182              "   Combination     Example\n"
 183              "   Union           [[:Greek:] [:letter:]]\n"
 184              "   Intersection    [[:Greek:] & [:letter:]]\n"
 185              "   Set Complement  [[:Greek:] - [:letter:]]\n"
 186              "   Complement      [^[:Greek:] [:letter:]]\n"
 187              "\n"
 188              "see: http://icu.sourceforge.net/userguide/unicodeSet.html\n"
 189              "\n"
 190              "Examples:\n"
 191              "   [:Punctuation:] Any-Remove\n"
 192              "   [:Cased-Letter:] Any-Upper\n"
 193              "   [:Control:] Any-Remove\n"
 194              "   [:Decimal_Number:] Any-Remove\n"
 195              "   [:Final_Punctuation:] Any-Remove\n"
 196              "   [:Georgian:] Any-Upper\n"
 197              "   [:Katakana:] Any-Remove\n"
 198              "   [:Arabic:] Any-Remove\n"
 199              "   [:Punctuation:] Remove\n"
 200              "   [[:Punctuation:]-[.,]] Remove\n"
 201              "   [:Line_Separator:] Any-Remove\n"
 202              "   [:Math_Symbol:] Any-Remove\n"
 203              "   Lower; [:^Letter:] Remove (word tokenization)\n"
 204              "   [:^Number:] Remove (numeric tokenization)\n"
 205              "   [:^Katagana:] Remove (remove everything except Katagana)\n"
 206              "   Lower;[[:WhiteSpace:][:Punctuation:]] Remove (word tokenization)\n"
 207              "   NFD; [:Nonspacing Mark:] Remove; NFC   (removes accents from characters)\n"
 208              "   [A-Za-z]; Lower(); Latin-Katakana; Katakana-Hiragana (transforms latin and katagana to hiragana)\n"
 209              "   [[:separator:][:start punctuation:][:initial punctuation:]] Remove \n"
 210              "\n"
 211              "see http://icu.sourceforge.net/userguide/Transform.html\n"
 212              "    http://www.unicode.org/Public/UNIDATA/UCD.html\n"
 213              "    http://icu.sourceforge.net/userguide/Transform.html\n"
 214              "    http://icu.sourceforge.net/userguide/TransformRule.html\n"
 215              );
 216
 217
 218   fprintf(config.outfile, "\n\n");
 219
 220     }
 221 }
 222
 223 static void print_icu_xml_locales(const struct config_t *p_config)
 224 {
 225   int32_t count;
 226   int32_t i;
 227   UErrorCode status = U_ZERO_ERROR;
 228
 229   UChar keyword[64];
 230   int32_t keyword_len = 0;
 231   char keyword_str[128];
 232   int32_t keyword_str_len = 0;
 233
 234   UChar language[64];
 235   int32_t language_len = 0;
 236   char lang_str[128];
 237   int32_t lang_str_len = 0;
 238
 239   UChar script[64];
 240   int32_t script_len = 0;
 241   char script_str[128];
 242   int32_t script_str_len = 0;
 243
 244   UChar location[64];
 245   int32_t location_len = 0;
 246   char location_str[128];
 247   int32_t location_str_len = 0;
 248
 249   UChar variant[64];
 250   int32_t variant_len = 0;
 251   char variant_str[128];
 252   int32_t variant_str_len = 0;
 253
 254   UChar name[64];
 255   int32_t name_len = 0;
 256   char name_str[128];
 257   int32_t name_str_len = 0;
 258
 259   UChar localname[64];
 260   int32_t localname_len = 0;
 261   char localname_str[128];
 262   int32_t localname_str_len = 0;
 263
 264   count = uloc_countAvailable() ;
 265
 266   if (p_config->xmloutput){
 267
 268       fprintf(config.outfile, "<locales count=\"%d\" default=\"%s\" collations=\"%d\">\n",
 269              count, uloc_getDefault(), ucol_countAvailable());
 270   }
 271
 272   for(i=0;i<count;i++)
 273   {
 274
 275     keyword_len
 276       = uloc_getDisplayKeyword(uloc_getAvailable(i), "en",
 277                                 keyword, 64,
 278                                 &status);
 279
 280     u_strToUTF8(keyword_str, 128, &keyword_str_len,
 281                 keyword, keyword_len,
 282                 &status);
 283
 284
 285     language_len
 286       = uloc_getDisplayLanguage(uloc_getAvailable(i), "en",
 287                                 language, 64,
 288                                 &status);
 289
 290     u_strToUTF8(lang_str, 128, &lang_str_len,
 291                 language, language_len,
 292                 &status);
 293
 294
 295     script_len
 296       = uloc_getDisplayScript(uloc_getAvailable(i), "en",
 297                                 script, 64,
 298                                 &status);
 299
 300     u_strToUTF8(script_str, 128, &script_str_len,
 301                 script, script_len,
 302                 &status);
 303
 304     location_len
 305       = uloc_getDisplayCountry(uloc_getAvailable(i), "en",
 306                                 location, 64,
 307                                 &status);
 308
 309     u_strToUTF8(location_str, 128, &location_str_len,
 310                 location, location_len,
 311                 &status);
 312
 313     variant_len
 314       = uloc_getDisplayVariant(uloc_getAvailable(i), "en",
 315                                 variant, 64,
 316                                 &status);
 317
 318     u_strToUTF8(variant_str, 128, &variant_str_len,
 319                 variant, variant_len,
 320                 &status);
 321
 322     name_len
 323       = uloc_getDisplayName(uloc_getAvailable(i), "en",
 324                                 name, 64,
 325                                 &status);
 326
 327     u_strToUTF8(name_str, 128, &name_str_len,
 328                 name, name_len,
 329                 &status);
 330
 331     localname_len
 332       = uloc_getDisplayName(uloc_getAvailable(i), uloc_getAvailable(i),
 333                                 localname, 64,
 334                                 &status);
 335
 336     u_strToUTF8(localname_str, 128, &localname_str_len,
 337                 localname, localname_len,
 338                 &status);
 339
 340
 341     if (p_config->xmloutput){
 342       fprintf(config.outfile, "<locale id=\"%s\"", uloc_getAvailable(i));
 343       /* fprintf(config.outfile, " locale=\"%s\"", uloc_getAvailable(i)); */
 344       /* if (strlen(keyword_str)) */
 345       /*   fprintf(config.outfile, " keyword=\"%s\"", keyword_str); */
 346       /* if (ucol_getAvailable(i)) */
 347       /*   fprintf(config.outfile, " collation=\"1\""); */
 348       if (strlen(lang_str))
 349         fprintf(config.outfile, " language=\"%s\"", lang_str);
 350       if (strlen(script_str))
 351         fprintf(config.outfile, " script=\"%s\"", script_str);
 352       if (strlen(location_str))
 353         fprintf(config.outfile, " location=\"%s\"", location_str);
 354       if (strlen(variant_str))
 355         fprintf(config.outfile, " variant=\"%s\"", variant_str);
 356       if (strlen(name_str))
 357         fprintf(config.outfile, " name=\"%s\"", name_str);
 358       if (strlen(localname_str))
 359         fprintf(config.outfile, " localname=\"%s\"", localname_str);
 360       fprintf(config.outfile, ">");
 361       if (strlen(localname_str))
 362         fprintf(config.outfile, "%s", localname_str);
 363       fprintf(config.outfile, "</locale>\n");
 364     }
 365     else if (1 == p_config->xmloutput){
 366       fprintf(config.outfile, "%s", uloc_getAvailable(i));
 367       fprintf(config.outfile, " | ");
 368       if (strlen(name_str))
 369         fprintf(config.outfile, "%s", name_str);
 370       fprintf(config.outfile, " | ");
 371       if (strlen(localname_str))
 372         fprintf(config.outfile, "%s", localname_str);
 373       fprintf(config.outfile, "\n");
 374     }
 375     else
 376       fprintf(config.outfile, "%s ", uloc_getAvailable(i));
 377   }
 378   if (p_config->xmloutput)
 379     fprintf(config.outfile, "</locales>\n");
 380   else
 381     fprintf(config.outfile, "\n");
 382
 383   if(U_FAILURE(status)) {
 384     fprintf(stderr, "ICU Error: %d %s\n", status, u_errorName(status));
 385     exit(status);
 386   }
 387 }
 388
 389
 390 static void print_info(const struct config_t *p_config)
 391 {
 392   if (p_config->xmloutput)
 393     fprintf(config.outfile, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
 394            "<icu>\n");
 395
 396     if ('c' == config.print[0])
 397         print_icu_converters(&config);
 398     else if ('l' == config.print[0])
 399         print_icu_xml_locales(&config);
 400     else if ('t' == config.print[0])
 401         print_icu_transliterators(&config);
 402     else {
 403         print_icu_converters(&config);
 404         print_icu_xml_locales(&config);
 405         print_icu_transliterators(&config);
 406     }
 407
 408     if (p_config->xmloutput)
 409         fprintf(config.outfile, "</icu>\n");
 410
 411   exit(0);
 412 };
 413
 414
 415
 416 static void process_text_file(const struct config_t *p_config)
 417 {
 418     char * line = 0;
 419     size_t line_cap = 0;
 420     ssize_t line_len;
 421
 422     xmlDoc *doc = xmlParseFile(config.conffile);
 423     xmlNode *xml_node = xmlDocGetRootElement(doc);
 424
 425     long unsigned int token_count = 0;
 426     long unsigned int line_count = 0;
 427
 428     UErrorCode status = U_ZERO_ERROR;
 429     int success = 0;
 430
 431
 432     config.chain = icu_chain_xml_config(xml_node, &status);
 433
 434     if (config.chain && U_SUCCESS(status))
 435         success = 1;
 436
 437     if (p_config->xmloutput)
 438         fprintf(config.outfile,
 439                 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
 440                 "<icu>\n"
 441                 "<tokens>\n");
 442
 443     // read input lines for processing
 444     while ((line_len = getline(&line, &line_cap, config.infile)) != -1) {
 445         success = icu_chain_assign_cstr(config.chain, line, &status);
 446         line_count++;
 447
 448         while (success && icu_chain_next_token(config.chain, &status)){
 449             if (U_FAILURE(status))
 450                 success = 0;
 451             else {
 452                 token_count++;
 453                 if (p_config->xmloutput)
 454                     fprintf(config.outfile,
 455                             "<token id=\%lu\" line=\"%lu\""
 456                             " norm=\"%s\" display=\"%s\"/>\n",
 457                             token_count,
 458                             line_count,
 459                             icu_chain_get_norm(config.chain),
 460                             icu_chain_get_display(config.chain));
 461                 else
 462                     fprintf(config.outfile, "%lu %lu '%s' '%s'\n",
 463                             token_count,
 464                             line_count,
 465                             icu_chain_get_norm(config.chain),
 466                             icu_chain_get_display(config.chain));
 467             }
 468         }
 469
 470     }
 471
 472    if (p_config->xmloutput)
 473         fprintf(config.outfile,
 474                 "</tokens>\n"
 475                 "</icu>\n");
 476
 477     icu_chain_destroy(config.chain);
 478     xmlFreeDoc(doc);
 479     if (line)
 480         free(line);
 481 };
 482
 483 #endif // HAVE_ICU
 484
 485
 486 int main(int argc, char **argv)
 487 {
 488
 489 #ifdef HAVE_ICU
 490
 491     read_params(argc, argv, &config);
 492
 493     if (config.conffile && strlen(config.conffile))
 494         process_text_file(&config);
 495
 496     if (config.print && strlen(config.print))
 497         print_info(&config);
 498
 499 #else // HAVE_ICU
 500
 501     printf("ICU not available on your system.\n"
 502            "Please install libicu36-dev and icu-doc or similar, "
 503            "re-configure and re-compile\n");
 504
 505
 506 #endif // HAVE_ICU
 507
 508     return(0);
 509 };
 510
 511
 512 /*
 513  * Local variables:
 514  * c-basic-offset: 4
 515  * indent-tabs-mode: nil
 516  * End:
 517  * vim: shiftwidth=4 tabstop=8 expandtab
 518  */
 519