Print ICU error when ICU can not be configured
[yaz-moved-to-github.git] / util / yaz-icu.c
1 /* This file is part of the YAZ toolkit.
2  * Copyright (C) 1995-2009 Index Data
3  * See the file LICENSE for details.
4  */
5
6 #if HAVE_CONFIG_H
7 #include "config.h"
8 #endif
9
10 #include <string.h>
11
12 #include <stdio.h>
13 #include <stdlib.h>
14
15 #include <yaz/options.h>
16
17 #if YAZ_HAVE_ICU
18
19 #include <unicode/ucnv.h>
20 #include <unicode/ustring.h>
21 #include <unicode/ucol.h> 
22 #include <unicode/ubrk.h>
23 #include <unicode/utrans.h>
24
25 #include <yaz/icu.h>
26 #include <yaz/wrbuf.h>
27
28 /* commando line and config parameters */
29 static struct config_t { 
30     char conffile[1024];
31     char print[1024];
32     int xmloutput;
33     int sortoutput;
34     yaz_icu_chain_t chain;
35     FILE * infile;
36     FILE * outfile;
37 } config;
38
39
40   
41 void print_option_error(const struct config_t *p_config)
42 {  
43     fprintf(stderr, "Calling error, valid options are :\n");
44     fprintf(stderr, "yaz-icu\n"
45             "   [-c (path/to/config/file.xml)]\n"
46             "   [-p (a|c|l|t)] print ICU info \n"
47             "   [-s] Show sort normalization key\n"
48             "   [-x] XML output\n"
49             "\n"
50             "Examples:\n"
51             "cat hugetextfile.txt | ./yaz-icu -c config.xml \n"
52             "./yaz-icu -p c\n"
53             "./yaz-icu -p l -x\n"
54             "./yaz-icu -p t -x\n"
55             "\n"
56             "Example ICU chain XML configuration file:\n"
57             "<icu_chain locale=\"en\">\n"
58             "  <transform rule=\"[:Control:] Any-Remove\"/>\n"
59             "  <tokenize rule=\"l\"/>\n"
60             "  <transform rule=\"[[:WhiteSpace:][:Punctuation:]] Remove\"/>\n"
61             "  <casemap rule=\"l\"/>\n"
62             "</icu_chain>\n"
63           );
64     exit(1);
65 }
66
67 void read_params(int argc, char **argv, struct config_t *p_config)
68 {    
69     char *arg;
70     int ret;
71     
72     /* set default parameters */
73     p_config->conffile[0] = 0;
74     p_config->print[0] = 0;
75     p_config->xmloutput = 0;
76     p_config->sortoutput = 0;
77     p_config->chain = 0;
78     p_config->infile = stdin;
79     p_config->outfile = stdout;
80     
81     /* set up command line parameters */
82     
83     while ((ret = options("c:p:xs", argv, argc, &arg)) != -2)
84     {
85         switch (ret)
86         {
87         case 'c':
88             strcpy(p_config->conffile, arg);
89             break;
90         case 'p':
91             strcpy(p_config->print, arg);
92             break;
93         case 's':
94             p_config->sortoutput = 1;
95             break;
96         case 'x':
97             p_config->xmloutput = 1;
98             break;
99         default:
100             printf("Got %d\n", ret);
101             print_option_error(p_config);
102         }
103     }
104     
105     if ((!strlen(p_config->conffile)
106          && !strlen(p_config->print))
107         || !config.infile
108         || !config.outfile)
109         
110         print_option_error(p_config);
111 }
112
113
114 /*     UConverter *conv; */
115 /*     conv = ucnv_open("utf-8", &status); */
116 /*     assert(U_SUCCESS(status)); */
117
118 /*     *ustr16_len  */
119 /*       = ucnv_toUChars(conv, ustr16, 1024,  */
120 /*                       (const char *) *xstr8, strlen((const char *) *xstr8), */
121 /*                       &status); */
122   
123
124
125 /*      ucnv_fromUChars(conv, */
126 /*                      (char *) *xstr8, strlen((const char *) *xstr8), */
127 /*                      ustr16, *ustr16_len, */
128 /*                      &status); */
129 /*      ucnv_close(conv); */
130
131
132 static void print_icu_converters(const struct config_t *p_config)
133 {
134     int32_t count;
135     int32_t i;
136
137     count = ucnv_countAvailable();
138     if (p_config->xmloutput)
139         fprintf(config.outfile, "<converters count=\"%d\" default=\"%s\">\n",
140                 count, ucnv_getDefaultName());
141     else {    
142         fprintf(config.outfile, "Available ICU converters: %d\n", count);
143         fprintf(config.outfile, "Default ICU Converter is: '%s'\n", 
144                 ucnv_getDefaultName());
145     }
146     
147     for(i=0;i<count;i++)
148     {
149         if (p_config->xmloutput)
150             fprintf(config.outfile, "<converter id=\"%s\"/>\n", 
151                     ucnv_getAvailableName(i));
152         else     
153             fprintf(config.outfile, "%s ", ucnv_getAvailableName(i));
154     }
155     
156     if (p_config->xmloutput)
157         fprintf(config.outfile, "</converters>\n");
158     else
159         fprintf(config.outfile, "\n");
160 }
161
162 static void print_icu_transliterators(const struct config_t *p_config)
163 {
164     int32_t buf_cap = 128;
165     char buf[128];
166     int32_t i;
167     int32_t count = utrans_countAvailableIDs();
168     
169     if (p_config->xmloutput)
170         fprintf(config.outfile, "<transliterators count=\"%d\">\n",  count);
171     else 
172         fprintf(config.outfile, "Available ICU transliterators: %d\n", count);
173     
174     for(i = 0; i <count; i++)
175     {
176         utrans_getAvailableID(i, buf, buf_cap);
177         if (p_config->xmloutput)
178             fprintf(config.outfile, "<transliterator id=\"%s\"/>\n", buf);
179         else
180             fprintf(config.outfile, " %s", buf);
181     }
182     
183     if (p_config->xmloutput)
184     {
185         fprintf(config.outfile, "</transliterators>\n");
186     }
187     else
188     {
189         fprintf(config.outfile, "\n\nUnicode Set Patterns:\n"
190                 "   Pattern         Description\n"
191                 "   Ranges          [a-z]       The lower case letters a through z\n"
192                 "   Named Chars     [abc123] The six characters a,b,c,1,2 and 3\n"
193                 "   String          [abc{def}] chars a, b and c, and string 'def'\n"
194                 "   Categories      [\\p{Letter}] Perl General Category 'Letter'.\n"
195                 "   Categories      [:Letter:] Posix General Category 'Letter'.\n"
196                 "\n"
197                 "   Combination     Example\n"
198                 "   Union           [[:Greek:] [:letter:]]\n"
199                 "   Intersection    [[:Greek:] & [:letter:]]\n"
200                 "   Set Complement  [[:Greek:] - [:letter:]]\n"
201                 "   Complement      [^[:Greek:] [:letter:]]\n"
202                 "\n"
203              "see: http://icu.sourceforge.net/userguide/unicodeSet.html\n"
204                 "\n"
205                 "Examples:\n"
206                 "   [:Punctuation:] Any-Remove\n"
207                 "   [:Cased-Letter:] Any-Upper\n"
208                 "   [:Control:] Any-Remove\n"
209                 "   [:Decimal_Number:] Any-Remove\n"
210                 "   [:Final_Punctuation:] Any-Remove\n"
211                 "   [:Georgian:] Any-Upper\n"
212                 "   [:Katakana:] Any-Remove\n"
213                 "   [:Arabic:] Any-Remove\n"
214                 "   [:Punctuation:] Remove\n"
215                 "   [[:Punctuation:]-[.,]] Remove\n"
216                 "   [:Line_Separator:] Any-Remove\n"
217                 "   [:Math_Symbol:] Any-Remove\n"
218                 "   Lower; [:^Letter:] Remove (word tokenization)\n"
219                 "   [:^Number:] Remove (numeric tokenization)\n"
220                 "   [:^Katagana:] Remove (remove everything except Katagana)\n"
221                 "   Lower;[[:WhiteSpace:][:Punctuation:]] Remove (word tokenization)\n"
222                 "   NFD; [:Nonspacing Mark:] Remove; NFC   (removes accents from characters)\n"
223                 "   [A-Za-z]; Lower(); Latin-Katakana; Katakana-Hiragana (transforms latin and katagana to hiragana)\n"
224                 "   [[:separator:][:start punctuation:][:initial punctuation:]] Remove \n"
225                 "\n"
226                 "see http://icu.sourceforge.net/userguide/Transform.html\n"
227                 "    http://www.unicode.org/Public/UNIDATA/UCD.html\n"
228                 "    http://icu.sourceforge.net/userguide/Transform.html\n"
229                 "    http://icu.sourceforge.net/userguide/TransformRule.html\n"
230             );
231         
232         
233         fprintf(config.outfile, "\n\n");
234         
235     }
236 }
237
238 static void print_icu_xml_locales(const struct config_t *p_config)
239 {
240     int32_t count;
241     int32_t i;
242     UErrorCode status = U_ZERO_ERROR;
243     
244     UChar keyword[64];
245     int32_t keyword_len = 0;
246     char keyword_str[128];
247     int32_t keyword_str_len = 0;
248
249     UChar language[64];
250     int32_t language_len = 0;
251     char lang_str[128];
252     int32_t lang_str_len = 0;
253
254     UChar script[64];
255     int32_t script_len = 0;
256     char script_str[128];
257     int32_t script_str_len = 0;
258
259     UChar location[64];
260     int32_t location_len = 0;
261     char location_str[128];
262     int32_t location_str_len = 0;
263
264     UChar variant[64];
265     int32_t variant_len = 0;
266     char variant_str[128];
267     int32_t variant_str_len = 0;
268
269     UChar name[64];
270     int32_t name_len = 0;
271     char name_str[128];
272     int32_t name_str_len = 0;
273
274     UChar localname[64];
275     int32_t localname_len = 0;
276     char localname_str[128];
277     int32_t localname_str_len = 0;
278
279     count = uloc_countAvailable() ;
280
281     if (p_config->xmloutput)
282     {
283         fprintf(config.outfile, "<locales count=\"%d\" default=\"%s\" collations=\"%d\">\n", 
284                 count, uloc_getDefault(), ucol_countAvailable());
285     }
286   
287     for(i=0;i<count;i++) 
288     {
289
290         keyword_len 
291             = uloc_getDisplayKeyword(uloc_getAvailable(i), "en", 
292                                      keyword, 64, 
293                                      &status);
294
295         u_strToUTF8(keyword_str, 128, &keyword_str_len,
296                     keyword, keyword_len,
297                     &status);
298     
299     
300         language_len 
301             = uloc_getDisplayLanguage(uloc_getAvailable(i), "en", 
302                                       language, 64, 
303                                       &status);
304
305         u_strToUTF8(lang_str, 128, &lang_str_len,
306                     language, language_len,
307                     &status);
308
309
310         script_len 
311             = uloc_getDisplayScript(uloc_getAvailable(i), "en", 
312                                     script, 64, 
313                                     &status);
314
315         u_strToUTF8(script_str, 128, &script_str_len,
316                     script, script_len,
317                     &status);
318
319         location_len 
320             = uloc_getDisplayCountry(uloc_getAvailable(i), "en", 
321                                      location, 64, 
322                                      &status);
323
324         u_strToUTF8(location_str, 128, &location_str_len,
325                     location, location_len,
326                     &status);
327
328         variant_len 
329             = uloc_getDisplayVariant(uloc_getAvailable(i), "en", 
330                                      variant, 64, 
331                                      &status);
332
333         u_strToUTF8(variant_str, 128, &variant_str_len,
334                     variant, variant_len,
335                     &status);
336
337         name_len 
338             = uloc_getDisplayName(uloc_getAvailable(i), "en", 
339                                   name, 64, 
340                                   &status);
341
342         u_strToUTF8(name_str, 128, &name_str_len,
343                     name, name_len,
344                     &status);
345
346         localname_len 
347             = uloc_getDisplayName(uloc_getAvailable(i), uloc_getAvailable(i), 
348                                   localname, 64, 
349                                   &status);
350
351         u_strToUTF8(localname_str, 128, &localname_str_len,
352                     localname, localname_len,
353                     &status);
354
355
356         if (p_config->xmloutput)
357         {
358             fprintf(config.outfile, "<locale id=\"%s\"", uloc_getAvailable(i)); 
359             /* fprintf(config.outfile, " locale=\"%s\"", uloc_getAvailable(i)); */
360             /* if (strlen(keyword_str)) */
361             /*   fprintf(config.outfile, " keyword=\"%s\"", keyword_str); */
362             /* if (ucol_getAvailable(i)) */
363             /*   fprintf(config.outfile, " collation=\"1\""); */
364             if (strlen(lang_str))
365                 fprintf(config.outfile, " language=\"%s\"", lang_str);
366             if (strlen(script_str))
367                 fprintf(config.outfile, " script=\"%s\"", script_str);
368             if (strlen(location_str))
369                 fprintf(config.outfile, " location=\"%s\"", location_str);
370             if (strlen(variant_str))
371                 fprintf(config.outfile, " variant=\"%s\"", variant_str);
372             if (strlen(name_str))
373                 fprintf(config.outfile, " name=\"%s\"", name_str);
374             if (strlen(localname_str))
375                 fprintf(config.outfile, " localname=\"%s\"", localname_str);
376             fprintf(config.outfile, ">");
377             if (strlen(localname_str))
378                 fprintf(config.outfile, "%s", localname_str);
379             fprintf(config.outfile, "</locale>\n"); 
380         }
381         else if (1 == p_config->xmloutput)
382         {
383             fprintf(config.outfile, "%s", uloc_getAvailable(i)); 
384             fprintf(config.outfile, " | ");
385             if (strlen(name_str))
386                 fprintf(config.outfile, "%s", name_str);
387             fprintf(config.outfile, " | ");
388             if (strlen(localname_str))
389                 fprintf(config.outfile, "%s", localname_str);
390             fprintf(config.outfile, "\n");
391         }
392         else
393             fprintf(config.outfile, "%s ", uloc_getAvailable(i));
394     }
395     if (p_config->xmloutput)
396         fprintf(config.outfile, "</locales>\n");
397     else
398         fprintf(config.outfile, "\n");
399
400     if(U_FAILURE(status))
401     {
402         fprintf(stderr, "ICU Error: %d %s\n", status, u_errorName(status));
403         exit(2);
404     }
405 }
406
407
408 static void print_info(const struct config_t *p_config)
409 {
410     if (p_config->xmloutput)
411         fprintf(config.outfile, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
412                 "<icu>\n");
413
414     if ('c' == config.print[0])
415         print_icu_converters(&config);
416     else if ('l' == config.print[0])
417         print_icu_xml_locales(&config);
418     else if ('t' == config.print[0])
419         print_icu_transliterators(&config);
420     else {
421         print_icu_converters(&config);
422         print_icu_xml_locales(&config);
423         print_icu_transliterators(&config);
424     }
425
426     if (p_config->xmloutput)
427         fprintf(config.outfile, "</icu>\n");
428
429     exit(0);
430 }
431
432
433
434 static void process_text_file(const struct config_t *p_config)
435 {
436     char *line = 0;
437     char linebuf[1024];
438  
439     xmlDoc *doc = xmlParseFile(config.conffile);  
440     xmlNode *xml_node = xmlDocGetRootElement(doc);
441
442     long unsigned int token_count = 0;    
443     long unsigned int line_count = 0;    
444     
445     UErrorCode status = U_ZERO_ERROR;
446     
447     if (!xml_node)
448     {   
449         printf("Could not parse XML config file '%s' \n",
450                 config.conffile);
451         exit(1);
452     }
453
454     config.chain = icu_chain_xml_config(xml_node, 1, &status);
455
456     if (!config.chain || !U_SUCCESS(status))
457     {   
458         printf("Could not set up ICU chain from config file '%s' \n",
459                 config.conffile);
460         if (!U_SUCCESS(status))
461             printf("ICU Error: %d %s\n", status, u_errorName(status));
462         exit(1);
463     }
464
465     if (p_config->xmloutput)
466         fprintf(config.outfile,
467                 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
468                 "<icu>\n"
469                 "<tokens>\n");
470     
471     /* read input lines for processing */
472     while ((line=fgets(linebuf, sizeof(linebuf)-1, config.infile)))
473     {
474         int success = icu_chain_assign_cstr(config.chain, line, &status);
475         line_count++;
476
477         while (success && icu_chain_next_token(config.chain, &status))
478         {
479             WRBUF sw = wrbuf_alloc();
480             if (U_FAILURE(status))
481                 success = 0;
482             else {
483                 const char *sortkey = icu_chain_token_sortkey(config.chain);
484                 wrbuf_rewind(sw);
485                 wrbuf_puts_escaped(sw, sortkey);
486                 token_count++;
487                 if (p_config->xmloutput)                    
488                 {
489                     /* should XML encode this. Bug #1902 */
490                     fprintf(config.outfile, 
491                             "<token id=\"%lu\" line=\"%lu\""
492                             " norm=\"%s\" display=\"%s\"",
493                             token_count,
494                             line_count,
495                             icu_chain_token_norm(config.chain),
496                             icu_chain_token_display(config.chain));
497                     if (p_config->sortoutput)
498                     {
499                         fprintf(config.outfile, " sortkey=\"%s\"",
500                                 wrbuf_cstr(sw));
501                     }
502                     fprintf(config.outfile, "/>\n");
503                 }
504                 else
505                 {
506                     fprintf(config.outfile, "%lu %lu '%s' '%s'",
507                             token_count,
508                             line_count,
509                             icu_chain_token_norm(config.chain),
510                             icu_chain_token_display(config.chain));
511                     if (p_config->sortoutput)
512                     {
513                         fprintf(config.outfile, " '%s'", wrbuf_cstr(sw));
514                     }
515                     fprintf(config.outfile, "\n");
516                 }
517             }
518             wrbuf_destroy(sw);
519         }
520         
521     }
522
523     if (p_config->xmloutput)
524         fprintf(config.outfile, 
525                 "</tokens>\n"
526                 "</icu>\n");
527
528     icu_chain_destroy(config.chain);
529     xmlFreeDoc(doc);
530     if (line)
531         free(line);
532 }
533
534 #endif /* YAZ_HAVE_ICU */
535
536
537 int main(int argc, char **argv) 
538 {
539
540 #if YAZ_HAVE_ICU
541
542     read_params(argc, argv, &config);
543
544     if (config.conffile && strlen(config.conffile))
545         process_text_file(&config);
546      
547     if (config.print && strlen(config.print))
548         print_info(&config);
549
550 #else /* YAZ_HAVE_ICU */
551
552     printf("ICU not available on your system.\n"
553            "Please install libicu36-dev and icu-doc or similar, "
554            "re-configure and re-compile\n");
555
556
557     exit(3);
558 #endif /* YAZ_HAVE_ICU */
559
560     return 0;
561 }
562
563
564 /*
565  * Local variables:
566  * c-basic-offset: 4
567  * c-file-style: "Stroustrup"
568  * indent-tabs-mode: nil
569  * End:
570  * vim: shiftwidth=4 tabstop=8 expandtab
571  */
572