X-Git-Url: http://git.indexdata.com/?p=yaz-moved-to-github.git;a=blobdiff_plain;f=util%2Fyaz-icu.c;h=f14b494a003aa38e9a62a49e7d14c1f534aa7bd8;hp=4247f80ae9e923ec04bce1f0cdeacc81b2b980ad;hb=e8d49a7dea8eec8a8d588fca7464f29f4db5e838;hpb=0ffb25b0bf78f53399267664b71206c88980c4da

diff --git a/util/yaz-icu.c b/util/yaz-icu.c
index 4247f80..f14b494 100644
--- a/util/yaz-icu.c
+++ b/util/yaz-icu.c
@@ -1,26 +1,10 @@
-/* $Id: yaz-icu.c,v 1.1 2007-10-22 12:21:40 adam Exp $
-   Copyright (c) 2006-2007, Index Data.
-
-This file is part of Pazpar2.
-
-Pazpar2 is free software; you can redistribute it and/or modify it under
-the terms of the GNU General Public License as published by the Free
-Software Foundation; either version 2, or (at your option) any later
-version.
-
-Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY
-WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
-for more details.
-
-You should have received a copy of the GNU General Public License
-along with Pazpar2; see the file LICENSE.  If not, write to the
-Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
-02111-1307, USA.
+/* This file is part of the YAZ toolkit.
+ * Copyright (C) 1995-2008 Index Data
+ * See the file LICENSE for details.
  */
 
 #if HAVE_CONFIG_H
-#include "cconfig.h"
+#include "config.h"
 #endif
 
 #include <string.h>
@@ -28,23 +12,25 @@ Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
 #include <stdio.h>
 #include <stdlib.h>
 
-//#include <yaz/xmalloc.h>
 #include <yaz/options.h>
 
-
-#ifdef HAVE_ICU
+#if YAZ_HAVE_ICU
 
 #include <unicode/ucnv.h>
 #include <unicode/ustring.h>
+#include <unicode/ucol.h> 
+#include <unicode/ubrk.h>
+#include <unicode/utrans.h>
 
-#include <yaz/icu_I18N.h>
+#include <yaz/icu.h>
+#include <yaz/wrbuf.h>
 
 /* commando line and config parameters */
 static struct config_t { 
     char conffile[1024];
     char print[1024];
     int xmloutput;
-    struct icu_chain * chain;
+    yaz_icu_chain_t chain;
     FILE * infile;
     FILE * outfile;
 } config;
@@ -66,14 +52,11 @@ void print_option_error(const struct config_t *p_config)
             "./yaz-icu -p t -x\n"
             "\n"
             "Example ICU chain XML configuration file:\n"
-            "<icu_chain id=\"en:word\" locale=\"en\">\n"
-            "  <normalize rule=\"[:Control:] Any-Remove\"/>\n"
+            "<icu_chain locale=\"en\">\n"
+            "  <transform rule=\"[:Control:] Any-Remove\"/>\n"
             "  <tokenize rule=\"l\"/>\n"
-            "  <normalize rule=\"[[:WhiteSpace:][:Punctuation:]] Remove\"/>\n"
-            "  <display/>\n"
+            "  <transform rule=\"[[:WhiteSpace:][:Punctuation:]] Remove\"/>\n"
             "  <casemap rule=\"l\"/>\n"
-            "  <index/>\n"
-            "  <sortkey/>\n"
             "</icu_chain>\n"
           );
     exit(1);
@@ -118,7 +101,7 @@ void read_params(int argc, char **argv, struct config_t *p_config)
         || !config.outfile)
         
         print_option_error(p_config);
-};
+}
 
 
 /*     UConverter *conv; */
@@ -154,7 +137,8 @@ static void print_icu_converters(const struct config_t *p_config)
                 ucnv_getDefaultName());
     }
     
-    for(i=0;i<count;i++){
+    for(i=0;i<count;i++)
+    {
         if (p_config->xmloutput)
             fprintf(config.outfile, "<converter id=\"%s\"/>\n", 
                     ucnv_getAvailableName(i));
@@ -170,13 +154,10 @@ static void print_icu_converters(const struct config_t *p_config)
 
 static void print_icu_transliterators(const struct config_t *p_config)
 {
-    int32_t count;
-    int32_t i;
-    
-    count = utrans_countAvailableIDs();
-    
     int32_t buf_cap = 128;
-    char buf[buf_cap];
+    char buf[128];
+    int32_t i;
+    int32_t count = utrans_countAvailableIDs();
     
     if (p_config->xmloutput)
         fprintf(config.outfile, "<transliterators count=\"%d\">\n",  count);
@@ -192,7 +173,8 @@ static void print_icu_transliterators(const struct config_t *p_config)
             fprintf(config.outfile, " %s", buf);
     }
     
-    if (p_config->xmloutput){
+    if (p_config->xmloutput)
+    {
         fprintf(config.outfile, "</transliterators>\n");
     }
     else
@@ -289,8 +271,8 @@ static void print_icu_xml_locales(const struct config_t *p_config)
 
     count = uloc_countAvailable() ;
 
-    if (p_config->xmloutput){
-    
+    if (p_config->xmloutput)
+    {
         fprintf(config.outfile, "<locales count=\"%d\" default=\"%s\" collations=\"%d\">\n", 
                 count, uloc_getDefault(), ucol_countAvailable());
     }
@@ -364,7 +346,8 @@ static void print_icu_xml_locales(const struct config_t *p_config)
                     &status);
 
 
-        if (p_config->xmloutput){
+        if (p_config->xmloutput)
+        {
             fprintf(config.outfile, "<locale id=\"%s\"", uloc_getAvailable(i)); 
             /* fprintf(config.outfile, " locale=\"%s\"", uloc_getAvailable(i)); */
             /* if (strlen(keyword_str)) */
@@ -388,7 +371,8 @@ static void print_icu_xml_locales(const struct config_t *p_config)
                 fprintf(config.outfile, "%s", localname_str);
             fprintf(config.outfile, "</locale>\n"); 
         }
-        else if (1 == p_config->xmloutput){
+        else if (1 == p_config->xmloutput)
+        {
             fprintf(config.outfile, "%s", uloc_getAvailable(i)); 
             fprintf(config.outfile, " | ");
             if (strlen(name_str))
@@ -406,9 +390,10 @@ static void print_icu_xml_locales(const struct config_t *p_config)
     else
         fprintf(config.outfile, "\n");
 
-    if(U_FAILURE(status)) {
+    if(U_FAILURE(status))
+    {
         fprintf(stderr, "ICU Error: %d %s\n", status, u_errorName(status));
-        exit(status);
+        exit(2);
     }
 }
 
@@ -435,7 +420,7 @@ static void print_info(const struct config_t *p_config)
         fprintf(config.outfile, "</icu>\n");
 
     exit(0);
-};
+}
 
 
 
@@ -453,14 +438,14 @@ static void process_text_file(const struct config_t *p_config)
     UErrorCode status = U_ZERO_ERROR;
     int success = 0;
     
-    if (! xml_node) {   
+    if (! xml_node)
+    {   
         printf("Could not parse XML config file '%s' \n",
                 config.conffile);
         exit (1);
     }
 
-    
-    config.chain = icu_chain_xml_config(xml_node, &status);
+    config.chain = icu_chain_xml_config(xml_node, 1, &status);
 
     if (config.chain && U_SUCCESS(status))
         success = 1;
@@ -469,39 +454,50 @@ static void process_text_file(const struct config_t *p_config)
                 config.conffile);
         exit (1);
     }
-    
+
     if (p_config->xmloutput)
         fprintf(config.outfile,
                 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
                 "<icu>\n"
                 "<tokens>\n");
     
-    // read input lines for processing
+    /* read input lines for processing */
     while ((line=fgets(linebuf, sizeof(linebuf)-1, config.infile)))
     {
         success = icu_chain_assign_cstr(config.chain, line, &status);
         line_count++;
 
-        while (success && icu_chain_next_token(config.chain, &status)){
+        while (success && icu_chain_next_token(config.chain, &status))
+        {
+            WRBUF sw = wrbuf_alloc();
             if (U_FAILURE(status))
                 success = 0;
             else {
+                const char *sortkey = icu_chain_token_sortkey(config.chain);
+                wrbuf_rewind(sw);
+                wrbuf_puts_escaped(sw, sortkey);
                 token_count++;
                 if (p_config->xmloutput)                    
+                {
+                    /* should XML encode this. Bug #1902 */
                     fprintf(config.outfile, 
-                            "<token id=\%lu\" line=\"%lu\""
-                            " norm=\"%s\" display=\"%s\"/>\n",
+                            "<token id=\"%lu\" line=\"%lu\""
+                            " norm=\"%s\" display=\"%s\" sortkey=\"%s\"/>\n",
                             token_count,
                             line_count,
-                            icu_chain_get_norm(config.chain),
-                            icu_chain_get_display(config.chain));
+                            icu_chain_token_norm(config.chain),
+                            icu_chain_token_display(config.chain),
+                            wrbuf_cstr(sw));
+                }
                 else
-                    fprintf(config.outfile, "%lu %lu '%s' '%s'\n",
+                    fprintf(config.outfile, "%lu %lu '%s' '%s' '%s'\n",
                             token_count,
                             line_count,
-                            icu_chain_get_norm(config.chain),
-                            icu_chain_get_display(config.chain));
+                            icu_chain_token_norm(config.chain),
+                            icu_chain_token_display(config.chain),
+                            wrbuf_cstr(sw));
             }
+            wrbuf_destroy(sw);
         }
         
     }
@@ -515,15 +511,15 @@ static void process_text_file(const struct config_t *p_config)
     xmlFreeDoc(doc);
     if (line)
         free(line);
-};
+}
 
-#endif // HAVE_ICU
+#endif /* YAZ_HAVE_ICU */
 
 
 int main(int argc, char **argv) 
 {
 
-#ifdef HAVE_ICU
+#if YAZ_HAVE_ICU
 
     read_params(argc, argv, &config);
 
@@ -533,17 +529,18 @@ int main(int argc, char **argv)
     if (config.print && strlen(config.print))
         print_info(&config);
 
-#else // HAVE_ICU
+#else /* YAZ_HAVE_ICU */
 
     printf("ICU not available on your system.\n"
            "Please install libicu36-dev and icu-doc or similar, "
            "re-configure and re-compile\n");
 
 
-#endif // HAVE_ICU
+    exit(3);
+#endif /* YAZ_HAVE_ICU */
 
     return(0);
-};
+}
 
 
 /*