Merge branch 'master' into stemming

Conflicts:
	src/icu_chain.c

Merge minor ICU change and test changes.

Fix includes to include config.h.
Fix Makefile: Include stemmer in libyaz_icu.

diff --git a/include/yaz/stemmer.h b/include/yaz/stemmer.h
index f5d23ab..6e51cae 100644 (file)
--- a/include/yaz/stemmer.h
+++ b/include/yaz/stemmer.h
@@ -34,6 +34,7 @@
 #define YAZ_STEMMER_H
 
 #include <yaz/yconfig.h>
+#include <yaz/xmltypes.h>
 #include <yaz/icu_I18N.h>
 #include <yaz/log.h>
 

diff --git a/src/Makefile.am b/src/Makefile.am
index 200ba5c..d8143f5 100644 (file)
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -77,7 +77,7 @@ libyaz_la_SOURCES=version.c options.c log.c \
   wrbuf.c oid_db.c errno.c \
   nmemsdup.c xmalloc.c readconf.c tpath.c nmem.c matchstr.c atoin.c \
   siconv.c iconv-p.h utf8.c ucs4.c iso5428.c advancegreek.c \
-  odr_bool.c ber_bool.c ber_len.c ber_tag.c odr_util.c facet.c stemmer.c \
+  odr_bool.c ber_bool.c ber_len.c ber_tag.c odr_util.c facet.c \
   odr_null.c ber_null.c odr_int.c ber_int.c odr_tag.c odr_cons.c \
   odr_seq.c odr_oct.c ber_oct.c odr_bit.c ber_bit.c odr_oid.c \
   ber_oid.c odr_use.c odr_choice.c odr_any.c ber_any.c odr.c odr_mem.c \
@@ -115,7 +115,7 @@ libyaz_server_la_SOURCES = statserv.c seshigh.c eventl.c \
 
 libyaz_server_la_LDFLAGS=-version-info $(YAZ_VERSION_INFO)
 
-libyaz_icu_la_SOURCES = icu_chain.c icu_utf16.c icu_utf8.c \
+libyaz_icu_la_SOURCES = icu_chain.c icu_utf16.c icu_utf8.c stemmer.c \
        icu_transform.c icu_casemap.c icu_tokenizer.c icu_sortkey.c
 libyaz_icu_la_LDFLAGS=-version-info $(YAZ_VERSION_INFO)
 

diff --git a/src/icu_chain.c b/src/icu_chain.c
index fb69d82..8cc9a64 100644 (file)
--- a/src/icu_chain.c
+++ b/src/icu_chain.c
@@ -20,7 +20,8 @@
 #include <yaz/stemmer.h>
 
 #include <yaz/log.h>
-
+#include <yaz/nmem.h>
+#include <yaz/nmem_xml.h>
 #include <string.h>
 #include <stdlib.h>
 #include <stdio.h>
@@ -44,12 +45,12 @@ struct icu_chain_step
     /* type and action object */
     enum icu_chain_step_type type;
     union {
-        struct icu_casemap   * casemap;
-        struct icu_transform * transform;
-        struct icu_tokenizer * tokenizer;
-        yaz_stemmer_p          stemmer;
+       struct icu_casemap   *casemap;
+       struct icu_transform *transform;
+       struct icu_tokenizer *tokenizer;  
+        yaz_stemmer_p         stemmer;
     } u;
-    struct icu_chain_step * previous;
+    struct icu_chain_step *previous;
 };
 
 struct icu_chain
@@ -58,10 +59,10 @@ struct icu_chain
     char *locale;
     int sort;
 
-    UCollator * coll;
+    UCollator *coll;
     
     /* linked list of chain steps */
-    struct icu_chain_step * csteps;
+    struct icu_chain_step *csteps;
 };
 
 int icu_check_status(UErrorCode status)
@@ -74,12 +75,11 @@ int icu_check_status(UErrorCode status)
     return 1;
 }
 
-static struct icu_chain_step *icu_chain_step_create(
-    struct icu_chain * chain,  enum icu_chain_step_type type,
-    const uint8_t * rule, 
-    UErrorCode *status)
+static struct icu_chain_step *icu_chain_insert_step(
+    struct icu_chain *chain, enum icu_chain_step_type type,
+    const uint8_t *rule, UErrorCode *status)
 {
-    struct icu_chain_step * step = 0;
+    struct icu_chain_step *step = 0;
     
     if (!chain || !type || !rule)
         return 0;
@@ -115,11 +115,14 @@ static struct icu_chain_step *icu_chain_step_create(
     default:
         break;
     }
+    step->previous = chain->csteps;
+    chain->csteps = step;
+
     return step;
 }
 
 
-static void icu_chain_step_destroy(struct icu_chain_step * step)
+static void icu_chain_step_destroy(struct icu_chain_step *step)
 {
     if (!step)
         return;
@@ -186,9 +189,9 @@ struct icu_chain_step *icu_chain_step_clone(struct icu_chain_step *old)
 }
 
 struct icu_chain *icu_chain_create(const char *locale, int sort,
-                                   UErrorCode * status)
+                                   UErrorCode *status)
 {
-    struct icu_chain * chain 
+    struct icu_chain *chain 
         = (struct icu_chain *) xmalloc(sizeof(*chain));
 
     *status = U_ZERO_ERROR;
@@ -208,7 +211,7 @@ struct icu_chain *icu_chain_create(const char *locale, int sort,
     return chain;
 }
 
-void icu_chain_destroy(struct icu_chain * chain)
+void icu_chain_destroy(struct icu_chain *chain)
 {
     if (chain)
     {
@@ -224,15 +227,17 @@ void icu_chain_destroy(struct icu_chain * chain)
 }
 
 static struct icu_chain_step *icu_chain_insert_step(
-    struct icu_chain * chain, enum icu_chain_step_type type,
-    const uint8_t * rule, UErrorCode *status);
+    struct icu_chain *chain, enum icu_chain_step_type type,
+    const uint8_t *rule, UErrorCode *status);
 
-struct icu_chain * icu_chain_xml_config(const xmlNode *xml_node, 
-                                        int sort,
-                                        UErrorCode * status)
+struct icu_chain *icu_chain_xml_config(const xmlNode *xml_node, 
+                                       int sort,
+                                       UErrorCode *status)
 {
     xmlNode *node = 0;
-    struct icu_chain * chain = 0;
+    int no_errors = 0;
+    struct icu_chain *chain = 0;
+    NMEM nmem = 0;
    
     *status = U_ZERO_ERROR;
 
@@ -240,8 +245,8 @@ struct icu_chain * icu_chain_xml_config(const xmlNode *xml_node,
         return 0;
     
     {
-        xmlChar * xml_locale = xmlGetProp((xmlNode *) xml_node, 
-                                          (xmlChar *) "locale");
+        xmlChar *xml_locale = xmlGetProp((xmlNode *) xml_node, 
+                                         (xmlChar *) "locale");
         
         if (xml_locale)
         {
@@ -253,40 +258,58 @@ struct icu_chain * icu_chain_xml_config(const xmlNode *xml_node,
     if (!chain)
         return 0;
 
+    nmem = nmem_create();
     for (node = xml_node->children; node; node = node->next)
     {
-        xmlChar *xml_rule;
-        struct icu_chain_step * step = 0;
+        char *rule = 0;
+        struct icu_chain_step *step = 0;
+        struct _xmlAttr *attr;
 
+        nmem_reset(nmem);
         if (node->type != XML_ELEMENT_NODE)
             continue;
 
-        xml_rule = xmlGetProp(node, (xmlChar *) "rule");
-
+        for (attr = node->properties; attr; attr = attr->next)
+        {
+            if (!strcmp((const char *) attr->name, "rule"))
+            {
+                rule = nmem_text_node_cdata(attr->children, nmem);
+            }
+            else
+            {
+                yaz_log(YLOG_WARN, "Unsupported attribute '%s' for "
+                        "element '%s'", attr->name, node->name);
+                no_errors++;
+                continue;
+            }
+        }
+        if (!rule && node->children)
+            rule = nmem_text_node_cdata(node->children, nmem);
+        
         if (!strcmp((const char *) node->name, "casemap"))
             step = icu_chain_insert_step(chain, ICU_chain_step_type_casemap, 
-                                         (const uint8_t *) xml_rule, status);
+                                         (const uint8_t *) rule, status);
         else if (!strcmp((const char *) node->name, "transform"))
             step = icu_chain_insert_step(chain, ICU_chain_step_type_transform, 
-                                         (const uint8_t *) xml_rule, status);
+                                         (const uint8_t *) rule, status);
         else if (!strcmp((const char *) node->name, "transliterate"))
             step = icu_chain_insert_step(chain, ICU_chain_step_type_transliterate, 
-                                         (const uint8_t *) xml_rule, status);
+                                         (const uint8_t *) rule, status);
         else if (!strcmp((const char *) node->name, "tokenize"))
             step = icu_chain_insert_step(chain, ICU_chain_step_type_tokenize, 
-                                         (const uint8_t *) xml_rule, status);
+                                         (const uint8_t *) rule, status);
         else if (!strcmp((const char *) node->name, "display"))
             step = icu_chain_insert_step(chain, ICU_chain_step_type_display, 
                                          (const uint8_t *) "", status);
         else if (!strcmp((const char *) node->name, "stemming"))
-            step = yaz_chain_insert_step(chain, YAZ_chain_step_type_stemming,
-                                         (const uint8_t *) xml_rule, status);
+            step = icu_chain_insert_step(chain, YAZ_chain_step_type_stemming,
+                                         (const uint8_t *) rule, status);
         else if (!strcmp((const char *) node->name, "normalize"))
         {
             yaz_log(YLOG_WARN, "Element %s is deprecated. "
                     "Use transform instead", node->name);
             step = icu_chain_insert_step(chain, ICU_chain_step_type_transform, 
-                                         (const uint8_t *) xml_rule, status);
+                                         (const uint8_t *) rule, status);
         }
         else if (!strcmp((const char *) node->name, "index")
                  || !strcmp((const char *) node->name, "sortkey"))
@@ -297,36 +320,22 @@ struct icu_chain * icu_chain_xml_config(const xmlNode *xml_node,
         else
         {
             yaz_log(YLOG_WARN, "Unknown element %s", node->name);
-            icu_chain_destroy(chain);
-            return 0;
+            no_errors++;
+            continue;
         }
-        xmlFree(xml_rule);
         if (step && U_FAILURE(*status))
         {
-            icu_chain_destroy(chain);
-            return 0;
+            no_errors++;
+            break;
         }
     }
-    return chain;
-}
-
-
-static struct icu_chain_step *icu_chain_insert_step(
-    struct icu_chain * chain, enum icu_chain_step_type type,
-    const uint8_t * rule, UErrorCode *status)
-{    
-    struct icu_chain_step * step = 0;
-    if (!chain || !type || !rule)
+    nmem_destroy(nmem);
+    if (no_errors)
+    {
+        icu_chain_destroy(chain);
         return 0;
-
-    /* create actual chain step with this buffer */
-    step = icu_chain_step_create(chain, type, rule,
-                                 status);
-
-    step->previous = chain->csteps;
-    chain->csteps = step;
-
-    return step;
+    }
+    return chain;
 }
 
 struct icu_iter {
@@ -520,7 +529,7 @@ int icu_iter_get_token_number(yaz_icu_iter_t iter)
     return iter->token_count;
 }
 
-int icu_chain_assign_cstr(struct icu_chain * chain, const char * src8cstr, 
+int icu_chain_assign_cstr(struct icu_chain *chain, const char *src8cstr, 
                           UErrorCode *status)
 {
     if (chain->iter)
@@ -530,34 +539,34 @@ int icu_chain_assign_cstr(struct icu_chain * chain, const char * src8cstr,
     return 1;
 }
 
-int icu_chain_next_token(struct icu_chain * chain, UErrorCode *status)
+int icu_chain_next_token(struct icu_chain *chain, UErrorCode *status)
 {
     *status = U_ZERO_ERROR;
     return icu_iter_next(chain->iter);
 }
 
-int icu_chain_token_number(struct icu_chain * chain)
+int icu_chain_token_number(struct icu_chain *chain)
 {
     if (chain && chain->iter)
         return chain->iter->token_count;
     return 0;
 }
 
-const char * icu_chain_token_display(struct icu_chain * chain)
+const char *icu_chain_token_display(struct icu_chain *chain)
 {
     if (chain->iter)
         return icu_iter_get_display(chain->iter);
     return 0;
 }
 
-const char * icu_chain_token_norm(struct icu_chain * chain)
+const char *icu_chain_token_norm(struct icu_chain *chain)
 {
     if (chain->iter)
         return icu_iter_get_norm(chain->iter);
     return 0;
 }
 
-const char * icu_chain_token_sortkey(struct icu_chain * chain)
+const char *icu_chain_token_sortkey(struct icu_chain *chain)
 {
     if (chain->iter)
         return icu_iter_get_sortkey(chain->iter);

diff --git a/src/stemmer.c b/src/stemmer.c
index 5cb3267..b544ebc 100644 (file)
--- a/src/stemmer.c
+++ b/src/stemmer.c
@@ -1,5 +1,12 @@
 
 
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#if YAZ_HAVE_ICU
+
+#include <yaz/yconfig.h>
 
 #include <yaz/stemmer.h>
 
@@ -31,3 +38,4 @@ void yaz_stemmer_destroy(yaz_stemmer_p stemmer) {
 
 }
 
+#endif /* YAZ_HAVE_ICU */

diff --git a/test/Makefile.am b/test/Makefile.am
index 23b2f12..1fb2c35 100644 (file)
--- a/test/Makefile.am
+++ b/test/Makefile.am
@@ -23,6 +23,7 @@ EXTRA_DIST = tstodr.asn test_odrcodec.c test_odrcodec.h cql2xcqlsample \
  marccol5.u8.marc marccol5.u8.1.lst marccol5.u8.2.lst \
  tsticu-0.xml tsticu-0.input tsticu-0.output \
  tsticu-1.xml tsticu-1.input tsticu-1.output \
+ tsticu-2.xml tsticu-2.input tsticu-2.output \
  test_record_conv.xsl test_xml_include.xml
 
 YAZCOMP = ../util/yaz-asncomp

diff --git a/test/test_icu.c b/test/test_icu.c
index 2ad005c..8be8869 100644 (file)
--- a/test/test_icu.c
+++ b/test/test_icu.c
@@ -450,6 +450,8 @@ static void check_icu_chain(void)
 
     xmlFreeDoc(doc);
     YAZ_CHECK(chain);
+    if (!chain)
+        return;
 
     YAZ_CHECK(icu_chain_assign_cstr(chain, en_str, &status));
 
@@ -507,6 +509,8 @@ static void check_bug_1140(void)
 
     xmlFreeDoc(doc);
     YAZ_CHECK(chain);
+    if (!chain)
+        return;
     
     YAZ_CHECK(icu_chain_assign_cstr(
                   chain,  "O Romeo, Romeo! wherefore art thou\t Romeo?",
@@ -773,6 +777,8 @@ static void check_icu_iter2(void)
 
     xmlFreeDoc(doc);
     YAZ_CHECK(chain);
+    if (!chain)
+        return;
     
     YAZ_CHECK(test_iter(chain, "Adobe Acrobat Reader, 1991-1999.",
                         "[adobe][acrobat][reader][1991][][1999][]"));

diff --git a/test/tsticu-2.input b/test/tsticu-2.input
new file mode 100644 (file)
index 0000000..579a976
--- /dev/null
+++ b/test/tsticu-2.input
@@ -0,0 +1,2 @@
+`a'
+``a''

diff --git a/test/tsticu-2.output b/test/tsticu-2.output
new file mode 100644 (file)
index 0000000..b5ee584
--- /dev/null
+++ b/test/tsticu-2.output
@@ -0,0 +1,4 @@
+1 1 '‘a'
+' ''
+2 2 '“a''
+' ''

diff --git a/test/tsticu-2.xml b/test/tsticu-2.xml
new file mode 100644 (file)
index 0000000..eea85bd
--- /dev/null
+++ b/test/tsticu-2.xml
@@ -0,0 +1,8 @@
+<icu_chain locale="en">
+ <transliterate>
+$back = \` ;
+$back $back > “ ; # generate right d.q.m. (double quotation mark)
+$back > ‘ ;
+ </transliterate>
+</icu_chain>
+