X-Git-Url: http://git.indexdata.com/?p=idzebra-moved-to-github.git;a=blobdiff_plain;f=util%2Fcharmap.c;h=d38be4f6d62254142e54ccd49db52137f1774f2a;hp=96a390ab14b8f230a87e33d0ee8a73fc4f2326aa;hb=ecb3935e78cd9bcfdebafdee0834cfb1060d7b5e;hpb=d9b85731ab7b965b1ae9bc1c283e39faf10a644a

diff --git a/util/charmap.c b/util/charmap.c
index 96a390a..d38be4f 100644
--- a/util/charmap.c
+++ b/util/charmap.c
@@ -1,6 +1,6 @@
-/* $Id: charmap.c,v 1.29 2004-07-28 09:47:42 adam Exp $
-   Copyright (C) 1995,1996,1997,1998,1999,2000,2001,2002,2003,2004
-   Index Data Aps
+/* $Id: charmap.c,v 1.40 2006-05-10 08:13:44 adam Exp $
+   Copyright (C) 1995-2005
+   Index Data ApS
 
 This file is part of the Zebra server.
 
@@ -29,17 +29,20 @@ Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
 
 #include <ctype.h>
 #include <string.h>
+#include <stdlib.h>
 #include <assert.h>
 
 typedef unsigned ucs4_t;
 
-#include <yaz/yaz-util.h>
 #include <charmap.h>
 
+#include <yaz/yaz-util.h>
 
 #define CHR_MAXSTR 1024
 #define CHR_MAXEQUIV 32
 
+const unsigned char CHR_FIELD_BEGIN = '^';
+
 const char *CHR_UNKNOWN = "\001";
 const char *CHR_SPACE   = "\002";
 const char *CHR_CUT     = "\003";
@@ -73,6 +76,15 @@ typedef struct chrwork
 } chrwork;
 
 /*
+ * Callback for equivalent stuff
+ */
+typedef struct
+{
+    NMEM nmem;
+    int no_eq;
+    char *eq[CHR_MAXEQUIV];
+} chr_equiv_work;
+/*
  * Add an entry to the character map.
  */
 static chr_t_entry *set_map_string(chr_t_entry *root, NMEM nmem,
@@ -89,13 +101,14 @@ static chr_t_entry *set_map_string(chr_t_entry *root, NMEM nmem,
     }
     if (!len)
     {
-	if (!root->target || !root->target[0] || strcmp(root->target[0], to))
+	if (!root->target || !root->target[0] || 
+	    strcmp((const char *) root->target[0], to))
 	{
             if (from_0 && 
                 root->target && root->target[0] && root->target[0][0] &&
-                strcmp (root->target[0], CHR_UNKNOWN))
+                strcmp((const char *) root->target[0], CHR_UNKNOWN))
             {
-                yaz_log (LOG_WARN, "duplicate entry for charmap from '%s'",
+                yaz_log (YLOG_WARN, "duplicate entry for charmap from '%s'",
                          from_0);
             }
 	    root->target = (unsigned char **)
@@ -142,7 +155,7 @@ static chr_t_entry *find_entry(chr_t_entry *t, const char **from, int len)
     return t->target ? t : 0;
 }
 
-static chr_t_entry *find_entry_x(chr_t_entry *t, const char **from, int *len)
+static chr_t_entry *find_entry_x(chr_t_entry *t, const char **from, int *len, int first)
 {
     chr_t_entry *res;
 
@@ -153,35 +166,49 @@ static chr_t_entry *find_entry_x(chr_t_entry *t, const char **from, int *len)
 	from++;
 	len++;
     }
-    if (*len > 0 && t->children && t->children[(unsigned char) **from])
+    if (*len > 0 && t->children)
     {
 	const char *old_from = *from;
 	int old_len = *len;
+
+	res = 0;
+
+	if (first && t->children[CHR_FIELD_BEGIN])
+	{
+	    if ((res = find_entry_x(t->children[CHR_FIELD_BEGIN], from, len, 0)) && res != t->children[CHR_FIELD_BEGIN])
+		return res;
+            else
+	        res = 0;
+	    /* otherwhise there was no match on beginning of field, move on */
+	} 
 	
-	(*len)--;
-	(*from)++;
-	if ((res = find_entry_x(t->children[(unsigned char) *old_from],
-				from, len)))
-	    return res;
-	/* no match */
-	*len = old_len;
-	*from = old_from;
+	if (!res && t->children[(unsigned char) **from])
+	{
+	    (*len)--;
+	    (*from)++;
+	    if ((res = find_entry_x(t->children[(unsigned char) *old_from],
+				    from, len, 0)))
+		return res;
+	    /* no match */
+	    *len = old_len;
+	    *from = old_from;
+	}
     }
     /* no children match. use ourselves, if we have a target */
     return t->target ? t : 0;
 }
 
-const char **chr_map_input_x(chrmaptab maptab, const char **from, int *len)
+const char **chr_map_input_x(chrmaptab maptab, const char **from, int *len, int first)
 {
     chr_t_entry *t = maptab->input;
     chr_t_entry *res;
 
-    if (!(res = find_entry_x(t, from, len)))
+    if (!(res = find_entry_x(t, from, len, first)))
 	abort();
     return (const char **) (res->target);
 }
 
-const char **chr_map_input(chrmaptab maptab, const char **from, int len)
+const char **chr_map_input(chrmaptab maptab, const char **from, int len, int first)
 {
     chr_t_entry *t = maptab->input;
     chr_t_entry *res;
@@ -189,11 +216,25 @@ const char **chr_map_input(chrmaptab maptab, const char **from, int len)
 
     len_tmp[0] = len;
     len_tmp[1] = -1;
-    if (!(res = find_entry_x(t, from, len_tmp)))
+    if (!(res = find_entry_x(t, from, len_tmp, first)))
 	abort();
     return (const char **) (res->target);
 }
 
+const char **chr_map_q_input(chrmaptab maptab,
+			     const char **from, int len, int first)
+{
+    chr_t_entry *t = maptab->q_input;
+    chr_t_entry *res;
+    int len_tmp[2];
+    
+    len_tmp[0] = len;
+    len_tmp[1] = -1;
+    if (!(res = find_entry_x(t, from, len_tmp, first)))
+	return 0;
+    return (const char **) (res->target);
+}
+
 const char *chr_map_output(chrmaptab maptab, const char **from, int len)
 {
     unsigned char c = ** (unsigned char **) from;
@@ -201,50 +242,6 @@ const char *chr_map_output(chrmaptab maptab, const char **from, int len)
     return (const char*) maptab->output[c];
 }
 
-unsigned char zebra_prim(char **s)
-{
-    unsigned char c;
-    unsigned int i = 0;
-
-    yaz_log (LOG_DEBUG, "prim %.3s", *s);
-    if (**s == '\\')
-    {
-        (*s)++;
-        c = **s;
-        switch (c)
-        {
-        case '\\': c = '\\'; (*s)++; break;
-        case 'r': c = '\r'; (*s)++; break;
-        case 'n': c = '\n'; (*s)++; break;
-        case 't': c = '\t'; (*s)++; break;
-        case 's': c = ' '; (*s)++; break;
-        case 'x': sscanf(*s, "x%2x", &i); c = i; *s += 3; break;
-        case '0':
-        case '1':
-        case '2':
-        case '3':
-        case '4':
-        case '5':
-        case '6':
-        case '7':
-        case '8':
-        case '9':
-            sscanf(*s, "%3o", &i);
-            c = i;
-            *s += 3;
-            break;
-        default:
-            (*s)++;
-        }
-    }
-    else
-    {
-        c = **s;
-        ++(*s);
-    }
-    return c;
-}
-
 static int zebra_ucs4_strlen(ucs4_t *s)
 {
     int i = 0;
@@ -259,8 +256,8 @@ ucs4_t zebra_prim_w(ucs4_t **s)
     ucs4_t i = 0;
     char fmtstr[8];
 
-    yaz_log (LOG_DEBUG, "prim %.3s", (char *) *s);
-    if (**s == '\\')
+    yaz_log (YLOG_DEBUG, "prim_w %.3s", (char *) *s);
+    if (**s == '\\' && 1[*s])
     {
 	(*s)++;
 	c = **s;
@@ -325,7 +322,7 @@ ucs4_t zebra_prim_w(ucs4_t **s)
         c = **s;
         ++(*s);
     }
-    yaz_log (LOG_DEBUG, "out %d", c);
+    yaz_log (YLOG_DEBUG, "out %d", c);
     return c;
 }
 
@@ -374,41 +371,38 @@ static void fun_mkstring(const char *s, void *data, int num)
     chrwork *arg = (chrwork *) data;
     const char **res, *p = s;
 
-    res = chr_map_input(arg->map, &s, strlen(s));
+    res = chr_map_input(arg->map, &s, strlen(s), 0);
     if (*res == (char*) CHR_UNKNOWN)
-	logf(LOG_WARN, "Map: '%s' has no mapping", p);
+	yaz_log(YLOG_WARN, "Map: '%s' has no mapping", p);
     strncat(arg->string, *res, CHR_MAXSTR - strlen(arg->string));
     arg->string[CHR_MAXSTR] = '\0';
 }
 
 /*
- * Add a map to the string contained in the argument.
+ * Create an unmodified string (scan_string handler).
  */
-static void fun_add_map(const char *s, void *data, int num)
+static void fun_add_equivalent_string(const char *s, void *data, int num)
 {
-    chrwork *arg = (chrwork *) data;
-
-    assert(arg->map->input);
-    logf (LOG_DEBUG, "set map %.*s", (int) strlen(s), s);
-    set_map_string(arg->map->input, arg->map->nmem, s, strlen(s), arg->string,
-                   0);
-    for (s = arg->string; *s; s++)
-	logf (LOG_DEBUG, " %3d", (unsigned char) *s);
+    chr_equiv_work *arg = (chr_equiv_work *) data;
+    
+    if (arg->no_eq == CHR_MAXEQUIV)
+	return;
+    arg->eq[arg->no_eq++] = nmem_strdup(arg->nmem, s);
 }
 
 /*
- * Add a query map to the string contained in the argument.
+ * Add a map to the string contained in the argument.
  */
-static void fun_add_qmap(const char *s, void *data, int num)
+static void fun_add_map(const char *s, void *data, int num)
 {
     chrwork *arg = (chrwork *) data;
 
-    assert(arg->map->q_input);
-    logf (LOG_DEBUG, "set qmap %.*s", (int) strlen(s), s);
-    set_map_string(arg->map->q_input, arg->map->nmem, s,
-		   strlen(s), arg->string, 0);
+    assert(arg->map->input);
+    yaz_log (YLOG_DEBUG, "set map %.*s", (int) strlen(s), s);
+    set_map_string(arg->map->input, arg->map->nmem, s, strlen(s), arg->string,
+                   0);
     for (s = arg->string; *s; s++)
-	logf (LOG_DEBUG, " %3d", (unsigned char) *s);
+	yaz_log (YLOG_DEBUG, " %3d", (unsigned char) *s);
 }
 
 static int scan_to_utf8 (yaz_iconv_t t, ucs4_t *from, size_t inlen,
@@ -425,9 +419,9 @@ static int scan_to_utf8 (yaz_iconv_t t, ucs4_t *from, size_t inlen,
         ret = yaz_iconv (t, &inbuf, &inbytesleft, &outbuf, &outbytesleft);
         if (ret == (size_t) (-1))
         {
-	    yaz_log(LOG_LOG, "from: %2X %2X %2X %2X",
+	    yaz_log(YLOG_LOG, "from: %2X %2X %2X %2X",
 		    from[0], from[1], from[2], from[3]);
-            yaz_log (LOG_WARN|LOG_ERRNO, "bad unicode sequence");
+            yaz_log (YLOG_WARN|YLOG_ERRNO, "bad unicode sequence");
             return -1;
         }
     }
@@ -443,6 +437,7 @@ static int scan_string(char *s_native,
     char str[1024];
 
     ucs4_t arg[512];
+    ucs4_t arg_prim[512];
     ucs4_t *s0, *s = arg;
     ucs4_t c, begin, end;
     size_t i;
@@ -477,14 +472,14 @@ static int scan_string(char *s_native,
 	    begin = zebra_prim_w(&s);
 	    if (*s != '-')
 	    {
-		logf(LOG_FATAL, "Bad range in char-map");
+		yaz_log(YLOG_FATAL, "Bad range in char-map");
 		return -1;
 	    }
 	    s++;
 	    end = zebra_prim_w(&s);
 	    if (end <= begin)
 	    {
-		logf(LOG_FATAL, "Bad range in char-map");
+		yaz_log(YLOG_FATAL, "Bad range in char-map");
 		return -1;
 	    }
 	    s++;
@@ -495,14 +490,13 @@ static int scan_string(char *s_native,
 		(*fun)(str, data, num ? (*num)++ : 0);
 	    }
 	    break;
-	case '[': s++; abort(); break;
 	case '(':
             ++s;
-            s0 = s;
-            while (*s != ')' || s[-1] == '\\')
-                s++;
-	    *s = 0;
-            if (scan_to_utf8 (t_utf8, s0, s - s0, str, sizeof(str)-1))
+	    s0 = s; i = 0;
+	    while (*s != ')' || s[-1] == '\\')
+		arg_prim[i++] = zebra_prim_w(&s);
+	    arg_prim[i] = 0;
+            if (scan_to_utf8 (t_utf8, arg_prim, zebra_ucs4_strlen(arg_prim), str, sizeof(str)-1))
                 return -1;
 	    (*fun)(str, data, num ? (*num)++ : 0);
 	    s++;
@@ -517,8 +511,8 @@ static int scan_string(char *s_native,
     return 0;
 }
 
-chrmaptab chrmaptab_create(const char *tabpath, const char *name, int map_only,
-                           const char *tabroot)
+chrmaptab chrmaptab_create(const char *tabpath, const char *name,
+			   const char *tabroot)
 {
     FILE *f;
     char line[512], *argv[50];
@@ -532,17 +526,18 @@ chrmaptab chrmaptab_create(const char *tabpath, const char *name, int map_only,
     unsigned endian = 31;
     const char *ucs4_native = "UCS-4";
 
+    yaz_log (YLOG_DEBUG, "maptab %s open", name);
+    if (!(f = yaz_fopen(tabpath, name, "r", tabroot)))
+    {
+	yaz_log(YLOG_WARN|YLOG_ERRNO, "%s", name);
+	return 0;
+    }
+
     if (*(char*) &endian == 31)      /* little endian? */
         ucs4_native = "UCS-4LE";
 
     t_utf8 = yaz_iconv_open ("UTF-8", ucs4_native);
 
-    logf (LOG_DEBUG, "maptab %s open", name);
-    if (!(f = yaz_fopen(tabpath, name, "r", tabroot)))
-    {
-	logf(LOG_WARN|LOG_ERRNO, "%s", name);
-	return 0;
-    }
     nmem = nmem_create ();
     res = (chrmaptab) nmem_malloc(nmem, sizeof(*res));
     res->nmem = nmem;
@@ -561,15 +556,7 @@ chrmaptab chrmaptab_create(const char *tabpath, const char *name, int map_only,
 	res->input->children[i]->target = (unsigned char **)
 	    nmem_malloc (res->nmem, 2 * sizeof(unsigned char *));
 	res->input->children[i]->target[1] = 0;
-	if (map_only)
-	{
-	    res->input->children[i]->target[0] = (unsigned char *)
-		nmem_malloc (res->nmem, 2 * sizeof(unsigned char));
-	    res->input->children[i]->target[0][0] = i;
-	    res->input->children[i]->target[0][1] = 0;
-	}
-	else
-	    res->input->children[i]->target[0] = (unsigned char*) CHR_UNKNOWN;
+	res->input->children[i]->target[0] = (unsigned char*) CHR_UNKNOWN;
     }
     res->q_input = (chr_t_entry *)
 	nmem_malloc(res->nmem, sizeof(*res->q_input));
@@ -583,17 +570,17 @@ chrmaptab chrmaptab_create(const char *tabpath, const char *name, int map_only,
     res->base_uppercase = 0;
 
     while (!errors && (argc = readconf_line(f, &lineno, line, 512, argv, 50)))
-	if (!map_only && !yaz_matchstr(argv[0], "lowercase"))
+	if (!yaz_matchstr(argv[0], "lowercase"))
 	{
 	    if (argc != 2)
 	    {
-		logf(LOG_FATAL, "Syntax error in charmap");
+		yaz_log(YLOG_FATAL, "Syntax error in charmap");
 		++errors;
 	    }
 	    if (scan_string(argv[1], t_unicode, t_utf8, fun_addentry,
                             res, &num) < 0)
 	    {
-		logf(LOG_FATAL, "Bad value-set specification");
+		yaz_log(YLOG_FATAL, "Bad value-set specification");
 		++errors;
 	    }
 	    res->base_uppercase = num;
@@ -601,50 +588,50 @@ chrmaptab chrmaptab_create(const char *tabpath, const char *name, int map_only,
 	    res->output[(int) *CHR_UNKNOWN + num] = (unsigned char*) "@";
 	    num = (int) *CHR_BASE;
 	}
-	else if (!map_only && !yaz_matchstr(argv[0], "uppercase"))
+	else if (!yaz_matchstr(argv[0], "uppercase"))
 	{
 	    if (!res->base_uppercase)
 	    {
-		logf(LOG_FATAL, "Uppercase directive with no lowercase set");
+		yaz_log(YLOG_FATAL, "Uppercase directive with no lowercase set");
 		++errors;
 	    }
 	    if (argc != 2)
 	    {
-		logf(LOG_FATAL, "Missing arg for uppercase directive");
+		yaz_log(YLOG_FATAL, "Missing arg for uppercase directive");
 		++errors;
 	    }
 	    if (scan_string(argv[1], t_unicode, t_utf8, fun_addentry,
                             res, &num) < 0)
 	    {
-		logf(LOG_FATAL, "Bad value-set specification");
+		yaz_log(YLOG_FATAL, "Bad value-set specification");
 		++errors;
 	    }
 	}
-	else if (!map_only && !yaz_matchstr(argv[0], "space"))
+	else if (!yaz_matchstr(argv[0], "space"))
 	{
 	    if (argc != 2)
 	    {
-		logf(LOG_FATAL, "Syntax error in charmap for space");
+		yaz_log(YLOG_FATAL, "Syntax error in charmap for space");
 		++errors;
 	    }
 	    if (scan_string(argv[1], t_unicode, t_utf8,
                             fun_addspace, res, 0) < 0)
 	    {
-		logf(LOG_FATAL, "Bad space specification");
+		yaz_log(YLOG_FATAL, "Bad space specification");
 		++errors;
 	    }
 	}
-	else if (!map_only && !yaz_matchstr(argv[0], "cut"))
+	else if (!yaz_matchstr(argv[0], "cut"))
 	{
 	    if (argc != 2)
 	    {
-		logf(LOG_FATAL, "Syntax error in charmap for cut");
+		yaz_log(YLOG_FATAL, "Syntax error in charmap for cut");
 		++errors;
 	    }
 	    if (scan_string(argv[1], t_unicode, t_utf8,
                             fun_addcut, res, 0) < 0)
 	    {
-		logf(LOG_FATAL, "Bad cut specification");
+		yaz_log(YLOG_FATAL, "Bad cut specification");
 		++errors;
 	    }
 	}
@@ -654,7 +641,7 @@ chrmaptab chrmaptab_create(const char *tabpath, const char *name, int map_only,
 
 	    if (argc != 3)
 	    {
-		logf(LOG_FATAL, "charmap directive map requires 2 args");
+		yaz_log(YLOG_FATAL, "charmap directive map requires 2 args");
 		++errors;
 	    }
 	    buf.map = res;
@@ -662,39 +649,68 @@ chrmaptab chrmaptab_create(const char *tabpath, const char *name, int map_only,
 	    if (scan_string(argv[2], t_unicode, t_utf8,
                             fun_mkstring, &buf, 0) < 0)
 	    {
-		logf(LOG_FATAL, "Bad map target");
+		yaz_log(YLOG_FATAL, "Bad map target");
 		++errors;
 	    }
 	    if (scan_string(argv[1], t_unicode, t_utf8,
                             fun_add_map, &buf, 0) < 0)
 	    {
-		logf(LOG_FATAL, "Bad map source");
+		yaz_log(YLOG_FATAL, "Bad map source");
 		++errors;
 	    }
 	}
-	else if (!yaz_matchstr(argv[0], "qmap"))
+	else if (!yaz_matchstr(argv[0], "equivalent"))
 	{
-	    chrwork buf;
+	    chr_equiv_work w;
 
-	    if (argc != 3)
+	    if (argc != 2)
 	    {
-		logf(LOG_FATAL, "charmap directive qmap requires 2 args");
+		yaz_log(YLOG_FATAL, "equivalent requires 1 argument");
 		++errors;
 	    }
-	    buf.map = res;
-	    buf.string[0] = '\0';
-	    if (scan_string(argv[2], t_unicode, t_utf8, 
-                            fun_mkstring, &buf, 0) < 0)
+	    w.nmem = res->nmem;
+	    w.no_eq = 0;
+	    if (scan_string(argv[1], t_unicode, t_utf8, 
+                            fun_add_equivalent_string, &w, 0) < 0)
 	    {
-		logf(LOG_FATAL, "Bad qmap target");
+		yaz_log(YLOG_FATAL, "equivalent: invalid string");
 		++errors;
 	    }
-	    if (scan_string(argv[1], t_unicode, t_utf8, 
-                            fun_add_qmap, &buf, 0) < 0)
+	    else if (w.no_eq == 0)
 	    {
-		logf(LOG_FATAL, "Bad qmap source");
+		yaz_log(YLOG_FATAL, "equivalent: no strings");
 		++errors;
 	    }
+	    else
+	    {
+		char *result_str;
+		int i, slen = 5;
+
+		/* determine length of regular expression */
+		for (i = 0; i<w.no_eq; i++)
+		    slen += strlen(w.eq[i]) + 1;
+		result_str = nmem_malloc(res->nmem, slen + 5);
+
+		/* build the regular expression */
+		*result_str = '\0';
+		slen = 0;
+		for (i = 0; i<w.no_eq; i++)
+		{
+		    result_str[slen++]  = i ? '|' : '(';
+		    strcpy(result_str + slen, w.eq[i]);
+		    slen += strlen(w.eq[i]);
+		}
+		result_str[slen++] = ')';
+		result_str[slen] = '\0';
+
+		/* each eq will map to this regular expression */
+		for (i = 0; i<w.no_eq; i++)
+		{
+		    set_map_string(res->q_input, res->nmem,
+				   w.eq[i], strlen(w.eq[i]),
+				   result_str, 0);
+		}
+	    }
 	}
         else if (!yaz_matchstr(argv[0], "encoding"))
         {
@@ -723,7 +739,7 @@ chrmaptab chrmaptab_create(const char *tabpath, const char *name, int map_only,
         }
 	else
 	{
-	    logf(LOG_WARN, "Syntax error at '%s' in %s", line, name);
+	    yaz_log(YLOG_WARN, "Syntax error at '%s' in %s", line, name);
 	}
     
     yaz_fclose(f);
@@ -732,7 +748,7 @@ chrmaptab chrmaptab_create(const char *tabpath, const char *name, int map_only,
 	chrmaptab_destroy(res);
 	res = 0;
     }
-    logf (LOG_DEBUG, "maptab %s close %d errors", name, errors);
+    yaz_log (YLOG_DEBUG, "maptab %s close %d errors", name, errors);
     if (t_utf8 != 0)
         yaz_iconv_close(t_utf8);
     if (t_unicode != 0)
@@ -747,3 +763,11 @@ void chrmaptab_destroy(chrmaptab tab)
 }
 
 
+/*
+ * Local variables:
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ * vim: shiftwidth=4 tabstop=8 expandtab
+ */
+