1 /* This file is part of the Zebra server.
2 Copyright (C) 1994-2011 Index Data
4 Zebra is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
9 Zebra is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
23 * \brief character conversions (.chr)
25 * Support module to handle character-conversions into and out of the
37 typedef unsigned ucs4_t;
41 #include <yaz/yaz-util.h>
43 #define CHR_MAXSTR 1024
44 #define CHR_MAXEQUIV 32
46 const unsigned char CHR_FIELD_BEGIN = '^';
48 const char *CHR_UNKNOWN = "\001";
49 const char *CHR_SPACE = "\002";
50 const char *CHR_CUT = "\003";
51 const char *CHR_BASE = "\005"; /* CHECK CHR_BASE_CHAR as well */
55 chr_t_entry *input; /* mapping table for input data */
56 chr_t_entry *q_input; /* mapping table for queries */
57 unsigned char *output[256]; /* return mapping - for display of registers */
58 int base_uppercase; /* Start of upper-case ordinals */
63 * Character map trie node.
67 chr_t_entry **children; /* array of children */
68 unsigned char **target; /* target for this node, if any */
72 * General argument structure for callback functions (internal use only)
74 typedef struct chrwork
77 char string[CHR_MAXSTR+1];
81 * Callback for equivalent stuff
87 char *eq[CHR_MAXEQUIV];
90 * Add an entry to the character map.
92 static chr_t_entry *set_map_string(chr_t_entry *root, NMEM nmem,
93 const char *from, int len, char *to,
100 root = (chr_t_entry *) nmem_malloc(nmem, sizeof(*root));
106 if (!root->target || !root->target[0] ||
107 strcmp((const char *) root->target[0], to))
110 root->target && root->target[0] && root->target[0][0] &&
111 strcmp((const char *) root->target[0], CHR_UNKNOWN))
113 yaz_log(YLOG_WARN, "duplicate entry for charmap from '%s'",
116 root->target = (unsigned char **)
117 nmem_malloc(nmem, sizeof(*root->target)*2);
118 root->target[0] = (unsigned char *) nmem_strdup(nmem, to);
128 root->children = (chr_t_entry **)
129 nmem_malloc(nmem, sizeof(chr_t_entry*) * 256);
130 for (i = 0; i < 256; i++)
131 root->children[i] = 0;
133 if (!(root->children[(unsigned char) *from] =
134 set_map_string(root->children[(unsigned char) *from], nmem,
135 from + 1, len - 1, to, from_0)))
141 static chr_t_entry *find_entry_x(chr_t_entry *t, const char **from, int *len, int first)
146 { /* switch to next buffer */
152 if (*len > 0 && t->children)
154 const char *old_from = *from;
159 if (first && t->children[CHR_FIELD_BEGIN])
161 if ((res = find_entry_x(t->children[CHR_FIELD_BEGIN], from, len, 0)) && res != t->children[CHR_FIELD_BEGIN])
165 /* otherwhise there was no match on beginning of field, move on */
168 if (!res && t->children[(unsigned char) **from])
172 if ((res = find_entry_x(t->children[(unsigned char) *old_from],
180 /* no children match. use ourselves, if we have a target */
181 return t->target ? t : 0;
184 const char **chr_map_input_x(chrmaptab maptab, const char **from, int *len, int first)
186 chr_t_entry *t = maptab->input;
189 if (!(res = find_entry_x(t, from, len, first)))
191 return (const char **) (res->target);
194 const char **chr_map_input(chrmaptab maptab, const char **from, int len, int first)
196 chr_t_entry *t = maptab->input;
202 if (!(res = find_entry_x(t, from, len_tmp, first)))
204 return (const char **) (res->target);
207 const char **chr_map_q_input(chrmaptab maptab,
208 const char **from, int len, int first)
210 chr_t_entry *t = maptab->q_input;
216 if (!(res = find_entry_x(t, from, len_tmp, first)))
218 return (const char **) (res->target);
221 const char *chr_map_output(chrmaptab maptab, const char **from, int len)
223 unsigned char c = ** (unsigned char **) from;
224 const char *out = (const char*) maptab->output[c];
231 static int zebra_ucs4_strlen(ucs4_t *s)
239 ucs4_t zebra_prim_w(ucs4_t **s)
245 if (**s == '\\' && 1[*s])
251 case '\\': c = '\\'; (*s)++; break;
252 case 'r': c = '\r'; (*s)++; break;
253 case 'n': c = '\n'; (*s)++; break;
254 case 't': c = '\t'; (*s)++; break;
255 case 's': c = ' '; (*s)++; break;
257 if (zebra_ucs4_strlen(*s) >= 3)
262 sscanf(fmtstr, "%x", &i);
277 if (zebra_ucs4_strlen(*s) >= 3)
283 sscanf(fmtstr, "%o", &i);
289 if (zebra_ucs4_strlen(*s) >= 5)
296 sscanf(fmtstr, "%x", &i);
310 yaz_log(YLOG_DEBUG, "out %d", c);
316 * Add an entry to the value space.
318 static void fun_addentry(const char *s, void *data, int num)
320 chrmaptab tab = (chrmaptab) data;
323 tmp[0] = num; tmp[1] = '\0';
324 tab->input = set_map_string(tab->input, tab->nmem, s, strlen(s), tmp, 0);
325 tab->output[num + tab->base_uppercase] =
326 (unsigned char *) nmem_strdup(tab->nmem, s);
331 * Add a space-entry to the value space.
333 static void fun_addspace(const char *s, void *data, int num)
335 chrmaptab tab = (chrmaptab) data;
336 tab->input = set_map_string(tab->input, tab->nmem, s, strlen(s),
337 (char*) CHR_SPACE, 0);
342 * Add a space-entry to the value space.
344 static void fun_addcut(const char *s, void *data, int num)
346 chrmaptab tab = (chrmaptab) data;
347 tab->input = set_map_string(tab->input, tab->nmem, s, strlen(s),
352 * Create a string containing the mapped characters provided.
354 static void fun_mkstring(const char *s, void *data, int num)
356 chrwork *arg = (chrwork *) data;
357 const char **res, *p = s;
359 res = chr_map_input(arg->map, &s, strlen(s), 0);
360 if (*res == (char*) CHR_UNKNOWN)
361 yaz_log(YLOG_WARN, "Map: '%s' has no mapping", p);
362 strncat(arg->string, *res, CHR_MAXSTR - strlen(arg->string));
363 arg->string[CHR_MAXSTR] = '\0';
367 * Create an unmodified string (scan_string handler).
369 static void fun_add_equivalent_string(const char *s, void *data, int num)
371 chr_equiv_work *arg = (chr_equiv_work *) data;
373 if (arg->no_eq == CHR_MAXEQUIV)
375 arg->eq[arg->no_eq++] = nmem_strdup(arg->nmem, s);
379 * Add a map to the string contained in the argument.
381 static void fun_add_map(const char *s, void *data, int num)
383 chrwork *arg = (chrwork *) data;
385 assert(arg->map->input);
386 yaz_log(YLOG_DEBUG, "set map %.*s", (int) strlen(s), s);
387 set_map_string(arg->map->input, arg->map->nmem, s, strlen(s), arg->string,
389 for (s = arg->string; *s; s++)
390 yaz_log(YLOG_DEBUG, " %3d", (unsigned char) *s);
393 static int scan_to_utf8(yaz_iconv_t t, ucs4_t *from, size_t inlen,
394 char *outbuf, size_t outbytesleft)
396 size_t inbytesleft = inlen * sizeof(ucs4_t);
397 char *inbuf = (char*) from;
401 *outbuf++ = *from; /* ISO-8859-1 is OK here */
404 ret = yaz_iconv(t, &inbuf, &inbytesleft, &outbuf, &outbytesleft);
405 if (ret != (size_t) (-1))
406 ret = yaz_iconv(t, 0, 0, &outbuf, &outbytesleft);
409 if (ret == (size_t) (-1))
411 yaz_log(YLOG_LOG, "from: %2X %2X %2X %2X",
412 from[0], from[1], from[2], from[3]);
413 yaz_log(YLOG_WARN|YLOG_ERRNO, "bad unicode sequence");
421 static int scan_string(char *s_native,
422 yaz_iconv_t t_unicode, yaz_iconv_t t_utf8,
423 void (*fun)(const char *c, void *data, int num),
424 void *data, int *num)
429 ucs4_t arg_prim[512];
430 ucs4_t *s0, *s = arg;
431 ucs4_t c, begin, end;
436 char *outbuf = (char *) arg;
437 char *inbuf = s_native;
438 size_t outbytesleft = sizeof(arg)-4;
439 size_t inbytesleft = strlen(s_native);
441 ret = yaz_iconv(t_unicode, &inbuf, &inbytesleft,
442 &outbuf, &outbytesleft);
443 if (ret != (size_t)(-1))
444 ret = yaz_iconv(t_unicode, 0, 0, &outbuf, &outbytesleft);
446 if (ret == (size_t)(-1))
448 i = (outbuf - (char*) arg)/sizeof(ucs4_t);
452 for (i = 0; s_native[i]; i++)
453 arg[i] = s_native[i] & 255; /* ISO-8859-1 conversion */
455 arg[i] = 0; /* terminate */
456 if (s[0] == 0xfeff || s[0] == 0xfeff) /* skip byte Order Mark */
464 begin = zebra_prim_w(&s);
467 yaz_log(YLOG_FATAL, "Bad range in char-map");
471 end = zebra_prim_w(&s);
474 yaz_log(YLOG_FATAL, "Bad range in char-map");
478 for (c = begin; c <= end; c++)
480 if (scan_to_utf8(t_utf8, &c, 1, str, sizeof(str)-1))
482 (*fun)(str, data, num ? (*num)++ : 0);
488 while (*s != ')' || s[-1] == '\\')
492 yaz_log(YLOG_FATAL, "Missing ) in charmap");
495 arg_prim[i++] = zebra_prim_w(&s);
498 if (scan_to_utf8(t_utf8, arg_prim, zebra_ucs4_strlen(arg_prim), str, sizeof(str)-1))
500 (*fun)(str, data, num ? (*num)++ : 0);
504 c = zebra_prim_w(&s);
505 if (scan_to_utf8(t_utf8, &c, 1, str, sizeof(str)-1))
507 (*fun)(str, data, num ? (*num)++ : 0);
513 chrmaptab chrmaptab_create(const char *tabpath, const char *name,
517 char line[512], *argv[50];
520 int no_directives = 0;
522 int argc, num = (int) *CHR_BASE, i;
524 yaz_iconv_t t_unicode = 0;
525 yaz_iconv_t t_utf8 = 0;
526 unsigned endian = 31;
527 const char *ucs4_native = "UCS-4";
529 yaz_log(YLOG_DEBUG, "maptab %s open", name);
530 if (!(f = yaz_fopen(tabpath, name, "r", tabroot)))
532 yaz_log(YLOG_WARN|YLOG_ERRNO, "%s", name);
536 if (*(char*) &endian == 31) /* little endian? */
537 ucs4_native = "UCS-4LE";
539 t_utf8 = yaz_iconv_open("UTF-8", ucs4_native);
541 nmem = nmem_create();
542 res = (chrmaptab) nmem_malloc(nmem, sizeof(*res));
544 res->input = (chr_t_entry *) nmem_malloc(res->nmem, sizeof(*res->input));
545 res->input->target = (unsigned char **)
546 nmem_malloc(res->nmem, sizeof(*res->input->target) * 2);
547 res->input->target[0] = (unsigned char*) CHR_UNKNOWN;
548 res->input->target[1] = 0;
549 res->input->children = (chr_t_entry **)
550 nmem_malloc(res->nmem, sizeof(res->input) * 256);
551 for (i = 0; i < 256; i++)
553 res->input->children[i] = (chr_t_entry *)
554 nmem_malloc(res->nmem, sizeof(*res->input));
555 res->input->children[i]->children = 0;
556 res->input->children[i]->target = (unsigned char **)
557 nmem_malloc(res->nmem, 2 * sizeof(unsigned char *));
558 res->input->children[i]->target[1] = 0;
559 res->input->children[i]->target[0] = (unsigned char*) CHR_UNKNOWN;
561 res->q_input = (chr_t_entry *)
562 nmem_malloc(res->nmem, sizeof(*res->q_input));
563 res->q_input->target = 0;
564 res->q_input->children = 0;
566 for (i = *CHR_BASE; i < 256; i++)
568 res->output[(int) *CHR_SPACE] = (unsigned char *) " ";
569 res->output[(int) *CHR_UNKNOWN] = (unsigned char*) "@";
570 res->base_uppercase = 0;
572 while (!errors && (argc = readconf_line(f, &lineno, line, 512, argv, 50)))
575 if (!yaz_matchstr(argv[0], "lowercase"))
579 yaz_log(YLOG_FATAL, "Syntax error in charmap");
582 if (scan_string(argv[1], t_unicode, t_utf8, fun_addentry,
585 yaz_log(YLOG_FATAL, "Bad value-set specification");
588 res->base_uppercase = num;
589 res->output[(int) *CHR_SPACE + num] = (unsigned char *) " ";
590 res->output[(int) *CHR_UNKNOWN + num] = (unsigned char*) "@";
591 num = (int) *CHR_BASE;
593 else if (!yaz_matchstr(argv[0], "uppercase"))
595 if (!res->base_uppercase)
597 yaz_log(YLOG_FATAL, "Uppercase directive with no lowercase set");
602 yaz_log(YLOG_FATAL, "Missing arg for uppercase directive");
605 if (scan_string(argv[1], t_unicode, t_utf8, fun_addentry,
608 yaz_log(YLOG_FATAL, "Bad value-set specification");
612 else if (!yaz_matchstr(argv[0], "space"))
616 yaz_log(YLOG_FATAL, "Syntax error in charmap for space");
619 if (scan_string(argv[1], t_unicode, t_utf8,
620 fun_addspace, res, 0) < 0)
622 yaz_log(YLOG_FATAL, "Bad space specification");
626 else if (!yaz_matchstr(argv[0], "cut"))
630 yaz_log(YLOG_FATAL, "Syntax error in charmap for cut");
633 if (scan_string(argv[1], t_unicode, t_utf8,
634 fun_addcut, res, 0) < 0)
636 yaz_log(YLOG_FATAL, "Bad cut specification");
640 else if (!yaz_matchstr(argv[0], "map"))
646 yaz_log(YLOG_FATAL, "charmap directive map requires 2 args");
650 buf.string[0] = '\0';
651 if (scan_string(argv[2], t_unicode, t_utf8,
652 fun_mkstring, &buf, 0) < 0)
654 yaz_log(YLOG_FATAL, "Bad map target");
657 if (scan_string(argv[1], t_unicode, t_utf8,
658 fun_add_map, &buf, 0) < 0)
660 yaz_log(YLOG_FATAL, "Bad map source");
664 else if (!yaz_matchstr(argv[0], "equivalent"))
670 yaz_log(YLOG_FATAL, "equivalent requires 1 argument");
675 if (scan_string(argv[1], t_unicode, t_utf8,
676 fun_add_equivalent_string, &w, 0) < 0)
678 yaz_log(YLOG_FATAL, "equivalent: invalid string");
681 else if (w.no_eq == 0)
683 yaz_log(YLOG_FATAL, "equivalent: no strings");
691 /* determine length of regular expression */
692 for (i = 0; i<w.no_eq; i++)
693 slen += strlen(w.eq[i]) + 1;
694 result_str = nmem_malloc(res->nmem, slen + 5);
696 /* build the regular expression */
699 for (i = 0; i<w.no_eq; i++)
701 result_str[slen++] = i ? '|' : '(';
702 strcpy(result_str + slen, w.eq[i]);
703 slen += strlen(w.eq[i]);
705 result_str[slen++] = ')';
706 result_str[slen] = '\0';
708 /* each eq will map to this regular expression */
709 for (i = 0; i<w.no_eq; i++)
711 set_map_string(res->q_input, res->nmem,
712 w.eq[i], strlen(w.eq[i]),
717 else if (!yaz_matchstr(argv[0], "encoding"))
720 yaz_iconv_close(t_unicode);
721 t_unicode = yaz_iconv_open(ucs4_native, argv[1]);
725 yaz_log(YLOG_WARN, "Syntax error at '%s' in %s", line, name);
730 if (no_directives == 0)
732 yaz_log(YLOG_WARN, "No directives in '%s'", name);
737 chrmaptab_destroy(res);
740 yaz_log(YLOG_DEBUG, "maptab %s num=%d close %d errors", name, num, errors);
742 yaz_iconv_close(t_utf8);
744 yaz_iconv_close(t_unicode);
748 void chrmaptab_destroy(chrmaptab tab)
751 nmem_destroy(tab->nmem);
758 * c-file-style: "Stroustrup"
759 * indent-tabs-mode: nil
761 * vim: shiftwidth=4 tabstop=8 expandtab