Exteded yaz_marc_write_marcxhange new parameters, format and type.
[yaz-moved-to-github.git] / src / siconv.c
1 /*
2  * Copyright (C) 1995-2006, Index Data ApS
3  * See the file LICENSE for details.
4  *
5  * $Id: siconv.c,v 1.21 2006-04-19 23:48:06 adam Exp $
6  */
7 /**
8  * \file siconv.c
9  * \brief Implements simple ICONV
10  *
11  * This implements an interface similar to that of iconv and
12  * is used by YAZ to interface with iconv (if present).
13  * For systems where iconv is not present, this layer
14  * provides a few important conversions: UTF-8, MARC-8, Latin-1.
15  */
16
17 #if HAVE_CONFIG_H
18 #include <config.h>
19 #endif
20
21 #include <errno.h>
22 #include <string.h>
23 #include <ctype.h>
24 #if HAVE_WCHAR_H
25 #include <wchar.h>
26 #endif
27
28 #if HAVE_ICONV_H
29 #include <iconv.h>
30 #endif
31
32 #include <yaz/yaz-util.h>
33
34 unsigned long yaz_marc8_1_conv(unsigned char *inp, size_t inbytesleft,
35                                size_t *no_read, int *combining);
36 unsigned long yaz_marc8_2_conv(unsigned char *inp, size_t inbytesleft,
37                                size_t *no_read, int *combining);
38 unsigned long yaz_marc8_3_conv(unsigned char *inp, size_t inbytesleft,
39                                size_t *no_read, int *combining);
40 unsigned long yaz_marc8_4_conv(unsigned char *inp, size_t inbytesleft,
41                                size_t *no_read, int *combining);
42 unsigned long yaz_marc8_5_conv(unsigned char *inp, size_t inbytesleft,
43                                size_t *no_read, int *combining);
44 unsigned long yaz_marc8_6_conv(unsigned char *inp, size_t inbytesleft,
45                                size_t *no_read, int *combining);
46 unsigned long yaz_marc8_7_conv(unsigned char *inp, size_t inbytesleft,
47                                size_t *no_read, int *combining);
48 unsigned long yaz_marc8_8_conv(unsigned char *inp, size_t inbytesleft,
49                                size_t *no_read, int *combining);
50 unsigned long yaz_marc8_9_conv(unsigned char *inp, size_t inbytesleft,
51                                size_t *no_read, int *combining);
52
53
54 unsigned long yaz_marc8r_1_conv(unsigned char *inp, size_t inbytesleft,
55                                 size_t *no_read, int *combining);
56 unsigned long yaz_marc8r_2_conv(unsigned char *inp, size_t inbytesleft,
57                                 size_t *no_read, int *combining);
58 unsigned long yaz_marc8r_3_conv(unsigned char *inp, size_t inbytesleft,
59                                 size_t *no_read, int *combining);
60 unsigned long yaz_marc8r_4_conv(unsigned char *inp, size_t inbytesleft,
61                                 size_t *no_read, int *combining);
62 unsigned long yaz_marc8r_5_conv(unsigned char *inp, size_t inbytesleft,
63                                 size_t *no_read, int *combining);
64 unsigned long yaz_marc8r_6_conv(unsigned char *inp, size_t inbytesleft,
65                                 size_t *no_read, int *combining);
66 unsigned long yaz_marc8r_7_conv(unsigned char *inp, size_t inbytesleft,
67                                 size_t *no_read, int *combining);
68 unsigned long yaz_marc8r_8_conv(unsigned char *inp, size_t inbytesleft,
69                                 size_t *no_read, int *combining);
70 unsigned long yaz_marc8r_9_conv(unsigned char *inp, size_t inbytesleft,
71                                 size_t *no_read, int *combining);
72
73 struct yaz_iconv_struct {
74     int my_errno;
75     int init_flag;
76     size_t (*init_handle)(yaz_iconv_t cd, unsigned char *inbuf,
77                           size_t inbytesleft, size_t *no_read);
78     unsigned long (*read_handle)(yaz_iconv_t cd, unsigned char *inbuf,
79                                  size_t inbytesleft, size_t *no_read);
80     size_t (*write_handle)(yaz_iconv_t cd, unsigned long x,
81                            char **outbuf, size_t *outbytesleft,
82                            int last);
83     int marc8_esc_mode;
84
85     int comb_offset;
86     int comb_size;
87     unsigned long comb_x[8];
88     size_t comb_no_read[8];
89     size_t no_read_x;
90     unsigned long unget_x;
91 #if HAVE_ICONV_H
92     iconv_t iconv_cd;
93 #endif
94     unsigned long compose_char;
95
96     unsigned long write_marc8_comb_ch[8];
97     size_t write_marc8_comb_no;
98     unsigned long write_marc8_last;
99     const char *write_marc8_page_chr;
100 };
101
102 static unsigned long yaz_read_ISO8859_1 (yaz_iconv_t cd, unsigned char *inp,
103                                          size_t inbytesleft, size_t *no_read)
104 {
105     unsigned long x = inp[0];
106     *no_read = 1;
107     return x;
108 }
109
110 static size_t yaz_init_UTF8 (yaz_iconv_t cd, unsigned char *inp,
111                              size_t inbytesleft, size_t *no_read)
112 {
113     if (inp[0] != 0xef)
114     {
115         *no_read = 0;
116         return 0;
117     }
118     if (inbytesleft < 3)
119     {
120         cd->my_errno = YAZ_ICONV_EINVAL;
121         return (size_t) -1;
122     }
123     if (inp[1] != 0xbb && inp[2] == 0xbf)
124         *no_read = 3;
125     else
126         *no_read = 0;
127     return 0;
128 }
129
130 static unsigned long yaz_read_UTF8 (yaz_iconv_t cd, unsigned char *inp,
131                                     size_t inbytesleft, size_t *no_read)
132 {
133     unsigned long x = 0;
134
135     if (inp[0] <= 0x7f)
136     {
137         x = inp[0];
138         *no_read = 1;
139     }
140     else if (inp[0] <= 0xbf || inp[0] >= 0xfe)
141     {
142         *no_read = 0;
143         cd->my_errno = YAZ_ICONV_EILSEQ;
144     }
145     else if (inp[0] <= 0xdf && inbytesleft >= 2)
146     {
147         x = ((inp[0] & 0x1f) << 6) | (inp[1] & 0x3f);
148         if (x >= 0x80)
149             *no_read = 2;
150         else
151         {
152             *no_read = 0;
153             cd->my_errno = YAZ_ICONV_EILSEQ;
154         }
155     }
156     else if (inp[0] <= 0xef && inbytesleft >= 3)
157     {
158         x = ((inp[0] & 0x0f) << 12) | ((inp[1] & 0x3f) << 6) |
159             (inp[2] & 0x3f);
160         if (x >= 0x800)
161             *no_read = 3;
162         else
163         {
164             *no_read = 0;
165             cd->my_errno = YAZ_ICONV_EILSEQ;
166         }
167     }
168     else if (inp[0] <= 0xf7 && inbytesleft >= 4)
169     {
170         x =  ((inp[0] & 0x07) << 18) | ((inp[1] & 0x3f) << 12) |
171             ((inp[2] & 0x3f) << 6) | (inp[3] & 0x3f);
172         if (x >= 0x10000)
173             *no_read = 4;
174         else
175         {
176             *no_read = 0;
177             cd->my_errno = YAZ_ICONV_EILSEQ;
178         }
179     }
180     else if (inp[0] <= 0xfb && inbytesleft >= 5)
181     {
182         x =  ((inp[0] & 0x03) << 24) | ((inp[1] & 0x3f) << 18) |
183             ((inp[2] & 0x3f) << 12) | ((inp[3] & 0x3f) << 6) |
184             (inp[4] & 0x3f);
185         if (x >= 0x200000)
186             *no_read = 5;
187         else
188         {
189             *no_read = 0;
190             cd->my_errno = YAZ_ICONV_EILSEQ;
191         }
192     }
193     else if (inp[0] <= 0xfd && inbytesleft >= 6)
194     {
195         x =  ((inp[0] & 0x01) << 30) | ((inp[1] & 0x3f) << 24) |
196             ((inp[2] & 0x3f) << 18) | ((inp[3] & 0x3f) << 12) |
197             ((inp[4] & 0x3f) << 6) | (inp[5] & 0x3f);
198         if (x >= 0x4000000)
199             *no_read = 6;
200         else
201         {
202             *no_read = 0;
203             cd->my_errno = YAZ_ICONV_EILSEQ;
204         }
205     }
206     else
207     {
208         *no_read = 0;
209         cd->my_errno = YAZ_ICONV_EINVAL;
210     }
211     return x;
212 }
213
214 static unsigned long yaz_read_UCS4 (yaz_iconv_t cd, unsigned char *inp,
215                                     size_t inbytesleft, size_t *no_read)
216 {
217     unsigned long x = 0;
218     
219     if (inbytesleft < 4)
220     {
221         cd->my_errno = YAZ_ICONV_EINVAL; /* incomplete input */
222         *no_read = 0;
223     }
224     else
225     {
226         x = (inp[0]<<24) | (inp[1]<<16) | (inp[2]<<8) | inp[3];
227         *no_read = 4;
228     }
229     return x;
230 }
231
232 static unsigned long yaz_read_UCS4LE (yaz_iconv_t cd, unsigned char *inp,
233                                       size_t inbytesleft, size_t *no_read)
234 {
235     unsigned long x = 0;
236     
237     if (inbytesleft < 4)
238     {
239         cd->my_errno = YAZ_ICONV_EINVAL; /* incomplete input */
240         *no_read = 0;
241     }
242     else
243     {
244         x = (inp[3]<<24) | (inp[2]<<16) | (inp[1]<<8) | inp[0];
245         *no_read = 4;
246     }
247     return x;
248 }
249
250 #if HAVE_WCHAR_H
251 static unsigned long yaz_read_wchar_t (yaz_iconv_t cd, unsigned char *inp,
252                                        size_t inbytesleft, size_t *no_read)
253 {
254     unsigned long x = 0;
255     
256     if (inbytesleft < sizeof(wchar_t))
257     {
258         cd->my_errno = YAZ_ICONV_EINVAL; /* incomplete input */
259         *no_read = 0;
260     }
261     else
262     {
263         wchar_t wch;
264         memcpy (&wch, inp, sizeof(wch));
265         x = wch;
266         *no_read = sizeof(wch);
267     }
268     return x;
269 }
270 #endif
271
272
273 static unsigned long yaz_read_marc8_comb (yaz_iconv_t cd, unsigned char *inp,
274                                           size_t inbytesleft, size_t *no_read,
275                                           int *comb);
276
277 static unsigned long yaz_read_marc8 (yaz_iconv_t cd, unsigned char *inp,
278                                      size_t inbytesleft, size_t *no_read)
279 {
280     unsigned long x;
281     if (cd->comb_offset < cd->comb_size)
282     {
283         *no_read = cd->comb_no_read[cd->comb_offset];
284         x = cd->comb_x[cd->comb_offset];
285
286         /* special case for double-diacritic combining characters, 
287            INVERTED BREVE and DOUBLE TILDE.
288            We'll increment the no_read counter by 1, since we want to skip over
289            the processing of the closing ligature character
290         */
291         /* this code is no longer necessary.. our handlers code in
292            yaz_marc8_?_conv (generated by charconv.tcl) now returns
293            0 and no_read=1 when a sequence does not match the input.
294            The SECOND HALFs in codetables.xml produces a non-existant
295            entry in the conversion trie.. Hence when met, the input byte is
296            skipped as it should (in yaz_iconv)
297         */
298 #if 0
299         if (x == 0x0361 || x == 0x0360)
300             *no_read += 1;
301 #endif
302         cd->comb_offset++;
303         return x;
304     }
305
306     cd->comb_offset = 0;
307     for (cd->comb_size = 0; cd->comb_size < 8; cd->comb_size++)
308     {
309         int comb = 0;
310         x = yaz_read_marc8_comb(cd, inp, inbytesleft, no_read, &comb);
311         if (!comb || !x)
312             break;
313         cd->comb_x[cd->comb_size] = x;
314         cd->comb_no_read[cd->comb_size] = *no_read;
315         inp += *no_read;
316         inbytesleft = inbytesleft - *no_read;
317     }
318     return x;
319 }
320
321 static unsigned long yaz_read_marc8_comb (yaz_iconv_t cd, unsigned char *inp,
322                                           size_t inbytesleft, size_t *no_read,
323                                           int *comb)
324 {
325     *no_read = 0;
326     while(inbytesleft >= 1 && inp[0] == 27)
327     {
328         size_t inbytesleft0 = inbytesleft;
329         inp++;
330         inbytesleft--;
331         while(inbytesleft > 0 && strchr("(,$!", *inp))
332         {
333             inbytesleft--;
334             inp++;
335         }
336         if (inbytesleft <= 0)
337         {
338             *no_read = 0;
339             cd->my_errno = YAZ_ICONV_EINVAL;
340             return 0;
341         }
342         cd->marc8_esc_mode = *inp++;
343         inbytesleft--;
344         (*no_read) += inbytesleft0 - inbytesleft;
345     }
346     if (inbytesleft <= 0)
347         return 0;
348     else
349     {
350         unsigned long x;
351         size_t no_read_sub = 0;
352         *comb = 0;
353
354         switch(cd->marc8_esc_mode)
355         {
356         case 'B':  /* Basic ASCII */
357         case 'E':  /* ANSEL */
358         case 's':  /* ASCII */
359             x = yaz_marc8_1_conv(inp, inbytesleft, &no_read_sub, comb);
360             break;
361         case 'g':  /* Greek */
362             x = yaz_marc8_2_conv(inp, inbytesleft, &no_read_sub, comb);
363             break;
364         case 'b':  /* Subscripts */
365             x = yaz_marc8_3_conv(inp, inbytesleft, &no_read_sub, comb);
366             break;
367         case 'p':  /* Superscripts */
368             x = yaz_marc8_4_conv(inp, inbytesleft, &no_read_sub, comb);
369             break;
370         case '2':  /* Basic Hebrew */
371             x = yaz_marc8_5_conv(inp, inbytesleft, &no_read_sub, comb);
372             break;
373         case 'N':  /* Basic Cyrillic */
374         case 'Q':  /* Extended Cyrillic */
375             x = yaz_marc8_6_conv(inp, inbytesleft, &no_read_sub, comb);
376             break;
377         case '3':  /* Basic Arabic */
378         case '4':  /* Extended Arabic */
379             x = yaz_marc8_7_conv(inp, inbytesleft, &no_read_sub, comb);
380             break;
381         case 'S':  /* Greek */
382             x = yaz_marc8_8_conv(inp, inbytesleft, &no_read_sub, comb);
383             break;
384         case '1':  /* Chinese, Japanese, Korean (EACC) */
385             x = yaz_marc8_9_conv(inp, inbytesleft, &no_read_sub, comb);
386             break;
387         default:
388             *no_read = 0;
389             cd->my_errno = YAZ_ICONV_EILSEQ;
390             return 0;
391         }
392         *no_read += no_read_sub;
393         return x;
394     }
395 }
396
397 static size_t yaz_write_UTF8 (yaz_iconv_t cd, unsigned long x,
398                               char **outbuf, size_t *outbytesleft,
399                               int last)
400 {
401     unsigned char *outp = (unsigned char *) *outbuf;
402
403     if (x <= 0x7f && *outbytesleft >= 1)
404     {
405         *outp++ = (unsigned char) x;
406         (*outbytesleft)--;
407     } 
408     else if (x <= 0x7ff && *outbytesleft >= 2)
409     {
410         *outp++ = (unsigned char) ((x >> 6) | 0xc0);
411         *outp++ = (unsigned char) ((x & 0x3f) | 0x80);
412         (*outbytesleft) -= 2;
413     }
414     else if (x <= 0xffff && *outbytesleft >= 3)
415     {
416         *outp++ = (unsigned char) ((x >> 12) | 0xe0);
417         *outp++ = (unsigned char) (((x >> 6) & 0x3f) | 0x80);
418         *outp++ = (unsigned char) ((x & 0x3f) | 0x80);
419         (*outbytesleft) -= 3;
420     }
421     else if (x <= 0x1fffff && *outbytesleft >= 4)
422     {
423         *outp++ = (unsigned char) ((x >> 18) | 0xf0);
424         *outp++ = (unsigned char) (((x >> 12) & 0x3f) | 0x80);
425         *outp++ = (unsigned char) (((x >> 6)  & 0x3f) | 0x80);
426         *outp++ = (unsigned char) ((x & 0x3f) | 0x80);
427         (*outbytesleft) -= 4;
428     }
429     else if (x <= 0x3ffffff && *outbytesleft >= 5)
430     {
431         *outp++ = (unsigned char) ((x >> 24) | 0xf8);
432         *outp++ = (unsigned char) (((x >> 18) & 0x3f) | 0x80);
433         *outp++ = (unsigned char) (((x >> 12) & 0x3f) | 0x80);
434         *outp++ = (unsigned char) (((x >> 6)  & 0x3f) | 0x80);
435         *outp++ = (unsigned char) ((x & 0x3f) | 0x80);
436         (*outbytesleft) -= 5;
437     }
438     else if (*outbytesleft >= 6)
439     {
440         *outp++ = (unsigned char) ((x >> 30) | 0xfc);
441         *outp++ = (unsigned char) (((x >> 24) & 0x3f) | 0x80);
442         *outp++ = (unsigned char) (((x >> 18) & 0x3f) | 0x80);
443         *outp++ = (unsigned char) (((x >> 12) & 0x3f) | 0x80);
444         *outp++ = (unsigned char) (((x >> 6)  & 0x3f) | 0x80);
445         *outp++ = (unsigned char) ((x & 0x3f) | 0x80);
446         (*outbytesleft) -= 6;
447     }
448     else 
449     {
450         cd->my_errno = YAZ_ICONV_E2BIG;  /* not room for output */
451         return (size_t)(-1);
452     }
453     *outbuf = (char *) outp;
454     return 0;
455 }
456
457
458 static size_t yaz_write_ISO8859_1 (yaz_iconv_t cd, unsigned long x,
459                                    char **outbuf, size_t *outbytesleft,
460                                    int last)
461 {
462     /* list of two char unicode sequence that, when combined, are
463        equivalent to single unicode chars that can be represented in
464        ISO-8859-1/Latin-1.
465        Regular iconv on Linux at least does not seem to convert these,
466        but since MARC-8 to UTF-8 generates these composed sequence
467        we get a better chance of a successful MARC-8 -> ISO-8859-1
468        conversion */
469     static struct {
470         unsigned long x1, x2;
471         unsigned y;
472     } latin1_comb[] = {
473         { 'A', 0x0300, 0xc0}, /* LATIN CAPITAL LETTER A WITH GRAVE */
474         { 'A', 0x0301, 0xc1}, /* LATIN CAPITAL LETTER A WITH ACUTE */
475         { 'A', 0x0302, 0xc2}, /* LATIN CAPITAL LETTER A WITH CIRCUMFLEX */
476         { 'A', 0x0303, 0xc3}, /* LATIN CAPITAL LETTER A WITH TILDE */
477         { 'A', 0x0308, 0xc4}, /* LATIN CAPITAL LETTER A WITH DIAERESIS */
478         { 'A', 0x030a, 0xc5}, /* LATIN CAPITAL LETTER A WITH RING ABOVE */
479         /* no need for 0xc6      LATIN CAPITAL LETTER AE */
480         { 'C', 0x0327, 0xc7}, /* LATIN CAPITAL LETTER C WITH CEDILLA */
481         { 'E', 0x0300, 0xc8}, /* LATIN CAPITAL LETTER E WITH GRAVE */
482         { 'E', 0x0301, 0xc9}, /* LATIN CAPITAL LETTER E WITH ACUTE */
483         { 'E', 0x0302, 0xca}, /* LATIN CAPITAL LETTER E WITH CIRCUMFLEX */
484         { 'E', 0x0308, 0xcb}, /* LATIN CAPITAL LETTER E WITH DIAERESIS */
485         { 'I', 0x0300, 0xcc}, /* LATIN CAPITAL LETTER I WITH GRAVE */
486         { 'I', 0x0301, 0xcd}, /* LATIN CAPITAL LETTER I WITH ACUTE */
487         { 'I', 0x0302, 0xce}, /* LATIN CAPITAL LETTER I WITH CIRCUMFLEX */
488         { 'I', 0x0308, 0xcf}, /* LATIN CAPITAL LETTER I WITH DIAERESIS */
489         { 'N', 0x0303, 0xd1}, /* LATIN CAPITAL LETTER N WITH TILDE */
490         { 'O', 0x0300, 0xd2}, /* LATIN CAPITAL LETTER O WITH GRAVE */
491         { 'O', 0x0301, 0xd3}, /* LATIN CAPITAL LETTER O WITH ACUTE */
492         { 'O', 0x0302, 0xd4}, /* LATIN CAPITAL LETTER O WITH CIRCUMFLEX */
493         { 'O', 0x0303, 0xd5}, /* LATIN CAPITAL LETTER O WITH TILDE */
494         { 'O', 0x0308, 0xd6}, /* LATIN CAPITAL LETTER O WITH DIAERESIS */
495         /* omitted:    0xd7      MULTIPLICATION SIGN */
496         /* omitted:    0xd8      LATIN CAPITAL LETTER O WITH STROKE */
497         { 'U', 0x0300, 0xd9}, /* LATIN CAPITAL LETTER U WITH GRAVE */
498         { 'U', 0x0301, 0xda}, /* LATIN CAPITAL LETTER U WITH ACUTE */
499         { 'U', 0x0302, 0xdb}, /* LATIN CAPITAL LETTER U WITH CIRCUMFLEX */
500         { 'U', 0x0308, 0xdc}, /* LATIN CAPITAL LETTER U WITH DIAERESIS */
501         { 'Y', 0x0301, 0xdd}, /* LATIN CAPITAL LETTER Y WITH ACUTE */
502         /* omitted:    0xde      LATIN CAPITAL LETTER THORN */
503         /* omitted:    0xdf      LATIN SMALL LETTER SHARP S */
504         { 'a', 0x0300, 0xe0}, /* LATIN SMALL LETTER A WITH GRAVE */
505         { 'a', 0x0301, 0xe1}, /* LATIN SMALL LETTER A WITH ACUTE */
506         { 'a', 0x0302, 0xe2}, /* LATIN SMALL LETTER A WITH CIRCUMFLEX */
507         { 'a', 0x0303, 0xe3}, /* LATIN SMALL LETTER A WITH TILDE */
508         { 'a', 0x0308, 0xe4}, /* LATIN SMALL LETTER A WITH DIAERESIS */
509         { 'a', 0x030a, 0xe5}, /* LATIN SMALL LETTER A WITH RING ABOVE */
510         /* omitted:    0xe6      LATIN SMALL LETTER AE */
511         { 'c', 0x0327, 0xe7}, /* LATIN SMALL LETTER C WITH CEDILLA */
512         { 'e', 0x0300, 0xe8}, /* LATIN SMALL LETTER E WITH GRAVE */
513         { 'e', 0x0301, 0xe9}, /* LATIN SMALL LETTER E WITH ACUTE */
514         { 'e', 0x0302, 0xea}, /* LATIN SMALL LETTER E WITH CIRCUMFLEX */
515         { 'e', 0x0308, 0xeb}, /* LATIN SMALL LETTER E WITH DIAERESIS */
516         { 'i', 0x0300, 0xec}, /* LATIN SMALL LETTER I WITH GRAVE */
517         { 'i', 0x0301, 0xed}, /* LATIN SMALL LETTER I WITH ACUTE */
518         { 'i', 0x0302, 0xee}, /* LATIN SMALL LETTER I WITH CIRCUMFLEX */
519         { 'i', 0x0308, 0xef}, /* LATIN SMALL LETTER I WITH DIAERESIS */
520         /* omitted:    0xf0      LATIN SMALL LETTER ETH */
521         { 'n', 0x0303, 0xf1}, /* LATIN SMALL LETTER N WITH TILDE */
522         { 'o', 0x0300, 0xf2}, /* LATIN SMALL LETTER O WITH GRAVE */
523         { 'o', 0x0301, 0xf3}, /* LATIN SMALL LETTER O WITH ACUTE */
524         { 'o', 0x0302, 0xf4}, /* LATIN SMALL LETTER O WITH CIRCUMFLEX */
525         { 'o', 0x0303, 0xf5}, /* LATIN SMALL LETTER O WITH TILDE */
526         { 'o', 0x0308, 0xf6}, /* LATIN SMALL LETTER O WITH DIAERESIS */
527         /* omitted:    0xf7      DIVISION SIGN */
528         /* omitted:    0xf8      LATIN SMALL LETTER O WITH STROKE */
529         { 'u', 0x0300, 0xf9}, /* LATIN SMALL LETTER U WITH GRAVE */
530         { 'u', 0x0301, 0xfa}, /* LATIN SMALL LETTER U WITH ACUTE */
531         { 'u', 0x0302, 0xfb}, /* LATIN SMALL LETTER U WITH CIRCUMFLEX */
532         { 'u', 0x0308, 0xfc}, /* LATIN SMALL LETTER U WITH DIAERESIS */
533         { 'y', 0x0301, 0xfd}, /* LATIN SMALL LETTER Y WITH ACUTE */
534         /* omitted:    0xfe      LATIN SMALL LETTER THORN */
535         { 'y', 0x0308, 0xff}, /* LATIN SMALL LETTER Y WITH DIAERESIS */
536         
537         { 0, 0, 0}
538     };
539     unsigned char *outp = (unsigned char *) *outbuf;
540
541     if (cd->compose_char)
542     {
543         int i;
544         for (i = 0; latin1_comb[i].x1; i++)
545             if (cd->compose_char == latin1_comb[i].x1 && x == latin1_comb[i].x2)
546             {
547                 x = latin1_comb[i].y;
548                 break;
549             }
550         if (*outbytesleft < 1)
551         {  /* no room. Retain compose_char and bail out */
552             cd->my_errno = YAZ_ICONV_E2BIG;
553             return (size_t)(-1);
554         }
555         if (!latin1_comb[i].x1) 
556         {   /* not found. Just write compose_char */
557             *outp++ = (unsigned char) cd->compose_char;
558             (*outbytesleft)--;
559             *outbuf = (char *) outp;
560         }
561         /* compose_char used so reset it. x now holds current char */
562         cd->compose_char = 0;
563     }
564
565     if (!last && x > 32 && x < 127 && cd->compose_char == 0)
566     {
567         cd->compose_char = x;
568         return 0;
569     }
570     else if (x > 255 || x < 1)
571     {
572         cd->my_errno = YAZ_ICONV_EILSEQ;
573         return (size_t) -1;
574     }
575     else if (*outbytesleft < 1)
576     {
577         cd->my_errno = YAZ_ICONV_E2BIG;
578         return (size_t)(-1);
579     }
580     *outp++ = (unsigned char) x;
581     (*outbytesleft)--;
582     *outbuf = (char *) outp;
583     return 0;
584 }
585
586
587 static size_t yaz_write_UCS4 (yaz_iconv_t cd, unsigned long x,
588                               char **outbuf, size_t *outbytesleft,
589                               int last)
590 {
591     unsigned char *outp = (unsigned char *) *outbuf;
592     if (*outbytesleft >= 4)
593     {
594         *outp++ = (unsigned char) (x>>24);
595         *outp++ = (unsigned char) (x>>16);
596         *outp++ = (unsigned char) (x>>8);
597         *outp++ = (unsigned char) x;
598         (*outbytesleft) -= 4;
599     }
600     else
601     {
602         cd->my_errno = YAZ_ICONV_E2BIG;
603         return (size_t)(-1);
604     }
605     *outbuf = (char *) outp;
606     return 0;
607 }
608
609 static size_t yaz_write_UCS4LE (yaz_iconv_t cd, unsigned long x,
610                                 char **outbuf, size_t *outbytesleft,
611                                 int last)
612 {
613     unsigned char *outp = (unsigned char *) *outbuf;
614     if (*outbytesleft >= 4)
615     {
616         *outp++ = (unsigned char) x;
617         *outp++ = (unsigned char) (x>>8);
618         *outp++ = (unsigned char) (x>>16);
619         *outp++ = (unsigned char) (x>>24);
620         (*outbytesleft) -= 4;
621     }
622     else
623     {
624         cd->my_errno = YAZ_ICONV_E2BIG;
625         return (size_t)(-1);
626     }
627     *outbuf = (char *) outp;
628     return 0;
629 }
630
631 static unsigned long lookup_marc8(yaz_iconv_t cd,
632                                   unsigned long x, int *comb,
633                                   const char **page_chr)
634 {
635     char utf8_buf[7];
636     char *utf8_outbuf = utf8_buf;
637     size_t utf8_outbytesleft = sizeof(utf8_buf)-1, r;
638
639     r = yaz_write_UTF8(cd, x, &utf8_outbuf, &utf8_outbytesleft, 0);
640     if (r == (size_t)(-1))
641     {
642         cd->my_errno = YAZ_ICONV_EILSEQ;
643         return 0;
644     }
645     else
646     {
647         unsigned char *inp;
648         size_t inbytesleft, no_read_sub = 0;
649         unsigned long x;
650
651         *utf8_outbuf = '\0';        
652         inp = (unsigned char *) utf8_buf;
653         inbytesleft = strlen(utf8_buf);
654         
655         x = yaz_marc8r_1_conv(inp, inbytesleft, &no_read_sub, comb);
656         if (x)
657         {
658             *page_chr = "\033(B";
659             return x;
660         }
661         x = yaz_marc8r_2_conv(inp, inbytesleft, &no_read_sub, comb);
662         if (x)
663         {
664             *page_chr = "\033g";
665             return x;
666         }
667         x = yaz_marc8r_3_conv(inp, inbytesleft, &no_read_sub, comb);
668         if (x)
669         {
670             *page_chr = "\033b";
671             return x;
672         }
673         x = yaz_marc8r_4_conv(inp, inbytesleft, &no_read_sub, comb);
674         if (x)
675         {
676             *page_chr = "\033p";
677             return x;
678         }
679         x = yaz_marc8r_5_conv(inp, inbytesleft, &no_read_sub, comb);
680         if (x)
681         {
682             *page_chr = "\033(2";
683             return x;
684         }
685         x = yaz_marc8r_6_conv(inp, inbytesleft, &no_read_sub, comb);
686         if (x)
687         {
688             *page_chr = "\033(N";
689             return x;
690         }
691         x = yaz_marc8r_7_conv(inp, inbytesleft, &no_read_sub, comb);
692         if (x)
693         {
694             *page_chr = "\033(3";
695             return x;
696         }
697         x = yaz_marc8r_8_conv(inp, inbytesleft, &no_read_sub, comb);
698         if (x)
699         {
700             *page_chr = "\033(S";
701             return x;
702         }
703         x = yaz_marc8r_9_conv(inp, inbytesleft, &no_read_sub, comb);
704         if (x)
705         {
706             *page_chr = "\033(1";
707             return x;
708         }
709         cd->my_errno = YAZ_ICONV_EILSEQ;
710         return x;
711     }
712 }
713
714 static size_t flush_combos(yaz_iconv_t cd,
715                            char **outbuf, size_t *outbytesleft)
716 {
717     unsigned long y = cd->write_marc8_last;
718     unsigned char byte, second_half = 0;
719     char out_buf[10];
720     size_t i, out_no = 0;
721
722     if (!y)
723         return 0;
724
725     byte = (unsigned char )((y>>16) & 0xff);
726     if (byte)
727         out_buf[out_no++] = byte;
728     byte = (unsigned char)((y>>8) & 0xff);
729     if (byte)
730         out_buf[out_no++] = byte;
731     byte = (unsigned char )(y & 0xff);
732     if (byte)
733         out_buf[out_no++] = byte;
734
735     if (out_no + cd->write_marc8_comb_no + 1 > *outbytesleft)
736     {
737         cd->my_errno = YAZ_ICONV_E2BIG;
738         return (size_t) (-1);
739     }
740
741     for (i = 0; i < cd->write_marc8_comb_no; i++)
742     {
743         /* all MARC-8 combined characters are simple bytes */
744         byte = (unsigned char )(cd->write_marc8_comb_ch[i]);
745         if (byte == 0xEB)
746             second_half = 0xEC;
747         else if (byte == 0xFA)
748             second_half = 0xFB;
749
750         *(*outbuf)++ = byte;
751         (*outbytesleft)--;
752     }
753     memcpy(*outbuf, out_buf, out_no);
754     *outbuf += out_no;
755     (*outbytesleft) -= out_no;
756     if (second_half)
757     {
758         *(*outbuf)++ = second_half;
759         (*outbytesleft)--;
760     }        
761
762     cd->write_marc8_last = 0;
763     cd->write_marc8_comb_no = 0;
764     return 0;
765 }
766
767 static size_t yaz_write_marc8(yaz_iconv_t cd, unsigned long x,
768                               char **outbuf, size_t *outbytesleft,
769                               int last)
770 {
771     int comb = 0;
772     const char *page_chr = 0;
773     unsigned long y = lookup_marc8(cd, x, &comb, &page_chr);
774
775     if (!y)
776         return (size_t) (-1);
777
778     if (comb)
779     {
780         if (cd->write_marc8_comb_no < 6)
781             cd->write_marc8_comb_ch[cd->write_marc8_comb_no++] = y;
782     }
783     else
784     {
785         size_t r = flush_combos(cd, outbuf, outbytesleft);
786         if (r)
787             return r;
788         if (strcmp(page_chr, cd->write_marc8_page_chr))
789         {
790             size_t plen = strlen(page_chr);
791
792             if (*outbytesleft < plen)
793             {
794                 cd->my_errno = YAZ_ICONV_E2BIG;
795                 return (size_t) (-1);
796             }
797             memcpy(*outbuf, page_chr, plen);
798             (*outbuf) += plen;
799             (*outbytesleft) -= plen;
800             cd->write_marc8_page_chr = page_chr;            
801         }
802         cd->write_marc8_last = y;
803     }
804     if (last)
805     {
806         size_t r = flush_combos(cd, outbuf, outbytesleft);
807         if (r)
808         {
809             if (comb)
810                 cd->write_marc8_comb_no--;
811             else
812                 cd->write_marc8_last = 0;
813             return r;
814         }
815     }
816     return 0;
817 }
818
819 #if HAVE_WCHAR_H
820 static size_t yaz_write_wchar_t (yaz_iconv_t cd, unsigned long x,
821                                  char **outbuf, size_t *outbytesleft,
822                                  int last)
823 {
824     unsigned char *outp = (unsigned char *) *outbuf;
825
826     if (*outbytesleft >= sizeof(wchar_t))
827     {
828         wchar_t wch = x;
829         memcpy(outp, &wch, sizeof(wch));
830         outp += sizeof(wch);
831         (*outbytesleft) -= sizeof(wch);
832     }
833     else
834     {
835         cd->my_errno = YAZ_ICONV_E2BIG;
836         return (size_t)(-1);
837     }
838     *outbuf = (char *) outp;
839     return 0;
840 }
841 #endif
842
843 int yaz_iconv_isbuiltin(yaz_iconv_t cd)
844 {
845     return cd->read_handle && cd->write_handle;
846 }
847
848 yaz_iconv_t yaz_iconv_open (const char *tocode, const char *fromcode)
849 {
850     yaz_iconv_t cd = (yaz_iconv_t) xmalloc (sizeof(*cd));
851
852     cd->write_handle = 0;
853     cd->read_handle = 0;
854     cd->init_handle = 0;
855     cd->my_errno = YAZ_ICONV_UNKNOWN;
856     cd->marc8_esc_mode = 'B';
857     cd->comb_offset = cd->comb_size = 0;
858     cd->compose_char = 0;
859
860     cd->write_marc8_comb_no = 0;
861     cd->write_marc8_last = 0;
862     cd->write_marc8_page_chr = "\033(B";
863
864     /* a useful hack: if fromcode has leading @,
865        the library not use YAZ's own conversions .. */
866     if (fromcode[0] == '@')
867         fromcode++;
868     else
869     {
870         if (!yaz_matchstr(fromcode, "UTF8"))
871         {
872             cd->read_handle = yaz_read_UTF8;
873             cd->init_handle = yaz_init_UTF8;
874         }
875         else if (!yaz_matchstr(fromcode, "ISO88591"))
876             cd->read_handle = yaz_read_ISO8859_1;
877         else if (!yaz_matchstr(fromcode, "UCS4"))
878             cd->read_handle = yaz_read_UCS4;
879         else if (!yaz_matchstr(fromcode, "UCS4LE"))
880             cd->read_handle = yaz_read_UCS4LE;
881         else if (!yaz_matchstr(fromcode, "MARC8"))
882             cd->read_handle = yaz_read_marc8;
883 #if HAVE_WCHAR_H
884         else if (!yaz_matchstr(fromcode, "WCHAR_T"))
885             cd->read_handle = yaz_read_wchar_t;
886 #endif
887         
888         if (!yaz_matchstr(tocode, "UTF8"))
889             cd->write_handle = yaz_write_UTF8;
890         else if (!yaz_matchstr(tocode, "ISO88591"))
891             cd->write_handle = yaz_write_ISO8859_1;
892         else if (!yaz_matchstr (tocode, "UCS4"))
893             cd->write_handle = yaz_write_UCS4;
894         else if (!yaz_matchstr(tocode, "UCS4LE"))
895             cd->write_handle = yaz_write_UCS4LE;
896         else if (!yaz_matchstr(tocode, "MARC8"))
897             cd->write_handle = yaz_write_marc8;
898 #if HAVE_WCHAR_H
899         else if (!yaz_matchstr(tocode, "WCHAR_T"))
900             cd->write_handle = yaz_write_wchar_t;
901 #endif
902     }
903 #if HAVE_ICONV_H
904     cd->iconv_cd = 0;
905     if (!cd->read_handle || !cd->write_handle)
906     {
907         cd->iconv_cd = iconv_open (tocode, fromcode);
908         if (cd->iconv_cd == (iconv_t) (-1))
909         {
910             xfree (cd);
911             return 0;
912         }
913     }
914 #else
915     if (!cd->read_handle || !cd->write_handle)
916     {
917         xfree (cd);
918         return 0;
919     }
920 #endif
921     cd->init_flag = 1;
922     return cd;
923 }
924
925 size_t yaz_iconv(yaz_iconv_t cd, char **inbuf, size_t *inbytesleft,
926                  char **outbuf, size_t *outbytesleft)
927 {
928     char *inbuf0;
929     size_t r = 0;
930
931 #if HAVE_ICONV_H
932     if (cd->iconv_cd)
933     {
934         size_t r =
935             iconv(cd->iconv_cd, inbuf, inbytesleft, outbuf, outbytesleft);
936         if (r == (size_t)(-1))
937         {
938             switch (yaz_errno())
939             {
940             case E2BIG:
941                 cd->my_errno = YAZ_ICONV_E2BIG;
942                 break;
943             case EINVAL:
944                 cd->my_errno = YAZ_ICONV_EINVAL;
945                 break;
946             case EILSEQ:
947                 cd->my_errno = YAZ_ICONV_EILSEQ;
948                 break;
949             default:
950                 cd->my_errno = YAZ_ICONV_UNKNOWN;
951             }
952         }
953         return r;
954     }
955 #endif
956     if (inbuf == 0 || *inbuf == 0)
957     {
958         cd->init_flag = 1;
959         cd->my_errno = YAZ_ICONV_UNKNOWN;
960         return 0;
961     }
962     inbuf0 = *inbuf;
963
964     if (cd->init_flag)
965     {
966         if (cd->init_handle)
967         {
968             size_t no_read;
969             size_t r = (cd->init_handle)(cd, (unsigned char *) *inbuf,
970                                          *inbytesleft, &no_read);
971             if (r)
972             {
973                 if (cd->my_errno == YAZ_ICONV_EINVAL)
974                     return r;
975                 cd->init_flag = 0;
976                 return r;
977             }
978             *inbytesleft -= no_read;
979             *inbuf += no_read;
980         }
981         cd->init_flag = 0;
982         cd->unget_x = 0;
983         cd->no_read_x = 0;
984     }
985     while (1)
986     {
987         unsigned long x;
988         size_t no_read;
989
990         if (*inbytesleft == 0)
991         {
992             r = *inbuf - inbuf0;
993             break;
994         }
995         if (!cd->unget_x)
996         {
997             x = (cd->read_handle)(cd, (unsigned char *) *inbuf, *inbytesleft,
998                                   &no_read);
999             if (no_read == 0)
1000             {
1001                 r = (size_t)(-1);
1002                 break;
1003             }
1004         }
1005         else
1006         {
1007             x = cd->unget_x;
1008             no_read = cd->no_read_x;
1009         }
1010         if (x)
1011         {
1012             r = (cd->write_handle)(cd, x, outbuf, outbytesleft,
1013                                    (*inbytesleft - no_read) == 0 ? 1 : 0);
1014             if (r)
1015             {
1016                 /* unable to write it. save it because read_handle cannot
1017                    rewind .. */
1018                 if (cd->my_errno == YAZ_ICONV_E2BIG)
1019                 {
1020                     cd->unget_x = x;
1021                     cd->no_read_x = no_read;
1022                     break;
1023                 }
1024             }
1025             cd->unget_x = 0;
1026         }
1027         *inbytesleft -= no_read;
1028         (*inbuf) += no_read;
1029     }
1030     return r;
1031 }
1032
1033 int yaz_iconv_error (yaz_iconv_t cd)
1034 {
1035     return cd->my_errno;
1036 }
1037
1038 int yaz_iconv_close (yaz_iconv_t cd)
1039 {
1040 #if HAVE_ICONV_H
1041     if (cd->iconv_cd)
1042         iconv_close (cd->iconv_cd);
1043 #endif
1044     xfree (cd);
1045     return 0;
1046 }
1047
1048 /*
1049  * Local variables:
1050  * c-basic-offset: 4
1051  * indent-tabs-mode: nil
1052  * End:
1053  * vim: shiftwidth=4 tabstop=8 expandtab
1054  */
1055