X-Git-Url: http://git.indexdata.com/?p=yaz-moved-to-github.git;a=blobdiff_plain;f=src%2Fsiconv.c;h=b6bcf0e466d31a5b09926e85173bdcfd4228110a;hp=9e5393b89cad8ddf4af040eabc5f90d0b8d8b553;hb=ee6ab2ee3a9ee1a8c65d7272ec7fba1d886f5af0;hpb=a51e14df3987e0c5328d0379c059b772aa4976fa

diff --git a/src/siconv.c b/src/siconv.c
index 9e5393b..b6bcf0e 100644
--- a/src/siconv.c
+++ b/src/siconv.c
@@ -1,23 +1,23 @@
-/*
- * Copyright (C) 1995-2005, Index Data ApS
+/* This file is part of the YAZ toolkit.
+ * Copyright (C) 1995-2008 Index Data
  * See the file LICENSE for details.
- *
- * $Id: siconv.c,v 1.15 2005-11-06 01:28:09 adam Exp $
  */
 /**
- * \file siconv.c
+ * \file
  * \brief Implements simple ICONV
  *
  * This implements an interface similar to that of iconv and
  * is used by YAZ to interface with iconv (if present).
  * For systems where iconv is not present, this layer
- * provides a few important conversion: UTF-8, MARC-8, Latin-1.
+ * provides a few important conversions: UTF-8, MARC-8, Latin-1.
+ *
  */
 
 #if HAVE_CONFIG_H
 #include <config.h>
 #endif
 
+#include <assert.h>
 #include <errno.h>
 #include <string.h>
 #include <ctype.h>
@@ -29,38 +29,32 @@
 #include <iconv.h>
 #endif
 
-#include <yaz/yaz-util.h>
-
-unsigned long yaz_marc8_1_conv (unsigned char *inp, size_t inbytesleft,
-                              size_t *no_read, int *combining);
-unsigned long yaz_marc8_2_conv (unsigned char *inp, size_t inbytesleft,
-                                size_t *no_read, int *combining);
-unsigned long yaz_marc8_3_conv (unsigned char *inp, size_t inbytesleft,
-                                size_t *no_read, int *combining);
-unsigned long yaz_marc8_4_conv (unsigned char *inp, size_t inbytesleft,
-                                size_t *no_read, int *combining);
-unsigned long yaz_marc8_5_conv (unsigned char *inp, size_t inbytesleft,
-                                size_t *no_read, int *combining);
-unsigned long yaz_marc8_6_conv (unsigned char *inp, size_t inbytesleft,
-                                size_t *no_read, int *combining);
-unsigned long yaz_marc8_7_conv (unsigned char *inp, size_t inbytesleft,
-                                size_t *no_read, int *combining);
-unsigned long yaz_marc8_8_conv (unsigned char *inp, size_t inbytesleft,
-                                size_t *no_read, int *combining);
-unsigned long yaz_marc8_9_conv (unsigned char *inp, size_t inbytesleft,
-                                size_t *no_read, int *combining);
-    
+#include <yaz/xmalloc.h>
+#include <yaz/nmem.h>
+#include "iconv-p.h"
+
+yaz_conv_func_t yaz_marc8_42_conv;
+yaz_conv_func_t yaz_marc8_45_conv;
+yaz_conv_func_t yaz_marc8_67_conv;
+yaz_conv_func_t yaz_marc8_62_conv;
+yaz_conv_func_t yaz_marc8_70_conv;
+yaz_conv_func_t yaz_marc8_32_conv;
+yaz_conv_func_t yaz_marc8_4E_conv;
+yaz_conv_func_t yaz_marc8_51_conv;
+yaz_conv_func_t yaz_marc8_33_conv;
+yaz_conv_func_t yaz_marc8_34_conv;
+yaz_conv_func_t yaz_marc8_53_conv;
+yaz_conv_func_t yaz_marc8_31_conv;
+
 struct yaz_iconv_struct {
     int my_errno;
     int init_flag;
     size_t (*init_handle)(yaz_iconv_t cd, unsigned char *inbuf,
-                          size_t inbytesleft, size_t *no_read);
+                            size_t inbytesleft, size_t *no_read);
     unsigned long (*read_handle)(yaz_iconv_t cd, unsigned char *inbuf,
                                  size_t inbytesleft, size_t *no_read);
-    size_t (*write_handle)(yaz_iconv_t cd, unsigned long x,
-                           char **outbuf, size_t *outbytesleft,
-                           int last);
-    int marc8_esc_mode;
+    int g0_mode;
+    int g1_mode;
 
     int comb_offset;
     int comb_size;
@@ -71,162 +65,21 @@ struct yaz_iconv_struct {
 #if HAVE_ICONV_H
     iconv_t iconv_cd;
 #endif
-    unsigned long compose_char;
+    struct yaz_iconv_encoder_s encoder;
 };
 
-static unsigned long yaz_read_ISO8859_1 (yaz_iconv_t cd, unsigned char *inp,
-                                         size_t inbytesleft, size_t *no_read)
+
+static unsigned long yaz_read_ISO8859_1(yaz_iconv_t cd, unsigned char *inp,
+                                        size_t inbytesleft, size_t *no_read)
 {
     unsigned long x = inp[0];
     *no_read = 1;
     return x;
 }
 
-static size_t yaz_init_UTF8 (yaz_iconv_t cd, unsigned char *inp,
-                             size_t inbytesleft, size_t *no_read)
-{
-    if (inp[0] != 0xef)
-    {
-        *no_read = 0;
-        return 0;
-    }
-    if (inbytesleft < 3)
-    {
-        cd->my_errno = YAZ_ICONV_EINVAL;
-        return (size_t) -1;
-    }
-    if (inp[1] != 0xbb || inp[2] != 0xbf)
-    {
-        cd->my_errno = YAZ_ICONV_EILSEQ;
-        return (size_t) -1;
-    }
-    *no_read = 3;
-    return 0;
-}
-
-static unsigned long yaz_read_UTF8 (yaz_iconv_t cd, unsigned char *inp,
-                                    size_t inbytesleft, size_t *no_read)
-{
-    unsigned long x = 0;
-
-    if (inp[0] <= 0x7f)
-    {
-        x = inp[0];
-        *no_read = 1;
-    }
-    else if (inp[0] <= 0xbf || inp[0] >= 0xfe)
-    {
-        *no_read = 0;
-        cd->my_errno = YAZ_ICONV_EILSEQ;
-    }
-    else if (inp[0] <= 0xdf && inbytesleft >= 2)
-    {
-        x = ((inp[0] & 0x1f) << 6) | (inp[1] & 0x3f);
-        if (x >= 0x80)
-            *no_read = 2;
-        else
-        {
-            *no_read = 0;
-            cd->my_errno = YAZ_ICONV_EILSEQ;
-        }
-    }
-    else if (inp[0] <= 0xef && inbytesleft >= 3)
-    {
-        x = ((inp[0] & 0x0f) << 12) | ((inp[1] & 0x3f) << 6) |
-            (inp[1] & 0x3f);
-        if (x >= 0x800)
-            *no_read = 3;
-        else
-        {
-            *no_read = 0;
-            cd->my_errno = YAZ_ICONV_EILSEQ;
-        }
-    }
-    else if (inp[0] <= 0xf7 && inbytesleft >= 4)
-    {
-        x =  ((inp[0] & 0x07) << 18) | ((inp[1] & 0x3f) << 12) |
-            ((inp[2] & 0x3f) << 6) | (inp[3] & 0x3f);
-        if (x >= 0x10000)
-            *no_read = 4;
-        else
-        {
-            *no_read = 0;
-            cd->my_errno = YAZ_ICONV_EILSEQ;
-        }
-    }
-    else if (inp[0] <= 0xfb && inbytesleft >= 5)
-    {
-        x =  ((inp[0] & 0x03) << 24) | ((inp[1] & 0x3f) << 18) |
-            ((inp[2] & 0x3f) << 12) | ((inp[3] & 0x3f) << 6) |
-            (inp[4] & 0x3f);
-        if (x >= 0x200000)
-            *no_read = 5;
-        else
-        {
-            *no_read = 0;
-            cd->my_errno = YAZ_ICONV_EILSEQ;
-        }
-    }
-    else if (inp[0] <= 0xfd && inbytesleft >= 6)
-    {
-        x =  ((inp[0] & 0x01) << 30) | ((inp[1] & 0x3f) << 24) |
-            ((inp[2] & 0x3f) << 18) | ((inp[3] & 0x3f) << 12) |
-            ((inp[4] & 0x3f) << 6) | (inp[5] & 0x3f);
-        if (x >= 0x4000000)
-            *no_read = 6;
-        else
-        {
-            *no_read = 0;
-            cd->my_errno = YAZ_ICONV_EILSEQ;
-        }
-    }
-    else
-    {
-        *no_read = 0;
-        cd->my_errno = YAZ_ICONV_EINVAL;
-    }
-    return x;
-}
-
-static unsigned long yaz_read_UCS4 (yaz_iconv_t cd, unsigned char *inp,
-                                    size_t inbytesleft, size_t *no_read)
-{
-    unsigned long x = 0;
-    
-    if (inbytesleft < 4)
-    {
-        cd->my_errno = YAZ_ICONV_EINVAL; /* incomplete input */
-        *no_read = 0;
-    }
-    else
-    {
-        x = (inp[0]<<24) | (inp[1]<<16) | (inp[2]<<8) | inp[3];
-        *no_read = 4;
-    }
-    return x;
-}
-
-static unsigned long yaz_read_UCS4LE (yaz_iconv_t cd, unsigned char *inp,
-                                      size_t inbytesleft, size_t *no_read)
-{
-    unsigned long x = 0;
-    
-    if (inbytesleft < 4)
-    {
-        cd->my_errno = YAZ_ICONV_EINVAL; /* incomplete input */
-        *no_read = 0;
-    }
-    else
-    {
-        x = (inp[3]<<24) | (inp[2]<<16) | (inp[1]<<8) | inp[0];
-        *no_read = 4;
-    }
-    return x;
-}
-
 #if HAVE_WCHAR_H
-static unsigned long yaz_read_wchar_t (yaz_iconv_t cd, unsigned char *inp,
-                                       size_t inbytesleft, size_t *no_read)
+static unsigned long yaz_read_wchar_t(yaz_iconv_t cd, unsigned char *inp,
+                                      size_t inbytesleft, size_t *no_read)
 {
     unsigned long x = 0;
     
@@ -238,7 +91,7 @@ static unsigned long yaz_read_wchar_t (yaz_iconv_t cd, unsigned char *inp,
     else
     {
         wchar_t wch;
-        memcpy (&wch, inp, sizeof(wch));
+        memcpy(&wch, inp, sizeof(wch));
         x = wch;
         *no_read = sizeof(wch);
     }
@@ -247,12 +100,12 @@ static unsigned long yaz_read_wchar_t (yaz_iconv_t cd, unsigned char *inp,
 #endif
 
 
-static unsigned long yaz_read_marc8_comb (yaz_iconv_t cd, unsigned char *inp,
-                                          size_t inbytesleft, size_t *no_read,
-                                          int *comb);
+static unsigned long yaz_read_marc8_comb(yaz_iconv_t cd, unsigned char *inp,
+                                         size_t inbytesleft, size_t *no_read,
+                                         int *comb);
 
-static unsigned long yaz_read_marc8 (yaz_iconv_t cd, unsigned char *inp,
-                                     size_t inbytesleft, size_t *no_read)
+static unsigned long yaz_read_marc8(yaz_iconv_t cd, unsigned char *inp,
+                                    size_t inbytesleft, size_t *no_read)
 {
     unsigned long x;
     if (cd->comb_offset < cd->comb_size)
@@ -265,6 +118,13 @@ static unsigned long yaz_read_marc8 (yaz_iconv_t cd, unsigned char *inp,
            We'll increment the no_read counter by 1, since we want to skip over
            the processing of the closing ligature character
         */
+        /* this code is no longer necessary.. our handlers code in
+           yaz_marc8_?_conv (generated by charconv.tcl) now returns
+           0 and no_read=1 when a sequence does not match the input.
+           The SECOND HALFs in codetables.xml produces a non-existant
+           entry in the conversion trie.. Hence when met, the input byte is
+           skipped as it should (in yaz_iconv)
+        */
 #if 0
         if (x == 0x0361 || x == 0x0360)
             *no_read += 1;
@@ -277,6 +137,14 @@ static unsigned long yaz_read_marc8 (yaz_iconv_t cd, unsigned char *inp,
     for (cd->comb_size = 0; cd->comb_size < 8; cd->comb_size++)
     {
         int comb = 0;
+
+        if (inbytesleft == 0 && cd->comb_size)
+        {
+            cd->my_errno = YAZ_ICONV_EINVAL;
+            x = 0;
+            *no_read = 0;
+            break;
+        }
         x = yaz_read_marc8_comb(cd, inp, inbytesleft, no_read, &comb);
         if (!comb || !x)
             break;
@@ -288,71 +156,119 @@ static unsigned long yaz_read_marc8 (yaz_iconv_t cd, unsigned char *inp,
     return x;
 }
 
-static unsigned long yaz_read_marc8_comb (yaz_iconv_t cd, unsigned char *inp,
-                                          size_t inbytesleft, size_t *no_read,
-                                          int *comb)
+static unsigned long yaz_read_marc8s(yaz_iconv_t cd, unsigned char *inp,
+                                     size_t inbytesleft, size_t *no_read)
+{
+    unsigned long x = yaz_read_marc8(cd, inp, inbytesleft, no_read);
+    if (x && cd->comb_size == 1)
+    {
+        if (yaz_iso_8859_1_lookup_x12(x, cd->comb_x[0], &x))
+        {
+            *no_read += cd->comb_no_read[0];
+            cd->comb_size = 0;
+        }
+    }
+    return x;
+}
+
+static unsigned long yaz_read_marc8_comb(yaz_iconv_t cd, unsigned char *inp,
+                                         size_t inbytesleft, size_t *no_read,
+                                         int *comb)
 {
     *no_read = 0;
-    while(inbytesleft >= 1 && inp[0] == 27)
+    while (inbytesleft > 0 && *inp == 27)
     {
+        int *modep = &cd->g0_mode;
         size_t inbytesleft0 = inbytesleft;
-        inp++;
+
         inbytesleft--;
-        while(inbytesleft > 0 && strchr("(,$!", *inp))
+        inp++;
+        if (inbytesleft == 0)
+            goto incomplete;
+        if (*inp == '$') /* set with multiple bytes */
         {
             inbytesleft--;
             inp++;
         }
-        if (inbytesleft <= 0)
+        if (inbytesleft == 0)
+            goto incomplete;
+        if (*inp == '(' || *inp == ',')  /* G0 */
         {
-            *no_read = 0;
-            cd->my_errno = YAZ_ICONV_EINVAL;
-            return 0;
+            inbytesleft--;
+            inp++;
+        }
+        else if (*inp == ')' || *inp == '-') /* G1 */
+        {
+            inbytesleft--;
+            inp++;
+            modep = &cd->g1_mode;
+        }
+        if (inbytesleft == 0)
+            goto incomplete;
+        if (*inp == '!') /* ANSEL is a special case */
+        {
+            inbytesleft--;
+            inp++;
         }
-        cd->marc8_esc_mode = *inp++;
+        if (inbytesleft == 0)
+            goto incomplete;
+        *modep = *inp++; /* Final character */
         inbytesleft--;
+
         (*no_read) += inbytesleft0 - inbytesleft;
     }
-    if (inbytesleft <= 0)
+    if (inbytesleft == 0)
         return 0;
+    else if (*inp == ' ')
+    {
+        *no_read += 1;
+        return ' ';
+    }
     else
     {
         unsigned long x;
         size_t no_read_sub = 0;
+        int mode = *inp < 128 ? cd->g0_mode : cd->g1_mode;
         *comb = 0;
 
-        switch(cd->marc8_esc_mode)
+        switch(mode)
         {
         case 'B':  /* Basic ASCII */
-        case 'E':  /* ANSEL */
         case 's':  /* ASCII */
-            x = yaz_marc8_1_conv(inp, inbytesleft, &no_read_sub, comb);
+            x = yaz_marc8_42_conv(inp, inbytesleft, &no_read_sub, comb, 127, 0);
+            break;
+        case 'E':  /* ANSEL */
+            x = yaz_marc8_45_conv(inp, inbytesleft, &no_read_sub, comb, 127, 128);
             break;
         case 'g':  /* Greek */
-            x = yaz_marc8_2_conv(inp, inbytesleft, &no_read_sub, comb);
+            x = yaz_marc8_67_conv(inp, inbytesleft, &no_read_sub, comb, 127, 0);
             break;
         case 'b':  /* Subscripts */
-            x = yaz_marc8_3_conv(inp, inbytesleft, &no_read_sub, comb);
+            x = yaz_marc8_62_conv(inp, inbytesleft, &no_read_sub, comb, 127, 0);
             break;
         case 'p':  /* Superscripts */
-            x = yaz_marc8_4_conv(inp, inbytesleft, &no_read_sub, comb);
+            x = yaz_marc8_70_conv(inp, inbytesleft, &no_read_sub, comb, 127, 0);
             break;
         case '2':  /* Basic Hebrew */
-            x = yaz_marc8_5_conv(inp, inbytesleft, &no_read_sub, comb);
+            x = yaz_marc8_32_conv(inp, inbytesleft, &no_read_sub, comb, 127, 0);
             break;
         case 'N':  /* Basic Cyrillic */
+            x = yaz_marc8_4E_conv(inp, inbytesleft, &no_read_sub, comb, 127, 0);
+            break;
         case 'Q':  /* Extended Cyrillic */
-            x = yaz_marc8_6_conv(inp, inbytesleft, &no_read_sub, comb);
+            x = yaz_marc8_51_conv(inp, inbytesleft, &no_read_sub, comb, 127, 0);
             break;
         case '3':  /* Basic Arabic */
+            x = yaz_marc8_33_conv(inp, inbytesleft, &no_read_sub, comb, 127, 0);
+            break;
         case '4':  /* Extended Arabic */
-            x = yaz_marc8_7_conv(inp, inbytesleft, &no_read_sub, comb);
+            x = yaz_marc8_34_conv(inp, inbytesleft, &no_read_sub, comb, 127, 0);
             break;
         case 'S':  /* Greek */
-            x = yaz_marc8_8_conv(inp, inbytesleft, &no_read_sub, comb);
+            x = yaz_marc8_53_conv(inp, inbytesleft, &no_read_sub, comb, 127, 0);
             break;
         case '1':  /* Chinese, Japanese, Korean (EACC) */
-            x = yaz_marc8_9_conv(inp, inbytesleft, &no_read_sub, comb);
+            x = yaz_marc8_31_conv(inp, inbytesleft, &no_read_sub, comb, 127, 0);
             break;
         default:
             *no_read = 0;
@@ -362,291 +278,52 @@ static unsigned long yaz_read_marc8_comb (yaz_iconv_t cd, unsigned char *inp,
         *no_read += no_read_sub;
         return x;
     }
-}
-
-static size_t yaz_write_UTF8 (yaz_iconv_t cd, unsigned long x,
-                              char **outbuf, size_t *outbytesleft,
-                              int last)
-{
-    unsigned char *outp = (unsigned char *) *outbuf;
-    if (x <= 0x7f && *outbytesleft >= 1)
-    {
-        *outp++ = (unsigned char) x;
-        (*outbytesleft)--;
-    } 
-    else if (x <= 0x7ff && *outbytesleft >= 2)
-    {
-        *outp++ = (unsigned char) ((x >> 6) | 0xc0);
-        *outp++ = (unsigned char) ((x & 0x3f) | 0x80);
-        (*outbytesleft) -= 2;
-    }
-    else if (x <= 0xffff && *outbytesleft >= 3)
-    {
-        *outp++ = (unsigned char) ((x >> 12) | 0xe0);
-        *outp++ = (unsigned char) (((x >> 6) & 0x3f) | 0x80);
-        *outp++ = (unsigned char) ((x & 0x3f) | 0x80);
-        (*outbytesleft) -= 3;
-    }
-    else if (x <= 0x1fffff && *outbytesleft >= 4)
-    {
-        *outp++ = (unsigned char) ((x >> 18) | 0xf0);
-        *outp++ = (unsigned char) (((x >> 12) & 0x3f) | 0x80);
-        *outp++ = (unsigned char) (((x >> 6)  & 0x3f) | 0x80);
-        *outp++ = (unsigned char) ((x & 0x3f) | 0x80);
-        (*outbytesleft) -= 4;
-    }
-    else if (x <= 0x3ffffff && *outbytesleft >= 5)
-    {
-        *outp++ = (unsigned char) ((x >> 24) | 0xf8);
-        *outp++ = (unsigned char) (((x >> 18) & 0x3f) | 0x80);
-        *outp++ = (unsigned char) (((x >> 12) & 0x3f) | 0x80);
-        *outp++ = (unsigned char) (((x >> 6)  & 0x3f) | 0x80);
-        *outp++ = (unsigned char) ((x & 0x3f) | 0x80);
-        (*outbytesleft) -= 5;
-    }
-    else if (*outbytesleft >= 6)
-    {
-        *outp++ = (unsigned char) ((x >> 30) | 0xfc);
-        *outp++ = (unsigned char) (((x >> 24) & 0x3f) | 0x80);
-        *outp++ = (unsigned char) (((x >> 18) & 0x3f) | 0x80);
-        *outp++ = (unsigned char) (((x >> 12) & 0x3f) | 0x80);
-        *outp++ = (unsigned char) (((x >> 6)  & 0x3f) | 0x80);
-        *outp++ = (unsigned char) ((x & 0x3f) | 0x80);
-        (*outbytesleft) -= 6;
-    }
-    else 
-    {
-        cd->my_errno = YAZ_ICONV_E2BIG;  /* not room for output */
-        return (size_t)(-1);
-    }
-    *outbuf = (char *) outp;
+incomplete:
+    *no_read = 0;
+    cd->my_errno = YAZ_ICONV_EINVAL;
     return 0;
 }
 
 
-static size_t yaz_write_ISO8859_1 (yaz_iconv_t cd, unsigned long x,
-                                   char **outbuf, size_t *outbytesleft,
-                                   int last)
-{
-    /* list of two char unicode sequence that, when combined, are
-       equivalent to single unicode chars that can be represented in
-       ISO-8859-1/Latin-1.
-       Regular iconv on Linux at least does not seem to convert these,
-       but since MARC-8 to UTF-8 generates these composed sequence
-       we get a better chance of a successful MARC-8 -> ISO-8859-1
-       conversion */
-    static struct {
-        unsigned long x1, x2;
-        unsigned y;
-    } comb[] = {
-        { 'A', 0x0300, 0xc0}, /* LATIN CAPITAL LETTER A WITH GRAVE */
-        { 'A', 0x0301, 0xc1}, /* LATIN CAPITAL LETTER A WITH ACUTE */
-        { 'A', 0x0302, 0xc2}, /* LATIN CAPITAL LETTER A WITH CIRCUMFLEX */
-        { 'A', 0x0303, 0xc3}, /* LATIN CAPITAL LETTER A WITH TILDE */
-        { 'A', 0x0308, 0xc4}, /* LATIN CAPITAL LETTER A WITH DIAERESIS */
-        { 'A', 0x030a, 0xc5}, /* LATIN CAPITAL LETTER A WITH RING ABOVE */
-        /* no need for 0xc6      LATIN CAPITAL LETTER AE */
-        { 'C', 0x0327, 0xc7}, /* LATIN CAPITAL LETTER C WITH CEDILLA */
-        { 'E', 0x0300, 0xc8}, /* LATIN CAPITAL LETTER E WITH GRAVE */
-        { 'E', 0x0301, 0xc9}, /* LATIN CAPITAL LETTER E WITH ACUTE */
-        { 'E', 0x0302, 0xca}, /* LATIN CAPITAL LETTER E WITH CIRCUMFLEX */
-        { 'E', 0x0308, 0xcb}, /* LATIN CAPITAL LETTER E WITH DIAERESIS */
-        { 'I', 0x0300, 0xcc}, /* LATIN CAPITAL LETTER I WITH GRAVE */
-        { 'I', 0x0301, 0xcd}, /* LATIN CAPITAL LETTER I WITH ACUTE */
-        { 'I', 0x0302, 0xce}, /* LATIN CAPITAL LETTER I WITH CIRCUMFLEX */
-        { 'I', 0x0308, 0xcf}, /* LATIN CAPITAL LETTER I WITH DIAERESIS */
-        { 'N', 0x0303, 0xd1}, /* LATIN CAPITAL LETTER N WITH TILDE */
-        { 'O', 0x0300, 0xd2}, /* LATIN CAPITAL LETTER O WITH GRAVE */
-        { 'O', 0x0301, 0xd3}, /* LATIN CAPITAL LETTER O WITH ACUTE */
-        { 'O', 0x0302, 0xd4}, /* LATIN CAPITAL LETTER O WITH CIRCUMFLEX */
-        { 'O', 0x0303, 0xd5}, /* LATIN CAPITAL LETTER O WITH TILDE */
-        { 'O', 0x0308, 0xd6}, /* LATIN CAPITAL LETTER O WITH DIAERESIS */
-        /* omitted:    0xd7      MULTIPLICATION SIGN */
-        /* omitted:    0xd8      LATIN CAPITAL LETTER O WITH STROKE */
-        { 'U', 0x0300, 0xd9}, /* LATIN CAPITAL LETTER U WITH GRAVE */
-        { 'U', 0x0301, 0xda}, /* LATIN CAPITAL LETTER U WITH ACUTE */
-        { 'U', 0x0302, 0xdb}, /* LATIN CAPITAL LETTER U WITH CIRCUMFLEX */
-        { 'U', 0x0308, 0xdc}, /* LATIN CAPITAL LETTER U WITH DIAERESIS */
-        { 'Y', 0x0301, 0xdd}, /* LATIN CAPITAL LETTER Y WITH ACUTE */
-        /* omitted:    0xde      LATIN CAPITAL LETTER THORN */
-        /* omitted:    0xdf      LATIN SMALL LETTER SHARP S */
-        { 'a', 0x0300, 0xe0}, /* LATIN SMALL LETTER A WITH GRAVE */
-        { 'a', 0x0301, 0xe1}, /* LATIN SMALL LETTER A WITH ACUTE */
-        { 'a', 0x0302, 0xe2}, /* LATIN SMALL LETTER A WITH CIRCUMFLEX */
-        { 'a', 0x0303, 0xe3}, /* LATIN SMALL LETTER A WITH TILDE */
-        { 'a', 0x0308, 0xe4}, /* LATIN SMALL LETTER A WITH DIAERESIS */
-        { 'a', 0x030a, 0xe5}, /* LATIN SMALL LETTER A WITH RING ABOVE */
-        /* omitted:    0xe6      LATIN SMALL LETTER AE */
-        { 'c', 0x0327, 0xe7}, /* LATIN SMALL LETTER C WITH CEDILLA */
-        { 'e', 0x0300, 0xe8}, /* LATIN SMALL LETTER E WITH GRAVE */
-        { 'e', 0x0301, 0xe9}, /* LATIN SMALL LETTER E WITH ACUTE */
-        { 'e', 0x0302, 0xea}, /* LATIN SMALL LETTER E WITH CIRCUMFLEX */
-        { 'e', 0x0308, 0xeb}, /* LATIN SMALL LETTER E WITH DIAERESIS */
-        { 'i', 0x0300, 0xec}, /* LATIN SMALL LETTER I WITH GRAVE */
-        { 'i', 0x0301, 0xed}, /* LATIN SMALL LETTER I WITH ACUTE */
-        { 'i', 0x0302, 0xee}, /* LATIN SMALL LETTER I WITH CIRCUMFLEX */
-        { 'i', 0x0308, 0xef}, /* LATIN SMALL LETTER I WITH DIAERESIS */
-        /* omitted:    0xf0      LATIN SMALL LETTER ETH */
-        { 'n', 0x0303, 0xf1}, /* LATIN SMALL LETTER N WITH TILDE */
-        { 'o', 0x0300, 0xf2}, /* LATIN SMALL LETTER O WITH GRAVE */
-        { 'o', 0x0301, 0xf3}, /* LATIN SMALL LETTER O WITH ACUTE */
-        { 'o', 0x0302, 0xf4}, /* LATIN SMALL LETTER O WITH CIRCUMFLEX */
-        { 'o', 0x0303, 0xf5}, /* LATIN SMALL LETTER O WITH TILDE */
-        { 'o', 0x0308, 0xf6}, /* LATIN SMALL LETTER O WITH DIAERESIS */
-        /* omitted:    0xf7      DIVISION SIGN */
-        /* omitted:    0xf8      LATIN SMALL LETTER O WITH STROKE */
-        { 'u', 0x0300, 0xf9}, /* LATIN SMALL LETTER U WITH GRAVE */
-        { 'u', 0x0301, 0xfa}, /* LATIN SMALL LETTER U WITH ACUTE */
-        { 'u', 0x0302, 0xfb}, /* LATIN SMALL LETTER U WITH CIRCUMFLEX */
-        { 'u', 0x0308, 0xfc}, /* LATIN SMALL LETTER U WITH DIAERESIS */
-        { 'y', 0x0301, 0xfd}, /* LATIN SMALL LETTER Y WITH ACUTE */
-        /* omitted:    0xfe      LATIN SMALL LETTER THORN */
-        { 'y', 0x0308, 0xff}, /* LATIN SMALL LETTER Y WITH DIAERESIS */
-        
-        { 0, 0, 0}
-    };
-    unsigned char *outp = (unsigned char *) *outbuf;
-
-    if (!last && x > 32 && x < 127 && cd->compose_char == 0)
-    {
-        cd->compose_char = x;
-        return 0;
-    }
-    else if (cd->compose_char)
-    {
-        int i;
-        for (i = 0; comb[i].x1; i++)
-            if (cd->compose_char == comb[i].x1 && x == comb[i].x2)
-            {
-                x = comb[i].y;
-                break;
-            }
-        if (!comb[i].x1) 
-        {   /* not found */
-            if (*outbytesleft >= 1)
-            {
-                *outp++ = (unsigned char) cd->compose_char;
-                (*outbytesleft)--;
-                *outbuf = (char *) outp;
-                if (!last && x > 32 && x < 127)
-                {
-                    cd->compose_char = x;
-                    return 0;
-                }
-            }
-            else
-            {
-                cd->my_errno = YAZ_ICONV_E2BIG;
-                return (size_t)(-1);
-            }
-        }
-        /* compose_char and old x combined to one new char: x */
-        cd->compose_char = 0;
-    }
-    if (x > 255 || x < 1)
-    {
-        cd->my_errno = YAZ_ICONV_EILSEQ;
-        return (size_t) -1;
-    }
-    else if (*outbytesleft >= 1)
-    {
-        *outp++ = (unsigned char) x;
-        (*outbytesleft)--;
-    }
-    else 
-    {
-        cd->my_errno = YAZ_ICONV_E2BIG;
-        return (size_t)(-1);
-    }
-    *outbuf = (char *) outp;
-    return 0;
-}
-
 
-static size_t yaz_write_UCS4 (yaz_iconv_t cd, unsigned long x,
-                              char **outbuf, size_t *outbytesleft,
-                              int last)
+int yaz_iconv_isbuiltin(yaz_iconv_t cd)
 {
-    unsigned char *outp = (unsigned char *) *outbuf;
-    if (*outbytesleft >= 4)
-    {
-        *outp++ = (unsigned char) (x>>24);
-        *outp++ = (unsigned char) (x>>16);
-        *outp++ = (unsigned char) (x>>8);
-        *outp++ = (unsigned char) x;
-        (*outbytesleft) -= 4;
-    }
-    else
-    {
-        cd->my_errno = YAZ_ICONV_E2BIG;
-        return (size_t)(-1);
-    }
-    *outbuf = (char *) outp;
-    return 0;
+    return cd->read_handle && cd->encoder.write_handle;
 }
 
-static size_t yaz_write_UCS4LE (yaz_iconv_t cd, unsigned long x,
-                                char **outbuf, size_t *outbytesleft,
-                                int last)
-{
-    unsigned char *outp = (unsigned char *) *outbuf;
-    if (*outbytesleft >= 4)
-    {
-        *outp++ = (unsigned char) x;
-        *outp++ = (unsigned char) (x>>8);
-        *outp++ = (unsigned char) (x>>16);
-        *outp++ = (unsigned char) (x>>24);
-        (*outbytesleft) -= 4;
-    }
-    else
-    {
-        cd->my_errno = YAZ_ICONV_E2BIG;
-        return (size_t)(-1);
-    }
-    *outbuf = (char *) outp;
-    return 0;
-}
 
-#if HAVE_WCHAR_H
-static size_t yaz_write_wchar_t (yaz_iconv_t cd, unsigned long x,
-                                 char **outbuf, size_t *outbytesleft,
-                                 int last)
+static int prepare_encoders(yaz_iconv_t cd, const char *tocode)
 {
-    unsigned char *outp = (unsigned char *) *outbuf;
-
-    if (*outbytesleft >= sizeof(wchar_t))
-    {
-        wchar_t wch = x;
-        memcpy(outp, &wch, sizeof(wch));
-        outp += sizeof(wch);
-        (*outbytesleft) -= sizeof(wch);
-    }
-    else
-    {
-        cd->my_errno = YAZ_ICONV_E2BIG;
-        return (size_t)(-1);
-    }
-    *outbuf = (char *) outp;
+    if (yaz_marc8_encoder(tocode, &cd->encoder))
+        return 1;
+    if (yaz_utf8_encoder(tocode, &cd->encoder))
+        return 1;
+    if (yaz_ucs4_encoder(tocode, &cd->encoder))
+        return 1;
+    if (yaz_iso_8859_1_encoder(tocode, &cd->encoder))
+        return 1;
+    if (yaz_iso_5428_encoder(tocode, &cd->encoder))
+        return 1;
+    if (yaz_advancegreek_encoder(tocode, &cd->encoder))
+        return 1;
+    if (yaz_wchar_encoder(tocode, &cd->encoder))
+        return 1;
     return 0;
 }
-#endif
 
-int yaz_iconv_isbuiltin(yaz_iconv_t cd)
-{
-    return cd->read_handle && cd->write_handle;
-}
-
-yaz_iconv_t yaz_iconv_open (const char *tocode, const char *fromcode)
+yaz_iconv_t yaz_iconv_open(const char *tocode, const char *fromcode)
 {
     yaz_iconv_t cd = (yaz_iconv_t) xmalloc (sizeof(*cd));
 
-    cd->write_handle = 0;
+    cd->encoder.data = 0;
+    cd->encoder.write_handle = 0;
+    cd->encoder.flush_handle = 0;
+    cd->encoder.init_handle = 0;
+    cd->encoder.destroy_handle = 0;
+
     cd->read_handle = 0;
     cd->init_handle = 0;
     cd->my_errno = YAZ_ICONV_UNKNOWN;
-    cd->marc8_esc_mode = 'B';
-    cd->comb_offset = cd->comb_size = 0;
-    cd->compose_char = 0;
 
     /* a useful hack: if fromcode has leading @,
        the library not use YAZ's own conversions .. */
@@ -667,42 +344,41 @@ yaz_iconv_t yaz_iconv_open (const char *tocode, const char *fromcode)
             cd->read_handle = yaz_read_UCS4LE;
         else if (!yaz_matchstr(fromcode, "MARC8"))
             cd->read_handle = yaz_read_marc8;
+        else if (!yaz_matchstr(fromcode, "MARC8s"))
+            cd->read_handle = yaz_read_marc8s;
+        else if (!yaz_matchstr(fromcode, "advancegreek"))
+            cd->read_handle = yaz_read_advancegreek;
+        else if (!yaz_matchstr(fromcode, "iso54281984"))
+            cd->read_handle = yaz_read_iso5428_1984;
+        else if (!yaz_matchstr(fromcode, "iso5428:1984"))
+            cd->read_handle = yaz_read_iso5428_1984;
 #if HAVE_WCHAR_H
         else if (!yaz_matchstr(fromcode, "WCHAR_T"))
             cd->read_handle = yaz_read_wchar_t;
 #endif
-        
-        if (!yaz_matchstr(tocode, "UTF8"))
-            cd->write_handle = yaz_write_UTF8;
-        else if (!yaz_matchstr(tocode, "ISO88591"))
-            cd->write_handle = yaz_write_ISO8859_1;
-        else if (!yaz_matchstr (tocode, "UCS4"))
-            cd->write_handle = yaz_write_UCS4;
-        else if (!yaz_matchstr(tocode, "UCS4LE"))
-            cd->write_handle = yaz_write_UCS4LE;
-#if HAVE_WCHAR_H
-        else if (!yaz_matchstr(tocode, "WCHAR_T"))
-            cd->write_handle = yaz_write_wchar_t;
-#endif
+        prepare_encoders(cd, tocode);
     }
+    if (cd->read_handle && cd->encoder.write_handle)
+    {
 #if HAVE_ICONV_H
-    cd->iconv_cd = 0;
-    if (!cd->read_handle || !cd->write_handle)
+        cd->iconv_cd = 0;
+#endif
+        ;
+    }
+    else
     {
-        cd->iconv_cd = iconv_open (tocode, fromcode);
+#if HAVE_ICONV_H
+        cd->iconv_cd = iconv_open(tocode, fromcode);
         if (cd->iconv_cd == (iconv_t) (-1))
         {
-            xfree (cd);
+            yaz_iconv_close(cd);
             return 0;
         }
-    }
 #else
-    if (!cd->read_handle || !cd->write_handle)
-    {
-        xfree (cd);
+        yaz_iconv_close(cd);
         return 0;
-    }
 #endif
+    }
     cd->init_flag = 1;
     return cd;
 }
@@ -710,8 +386,9 @@ yaz_iconv_t yaz_iconv_open (const char *tocode, const char *fromcode)
 size_t yaz_iconv(yaz_iconv_t cd, char **inbuf, size_t *inbytesleft,
                  char **outbuf, size_t *outbytesleft)
 {
-    char *inbuf0;
+    char *inbuf0 = 0;
     size_t r = 0;
+
 #if HAVE_ICONV_H
     if (cd->iconv_cd)
     {
@@ -737,19 +414,27 @@ size_t yaz_iconv(yaz_iconv_t cd, char **inbuf, size_t *inbytesleft,
         return r;
     }
 #endif
-    if (inbuf == 0 || *inbuf == 0)
-    {
-        cd->init_flag = 1;
-        cd->my_errno = YAZ_ICONV_UNKNOWN;
-        return 0;
-    }
-    inbuf0 = *inbuf;
+
+    if (inbuf)
+        inbuf0 = *inbuf;
 
     if (cd->init_flag)
     {
-        if (cd->init_handle)
+        cd->my_errno = YAZ_ICONV_UNKNOWN;
+        cd->g0_mode = 'B';
+        cd->g1_mode = 'E';
+        
+        cd->comb_offset = cd->comb_size = 0;
+
+        if (cd->encoder.init_handle)
+            (*cd->encoder.init_handle)(&cd->encoder);
+        
+        cd->unget_x = 0;
+        cd->no_read_x = 0;
+
+        if (cd->init_handle && inbuf && *inbuf)
         {
-            size_t no_read;
+            size_t no_read = 0;
             size_t r = (cd->init_handle)(cd, (unsigned char *) *inbuf,
                                          *inbytesleft, &no_read);
             if (r)
@@ -762,46 +447,64 @@ size_t yaz_iconv(yaz_iconv_t cd, char **inbuf, size_t *inbytesleft,
             *inbytesleft -= no_read;
             *inbuf += no_read;
         }
-        cd->init_flag = 0;
+    }
+    cd->init_flag = 0;
+
+    if (!inbuf || !*inbuf)
+    {
+        if (outbuf && *outbuf)
+        {
+            if (cd->unget_x)
+                r = (*cd->encoder.write_handle)(cd, &cd->encoder,
+                                                cd->unget_x, outbuf, outbytesleft);
+            if (cd->encoder.flush_handle)
+                r = (*cd->encoder.flush_handle)(cd, &cd->encoder,
+                                                outbuf, outbytesleft);
+        }
+        if (r == 0)
+            cd->init_flag = 1;
         cd->unget_x = 0;
-        cd->no_read_x = 0;
+        return r;
     }
     while (1)
     {
         unsigned long x;
         size_t no_read;
 
-        if (*inbytesleft == 0)
+        if (cd->unget_x)
         {
-            r = *inbuf - inbuf0;
-            break;
+            x = cd->unget_x;
+            no_read = cd->no_read_x;
         }
-        if (!cd->unget_x)
+        else
         {
-            x = (cd->read_handle)(cd, (unsigned char *) *inbuf, *inbytesleft,
-                                  &no_read);
+            if (*inbytesleft == 0)
+            {
+                r = *inbuf - inbuf0;
+                break;
+            }
+            x = (*cd->read_handle)(cd, (unsigned char *) *inbuf, *inbytesleft,
+                                   &no_read);
             if (no_read == 0)
             {
                 r = (size_t)(-1);
                 break;
             }
         }
-        else
-        {
-            x = cd->unget_x;
-            no_read = cd->no_read_x;
-        }
         if (x)
         {
-            r = (cd->write_handle)(cd, x, outbuf, outbytesleft,
-                                   (*inbytesleft - no_read) == 0 ? 1 : 0);
+            r = (*cd->encoder.write_handle)(cd, &cd->encoder,
+                                            x, outbuf, outbytesleft);
             if (r)
             {
                 /* unable to write it. save it because read_handle cannot
                    rewind .. */
-                cd->unget_x = x;
-                cd->no_read_x = no_read;
-                break;
+                if (cd->my_errno == YAZ_ICONV_E2BIG)
+                {
+                    cd->unget_x = x;
+                    cd->no_read_x = no_read;
+                    break;
+                }
             }
             cd->unget_x = 0;
         }
@@ -811,22 +514,28 @@ size_t yaz_iconv(yaz_iconv_t cd, char **inbuf, size_t *inbytesleft,
     return r;
 }
 
-int yaz_iconv_error (yaz_iconv_t cd)
+int yaz_iconv_error(yaz_iconv_t cd)
 {
     return cd->my_errno;
 }
 
-int yaz_iconv_close (yaz_iconv_t cd)
+int yaz_iconv_close(yaz_iconv_t cd)
 {
 #if HAVE_ICONV_H
     if (cd->iconv_cd)
-        iconv_close (cd->iconv_cd);
+        iconv_close(cd->iconv_cd);
 #endif
-    xfree (cd);
+    if (cd->encoder.destroy_handle)
+        (*cd->encoder.destroy_handle)(&cd->encoder);
+    xfree(cd);
     return 0;
 }
 
-    
+void yaz_iconv_set_errno(yaz_iconv_t cd, int no)
+{
+    cd->my_errno = no;
+}
+
 /*
  * Local variables:
  * c-basic-offset: 4
@@ -834,4 +543,3 @@ int yaz_iconv_close (yaz_iconv_t cd)
  * End:
  * vim: shiftwidth=4 tabstop=8 expandtab
  */
-