From: Adam Dickmeiss <adam@indexdata.dk>
Date: Mon, 16 Dec 2002 13:13:53 +0000 (+0000)
Subject: MARC8 decoding for yaz_iconv.
X-Git-Tag: YAZ.1.9.2.oleg
X-Git-Url: http://git.indexdata.com/?p=yaz-moved-to-github.git;a=commitdiff_plain;h=1121eeb134f61c6c2510143858db59045c1b6757

MARC8 decoding for yaz_iconv.
New MARC API. Utility marcdump allows you to specify source and
destination encoding.
YAZ client tries to convert MARC characters to native character
encoding for output terminal (some unices only).
---

diff --git a/CHANGELOG b/CHANGELOG
index 48d6b66..0ecad56 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,5 +1,20 @@
 Possible compatibility problems with earlier versions marked with '*'.
 
+--- 1.9.3 2002/MM/DD
+
+New MARC decode API. All new functions operate on a yaz_marc_t
+handle. The most important new functions are: yaz_marc_create,
+yaz_marc_decode_{buf,wrbuf}, yaz_marc_destroy, .. to create handler,
+decode and destroy respectively. Decoder can produce formats
+MARC line, simple XML, OAI MARC and MARC XML (LoC). See
+include/yaz/marcdisp.h.
+
+YAZ Iconv utility now supports MARC8 decoding (marc8.c). Converts
+to UTF-8, UCS-32, wchar_t or Latin-1.
+
+* Prototypes for yaz_iconv_-functions moved to separate header 
+include/yaz/yaz-iconv.h.
+
 Make a few private functions 'static' in unix.c. This prevents
 duplicate unix_close in PHP with YAZ and imap. PHP Bug 20977.
 
@@ -15,9 +30,6 @@ strerror/strerror_r/GetLastMessage.
 * ZOOM_record_get supports type "xml" in which case MARC XML (from LOC)
 is returned for MARC. If type is "oai", then OAI MARC is returned.
 
-yaz_marc_decode supports MARC XML as well. The xml parameter
-specifies type. See include/yaz/marcdisp.h
-
 Fix creation of lib/yaz-config so it works if srcdir != objdir. Patch
 from Kang-Jin Lee.
 
diff --git a/client/client.c b/client/client.c
index 4e77d74..3cfa3d5 100644
--- a/client/client.c
+++ b/client/client.c
@@ -2,11 +2,19 @@
  * Copyright (c) 1995-2002, Index Data
  * See the file LICENSE for details.
  *
- * $Id: client.c,v 1.175 2002-12-05 12:19:23 adam Exp $
+ * $Id: client.c,v 1.176 2002-12-16 13:13:53 adam Exp $
  */
 
 #include <stdio.h>
 #include <stdlib.h>
+#if HAVE_LOCALE_H
+#include <locale.h>
+#endif
+
+#if HAVE_LANGINFO_H
+#include <langinfo.h>
+#endif
+
 #include <time.h>
 #include <ctype.h>
 
@@ -49,6 +57,8 @@
 
 #define C_PROMPT "Z> "
 
+static char *codeset = 0;               /* character set for output */
+
 static ODR out, in, print;              /* encoding and decoding streams */
 static FILE *apdu_file = 0;
 static COMSTACK conn = 0;               /* our z-association */
@@ -156,8 +166,6 @@ void do_hex_dump(char* buf,int len)
 #endif
 }
 
-
-
 void add_otherInfos(Z_APDU *a) 
 {
     Z_OtherInformation **oi;
@@ -651,13 +659,65 @@ static void display_record(Z_External *r)
 #endif
                 )
             {
-                if (marc_display_exl (octet_buf, NULL, 0 /* debug */,
-                                      r->u.octet_aligned->len) <= 0)
+                char *result;
+                int rlen;
+                yaz_iconv_t cd = 0;
+                yaz_marc_t mt = yaz_marc_create();
+                    
+                if (yaz_marc_decode_buf(mt, octet_buf,r->u.octet_aligned->len,
+                                        &result, &rlen)> 0)
+                {
+                    char *from = 0;
+                    if (ent->value == VAL_USMARC)
+                    {
+                        if (octet_buf[9] == 'a')
+                            from = "UTF-8";
+                        else
+                            from = "MARC8";
+                    }
+                    else
+                        from = "ISO-8859-1";
+
+                    if (codeset && from)
+                    {   
+                        printf ("convert from %s to %s\n", from, codeset);
+                        cd = yaz_iconv_open(codeset, from);
+                    }
+                    if (!cd)
+                        fwrite (result, 1, rlen, stdout);
+                    else
+                    {
+                        char outbuf[12];
+                        size_t inbytesleft = rlen;
+                        const char *inp = result;
+                        
+                        while (inbytesleft)
+                        {
+                            int i;
+                            size_t outbytesleft = sizeof(outbuf);
+                            char *outp = outbuf;
+                            size_t r = yaz_iconv (cd, (char**) &inp,
+                                                  &inbytesleft, 
+                                                  &outp, &outbytesleft);
+                            if (r == (size_t) (-1))
+                            {
+                                int e = yaz_iconv_error(cd);
+                                if (e != YAZ_ICONV_E2BIG)
+                                    break;
+                            }
+                            fwrite (outbuf, outp - outbuf, 1, stdout);
+                        }
+                    }
+                }
+		else
                 {
                     printf ("bad MARC. Dumping as it is:\n");
                     print_record((const unsigned char*) octet_buf,
-                                 r->u.octet_aligned->len);
-                }
+                                  r->u.octet_aligned->len);
+		}	
+                yaz_marc_destroy(mt);
+                if (cd)
+                    yaz_iconv_close(cd);
             }
             else
             {
@@ -3239,7 +3299,15 @@ int main(int argc, char **argv)
     char *arg;
     int ret;
     
-    while ((ret = options("k:c:a:m:v:p:u:", argv, argc, &arg)) != -2)
+#if HAVE_LOCALE_H
+    if (!setlocale(LC_CTYPE, ""))
+        fprintf (stderr, "setlocale failed\n");
+#endif
+#if HAVE_LANGINFO_H
+    codeset = nl_langinfo(CODESET);
+#endif
+
+    while ((ret = options("k:c:a:m:v:p:u:t:", argv, argc, &arg)) != -2)
     {
         switch (ret)
         {
@@ -3261,19 +3329,22 @@ int main(int argc, char **argv)
                 exit (1);
             }
             break;
-		case 'c':
-			strncpy (ccl_fields, arg, sizeof(ccl_fields)-1);
-			ccl_fields[sizeof(ccl_fields)-1] = '\0';
-			break;
+        case 't':
+            codeset = arg;
+            break;
+        case 'c':
+            strncpy (ccl_fields, arg, sizeof(ccl_fields)-1);
+            ccl_fields[sizeof(ccl_fields)-1] = '\0';
+            break;
         case 'a':
             if (!strcmp(arg, "-"))
                 apdu_file=stderr;
             else
                 apdu_file=fopen(arg, "a");
             break;
-		case 'p':
-			yazProxy=strdup(arg);
-			break;
+        case 'p':
+            yazProxy=strdup(arg);
+            break;
         case 'u':
             if (!auth_command)
             {
diff --git a/configure.in b/configure.in
index 6bea66e..41cf8d1 100644
--- a/configure.in
+++ b/configure.in
@@ -1,6 +1,6 @@
 dnl YAZ Toolkit, Index Data 1994-2002
 dnl See the file LICENSE for details.
-dnl $Id: configure.in,v 1.98 2002-11-29 15:41:00 adam Exp $
+dnl $Id: configure.in,v 1.99 2002-12-16 13:13:53 adam Exp $
 AC_INIT(include/yaz/yaz-version.h)
 AM_INIT_AUTOMAKE(yaz, 1.9.2)
 dnl
@@ -236,7 +236,7 @@ if test "$enable_tcpd" != ""; then
 fi
 dnl
 dnl ------ Headers
-AC_CHECK_HEADERS(fnmatch.h)
+AC_CHECK_HEADERS(fnmatch.h wchar.h locale.h langinfo.h)
 AC_STDC_HEADERS
 if test "$ac_cv_header_stdc" = "no"; then
 	AC_MSG_WARN(Your system doesn't seem to support ANSI C)
diff --git a/include/yaz/Makefile.am b/include/yaz/Makefile.am
index 22a540e..275489a 100644
--- a/include/yaz/Makefile.am
+++ b/include/yaz/Makefile.am
@@ -1,14 +1,14 @@
-## $Id: Makefile.am,v 1.17 2002-10-22 12:51:18 adam Exp $
+## $Id: Makefile.am,v 1.18 2002-12-16 13:13:53 adam Exp $
 
 pkginclude_HEADERS= backend.h ccl.h comstack.h \
  diagbib1.h sortspec.h log.h logrpn.h marcdisp.h nmem.h odr.h oid.h \
  options.h otherinfo.h pquery.h prt-ext.h readconf.h statserv.h \
  tcpip.h unix.h tpath.h wrbuf.h xmalloc.h \
- yaz-ccl.h yaz-util.h yaz-version.h yconfig.h proto.h \
+ yaz-ccl.h yaz-iconv.h yaz-util.h yaz-version.h yconfig.h proto.h \
  \
  ill.h ill-core.h item-req.h z-accdes1.h z-accform1.h \
  z-acckrb1.h z-core.h z-date.h z-diag1.h z-espec1.h z-estask.h z-exp.h \
- z-grs.h z-opac.h z-rrf1.h z-rrf2.h z-sum.h z-sutrs.h z-uifr1.h \
+ z-grs.h z-mterm2.h z-opac.h z-rrf1.h z-rrf2.h z-sum.h z-sutrs.h z-uifr1.h \
  z-univ.h zes-expi.h zes-exps.h zes-order.h zes-pquery.h \
  zes-psched.h zes-admin.h zes-pset.h zes-update.h zes-update0.h \
  zoom.h z-charneg.h charneg.h
diff --git a/include/yaz/marcdisp.h b/include/yaz/marcdisp.h
index 1668836..3760a2f 100644
--- a/include/yaz/marcdisp.h
+++ b/include/yaz/marcdisp.h
@@ -23,34 +23,60 @@
  * LIABILITY, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
  * OF THIS SOFTWARE.
  *
- * $Id: marcdisp.h,v 1.6 2002-12-03 10:03:27 adam Exp $
+ * $Id: marcdisp.h,v 1.7 2002-12-16 13:13:53 adam Exp $
  */
 
 #ifndef MARCDISP_H
 #define MARCDISP_H
 
 #include <yaz/yconfig.h>
+#include <stdio.h>
 #include <yaz/wrbuf.h>
 
 YAZ_BEGIN_CDECL
 
+typedef struct yaz_marc_t_ *yaz_marc_t;
+
+/* create handler */
+YAZ_EXPORT yaz_marc_t yaz_marc_create(void);
+/* destroy */
+YAZ_EXPORT void yaz_marc_destroy(yaz_marc_t mt);
+
+/* set XML mode YAZ_MARC_LINE, YAZ_MARC_SIMPLEXML, ... */
+YAZ_EXPORT void yaz_marc_xml(yaz_marc_t mt, int xmlmode);
+#define YAZ_MARC_LINE      0
+#define YAZ_MARC_SIMPLEXML 1
+#define YAZ_MARC_OAIMARC   2
+#define YAZ_MARC_MARCXML   3
+
+/* set debug level, 0=none, 1=more, 2=even more, .. */
+YAZ_EXPORT void yaz_marc_debug(yaz_marc_t mt, int level);
+
+/* decode MARC in buf of size bsize. Returns >0 on success; <=0 on failure.
+   On success, result in *result with size *rsize. */
+YAZ_EXPORT int yaz_marc_decode_buf (yaz_marc_t mt, const char *buf, int bsize,
+                                    char **result, int *rsize);
+
+/* decode MARC in buf of size bsize. Returns >0 on success; <=0 on failure.
+   On success, result in WRBUF */
+YAZ_EXPORT int yaz_marc_decode_wrbuf (yaz_marc_t mt, const char *buf,
+                                      int bsize, WRBUF wrbuf);
+
+/* old functions (depricated) */
 YAZ_EXPORT int marc_display (const char *buf, FILE *outf);
 YAZ_EXPORT int marc_display_ex (const char *buf, FILE *outf, int debug);
 YAZ_EXPORT int marc_display_exl (const char *buf, FILE *outf, int debug,
                                  int length);
-YAZ_EXPORT int atoi_n (const char *buf, int len);
-
 YAZ_EXPORT int marc_display_wrbuf (const char *buf, WRBUF wr, int debug,
 				   int bsize);
+YAZ_EXPORT int yaz_marc_decode(const char *buf, WRBUF wr,
+                               int debug, int bsize, int xml);
 
-YAZ_EXPORT int yaz_marc_decode (const char *buf, WRBUF wr, int debug,
-                                int bsize, int xml);
 
-#define YAZ_MARC_LINE    0
-#define YAZ_MARC_XML     1
-#define YAZ_MARC_OAIMARC 2
-#define YAZ_MARC_MARCXML 3
+/* like atoi except that it reads exactly len characters */
+YAZ_EXPORT int atoi_n (const char *buf, int len);
 
+/* MARC control characters */
 #define ISO2709_RS 035
 #define ISO2709_FS 036
 #define ISO2709_IDFS 037
diff --git a/include/yaz/yaz-iconv.h b/include/yaz/yaz-iconv.h
new file mode 100644
index 0000000..5b4dd5f
--- /dev/null
+++ b/include/yaz/yaz-iconv.h
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 1995-2002, Index Data.
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and
+ * its documentation, in whole or in part, for any purpose, is hereby granted,
+ * provided that:
+ *
+ * 1. This copyright and permission notice appear in all copies of the
+ * software and its documentation. Notices of copyright or attribution
+ * which appear at the beginning of any file must remain unchanged.
+ *
+ * 2. The name of Index Data or the individual authors may not be used to
+ * endorse or promote products derived from this software without specific
+ * prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS, IMPLIED, OR OTHERWISE, INCLUDING WITHOUT LIMITATION, ANY
+ * WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
+ * IN NO EVENT SHALL INDEX DATA BE LIABLE FOR ANY SPECIAL, INCIDENTAL,
+ * INDIRECT OR CONSEQUENTIAL DAMAGES OF ANY KIND, OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER OR
+ * NOT ADVISED OF THE POSSIBILITY OF DAMAGE, AND ON ANY THEORY OF
+ * LIABILITY, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+ * OF THIS SOFTWARE.
+ *
+ * $Id: yaz-iconv.h,v 1.1 2002-12-16 13:13:53 adam Exp $
+ */
+
+#ifndef YAZ_ICONV_H
+#define YAZ_ICONV_H
+
+YAZ_BEGIN_CDECL
+
+typedef struct yaz_iconv_struct *yaz_iconv_t;
+#define YAZ_ICONV_UNKNOWN 1
+#define YAZ_ICONV_E2BIG 2
+#define YAZ_ICONV_EILSEQ 3
+#define YAZ_ICONV_EINVAL 4
+
+YAZ_EXPORT yaz_iconv_t yaz_iconv_open (const char *tocode,
+                                       const char *fromcode);
+YAZ_EXPORT size_t yaz_iconv (yaz_iconv_t cd, char **inbuf, size_t *inbytesleft,
+                             char **outbuf, size_t *outbytesleft);
+YAZ_EXPORT int yaz_iconv_error (yaz_iconv_t cd);
+
+YAZ_EXPORT int yaz_iconv_close (yaz_iconv_t cd);
+
+YAZ_EXPORT int yaz_iconv_isbuiltin(yaz_iconv_t cd);
+
+YAZ_EXPORT int yaz_matchstr(const char *s1, const char *s2);
+
+YAZ_END_CDECL
+
+#endif
diff --git a/include/yaz/yaz-util.h b/include/yaz/yaz-util.h
index 5139f37..bbba458 100644
--- a/include/yaz/yaz-util.h
+++ b/include/yaz/yaz-util.h
@@ -3,7 +3,7 @@
  * See the file LICENSE for details.
  * Sebastian Hammer, Adam Dickmeiss
  *
- * $Id: yaz-util.h,v 1.6 2002-12-10 10:59:28 adam Exp $
+ * $Id: yaz-util.h,v 1.7 2002-12-16 13:13:53 adam Exp $
  */
 
 #ifndef YAZ_UTIL_H
@@ -18,27 +18,6 @@
 #include <yaz/nmem.h>
 #include <yaz/readconf.h>
 #include <yaz/marcdisp.h>
-
-YAZ_BEGIN_CDECL
-
-typedef struct yaz_iconv_struct *yaz_iconv_t;
-#define YAZ_ICONV_UNKNOWN 1
-#define YAZ_ICONV_E2BIG 2
-#define YAZ_ICONV_EILSEQ 3
-#define YAZ_ICONV_EINVAL 4
-
-YAZ_EXPORT yaz_iconv_t yaz_iconv_open (const char *tocode,
-                                       const char *fromcode);
-YAZ_EXPORT size_t yaz_iconv (yaz_iconv_t cd, char **inbuf, size_t *inbytesleft,
-                             char **outbuf, size_t *outbytesleft);
-YAZ_EXPORT int yaz_iconv_error (yaz_iconv_t cd);
-
-YAZ_EXPORT int yaz_iconv_close (yaz_iconv_t cd);
-
-YAZ_EXPORT int yaz_iconv_isbuiltin(yaz_iconv_t cd);
-
-YAZ_EXPORT int yaz_matchstr(const char *s1, const char *s2);
-
-YAZ_END_CDECL
+#include <yaz/yaz-iconv.h>
 
 #endif
diff --git a/util/Makefile.am b/util/Makefile.am
index 781da6e..d2bb9bf 100644
--- a/util/Makefile.am
+++ b/util/Makefile.am
@@ -1,6 +1,6 @@
-## Copyright (C) 1994-2001, Index Data
+## Copyright (C) 1994-2002, Index Data
 ## All rights reserved.
-## $Id: Makefile.am,v 1.13 2002-09-11 21:25:57 adam Exp $
+## $Id: Makefile.am,v 1.14 2002-12-16 13:13:53 adam Exp $
 
 noinst_LTLIBRARIES = libutil.la
 
@@ -8,7 +8,7 @@ noinst_LTLIBRARIES = libutil.la
 
 bin_SCRIPTS = yaz-comp
 
-EXTRA_DIST = yaz-comp cvs-date.tcl
+EXTRA_DIST = yaz-comp cvs-date.tcl charconv.tcl charconv.sgm
 
 AM_CPPFLAGS=-I$(top_srcdir)/include
 
@@ -20,7 +20,9 @@ marcdump_SOURCES = marcdump.c
 yaziconv_LDADD = libutil.la 
 yaziconv_SOURCES = siconvtst.c
 
+marc8.c: charconv.sgm charconv.tcl
+	cd $(srcdir); ./charconv.tcl -p marc8 -s 50 charconv.sgm marc8.c
+
 libutil_la_SOURCES=options.c log.c marcdisp.c oid.c wrbuf.c nmemsdup.c \
-   xmalloc.c readconf.c tpath.c nmem.c matchstr.c atoin.c siconv.c
+   xmalloc.c readconf.c tpath.c nmem.c matchstr.c atoin.c siconv.c marc8.c
 
-#libyazthread_la_SOURCES=nmemthread.c xmalloc.c log.c
diff --git a/util/charconv.tcl b/util/charconv.tcl
new file mode 100755
index 0000000..befad0f
--- /dev/null
+++ b/util/charconv.tcl
@@ -0,0 +1,278 @@
+#!/bin/sh
+# the next line restats using tclsh \
+exec tclsh "$0" "$@"
+#
+# $Id: charconv.tcl,v 1.1 2002-12-16 13:13:53 adam Exp $
+
+proc usage {} {
+    puts {charconv.tcl: [-p prefix] [-s split] input output}
+    exit 1
+}
+
+proc ins_trie {from to} {
+    global trie
+    if {![info exists trie(no)]} {
+        set trie(no) 1
+        set trie(size) 0
+    }
+    incr trie(size)
+    ins_trie_r [split $from] $to 0
+}
+
+proc split_trie {this} {
+    global trie
+    set trie($this,type) d
+    foreach e $trie($this,content) {
+        set from [lindex $e 0]
+        set to [lindex $e 1]
+        
+        set ch [lindex $from 0]
+        set rest [lrange $from 1 end]
+        
+        if {[llength $rest]} {
+            if {![info exist trie($this,ptr,$ch)]} {
+                set trie($this,ptr,$ch) $trie(no)
+                incr trie(no)
+            }
+            ins_trie_r $rest $to $trie($this,ptr,$ch)
+        } else {
+            set trie($this,to,$ch) $to
+        }
+    }
+    set trie($this,content) missing
+}
+
+proc ins_trie_r {from to this} {
+    global trie
+
+    if {![info exist trie($this,type)]} {
+        set trie($this,type) f
+    }
+    if {$trie($this,type) == "f"} {
+        lappend trie($this,content) [list $from $to]
+        
+        # split ?
+        if {[llength $trie($this,content)] > $trie(split)} {
+            split_trie $this
+            return [ins_trie_r $from $to $this]
+        }
+    } else {
+        set ch [lindex $from 0]
+        set rest [lrange $from 1 end]
+
+        if {[llength $rest]} {
+            if {![info exist trie($this,ptr,$ch)]} {
+                set trie($this,ptr,$ch) $trie(no)
+                incr trie(no)
+            }
+            ins_trie_r $rest $to $trie($this,ptr,$ch)
+        } else {
+            set trie($this,to,$ch) $to
+        }
+    }
+}
+
+proc dump_trie {ofile} {
+    global trie
+
+    set f [open $ofile w]
+
+    puts $f "/* TRIE: size $trie(size) */"
+    puts $f "\#include <string.h>"
+    puts $f {
+        struct yaz_iconv_trie_flat {
+            char *from;
+            int to;
+        };
+        struct yaz_iconv_trie_dir {
+            struct yaz_iconv_trie *ptr;
+            int to;
+        };
+        
+        struct yaz_iconv_trie {
+            struct yaz_iconv_trie_flat *flat;
+            struct yaz_iconv_trie_dir *dir;
+        };
+    }
+
+    set this $trie(no)
+    while { [incr this -1] >= 0 } {
+        puts $f "/* PAGE $this */"
+        if {$trie($this,type) == "f"} {
+            puts $f "struct yaz_iconv_trie_flat $trie(prefix)page${this}_flat\[\] = \{"
+            foreach m $trie($this,content) {
+                puts -nonewline $f "  \{\""
+                foreach d [lindex $m 0] {
+                    puts -nonewline $f "\\0x$d"
+                }
+                puts -nonewline $f "\", 0x[lindex $m 1]"
+                puts $f "\},"
+            }
+            puts $f "  \{0, 0\}"
+            puts $f "\};"
+            puts $f "struct yaz_iconv_trie $trie(prefix)page${this} = \{"
+            puts $f "  $trie(prefix)page${this}_flat, 0"
+            puts $f "\};"
+        } else {
+            puts $f "struct yaz_iconv_trie_dir $trie(prefix)page${this}_dir\[256\] = \{"
+            for {set i 0} {$i < 256} {incr i} {
+                puts -nonewline $f "  \{"
+                set ch [format %02X $i]
+                set null 1
+                if {[info exist trie($this,ptr,$ch)]} {
+                    puts -nonewline $f "&$trie(prefix)page$trie($this,ptr,$ch), "
+                    set null 0
+                } else {
+                    puts -nonewline $f "0, "
+                }
+                if {[info exist trie($this,to,$ch)]} {
+                    puts -nonewline $f "0x$trie($this,to,$ch)\}"
+                    set null 0
+                } else {
+                    puts -nonewline $f "0\}"
+                }
+                if {!$null} {
+                    puts -nonewline $f " /* $ch */"
+                }
+                if {$i < 255} {
+                    puts $f ","
+                } else {
+                    puts $f ""
+                }
+            }
+            puts $f "\};"
+            puts $f "struct yaz_iconv_trie $trie(prefix)page${this} = \{"
+            puts $f "  0, $trie(prefix)page${this}_dir"
+            puts $f "\};"
+        }
+    }
+    puts $f {
+        static unsigned long lookup(struct yaz_iconv_trie *t, unsigned char *inp,
+                                    size_t inbytesleft, size_t *no_read)
+        {
+            if (!t || inbytesleft < 1)
+            return 0;
+            if (t->dir)
+            {
+                size_t ch = inp[0] & 0xff;
+                unsigned long code =
+                lookup(t->dir[ch].ptr, inp+1, inbytesleft-1, no_read);
+                if (code)
+                {
+                    (*no_read)++;
+                    return code;
+                }
+                if (t->dir[ch].to)
+                {
+                    code = t->dir[ch].to;
+                    *no_read = 1;
+                    return code;
+                }
+            }
+            else
+            {
+                struct yaz_iconv_trie_flat *flat = t->flat;
+                while (flat->from)
+                {
+                    size_t len = strlen(flat->from);
+                    if (len >= inbytesleft)
+                    {
+                        if (memcmp(flat->from, inp, len) == 0)
+                        {
+                            *no_read = len;
+                            return flat->to;
+                        }
+                    }
+                    flat++;
+                }
+            }
+            return 0;
+        }
+        
+    }
+    puts $f "unsigned long yaz_$trie(prefix)_conv
+            (unsigned char *inp, size_t inbytesleft, size_t *no_read)
+        {
+            unsigned long code;
+            
+            code = lookup(&$trie(prefix)page0, inp, inbytesleft, no_read);
+            if (!code)
+            {
+                *no_read = 1;
+                code = *inp;
+            }
+            return code;
+        }
+    "
+    close $f
+}
+
+proc readfile {fname} {
+    set lineno 0
+    set f [open $fname r]
+    while {1} {
+        incr lineno
+        set cnt [gets $f line]
+        if {$cnt < 0} {
+            break
+        }
+        set hex {}
+        set uni {}
+        regexp {<character hex="([^\"]*)".*<unientity>([0-9A-Z]*)</unientity>} $line s hex uni
+        # puts "$lineno hex=$hex uni=$uni $line"
+        if {[string length $uni]} {
+            ins_trie $hex $uni
+        }
+    }
+    close $f
+}
+
+set verbose 0
+set ifile {}
+set ofile {}
+set trie(split) 40
+set trie(prefix) {}
+# Parse command line
+set l [llength $argv]
+set i 0
+while {$i < $l} {
+    set arg [lindex $argv $i]
+    switch -glob -- $arg {
+        -v {
+            incr verbose
+        }
+        -s {
+            if {[string length $arg]} {
+                set arg [lindex $argv [incr i]]
+            }
+            set trie(split) $arg
+        }
+        -p {
+            if {[string length $arg]} {
+                set arg [lindex $argv [incr i]]
+            }
+            set trie(prefix) $arg
+        }
+        default {
+            if {![string length $ifile]} {
+                set ifile $arg
+            } elseif {![string length $ofile]} {
+                set ofile $arg
+            } else {
+                puts "charconv.tcl: too many files given"
+                usage
+            }
+        }
+    }
+    incr i
+}
+if {![string length $ifile]} {
+    puts "charconv.tcl: missing input file"
+    usage
+}
+if {![string length $ofile]} {
+    puts "charconv.tcl: missing output file"
+    usage
+}
+readfile $ifile
+dump_trie $ofile
diff --git a/util/marcdisp.c b/util/marcdisp.c
index 3e4e9d2..ede1c72 100644
--- a/util/marcdisp.c
+++ b/util/marcdisp.c
@@ -2,7 +2,7 @@
  * Copyright (c) 1995-2002, Index Data
  * See the file LICENSE for details.
  *
- * $Id: marcdisp.c,v 1.25 2002-12-09 23:32:29 adam Exp $
+ * $Id: marcdisp.c,v 1.26 2002-12-16 13:13:53 adam Exp $
  */
 
 #if HAVE_CONFIG_H
@@ -16,7 +16,89 @@
 #include <yaz/wrbuf.h>
 #include <yaz/yaz-util.h>
 
-int yaz_marc_decode (const char *buf, WRBUF wr, int debug, int bsize, int xml)
+struct yaz_marc_t_ {
+    WRBUF wr;
+    WRBUF own_wr;
+    WRBUF user_wr;
+    int xml;
+    int debug;
+};
+
+yaz_marc_t yaz_marc_create(void)
+{
+    yaz_marc_t mt = xmalloc(sizeof(*mt));
+    mt->xml = YAZ_MARC_LINE;
+    mt->debug = 0;
+    mt->wr = 0;
+    mt->own_wr = wrbuf_alloc();
+    mt->user_wr = 0;
+    return mt;
+}
+
+void yaz_marc_destroy(yaz_marc_t mt)
+{
+    if (!mt)
+        return ;
+    wrbuf_free (mt->own_wr, 1);
+    xfree (mt);
+}
+
+static void marc_cdata (yaz_marc_t mt, const char *buf, size_t len)
+{
+    size_t i;
+    for (i = 0; i<len; i++)
+    {
+        if (mt->xml)
+        {
+            switch (buf[i]) {
+            case '<':
+                wrbuf_puts(mt->wr, "&lt;");
+                break;
+            case '>':
+                wrbuf_puts(mt->wr, "&gt;");
+                break;
+            case '&':
+                wrbuf_puts(mt->wr, "&amp;");
+                break;
+            default:
+                wrbuf_putc(mt->wr, buf[i]);
+            }
+        }
+        else
+            wrbuf_putc(mt->wr, buf[i]);
+    }
+}
+
+#if 0
+static void marc_cdata (yaz_marc_t mt, const char *buf, size_t len)
+{
+    if (!mt->cd)
+        marc_cdata2 (mt, buf, len);
+    else
+    {
+        char outbuf[12];
+        size_t inbytesleft = len;
+        const char *inp = buf;
+        
+        while (inbytesleft)
+        {
+            size_t outbytesleft = sizeof(outbuf);
+            char *outp = outbuf;
+            size_t r = yaz_iconv (mt->cd, (char**) &inp, &inbytesleft, 
+                                  &outp, &outbytesleft);
+            if (r == (size_t) (-1))
+            {
+                int e = yaz_iconv_error(mt->cd);
+                if (e != YAZ_ICONV_E2BIG)
+                    break;
+            }
+            marc_cdata2 (mt, outbuf, outp - outbuf);
+        }
+    }
+}
+#endif
+
+static int yaz_marc_decode_int (yaz_marc_t mt, const char *buf, int bsize)
 {
     int entry_p;
     int record_length;
@@ -27,15 +109,20 @@ int yaz_marc_decode (const char *buf, WRBUF wr, int debug, int bsize, int xml)
     int length_starting;
     int length_implementation;
 
+    if (!mt->wr)
+        mt->wr = mt->own_wr;
+
+    wrbuf_rewind(mt->wr);
+
     record_length = atoi_n (buf, 5);
     if (record_length < 25)
     {
-	if (debug)
+	if (mt->debug)
 	{
 	    char str[40];
 	    
 	    sprintf (str, "Record length %d - aborting\n", record_length);
-	    wrbuf_puts (wr, str);
+	    wrbuf_puts (mt->wr, str);
 	}
         return -1;
     }
@@ -56,28 +143,28 @@ int yaz_marc_decode (const char *buf, WRBUF wr, int debug, int bsize, int xml)
     length_starting = atoi_n (buf+21, 1);
     length_implementation = atoi_n (buf+22, 1);
 
-    if (xml)
+    if (mt->xml)
     {
         char str[80];
         int i;
-        switch(xml)
+        switch(mt->xml)
         {
-        case YAZ_MARC_XML:
-            wrbuf_puts (wr, "<iso2709\n");
+        case YAZ_MARC_SIMPLEXML:
+            wrbuf_puts (mt->wr, "<iso2709\n");
             sprintf (str, " RecordStatus=\"%c\"\n", buf[5]);
-            wrbuf_puts (wr, str);
+            wrbuf_puts (mt->wr, str);
             sprintf (str, " TypeOfRecord=\"%c\"\n", buf[6]);
-            wrbuf_puts (wr, str);
+            wrbuf_puts (mt->wr, str);
             for (i = 1; i<=19; i++)
             {
                 sprintf (str, " ImplDefined%d=\"%c\"\n", i, buf[6+i]);
-                wrbuf_puts (wr, str);
+                wrbuf_puts (mt->wr, str);
             }
-            wrbuf_puts (wr, ">\n");
+            wrbuf_puts (mt->wr, ">\n");
             break;
         case YAZ_MARC_OAIMARC:
             wrbuf_puts(
-                wr,
+                mt->wr,
                 "<oai_marc xmlns=\"http://www.openarchives.org/OIA/oai_marc\""
                 "\n"
                 " xmlns:xsi=\"http://www.w3.org/2000/10/XMLSchema-instance\""
@@ -88,38 +175,38 @@ int yaz_marc_decode (const char *buf, WRBUF wr, int debug, int bsize, int xml)
             
             sprintf (str, " status=\"%c\" type=\"%c\" catForm=\"%c\">\n",
                      buf[5], buf[6], buf[7]);
-            wrbuf_puts (wr, str);
+            wrbuf_puts (mt->wr, str);
             break;
         case YAZ_MARC_MARCXML:
             wrbuf_printf(
-                wr,
+                mt->wr,
                 "<record xmlns=\"http://www.loc.gov/MARC21/slim\">\n"
                 "  <leader>%.24s</leader>\n", buf);
             break;
         }
     }
-    if (debug)
+    if (mt->debug)
     {
 	char str[40];
 
-        if (xml)
-            wrbuf_puts (wr, "<!--\n");
+        if (mt->xml)
+            wrbuf_puts (mt->wr, "<!--\n");
 	sprintf (str, "Record length         %5d\n", record_length);
-	wrbuf_puts (wr, str);
+	wrbuf_puts (mt->wr, str);
 	sprintf (str, "Indicator length      %5d\n", indicator_length);
-	wrbuf_puts (wr, str);
+	wrbuf_puts (mt->wr, str);
 	sprintf (str, "Identifier length     %5d\n", identifier_length);
-	wrbuf_puts (wr, str);
+	wrbuf_puts (mt->wr, str);
 	sprintf (str, "Base address          %5d\n", base_address);
-	wrbuf_puts (wr, str);
+	wrbuf_puts (mt->wr, str);
 	sprintf (str, "Length data entry     %5d\n", length_data_entry);
-	wrbuf_puts (wr, str);
+	wrbuf_puts (mt->wr, str);
 	sprintf (str, "Length starting       %5d\n", length_starting);
-	wrbuf_puts (wr, str);
+	wrbuf_puts (mt->wr, str);
 	sprintf (str, "Length implementation %5d\n", length_implementation);
-	wrbuf_puts (wr, str);
-        if (xml)
-            wrbuf_puts (wr, "-->\n");
+	wrbuf_puts (mt->wr, str);
+        if (mt->xml)
+            wrbuf_puts (mt->wr, "-->\n");
     }
 
     for (entry_p = 24; buf[entry_p] != ISO2709_FS; )
@@ -156,195 +243,235 @@ int yaz_marc_decode (const char *buf, WRBUF wr, int debug, int bsize, int xml)
         else if (!memcmp (tag, "00", 2))
             identifier_flag = 0;
         
-        switch(xml)
+        switch(mt->xml)
         {
         case YAZ_MARC_LINE:
-            if (debug)
-                wrbuf_puts (wr, "Tag: ");
-            wrbuf_puts (wr, tag);
-            wrbuf_puts (wr, " ");
+            if (mt->debug)
+                wrbuf_puts (mt->wr, "Tag: ");
+            wrbuf_puts (mt->wr, tag);
+            wrbuf_puts (mt->wr, " ");
             break;
-        case YAZ_MARC_XML:
-            wrbuf_printf (wr, "<field tag=\"%s\"", tag);
+        case YAZ_MARC_SIMPLEXML:
+            wrbuf_printf (mt->wr, "<field tag=\"%s\"", tag);
             break;
         case YAZ_MARC_OAIMARC:
             if (identifier_flag)
-                wrbuf_printf (wr, "  <varfield id=\"%s\"", tag);
+                wrbuf_printf (mt->wr, "  <varfield id=\"%s\"", tag);
             else
-                wrbuf_printf (wr, "  <fixfield id=\"%s\"", tag);
+                wrbuf_printf (mt->wr, "  <fixfield id=\"%s\"", tag);
             break;
         case YAZ_MARC_MARCXML:
             if (identifier_flag)
-                wrbuf_printf (wr, "  <datafield tag=\"%s\"", tag);
+                wrbuf_printf (mt->wr, "  <datafield tag=\"%s\"", tag);
             else
-                wrbuf_printf (wr, "  <controlfield tag=\"%s\"", tag);
+                wrbuf_printf (mt->wr, "  <controlfield tag=\"%s\"", tag);
         }
         
         if (identifier_flag)
 	{
             for (j = 0; j<indicator_length; j++, i++)
             {
-                switch(xml)
+                switch(mt->xml)
                 {
                 case YAZ_MARC_LINE:
-                    if (debug)
-                        wrbuf_puts (wr, " Ind: ");
-                    wrbuf_putc (wr, buf[i]);
+                    if (mt->debug)
+                        wrbuf_puts (mt->wr, " Ind: ");
+                    wrbuf_putc (mt->wr, buf[i]);
                     break;
-                case YAZ_MARC_XML:
-                    wrbuf_printf (wr, " Indicator%d=\"%c\"", j+1, buf[i]);
+                case YAZ_MARC_SIMPLEXML:
+                    wrbuf_printf (mt->wr, " Indicator%d=\"%c\"", j+1, buf[i]);
                     break;
                 case YAZ_MARC_OAIMARC:
-                    wrbuf_printf (wr, " i%d=\"%c\"", j+1, buf[i]);
+                    wrbuf_printf (mt->wr, " i%d=\"%c\"", j+1, buf[i]);
                     break;
                 case YAZ_MARC_MARCXML:
-                    wrbuf_printf (wr, " ind%d=\"%c\"", j+1, buf[i]);
+                    wrbuf_printf (mt->wr, " ind%d=\"%c\"", j+1, buf[i]);
                 }
             }
 	}
-        if (xml)
+        if (mt->xml)
         {
-            wrbuf_puts (wr, ">");
+            wrbuf_puts (mt->wr, ">");
             if (identifier_flag)
-                wrbuf_puts (wr, "\n");
+                wrbuf_puts (mt->wr, "\n");
         }
         else
         {
-            if (debug && !xml)
-                wrbuf_puts (wr, " Fields: ");
+            if (mt->debug && !mt->xml)
+                wrbuf_puts (mt->wr, " Fields: ");
         }
-	while (buf[i] != ISO2709_RS && buf[i] != ISO2709_FS && i < end_offset)
-	{
-            if (identifier_flag)
-	    {
-	        i++;
-                switch(xml)
+        if (identifier_flag)
+        {
+            while (buf[i] != ISO2709_RS && buf[i] != ISO2709_FS && i < end_offset)
+            {
+                int i0;
+                i++;
+                switch(mt->xml)
                 {
                 case YAZ_MARC_LINE: 
-                    wrbuf_puts (wr, " $"); 
+                    wrbuf_puts (mt->wr, " $"); 
                     for (j = 1; j<identifier_length; j++, i++)
-                        wrbuf_putc (wr, buf[i]);
-                    wrbuf_putc (wr, ' ');
+                        wrbuf_putc (mt->wr, buf[i]);
+                    wrbuf_putc (mt->wr, ' ');
                     break;
-                case YAZ_MARC_XML:
-                        wrbuf_puts (wr, "  <subfield code=\"");
+                case YAZ_MARC_SIMPLEXML:
+                    wrbuf_puts (mt->wr, "  <subfield code=\"");
                     for (j = 1; j<identifier_length; j++, i++)
-                        wrbuf_putc (wr, buf[i]);
-                    wrbuf_puts (wr, "\">");
+                        wrbuf_putc (mt->wr, buf[i]);
+                    wrbuf_puts (mt->wr, "\">");
                     break;
                 case YAZ_MARC_OAIMARC:
-                    wrbuf_puts (wr, "    <subfield label=\"");
+                    wrbuf_puts (mt->wr, "    <subfield label=\"");
                     for (j = 1; j<identifier_length; j++, i++)
-                        wrbuf_putc (wr, buf[i]);
-                    wrbuf_puts (wr, "\">");
+                        wrbuf_putc (mt->wr, buf[i]);
+                    wrbuf_puts (mt->wr, "\">");
                     break;
                 case YAZ_MARC_MARCXML:
-                    wrbuf_puts (wr, "    <subfield code=\"");
+                    wrbuf_puts (mt->wr, "    <subfield code=\"");
                     for (j = 1; j<identifier_length; j++, i++)
-                        wrbuf_putc (wr, buf[i]);
-                    wrbuf_puts (wr, "\">");
+                        wrbuf_putc (mt->wr, buf[i]);
+                    wrbuf_puts (mt->wr, "\">");
                     break;
                 }
-	        while (buf[i] != ISO2709_RS && buf[i] != ISO2709_IDFS &&
-	               buf[i] != ISO2709_FS && i < end_offset)
-		{
-                    if (xml && buf[i] == '<')
-                        wrbuf_puts(wr, "&#x003C;");
-                    else if (xml && buf[i] == '&')
-                        wrbuf_puts(wr, "&#x0026;");
-                    else
-                        wrbuf_putc (wr, buf[i]);
-		    i++;
-		}
-                if (xml)
-                    wrbuf_puts (wr, "</subfield>\n");
-	    }
-	    else
-	    {
-                if (xml && buf[i] == '<')
-                    wrbuf_puts(wr, "&lt;");
-                else if (xml && buf[i] == '&')
-                    wrbuf_puts(wr, "&amp;");
-                else if (xml && buf[i] == '"')
-                    wrbuf_puts(wr, "&quot;");
-                else
-                    wrbuf_putc (wr, buf[i]);
-		i++;
-	    }
+                i0 = i;
+                while (buf[i] != ISO2709_RS && buf[i] != ISO2709_IDFS &&
+                       buf[i] != ISO2709_FS && i < end_offset)
+                    i++;
+                marc_cdata(mt, buf + i0, i - i0);
+                
+                if (mt->xml)
+                    wrbuf_puts (mt->wr, "</subfield>\n");
+            }
+        }
+        else
+        {
+            int i0 = i;
+            while (buf[i] != ISO2709_RS && buf[i] != ISO2709_FS && i < end_offset)
+                i++;
+            marc_cdata(mt, buf + i0, i - i0);
 	}
-        if (!xml)
-            wrbuf_putc (wr, '\n');
+        if (!mt->xml)
+            wrbuf_putc (mt->wr, '\n');
 	if (i < end_offset)
-	    wrbuf_puts (wr, "  <!-- separator but not at end of field -->\n");
+	    wrbuf_puts (mt->wr, "  <!-- separator but not at end of field -->\n");
 	if (buf[i] != ISO2709_RS && buf[i] != ISO2709_FS)
-	    wrbuf_puts (wr, "  <!-- no separator at end of field -->\n");
-        switch(xml)
+	    wrbuf_puts (mt->wr, "  <!-- no separator at end of field -->\n");
+        switch(mt->xml)
         {
-        case YAZ_MARC_LINE: 
-            break;
-        case YAZ_MARC_XML:
-            wrbuf_puts (wr, "</field>\n");
+        case YAZ_MARC_SIMPLEXML:
+            wrbuf_puts (mt->wr, "</field>\n");
             break;
         case YAZ_MARC_OAIMARC:
             if (identifier_flag)
-                wrbuf_puts (wr, "  </varfield>\n");
+                wrbuf_puts (mt->wr, "  </varfield>\n");
             else
-                wrbuf_puts (wr, "  </fixfield>\n");
+                wrbuf_puts (mt->wr, "  </fixfield>\n");
             break;
         case YAZ_MARC_MARCXML:
             if (identifier_flag)
-                wrbuf_puts (wr, "  </datafield>\n");
+                wrbuf_puts (mt->wr, "  </datafield>\n");
             else
-                wrbuf_puts (wr, "  </controlfield>\n");
+                wrbuf_puts (mt->wr, "  </controlfield>\n");
             break;
         }
     }
-    switch(xml)
+    switch (mt->xml)
     {
     case YAZ_MARC_LINE:
-        wrbuf_puts (wr, "");
+        wrbuf_puts (mt->wr, "");
         break;
-    case YAZ_MARC_XML:
-        wrbuf_puts (wr, "</iso2709>\n");
+    case YAZ_MARC_SIMPLEXML:
+        wrbuf_puts (mt->wr, "</iso2709>\n");
         break;
     case YAZ_MARC_OAIMARC:
-        wrbuf_puts (wr, "</oai_marc>\n");
+        wrbuf_puts (mt->wr, "</oai_marc>\n");
         break;
     case YAZ_MARC_MARCXML:
-        wrbuf_puts (wr, "</record>\n");
+        wrbuf_puts (mt->wr, "</record>\n");
         break;
     }
     return record_length;
 }
 
-int marc_display_wrbuf (const char *buf, WRBUF wr, int debug,
-                        int bsize)
+int yaz_marc_decode_buf (yaz_marc_t mt, const char *buf, int bsize,
+                         char **result, int *rsize)
 {
-    return yaz_marc_decode (buf, wr, debug, bsize, 0);
+    int r = yaz_marc_decode_int(mt, buf, bsize);
+    if (r > 0)
+    {
+        if (result)
+            *result = wrbuf_buf(mt->wr);
+        if (rsize)
+            *rsize = wrbuf_len(mt->wr);
+    }
+    return r;
 }
 
-int marc_display_exl (const char *buf, FILE *outf, int debug, int length)
+int yaz_marc_decode_wrbuf (yaz_marc_t mt, const char *buf,
+                           int bsize, WRBUF wrbuf)
 {
-    int record_length;
+    mt->user_wr = wrbuf;
+    return yaz_marc_decode_int (mt, buf, bsize);
+}
+
 
-    WRBUF wrbuf = wrbuf_alloc ();
-    record_length = marc_display_wrbuf (buf, wrbuf, debug, length);
+void yaz_marc_xml(yaz_marc_t mt, int xmlmode)
+{
+    if (mt)
+        mt->xml = xmlmode;
+}
+
+void yaz_marc_debug(yaz_marc_t mt, int level)
+{
+    if (mt)
+        mt->debug = level;
+}
+
+/* depricated */
+int yaz_marc_decode(const char *buf, WRBUF wr, int debug, int bsize, int xml)
+{
+    yaz_marc_t mt = yaz_marc_create();
+    int r;
+
+    mt->debug = debug;
+    mt->user_wr = wr;
+    mt->xml = xml;
+    r = yaz_marc_decode_int(mt, buf, bsize);
+    yaz_marc_destroy(mt);
+    return r;
+}
+
+/* depricated */
+int marc_display_wrbuf (const char *buf, WRBUF wr, int debug, int bsize)
+{
+    return yaz_marc_decode(buf, wr, debug, bsize, 0);
+}
+
+/* depricated */
+int marc_display_exl (const char *buf, FILE *outf, int debug, int bsize)
+{
+    yaz_marc_t mt = yaz_marc_create();
+    int r;
+
+    mt->debug = debug;
+    r = yaz_marc_decode_int (mt, buf, bsize);
     if (!outf)
 	outf = stdout;
-    if (record_length > 0)
-	fwrite (wrbuf_buf(wrbuf), 1, wrbuf_len(wrbuf), outf);
-    wrbuf_free (wrbuf, 1);
-    return record_length;
+    if (r > 0)
+	fwrite (wrbuf_buf(mt->wr), 1, wrbuf_len(mt->wr), outf);
+    yaz_marc_destroy(mt);
+    return r;
 }
 
+/* depricated */
 int marc_display_ex (const char *buf, FILE *outf, int debug)
 {
     return marc_display_exl (buf, outf, debug, -1);
 }
 
+/* depricated */
 int marc_display (const char *buf, FILE *outf)
 {
     return marc_display_ex (buf, outf, 0);
 }
 
-
diff --git a/util/marcdump.c b/util/marcdump.c
index 076628e..f28a99b 100644
--- a/util/marcdump.c
+++ b/util/marcdump.c
@@ -3,7 +3,7 @@
  * See the file LICENSE for details.
  * Sebastian Hammer, Adam Dickmeiss
  *
- * $Id: marcdump.c,v 1.18 2002-12-03 10:03:27 adam Exp $
+ * $Id: marcdump.c,v 1.19 2002-12-16 13:13:53 adam Exp $
  */
 
 #if HAVE_CONFIG_H
@@ -14,6 +14,14 @@
 #include <stdlib.h>
 #include <string.h>
 #include <errno.h>
+
+#if HAVE_LOCALE_H
+#include <locale.h>
+#endif
+#if HAVE_LANGINFO_H
+#include <langinfo.h>
+#endif
+
 #include <yaz/marcdisp.h>
 #include <yaz/yaz-util.h>
 #include <yaz/xmalloc.h>
@@ -28,7 +36,7 @@
 
 static void usage(const char *prog)
 {
-    fprintf (stderr, "Usage: %s [-c cfile] [-x] [-O] [-X] [-v] file...\n",
+    fprintf (stderr, "Usage: %s [-c cfile] [-f from] [-t to] [-x] [-O] [-X] [-v] file...\n",
              prog);
 } 
 
@@ -43,20 +51,35 @@ int main (int argc, char **argv)
     int no = 0;
     int xml = 0;
     FILE *cfile = 0;
+    char *from = 0, *to = 0;
 
-    while ((r = options("vc:xOX", argv, argc, &arg)) != -2)
+    
+#if HAVE_LOCALE_H
+    setlocale(LC_CTYPE, "");
+#endif
+#if HAVE_LANGINFO_H
+    to = nl_langinfo(CODESET);
+#endif
+
+    while ((r = options("vc:xOXf:t:", argv, argc, &arg)) != -2)
     {
 	int count;
 	no++;
         switch (r)
         {
+        case 'f':
+            from = arg;
+            break;
+        case 't':
+            to = arg;
+            break;
 	case 'c':
 	    if (cfile)
 		fclose (cfile);
 	    cfile = fopen (arg, "w");
 	    break;
         case 'x':
-            xml = YAZ_MARC_XML;
+            xml = YAZ_MARC_SIMPLEXML;
             break;
         case 'O':
             xml = YAZ_MARC_OAIMARC;
@@ -71,50 +94,95 @@ int main (int argc, char **argv)
 	    {
 		fprintf (stderr, "%s: cannot open %s:%s\n",
 			 prog, arg, strerror (errno));
-		exit (1);
+		exit(1);
 	    }
 	    if (cfile)
 		fprintf (cfile, "char *marc_records[] = {\n");
-	    while (1)
-	    {
-                WRBUF wr = wrbuf_alloc();
+            if (1)
+            {
+                yaz_marc_t mt = yaz_marc_create();
+                yaz_iconv_t cd = 0;
+
+                if (from && to)
+                {
+                    cd = yaz_iconv_open(to, from);
+                    if (!cd)
+                    {
+                        fprintf(stderr, "conversion from %s to %s "
+                                "unsupported\n", from, to);
+                        exit(2);
+                    }
+                }
+                yaz_marc_xml(mt, xml);
+                yaz_marc_debug(mt, verbose);
+                while (1)
+                {
+                    int len;
+                    char *result;
+                    int rlen;
+                    
+                    r = fread (buf, 1, 5, inf);
+                    if (r < 5)
+                        break;
+                    len = atoi_n(buf, 5);
+                    if (len < 25 || len > 100000)
+                        break;
+                    len = len - 5;
+                    r = fread (buf + 5, 1, len, inf);
+                    if (r < len)
+                        break;
+                    r = yaz_marc_decode_buf (mt, buf, -1, &result, &rlen);
+                    if (r <= 0)
+                        break;
+                    if (!cd)
+                        fwrite (result, rlen, 1, stdout);
+                    else
+                    {
+                        char outbuf[12];
+                        size_t inbytesleft = rlen;
+                        const char *inp = result;
+                        
+                        while (inbytesleft)
+                        {
+                            int i;
+                            size_t outbytesleft = sizeof(outbuf);
+                            char *outp = outbuf;
+                            size_t r = yaz_iconv (cd, (char**) &inp,
+                                                  &inbytesleft, 
+                                                  &outp, &outbytesleft);
+                            if (r == (size_t) (-1))
+                            {
+                                int e = yaz_iconv_error(cd);
+                                if (e != YAZ_ICONV_E2BIG)
+                                    break;
+                            }
+                            fwrite (outbuf, outp - outbuf, 1, stdout);
+                        }
+                    }
 
-		int len;
-		
-		r = fread (buf, 1, 5, inf);
-		if (r < 5)
-		    break;
-		len = atoi_n(buf, 5);
-		if (len < 25 || len > 100000)
-		    break;
-		len = len - 5;
-		r = fread (buf + 5, 1, len, inf);
-		if (r < len)
-		    break;
-                r = yaz_marc_decode (buf, wr, verbose, -1, xml);
-		if (r <= 0)
-		    break;
-                fwrite (wrbuf_buf(wr), wrbuf_len(wr), 1, stdout);
-		if (cfile)
-		{
-		    char *p = buf;
-		    int i;
-		    if (count)
-			fprintf (cfile, ",");
-		    fprintf (cfile, "\n");
-		    for (i = 0; i < r; i++)
-		    {
-			if ((i & 15) == 0)
-			    fprintf (cfile, "  \"");
-			fprintf (cfile, "\\x%02X", p[i] & 255);
-			
-			if (i < r - 1 && (i & 15) == 15)
-			    fprintf (cfile, "\"\n");
-			
+                    if (cfile)
+                    {
+                        char *p = buf;
+                        int i;
+                        if (count)
+                            fprintf (cfile, ",");
+                        fprintf (cfile, "\n");
+                        for (i = 0; i < r; i++)
+                        {
+                            if ((i & 15) == 0)
+                                fprintf (cfile, "  \"");
+                            fprintf (cfile, "\\x%02X", p[i] & 255);
+                            
+                            if (i < r - 1 && (i & 15) == 15)
+                                fprintf (cfile, "\"\n");
+                            
 			}
-		    fprintf (cfile, "\"\n");
-		}
-		count++;
+                        fprintf (cfile, "\"\n");
+                    }
+                }
+                count++;
+                if (cd)
+                    yaz_iconv_close(cd);
 	    }
 	    if (cfile)
 		fprintf (cfile, "};\n");
diff --git a/util/siconv.c b/util/siconv.c
index 75c5979..b0e5611 100644
--- a/util/siconv.c
+++ b/util/siconv.c
@@ -2,7 +2,7 @@
  * Copyright (c) 1997-2002, Index Data
  * See the file LICENSE for details.
  *
- * $Id: siconv.c,v 1.7 2002-12-10 10:59:28 adam Exp $
+ * $Id: siconv.c,v 1.8 2002-12-16 13:13:53 adam Exp $
  */
 
 /* mini iconv and wrapper for system iconv library (if present) */
@@ -14,6 +14,9 @@
 #include <errno.h>
 #include <string.h>
 #include <ctype.h>
+#if HAVE_WCHAR_H
+#include <wchar.h>
+#endif
 
 #if HAVE_ICONV_H
 #include <iconv.h>
@@ -21,6 +24,9 @@
 
 #include <yaz/yaz-util.h>
 
+unsigned long yaz_marc8_conv (unsigned char *inp, size_t inbytesleft,
+                              size_t *no_read);
+    
 struct yaz_iconv_struct {
     int my_errno;
     int init_flag;
@@ -185,6 +191,34 @@ static unsigned long yaz_read_UCS4LE (yaz_iconv_t cd, unsigned char *inp,
     return x;
 }
 
+#if HAVE_WCHAR_H
+static unsigned long yaz_read_wchar_t (yaz_iconv_t cd, unsigned char *inp,
+                                       size_t inbytesleft, size_t *no_read)
+{
+    unsigned long x = 0;
+    
+    if (inbytesleft < sizeof(wchar_t))
+    {
+        cd->my_errno = YAZ_ICONV_EINVAL; /* incomplete input */
+        *no_read = 0;
+    }
+    else
+    {
+        wchar_t wch;
+        memcpy (&wch, inp, sizeof(wch));
+        x = wch;
+        *no_read = sizeof(wch);
+    }
+    return x;
+}
+#endif
+
+static unsigned long yaz_read_marc8 (yaz_iconv_t cd, unsigned char *inp,
+                                     size_t inbytesleft, size_t *no_read)
+{
+    return yaz_marc8_conv(inp, inbytesleft, no_read);
+}
+
 static size_t yaz_write_UTF8 (yaz_iconv_t cd, unsigned long x,
                               char **outbuf, size_t *outbytesleft)
 {
@@ -309,6 +343,29 @@ static size_t yaz_write_UCS4LE (yaz_iconv_t cd, unsigned long x,
     return 0;
 }
 
+#if HAVE_WCHAR_H
+static size_t yaz_write_wchar_t (yaz_iconv_t cd, unsigned long x,
+                                 char **outbuf, size_t *outbytesleft)
+{
+    unsigned char *outp = (unsigned char *) *outbuf;
+
+    if (*outbytesleft >= sizeof(wchar_t))
+    {
+        wchar_t wch = x;
+        memcpy(outp, &wch, sizeof(wch));
+        outp += sizeof(wch);
+        (*outbytesleft) -= sizeof(wch);
+    }
+    else
+    {
+        cd->my_errno = YAZ_ICONV_E2BIG;
+        return (size_t)(-1);
+    }
+    *outbuf = (char *) outp;
+    return 0;
+}
+#endif
+
 int yaz_iconv_isbuiltin(yaz_iconv_t cd)
 {
     return cd->read_handle && cd->write_handle;
@@ -340,6 +397,12 @@ yaz_iconv_t yaz_iconv_open (const char *tocode, const char *fromcode)
             cd->read_handle = yaz_read_UCS4;
         else if (!yaz_matchstr(fromcode, "UCS4LE"))
             cd->read_handle = yaz_read_UCS4LE;
+        else if (!yaz_matchstr(fromcode, "MARC8"))
+            cd->read_handle = yaz_read_marc8;
+#if HAVE_WCHAR_H
+        else if (!yaz_matchstr(fromcode, "WCHAR_T"))
+            cd->read_handle = yaz_read_wchar_t;
+#endif
         
         if (!yaz_matchstr(tocode, "UTF8"))
             cd->write_handle = yaz_write_UTF8;
@@ -349,6 +412,10 @@ yaz_iconv_t yaz_iconv_open (const char *tocode, const char *fromcode)
             cd->write_handle = yaz_write_UCS4;
         else if (!yaz_matchstr(tocode, "UCS4LE"))
             cd->write_handle = yaz_write_UCS4LE;
+#if HAVE_WCHAR_H
+        else if (!yaz_matchstr(tocode, "WCHAR_T"))
+            cd->write_handle = yaz_write_wchar_t;
+#endif
     }
 #if HAVE_ICONV_H
     cd->iconv_cd = 0;
diff --git a/util/siconvtst.c b/util/siconvtst.c
index 072997b..bc394b8 100644
--- a/util/siconvtst.c
+++ b/util/siconvtst.c
@@ -2,7 +2,7 @@
  * Copyright (c) 1997-2002, Index Data
  * See the file LICENSE for details.
  *
- * $Id: siconvtst.c,v 1.6 2002-12-10 10:59:28 adam Exp $
+ * $Id: siconvtst.c,v 1.7 2002-12-16 13:13:53 adam Exp $
  */
 
 #if HAVE_CONFIG_H
@@ -18,7 +18,7 @@
 #define CHUNK_IN 64
 #define CHUNK_OUT 64
 
-void convert (FILE *inf, yaz_iconv_t cd)
+void convert (FILE *inf, yaz_iconv_t cd, int verbose)
 {
     char inbuf0[CHUNK_IN], *inbuf = inbuf0;
     char outbuf0[CHUNK_OUT], *outbuf = outbuf0;
@@ -48,6 +48,12 @@ void convert (FILE *inf, yaz_iconv_t cd)
                 inbytesleft = r;
             }
         }
+        if (verbose > 1)
+        {
+            fprintf (stderr, "yaz_iconv: inbytesleft=%d outbytesleft=%d\n",
+                     inbytesleft, outbytesleft);
+
+        }
         r = yaz_iconv (cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft);
         if (r == (size_t)(-1))
         {
@@ -169,7 +175,7 @@ int main (int argc, char **argv)
                      yaz_iconv_isbuiltin(cd) ? "YAZ" : "iconv");
         }
     }
-    convert (inf, cd);
+    convert (inf, cd, verbose);
     yaz_iconv_close (cd);
     return 0;
 }
diff --git a/ztest/read-marc.c b/ztest/read-marc.c
index 6333a40..cc9ca29 100644
--- a/ztest/read-marc.c
+++ b/ztest/read-marc.c
@@ -2,15 +2,16 @@
  * Copyright (c) 2002, Index Data.
  * See the file LICENSE for details.
  *
- * $Id: read-marc.c,v 1.3 2002-09-24 08:05:41 adam Exp $
+ * $Id: read-marc.c,v 1.4 2002-12-16 13:13:53 adam Exp $
  */
 
 /*
  * Little toy-thing to read a MARC records from a fixed array.
  */
 #include <ctype.h>
-#include <yaz/odr.h>
 #include <yaz/wrbuf.h>
+#include <yaz/marcdisp.h>
+#include <yaz/odr.h>
 
 char *marc_records[] = {
 
@@ -1600,20 +1601,19 @@ char *dummy_marc_record (int num, ODR odr)
 /* read MARC record and convert to XML */
 char *dummy_xml_record (int num, ODR odr)
 {
-    WRBUF wr = wrbuf_alloc ();
+    yaz_marc_t mt = yaz_marc_create();
+    char *result;
+    int rlen;
     char *rec = dummy_marc_record (num, odr);
     int len;
+
     if (!rec)
         return 0;
 
-    len = yaz_marc_decode (rec, wr, 0, -1, 1);
+    yaz_marc_xml(mt, YAZ_MARC_MARCXML);
+    len = yaz_marc_decode_buf (mt, rec, -1, &result, &rlen);
     if (len > 1)
-    {
-        len = wrbuf_len(wr);
-        rec = (char *) odr_malloc (odr, len+1);
-        memcpy (rec, wrbuf_buf(wr), len+1);
-        rec[len] = 0;
-    }
-    wrbuf_free (wr, 1);
+        rec = (char *) odr_strdup(odr, result);
+    yaz_marc_destroy(mt);
     return rec;
 }
diff --git a/zutil/zoom-c.c b/zutil/zoom-c.c
index dc04658..8506a26 100644
--- a/zutil/zoom-c.c
+++ b/zutil/zoom-c.c
@@ -2,7 +2,7 @@
  * Copyright (c) 2000-2002, Index Data
  * See the file LICENSE for details.
  *
- * $Id: zoom-c.c,v 1.11 2002-12-10 13:14:14 adam Exp $
+ * $Id: zoom-c.c,v 1.12 2002-12-16 13:13:54 adam Exp $
  *
  * ZOOM layer for C, connections, result sets, queries.
  */
@@ -1094,6 +1094,7 @@ ZOOM_record_get (ZOOM_record rec, const char *type, int *len)
         }
         else if (r->which == Z_External_octet)
         {
+            yaz_marc_t mt;
             switch (ent->value)
             {
             case VAL_SOIF:
@@ -1106,18 +1107,23 @@ ZOOM_record_get (ZOOM_record rec, const char *type, int *len)
             default:
                 if (!rec->wrbuf_marc)
                     rec->wrbuf_marc = wrbuf_alloc();
+
+                mt = yaz_marc_create();
                 wrbuf_rewind (rec->wrbuf_marc);
-                if (yaz_marc_decode ((const char *)
-                                     r->u.octet_aligned->buf,
-                                     rec->wrbuf_marc, 0,
-                                     r->u.octet_aligned->len,
-                                     0) > 0)
+                if (yaz_marc_decode_wrbuf (
+                        mt, (const char *) r->u.octet_aligned->buf,
+                        r->u.octet_aligned->len,
+                        rec->wrbuf_marc) > 0)
                 {
-                    if (len) *len = wrbuf_len(rec->wrbuf_marc);
+                    if (len)
+                        *len = wrbuf_len(rec->wrbuf_marc);
+                    yaz_marc_destroy(mt);
                     return wrbuf_buf(rec->wrbuf_marc);
                 }
+                yaz_marc_destroy(mt);
             }
-            if (len) *len = r->u.octet_aligned->len;
+            if (len) 
+                *len = r->u.octet_aligned->len;
             return (const char *) r->u.octet_aligned->buf;
         }
         else if (r->which == Z_External_grs1)
@@ -1145,6 +1151,7 @@ ZOOM_record_get (ZOOM_record rec, const char *type, int *len)
         }
         else if (r->which == Z_External_octet)
         {
+            yaz_marc_t mt;
             int marc_decode_type = YAZ_MARC_MARCXML;
 
             if (!strcmp(type, "oai"))
@@ -1162,15 +1169,20 @@ ZOOM_record_get (ZOOM_record rec, const char *type, int *len)
                 if (!rec->wrbuf_marc)
                     rec->wrbuf_marc = wrbuf_alloc();
                 wrbuf_rewind (rec->wrbuf_marc);
-                if (yaz_marc_decode ((const char *)
-                                     r->u.octet_aligned->buf,
-                                     rec->wrbuf_marc, 0,
-                                     r->u.octet_aligned->len,
-                                     marc_decode_type) > 0)
+                mt = yaz_marc_create();
+
+                yaz_marc_xml(mt, YAZ_MARC_MARCXML);
+                if (yaz_marc_decode_wrbuf (
+                        mt, (const char *) r->u.octet_aligned->buf,
+                        r->u.octet_aligned->len,
+                        rec->wrbuf_marc) > 0)
                 {
-                    if (len) *len = wrbuf_len(rec->wrbuf_marc);
+                    if (len) 
+                        *len = wrbuf_len(rec->wrbuf_marc);
+                    yaz_marc_destroy(mt);
                     return wrbuf_buf(rec->wrbuf_marc);
                 }
+                yaz_marc_destroy(mt);
             }
             if (len) *len = r->u.octet_aligned->len;
             return (const char *) r->u.octet_aligned->buf;