From 95f9f397e64bd7a933482008f7afe647e56648a2 Mon Sep 17 00:00:00 2001 From: Adam Dickmeiss Date: Sat, 7 Aug 2004 08:06:57 +0000 Subject: [PATCH] For MARCXML, set leader pos 9 to a to signal unicode --- src/Makefile.am | 4 ++-- src/charconv.tcl | 49 +++++++++++++++++++++++++++++++++---------------- src/marcdisp.c | 10 ++++++++-- 3 files changed, 43 insertions(+), 20 deletions(-) diff --git a/src/Makefile.am b/src/Makefile.am index 1ef99f4..8ba3ba7 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1,6 +1,6 @@ ## Copyright (C) 1994-2004, Index Data ## All rights reserved. -## $Id: Makefile.am,v 1.13 2004-05-01 23:32:20 adam Exp $ +## $Id: Makefile.am,v 1.14 2004-08-07 08:06:57 adam Exp $ if ISTHR thrlib=libyazthread.la @@ -31,7 +31,7 @@ THREADED_FLAGS = @CFLAGSTHREADS@ # MARC8 conversion is generated from charconv.sgm + codetables.xml marc8.c: charconv.tcl charconv.sgm codetables.xml - cd $(srcdir); ./charconv.tcl -O 1 -p marc8 charconv.sgm codetables.xml -o marc8.c + cd $(srcdir); ./charconv.tcl -p marc8 codetables.xml -o marc8.c libyaz_la_SOURCES=version.c options.c log.c marcdisp.c oid.c wrbuf.c \ nmemsdup.c xmalloc.c readconf.c tpath.c nmem.c matchstr.c atoin.c \ diff --git a/src/charconv.tcl b/src/charconv.tcl index fcc83fd..d49f238 100755 --- a/src/charconv.tcl +++ b/src/charconv.tcl @@ -2,7 +2,7 @@ # the next line restats using tclsh \ exec tclsh "$0" "$@" # -# $Id: charconv.tcl,v 1.6 2004-03-20 07:16:25 adam Exp $ +# $Id: charconv.tcl,v 1.7 2004-08-07 08:06:57 adam Exp $ proc usage {} { puts {charconv.tcl: [-p prefix] [-s split] [-o ofile] file ... } @@ -18,10 +18,12 @@ proc preamble_trie {ofilehandle} { puts $f " struct yaz_iconv_trie_flat { char from\[6\]; + unsigned combining : 1; $totype to; }; struct yaz_iconv_trie_dir { - short ptr; + short ptr : 15; + unsigned combining : 1; $totype to; }; @@ -32,7 +34,7 @@ proc preamble_trie {ofilehandle} { " puts $f { static unsigned long lookup(struct yaz_iconv_trie **ptrs, int ptr, unsigned char *inp, - size_t inbytesleft, size_t *no_read) + size_t inbytesleft, size_t *no_read, int *combining) { struct yaz_iconv_trie *t = (ptr >= 0) ? ptrs[ptr] : 0; if (!t || inbytesleft < 1) @@ -41,7 +43,7 @@ proc preamble_trie {ofilehandle} { { size_t ch = inp[0] & 0xff; unsigned long code = - lookup(ptrs, t->dir[ch].ptr, inp+1, inbytesleft-1, no_read); + lookup(ptrs, t->dir[ch].ptr, inp+1, inbytesleft-1, no_read, combining); if (code) { (*no_read)++; @@ -50,6 +52,7 @@ proc preamble_trie {ofilehandle} { if (t->dir[ch].to) { code = t->dir[ch].to; + *combining = t->dir[ch].combining; *no_read = 1; return code; } @@ -65,6 +68,7 @@ proc preamble_trie {ofilehandle} { if (memcmp(flat->from, inp, len) == 0) { *no_read = len; + *combining = flat->combining; return flat->to; } } @@ -90,7 +94,7 @@ proc reset_trie {} { set trie(prefix) {} } -proc ins_trie {from to} { +proc ins_trie {from to combining} { global trie if {![info exists trie(no)]} { set trie(no) 1 @@ -101,7 +105,7 @@ proc ins_trie {from to} { set trie(max) $to } incr trie(size) - ins_trie_r [split $from] $to 0 + ins_trie_r [split $from] $to $combining 0 } proc split_trie {this} { @@ -110,6 +114,7 @@ proc split_trie {this} { foreach e $trie($this,content) { set from [lindex $e 0] set to [lindex $e 1] + set combining [lindex $e 2] set ch [lindex $from 0] set rest [lrange $from 1 end] @@ -119,27 +124,28 @@ proc split_trie {this} { set trie($this,ptr,$ch) $trie(no) incr trie(no) } - ins_trie_r $rest $to $trie($this,ptr,$ch) + ins_trie_r $rest $to $combining $trie($this,ptr,$ch) } else { set trie($this,to,$ch) $to + set trie($this,combining,$ch) $combining } } set trie($this,content) missing } -proc ins_trie_r {from to this} { +proc ins_trie_r {from to combining this} { global trie if {![info exist trie($this,type)]} { set trie($this,type) f } if {$trie($this,type) == "f"} { - lappend trie($this,content) [list $from $to] + lappend trie($this,content) [list $from $to $combining] # split ? if {[llength $trie($this,content)] > $trie(split)} { split_trie $this - return [ins_trie_r $from $to $this] + return [ins_trie_r $from $to $combining $this] } } else { set ch [lindex $from 0] @@ -150,9 +156,10 @@ proc ins_trie_r {from to this} { set trie($this,ptr,$ch) $trie(no) incr trie(no) } - ins_trie_r $rest $to $trie($this,ptr,$ch) + ins_trie_r $rest $to $combining $trie($this,ptr,$ch) } else { set trie($this,to,$ch) $to + set trie($this,combining,$ch) $combining } } } @@ -174,7 +181,7 @@ proc dump_trie {ofilehandle} { foreach d [lindex $m 0] { puts -nonewline $f "\\x$d" } - puts -nonewline $f "\", 0x[lindex $m 1]" + puts -nonewline $f "\", [lindex $m 2], 0x[lindex $m 1]" puts $f "\}," } puts $f " \{\"\", 0\}" @@ -194,6 +201,11 @@ proc dump_trie {ofilehandle} { } else { puts -nonewline $f "-1, " } + if {[info exist trie($this,combining,$ch)]} { + puts -nonewline $f "$trie($this,combining,$ch), " + } else { + puts -nonewline $f "0, " + } if {[info exist trie($this,to,$ch)]} { puts -nonewline $f "0x$trie($this,to,$ch)\}" set null 0 @@ -224,11 +236,11 @@ proc dump_trie {ofilehandle} { puts $f "" puts $f "unsigned long yaz_$trie(prefix)_conv - (unsigned char *inp, size_t inbytesleft, size_t *no_read) + (unsigned char *inp, size_t inbytesleft, size_t *no_read, int *combining) { unsigned long code; - code = lookup($trie(prefix)ptrs, 0, inp, inbytesleft, no_read); + code = lookup($trie(prefix)ptrs, 0, inp, inbytesleft, no_read, combining); if (!code) { *no_read = 1; @@ -247,6 +259,7 @@ proc readfile {fname ofilehandle prefix omits} { set lineno 0 set f [open $fname r] set tablenumber x + set combining 0 while {1} { incr lineno set cnt [gets $f line] @@ -259,11 +272,12 @@ proc readfile {fname ofilehandle prefix omits} { } elseif {[regexp {} $line s]} { dump_trie $ofilehandle } elseif {[regexp {([0-9A-Fa-f]*)} $line s hex ucs]} { - ins_trie $hex $ucs + ins_trie $hex $ucs $combining unset hex } elseif {[regexp {} $line s]} { if {[lsearch $omits $tablenumber] == -1} { dump_trie $ofilehandle @@ -274,13 +288,16 @@ proc readfile {fname ofilehandle prefix omits} { lappend hex [string range $marc $i [expr $i+1]] } # puts "ins_trie $hex $ucs" - ins_trie $hex $ucs + ins_trie $hex $ucs $combining unset hex } set marc {} set uni {} + set combining 0 } elseif {[regexp {([0-9A-Fa-f]*)} $line s marc]} { incr marc_lines + } elseif {[regexp {true} $line s]} { + set combining 1 } elseif {[regexp {([0-9A-Fa-f]*)} $line s ucs]} { incr ucs_lines } diff --git a/src/marcdisp.c b/src/marcdisp.c index a544fcf..38ac2ca 100644 --- a/src/marcdisp.c +++ b/src/marcdisp.c @@ -2,7 +2,7 @@ * Copyright (c) 1995-2004, Index Data * See the file LICENSE for details. * - * $Id: marcdisp.c,v 1.5 2004-03-16 13:15:58 adam Exp $ + * $Id: marcdisp.c,v 1.6 2004-08-07 08:07:00 adam Exp $ */ #if HAVE_CONFIG_H @@ -134,7 +134,13 @@ int yaz_marc_decode_wrbuf (yaz_marc_t mt, const char *buf, int bsize, WRBUF wr) wr, "\n" " "); - marc_cdata(mt, buf, 24, wr); +#if 1 + marc_cdata(mt, buf, 9, wr); + marc_cdata(mt, "a", 1, wr); /* set leader to signal unicode */ + marc_cdata(mt, buf+10, 14, wr); +#else + marc_cdata(mt, buf, 24, wr); /* leave header as is .. */ +#endif wrbuf_printf(wr, "\n"); break; } -- 1.7.10.4