Update TurboMARC definition and document it

Document TurboMARC in the tools chapter of the YAZ manual. Update
definition format mode to YAZ_MARC_TURBOMARC. The string as used in
yaz-marcdump and other tools is called "turbomarc". Rename function
yaz_marc_write_turbo_xml to yaz_marc_write_turbomarc.

diff --git a/doc/tools.xml b/doc/tools.xml
index ec97f06..b487442 100644 (file)
--- a/doc/tools.xml
+++ b/doc/tools.xml
@@ -1912,9 +1912,9 @@ void cql_to_xml_stdio(struct cql_node *cn, FILE *f);
   <sect1 id="marc"><title>MARC</title>
    
    <para>
   <sect1 id="marc"><title>MARC</title>
    
    <para>

-    YAZ provides a fast utility that decodes MARC records and
-    encodes to a varity of output formats. The MARC records must
-    be encoded in ISO2709.
+    YAZ provides a fast utility for working with MARC records.
+    Early versions of the MARC utility only allowed decoding of ISO2709.
+    Today the utility may both encode - and decode to a varity of formats.

    </para>
    <synopsis><![CDATA[
     #include <yaz/marcdisp.h>
    </para>
    <synopsis><![CDATA[
     #include <yaz/marcdisp.h>

@@ -1932,6 +1932,8 @@ void cql_to_xml_stdio(struct cql_node *cn, FILE *f);
     #define YAZ_MARC_MARCXML   3
     #define YAZ_MARC_ISO2709   4
     #define YAZ_MARC_XCHANGE   5
     #define YAZ_MARC_MARCXML   3
     #define YAZ_MARC_ISO2709   4
     #define YAZ_MARC_XCHANGE   5

+    #define YAZ_MARC_CHECK     6
+    #define YAZ_MARC_TURBOMARC 7

 
     /* supply iconv handle for character set conversion .. */
     void yaz_marc_iconv(yaz_marc_t mt, yaz_iconv_t cd);
 
     /* supply iconv handle for character set conversion .. */
     void yaz_marc_iconv(yaz_marc_t mt, yaz_iconv_t cd);

@@ -1941,15 +1943,22 @@ void cql_to_xml_stdio(struct cql_node *cn, FILE *f);
 
     /* decode MARC in buf of size bsize. Returns >0 on success; <=0 on failure.
     On success, result in *result with size *rsize. */
 
     /* decode MARC in buf of size bsize. Returns >0 on success; <=0 on failure.
     On success, result in *result with size *rsize. */

-    int yaz_marc_decode_buf (yaz_marc_t mt, const char *buf, int bsize,
-                             char **result, int *rsize);
+    int yaz_marc_decode_buf(yaz_marc_t mt, const char *buf, int bsize,
+                            const char **result, size_t *rsize);

 
     /* decode MARC in buf of size bsize. Returns >0 on success; <=0 on failure.
        On success, result in WRBUF */
 
     /* decode MARC in buf of size bsize. Returns >0 on success; <=0 on failure.
        On success, result in WRBUF */

-    int yaz_marc_decode_wrbuf (yaz_marc_t mt, const char *buf,
-                               int bsize, WRBUF wrbuf);
+    int yaz_marc_decode_wrbuf(yaz_marc_t mt, const char *buf,
+                              int bsize, WRBUF wrbuf);

 ]]>
    </synopsis>
 ]]>
    </synopsis>

+   <note>
+    <para>
+     The synopsis is just a basic subset of all functionality. Refer
+     to the actual header file <filename>marcdisp.h</filename> for
+     details.
+    </para>
+   </note>

    <para>
     A MARC conversion handle must be created by using
     <function>yaz_marc_create</function> and destroyed
    <para>
     A MARC conversion handle must be created by using
     <function>yaz_marc_create</function> and destroyed

@@ -1974,7 +1983,7 @@ void cql_to_xml_stdio(struct cql_node *cn, FILE *f);
       <term>YAZ_MARC_MARCXML</term>
       <listitem>
        <para>
       <term>YAZ_MARC_MARCXML</term>
       <listitem>
        <para>

-       The resulting record is converted to MARCXML.
+       <ulink url="&url.marcxml;">MARCXML</ulink>.

        </para>
       </listitem>
      </varlistentry>
        </para>
       </listitem>
      </varlistentry>

@@ -1983,10 +1992,41 @@ void cql_to_xml_stdio(struct cql_node *cn, FILE *f);
       <term>YAZ_MARC_ISO2709</term>
       <listitem>
        <para>
       <term>YAZ_MARC_ISO2709</term>
       <listitem>
        <para>

-       The resulting record is converted to ISO2709 (MARC).
+       ISO2709 (sometimes just referred to as "MARC").
+       </para>
+      </listitem>
+     </varlistentry>
+
+     <varlistentry>
+      <term>YAZ_MARC_XCHANGE</term>
+      <listitem>
+       <para>
+       <ulink url="&url.marcxchange;">MarcXchange</ulink>.

        </para>
       </listitem>
      </varlistentry>
        </para>
       </listitem>
      </varlistentry>

+
+     <varlistentry>
+      <term>YAZ_MARC_CHECK</term>
+      <listitem>
+       <para>
+       Pseudo format for validation only. Does not generate
+       any real output except diagnostics.
+       </para>
+      </listitem>
+     </varlistentry>
+
+     <varlistentry>
+      <term>YAZ_MARC_TURBOMARC</term>
+      <listitem>
+       <para>
+       XML format with same semantics as MARCXML but more compact
+       and geared towards fast processing with XSLT. Refer to
+       <xref linkend="tools.turbomarc"/> for more information.
+       </para>
+      </listitem>
+     </varlistentry>
+

     </variablelist>
    </para>
    <para>
     </variablelist>
    </para>
    <para>

@@ -2000,13 +2040,13 @@ void cql_to_xml_stdio(struct cql_node *cn, FILE *f);
    <example id="example.marc.display">
     <title>Display of MARC record</title>
     <para>
    <example id="example.marc.display">
     <title>Display of MARC record</title>
     <para>

-     The followint program snippet illustrates how the MARC API may
+     The following program snippet illustrates how the MARC API may

      be used to convert a MARC record to the line-by-line format:
      <programlisting><![CDATA[
       void print_marc(const char *marc_buf, int marc_buf_size)
       {
          char *result;      /* for result buf */
      be used to convert a MARC record to the line-by-line format:
      <programlisting><![CDATA[
       void print_marc(const char *marc_buf, int marc_buf_size)
       {
          char *result;      /* for result buf */

-         int result_len;    /* for size of result */
+         size_t result_len;    /* for size of result */

          yaz_marc_t mt = yaz_marc_create();
          yaz_marc_xml(mt, YAZ_MARC_LINE);
          yaz_marc_decode_buf(mt, marc_buf, marc_buf_size,
          yaz_marc_t mt = yaz_marc_create();
          yaz_marc_xml(mt, YAZ_MARC_LINE);
          yaz_marc_decode_buf(mt, marc_buf, marc_buf_size,

@@ -2018,6 +2058,71 @@ void cql_to_xml_stdio(struct cql_node *cn, FILE *f);
       </programlisting>
     </para>
    </example>
       </programlisting>
     </para>
    </example>

+   <sect2 id="tools.turbomarc">
+    <title>TurboMARC</title>
+    <para>
+     TurboMARC is yet another XML encoding of a MARC record. The format
+     was designed for fast processing with XSLT.
+    </para>
+    <para>
+     Applications like
+     Pazpar2 uses XSLT to convert an XML encode MARC record to an internal
+     representation. This conversion mostly check the tag of a MARC field
+     to determine the basic rules in the conversion. This check is
+     costly when that is tag is encoded as an attribute in MARCXML.
+     By having the tag value as the element instead, makes processing
+     many times faster (at least for Libxslt).
+    </para>
+    <para>
+     TurboMARC is encoded as follows:
+     <itemizedlist>
+      <listitem><para>
+       Record elements is part of namespace
+       "<literal>http://www.indexdata.com/MARC21/turboxml</literal>".
+       </para></listitem>
+      <listitem><para>
+       A record is enclosed in element <literal>r</literal>.
+       </para></listitem>
+      <listitem><para>
+       A collection of records is enclosed in element
+       <literal>collection</literal>.
+       </para></listitem>
+      <listitem><para>
+       The leader is encoded as element <literal>l</literal> with the 
+       leader content as its (text) value.
+       </para></listitem>
+      <listitem><para>
+       A control field is encoded as element <literal>c</literal> concatenated
+       with the tag value of the control field if the tag value
+       matches the regular expression <literal>[a-zA-Z0-9]*</literal>.
+       If the tag value do not match the regular expression
+       <literal>[a-zA-Z0-9]*</literal> the control field is encoded
+       as element <literal>c</literal> and attribute <literal>code</literal>
+       will hold the tag value.
+       This rule ensure that in the rare cases where a tag value might
+       result in a non-wellformed XML YAZ encode it as a coded attribute
+       (as in MARCXML).
+       </para>
+       <para>
+       The control field content is the the text value of this element.
+       Indicators are encoded as attribute names
+       <literal>i1</literal>, <literal>i2</literal>, etc.. and
+       corresponding values for each indicator.
+       </para></listitem>
+      <listitem><para>
+       A data field is encoded as element <literal>d</literal> concatenated
+       with the tag value of the data field or using the attribute
+       <literal>code</literal> as described in the rules for control fields.
+       The children of the data field element is subfield elements.
+       Each subfield element is encoded as <literal>s</literal>
+       concatenated with the sub field code.
+       The text of the subfield element is the contents of the subfield.
+       Indicators are encoded as attributes for the data field element similar
+       to the encoding for control fields.
+       </para></listitem>
+     </itemizedlist>
+    </para>
+   </sect2>

   </sect1>
 
   <sect1 id="tools.retrieval">
   </sect1>
 
   <sect1 id="tools.retrieval">

diff --git a/doc/yaz-marcdump-man.xml b/doc/yaz-marcdump-man.xml
index 51ebcfb..dbc4060 100644 (file)
--- a/doc/yaz-marcdump-man.xml
+++ b/doc/yaz-marcdump-man.xml
@@ -79,7 +79,8 @@
       Specifies input format. Must be one of
       <literal>marcxml</literal>, <literal>marc</literal> (ISO2709),
       <literal>marcxchange</literal> (ISO25577),
       Specifies input format. Must be one of
       <literal>marcxml</literal>, <literal>marc</literal> (ISO2709),
       <literal>marcxchange</literal> (ISO25577),

-      <literal>line</literal> (line mode MARC).
+      <literal>line</literal> (line mode MARC),
+      or <literal>turbomarc</literal> (Turbo MARC).

      </para></listitem>
    </varlistentry>
 
      </para></listitem>
    </varlistentry>
 

@@ -89,7 +90,8 @@
       Specifies output format. Must be one of
       <literal>marcxml</literal>, <literal>marc</literal> (ISO2709),
       <literal>marcxchange</literal> (ISO25577),
       Specifies output format. Must be one of
       <literal>marcxml</literal>, <literal>marc</literal> (ISO2709),
       <literal>marcxchange</literal> (ISO25577),

-      <literal>line</literal> (line mode MARC).
+      <literal>line</literal> (line mode MARC),
+      or <literal>turbomarc</literal> (Turbo MARC).

      </para></listitem>
    </varlistentry>
 
      </para></listitem>
    </varlistentry>
 

@@ -191,6 +193,16 @@
     yaz-marcdump -f MARC-8 -t UTF-8 -o marcxml marc21.raw >marcxml.xml
    </screen>
   </para>
     yaz-marcdump -f MARC-8 -t UTF-8 -o marcxml marc21.raw >marcxml.xml
    </screen>
   </para>

+
+  <para>
+   Turbo MARC is a compact XML notation with same semantics as
+   MARCXML, but which allows for faster processing via XSLT. In order
+   to generate Turbo MARC records encoded in UTF-8 from MARC21 (ISO), one
+   could use:
+   <screen>
+    yaz-marcdump -f MARC8 -t UTF8 -o turbomarc -i marc marc21.raw >out.xml
+   </screen>
+  </para>

  </refsect1> 
 
  <refsect1><title>FILES</title>
  </refsect1> 
 
  <refsect1><title>FILES</title>

diff --git a/include/yaz/marcdisp.h b/include/yaz/marcdisp.h
index a970de7..f472aba 100644 (file)
--- a/include/yaz/marcdisp.h
+++ b/include/yaz/marcdisp.h
@@ -73,9 +73,9 @@ YAZ_EXPORT void yaz_marc_xml(yaz_marc_t mt, int xmlmode);
 /** \brief Output format: MarcXchange (ISO25577) */
 #define YAZ_MARC_XCHANGE   5
 /** \brief Output format: check only (no marc output) */
 /** \brief Output format: MarcXchange (ISO25577) */
 #define YAZ_MARC_XCHANGE   5
 /** \brief Output format: check only (no marc output) */

-#define YAZ_MARC_CHECK  6
-/** \brief Output format: Turbo MARCXML Index Data format*/
-#define YAZ_MARC_TMARCXML   7
+#define YAZ_MARC_CHECK     6
+/** \brief Output format: Turbo MARC Index Data format (XML based) */
+#define YAZ_MARC_TURBOMARC 7

 
 /** \brief set iconv handle for character set conversion */
 YAZ_EXPORT void yaz_marc_iconv(yaz_marc_t mt, yaz_iconv_t cd);
 
 /** \brief set iconv handle for character set conversion */
 YAZ_EXPORT void yaz_marc_iconv(yaz_marc_t mt, yaz_iconv_t cd);

@@ -177,7 +177,7 @@ int yaz_marc_read_line(yaz_marc_t mt,
                        void *client_data);
 
 #if YAZ_HAVE_XML2
                        void *client_data);
 
 #if YAZ_HAVE_XML2

-/** \brief parses MARCXML/MarcXchange record from xmlNode pointer 
+/** \brief parses MARCXML/MarcXchange/TurboMARC record from xmlNode pointer 

     \param mt handle
     \param ptr is a pointer to root xml node 
     \retval 0 OK
     \param mt handle
     \param ptr is a pointer to root xml node 
     \retval 0 OK

@@ -204,13 +204,13 @@ YAZ_EXPORT int yaz_marc_write_line(yaz_marc_t mt, WRBUF wrbuf);
 */
 YAZ_EXPORT int yaz_marc_write_marcxml(yaz_marc_t mt, WRBUF wrbuf);
 
 */
 YAZ_EXPORT int yaz_marc_write_marcxml(yaz_marc_t mt, WRBUF wrbuf);
 

-/** \brief writes record in TMARCXML format
+/** \brief writes record in TurboMARC format

     \param mt handle
     \param wrbuf WRBUF for output
     \retval 0 OK
     \retval -1 ERROR
 */
     \param mt handle
     \param wrbuf WRBUF for output
     \retval 0 OK
     \retval -1 ERROR
 */

-YAZ_EXPORT int yaz_marc_write_turbo_xml(yaz_marc_t mt, WRBUF wrbuf);
+YAZ_EXPORT int yaz_marc_write_turbomarc(yaz_marc_t mt, WRBUF wrbuf);

 
 /** \brief writes record in MarcXchange XML (ISO25577)
     \param mt handle
 
 /** \brief writes record in MarcXchange XML (ISO25577)
     \param mt handle

diff --git a/src/marc_read_xml.c b/src/marc_read_xml.c
index 59ffb00..6b3e3dd 100644 (file)
--- a/src/marc_read_xml.c
+++ b/src/marc_read_xml.c
@@ -377,7 +377,7 @@ int yaz_marc_read_xml(yaz_marc_t mt, const xmlNode *ptr)
             }
             else if (!strcmp((const char *) ptr->name, "r"))
             {
             }
             else if (!strcmp((const char *) ptr->name, "r"))
             {

-                format = YAZ_MARC_TMARCXML;
+                format = YAZ_MARC_TURBOMARC;

                 break;
             }
             else
                 break;
             }
             else

@@ -402,7 +402,7 @@ int yaz_marc_read_xml(yaz_marc_t mt, const xmlNode *ptr)
     {
     case YAZ_MARC_MARCXML:
         return yaz_marc_read_xml_fields(mt, ptr->next);
     {
     case YAZ_MARC_MARCXML:
         return yaz_marc_read_xml_fields(mt, ptr->next);

-    case YAZ_MARC_TMARCXML:
+    case YAZ_MARC_TURBOMARC:

         return yaz_marc_read_turbo_xml_fields(mt, ptr->next);
     }
     return -1;
         return yaz_marc_read_turbo_xml_fields(mt, ptr->next);
     }
     return -1;

diff --git a/src/marcdisp.c b/src/marcdisp.c
index 10fa7b9..d4fdf6c 100644 (file)
--- a/src/marcdisp.c
+++ b/src/marcdisp.c
@@ -569,7 +569,7 @@ int yaz_marc_write_trailer(yaz_marc_t mt, WRBUF wr)
         switch(mt->output_format)
         {
         case YAZ_MARC_MARCXML:
         switch(mt->output_format)
         {
         case YAZ_MARC_MARCXML:

-        case YAZ_MARC_TMARCXML:
+        case YAZ_MARC_TURBOMARC:

             wrbuf_printf(wr, "</collection>\n");
             break;
         case YAZ_MARC_XCHANGE:
             wrbuf_printf(wr, "</collection>\n");
             break;
         case YAZ_MARC_XCHANGE:

@@ -593,8 +593,8 @@ int yaz_marc_write_mode(yaz_marc_t mt, WRBUF wr)
         return yaz_marc_write_line(mt, wr);
     case YAZ_MARC_MARCXML:
         return yaz_marc_write_marcxml(mt, wr);
         return yaz_marc_write_line(mt, wr);
     case YAZ_MARC_MARCXML:
         return yaz_marc_write_marcxml(mt, wr);

-    case YAZ_MARC_TMARCXML:
-        return yaz_marc_write_turbo_xml(mt, wr);
+    case YAZ_MARC_TURBOMARC:
+        return yaz_marc_write_turbomarc(mt, wr);

     case YAZ_MARC_XCHANGE:
         return yaz_marc_write_marcxchange(mt, wr, 0, 0); /* no format, type */
     case YAZ_MARC_ISO2709:
     case YAZ_MARC_XCHANGE:
         return yaz_marc_write_marcxchange(mt, wr, 0, 0); /* no format, type */
     case YAZ_MARC_ISO2709:

@@ -816,7 +816,7 @@ int yaz_marc_write_marcxml(yaz_marc_t mt, WRBUF wr)
                                      0, 0, 0);
 }
 
                                      0, 0, 0);
 }
 

-int yaz_marc_write_turbo_xml(yaz_marc_t mt, WRBUF wr)
+int yaz_marc_write_turbomarc(yaz_marc_t mt, WRBUF wr)

 {
     /* set leader 09 to 'a' for UNICODE */
     /* http://www.loc.gov/marc/bibliographic/ecbdldrd.html#mrcblea */
 {
     /* set leader 09 to 'a' for UNICODE */
     /* http://www.loc.gov/marc/bibliographic/ecbdldrd.html#mrcblea */

@@ -1336,8 +1336,8 @@ int yaz_marc_decode_formatstr(const char *arg)
         mode = YAZ_MARC_ISO2709;
     if (!strcmp(arg, "marcxml"))
         mode = YAZ_MARC_MARCXML;
         mode = YAZ_MARC_ISO2709;
     if (!strcmp(arg, "marcxml"))
         mode = YAZ_MARC_MARCXML;

-    if (!strcmp(arg, "tmarcxml"))
-        mode = YAZ_MARC_TMARCXML;
+    if (!strcmp(arg, "turbomarc"))
+        mode = YAZ_MARC_TURBOMARC;

     if (!strcmp(arg, "marcxchange"))
         mode = YAZ_MARC_XCHANGE;
     if (!strcmp(arg, "line"))
     if (!strcmp(arg, "marcxchange"))
         mode = YAZ_MARC_XCHANGE;
     if (!strcmp(arg, "line"))

diff --git a/src/record_conv.c b/src/record_conv.c
index 497203e..695f088 100644 (file)
--- a/src/record_conv.c
+++ b/src/record_conv.c
@@ -304,9 +304,9 @@ static int conv_marc(yaz_record_conv_t p, const xmlNode *ptr)
         if (input_charset && !output_charset)
             output_charset = "utf-8";
     }
         if (input_charset && !output_charset)
             output_charset = "utf-8";
     }

-    else if (!strcmp(output_format, "tmarcxml"))
+    else if (!strcmp(output_format, "turbomarc"))

     {
     {

-        output_format_mode = YAZ_MARC_TMARCXML;
+        output_format_mode = YAZ_MARC_TURBOMARC;

         if (input_charset && !output_charset)
             output_charset = "utf-8";
     }
         if (input_charset && !output_charset)
             output_charset = "utf-8";
     }

@@ -479,7 +479,7 @@ static int yaz_record_conv_record_rule(yaz_record_conv_t p,
                     ret = -1;
             }
             else if (r->u.marc.input_format == YAZ_MARC_MARCXML ||
                     ret = -1;
             }
             else if (r->u.marc.input_format == YAZ_MARC_MARCXML ||

-                     r->u.marc.input_format == YAZ_MARC_TMARCXML)
+                     r->u.marc.input_format == YAZ_MARC_TURBOMARC)

             {
                 xmlDocPtr doc = xmlParseMemory(wrbuf_buf(record),
                                                wrbuf_len(record));
             {
                 xmlDocPtr doc = xmlParseMemory(wrbuf_buf(record),
                                                wrbuf_len(record));

diff --git a/src/zoom-c.c b/src/zoom-c.c
index 24e6ad4..c579957 100644 (file)
--- a/src/zoom-c.c
+++ b/src/zoom-c.c
@@ -2196,7 +2196,7 @@ ZOOM_API(const char *)
     }
     else if (!strcmp(type, "txml"))
     {
     }
     else if (!strcmp(type, "txml"))
     {

-        return get_record_format(rec, len, npr, YAZ_MARC_TMARCXML, charset,
+        return get_record_format(rec, len, npr, YAZ_MARC_TURBOMARC, charset,

                                  format);
     }
     else if (!strcmp(type, "raw"))
                                  format);
     }
     else if (!strcmp(type, "raw"))

diff --git a/test/tstmarc.sh b/test/tstmarc.sh
index f1ac177..2996bb4 100755 (executable)
--- a/test/tstmarc.sh
+++ b/test/tstmarc.sh
@@ -101,12 +101,12 @@ binmarc_convert "xml,marcxml" "marcxml" "xml2"
 echo "binmarc -> marcxml(libxml2): $?" 
 fi
 
 echo "binmarc -> marcxml(libxml2): $?" 
 fi
 

-binmarc_convert "tmarcxml"  "tmarcxml" "t" 
-echo "binmarc -> tmarcxml: $?" 
+binmarc_convert "turbomarc"  "turbomarc" "t" 
+echo "binmarc -> turbomarc: $?" 

 
 if test -z "$noxmlwrite"; then
 
 if test -z "$noxmlwrite"; then

-binmarc_convert "xml,tmarcxml"  "tmarcxml" "xml2t" 
-echo "binmarc -> tmarcxml(libxml2): $?" 
+binmarc_convert "xml,turbomarc"  "turbomarc" "xml2t" 
+echo "binmarc -> turbomarc(libxml2): $?" 

 fi
 
 exit $ecode
 fi
 
 exit $ecode

diff --git a/util/marcdump.c b/util/marcdump.c
index c30623a..0696df8 100644 (file)
--- a/util/marcdump.c
+++ b/util/marcdump.c
@@ -223,7 +223,7 @@ static void dump(const char *fname, const char *from, const char *to,
     yaz_marc_write_using_libxml2(mt, write_using_libxml2);
     yaz_marc_debug(mt, verbose);
 
     yaz_marc_write_using_libxml2(mt, write_using_libxml2);
     yaz_marc_debug(mt, verbose);
 

-    if (input_format == YAZ_MARC_MARCXML || input_format == YAZ_MARC_TMARCXML || input_format == YAZ_MARC_XCHANGE)
+    if (input_format == YAZ_MARC_MARCXML || input_format == YAZ_MARC_TURBOMARC || input_format == YAZ_MARC_XCHANGE)

     {
 #if YAZ_HAVE_XML2
         marcdump_read_xml(mt, fname);
     {
 #if YAZ_HAVE_XML2
         marcdump_read_xml(mt, fname);