util/marcdump.c

   1 /* This file is part of the YAZ toolkit.
   2  * Copyright (C) 1995-2009 Index Data
   3  * See the file LICENSE for details.
   4  */
   5
   6 #define _FILE_OFFSET_BITS 64
   7
   8 #if HAVE_CONFIG_H
   9 #include <config.h>
  10 #endif
  11
  12 #if YAZ_HAVE_XML2
  13 #include <libxml/parser.h>
  14 #include <libxml/tree.h>
  15 #include <libxml/xpath.h>
  16 #include <libxml/xpathInternals.h>
  17
  18 /* Libxml2 version < 2.6.15. xmlreader not reliable/present */
  19 #if LIBXML_VERSION < 20615
  20 #define USE_XMLREADER 0
  21 #else
  22 #define USE_XMLREADER 1
  23 #endif
  24
  25 #if USE_XMLREADER
  26 #include <libxml/xmlreader.h>
  27 #endif
  28
  29 #endif
  30
  31 #include <stdio.h>
  32 #include <stdlib.h>
  33 #include <string.h>
  34 #include <errno.h>
  35 #include <assert.h>
  36
  37 #if HAVE_LOCALE_H
  38 #include <locale.h>
  39 #endif
  40 #if HAVE_LANGINFO_H
  41 #include <langinfo.h>
  42 #endif
  43
  44 #include <yaz/marcdisp.h>
  45 #include <yaz/yaz-util.h>
  46 #include <yaz/xmalloc.h>
  47 #include <yaz/options.h>
  48
  49 #ifndef SEEK_SET
  50 #define SEEK_SET 0
  51 #endif
  52 #ifndef SEEK_END
  53 #define SEEK_END 2
  54 #endif
  55
  56
  57 static char *prog;
  58
  59 static void usage(const char *prog)
  60 {
  61     fprintf (stderr, "Usage: %s [-i format] [-o format] [-f from] [-t to] "
  62              "[-l pos=value] [-c cfile] [-s prefix] [-C size] [-n] "
  63              "[-p] [-v] [-V] file...\n",
  64              prog);
  65 }
  66
  67 static void show_version(void)
  68 {
  69     char vstr[20], sha1_str[41];
  70
  71     yaz_version(vstr, sha1_str);
  72     printf("YAZ version: %s %s\n", YAZ_VERSION, YAZ_VERSION_SHA1);
  73     if (strcmp(sha1_str, YAZ_VERSION_SHA1))
  74         printf("YAZ DLL/SO: %s %s\n", vstr, sha1_str);
  75     exit(0);
  76 }
  77
  78 static int getbyte_stream(void *client_data)
  79 {
  80     FILE *f = (FILE*) client_data;
  81
  82     int c = fgetc(f);
  83     if (c == EOF)
  84         return 0;
  85     return c;
  86 }
  87
  88 static void ungetbyte_stream(int c, void *client_data)
  89 {
  90     FILE *f = (FILE*) client_data;
  91
  92     if (c == 0)
  93         c = EOF;
  94     ungetc(c, f);
  95 }
  96
  97 static void marcdump_read_line(yaz_marc_t mt, const char *fname)
  98 {
  99     FILE *inf = fopen(fname, "rb");
 100     if (!inf)
 101     {
 102         fprintf (stderr, "%s: cannot open %s:%s\n",
 103                  prog, fname, strerror (errno));
 104         exit(1);
 105     }
 106
 107     while (yaz_marc_read_line(mt, getbyte_stream,
 108                               ungetbyte_stream, inf) == 0)
 109     {
 110         WRBUF wrbuf = wrbuf_alloc();
 111         yaz_marc_write_mode(mt, wrbuf);
 112         fputs(wrbuf_cstr(wrbuf), stdout);
 113         wrbuf_destroy(wrbuf);
 114     }
 115     fclose(inf);
 116 }
 117
 118 #if YAZ_HAVE_XML2
 119 static void marcdump_read_xml(yaz_marc_t mt, const char *fname)
 120 {
 121     WRBUF wrbuf = wrbuf_alloc();
 122 #if USE_XMLREADER
 123     xmlTextReaderPtr reader = xmlReaderForFile(fname, 0 /* encoding */,
 124                                                0 /* options */);
 125
 126     if (reader)
 127     {
 128         int ret;
 129         while ((ret = xmlTextReaderRead(reader)) == 1)
 130         {
 131             int type = xmlTextReaderNodeType(reader);
 132             if (type == XML_READER_TYPE_ELEMENT)
 133             {
 134                 const char *name = (const char *)
 135                     xmlTextReaderLocalName(reader);
 136                 if (!strcmp(name, "record"))
 137                 {
 138                     xmlNodePtr ptr = xmlTextReaderExpand(reader);
 139
 140                     int r = yaz_marc_read_xml(mt, ptr);
 141                     if (r)
 142                         fprintf(stderr, "yaz_marc_read_xml failed\n");
 143                     else
 144                     {
 145                         yaz_marc_write_mode(mt, wrbuf);
 146
 147                         fputs(wrbuf_cstr(wrbuf), stdout);
 148                         wrbuf_rewind(wrbuf);
 149                     }
 150                 }
 151             }
 152         }
 153     }
 154 #else
 155     xmlDocPtr doc = xmlParseFile(fname);
 156     if (doc)
 157     {
 158         xmlNodePtr ptr = xmlDocGetRootElement(doc);
 159         for (; ptr; ptr = ptr->next)
 160         {
 161             if (ptr->type == XML_ELEMENT_NODE)
 162             {
 163                 if (!strcmp((const char *) ptr->name, "collection"))
 164                 {
 165                     ptr = ptr->children;
 166                     continue;
 167                 }
 168                 if (!strcmp((const char *) ptr->name, "record"))
 169                 {
 170                     int r = yaz_marc_read_xml(mt, ptr);
 171                     if (r)
 172                         fprintf(stderr, "yaz_marc_read_xml failed\n");
 173                     else
 174                     {
 175                         yaz_marc_write_mode(mt, wrbuf);
 176
 177                         fputs(wrbuf_cstr(wrbuf), stdout);
 178                         wrbuf_rewind(wrbuf);
 179                     }
 180                 }
 181             }
 182         }
 183         xmlFreeDoc(doc);
 184     }
 185 #endif
 186     fputs(wrbuf_cstr(wrbuf), stdout);
 187     wrbuf_destroy(wrbuf);
 188 }
 189 #endif
 190
 191 static void dump(const char *fname, const char *from, const char *to,
 192                  int input_format, int output_format,
 193                  int write_using_libxml2,
 194                  int print_offset, const char *split_fname, int split_chunk,
 195                  int verbose, FILE *cfile, const char *leader_spec)
 196 {
 197     yaz_marc_t mt = yaz_marc_create();
 198     yaz_iconv_t cd = 0;
 199
 200     if (yaz_marc_leader_spec(mt, leader_spec))
 201     {
 202         fprintf(stderr, "bad leader spec: %s\n", leader_spec);
 203         yaz_marc_destroy(mt);
 204         exit(2);
 205     }
 206     if (from && to)
 207     {
 208         cd = yaz_iconv_open(to, from);
 209         if (!cd)
 210         {
 211             fprintf(stderr, "conversion from %s to %s "
 212                     "unsupported\n", from, to);
 213             yaz_marc_destroy(mt);
 214             exit(2);
 215         }
 216         yaz_marc_iconv(mt, cd);
 217     }
 218     yaz_marc_xml(mt, output_format);
 219     yaz_marc_enable_collection(mt);
 220     yaz_marc_write_using_libxml2(mt, write_using_libxml2);
 221     yaz_marc_debug(mt, verbose);
 222
 223     if (input_format == YAZ_MARC_MARCXML || input_format == YAZ_MARC_XCHANGE)
 224     {
 225 #if YAZ_HAVE_XML2
 226         marcdump_read_xml(mt, fname);
 227 #endif
 228     }
 229     else if (input_format == YAZ_MARC_LINE)
 230     {
 231         marcdump_read_line(mt, fname);
 232     }
 233     else if (input_format == YAZ_MARC_ISO2709)
 234     {
 235         FILE *inf = fopen(fname, "rb");
 236         int num = 1;
 237         int marc_no = 0;
 238         int split_file_no = -1;
 239         if (!inf)
 240         {
 241             fprintf (stderr, "%s: cannot open %s:%s\n",
 242                      prog, fname, strerror (errno));
 243             exit(1);
 244         }
 245         if (cfile)
 246             fprintf (cfile, "char *marc_records[] = {\n");
 247         for(;; marc_no++)
 248         {
 249             const char *result = 0;
 250             size_t len;
 251             size_t rlen;
 252             size_t len_result;
 253             size_t r;
 254             char buf[100001];
 255
 256             r = fread (buf, 1, 5, inf);
 257             if (r < 5)
 258             {
 259                 if (r && print_offset && verbose)
 260                     printf ("<!-- Extra %ld bytes at end of file -->\n",
 261                             (long) r);
 262                 break;
 263             }
 264             while (*buf < '0' || *buf > '9')
 265             {
 266                 int i;
 267                 long off = ftell(inf) - 5;
 268                 if (verbose || print_offset)
 269                     printf("<!-- Skipping bad byte %d (0x%02X) at offset "
 270                            "%ld (0x%lx) -->\n",
 271                            *buf & 0xff, *buf & 0xff,
 272                            off, off);
 273                 for (i = 0; i<4; i++)
 274                     buf[i] = buf[i+1];
 275                 r = fread(buf+4, 1, 1, inf);
 276                 if (r < 1)
 277                     break;
 278             }
 279             if (r < 1)
 280             {
 281                 if (verbose || print_offset)
 282                     printf ("<!-- End of file with data -->\n");
 283                 break;
 284             }
 285             if (print_offset)
 286             {
 287                 long off = ftell(inf) - 5;
 288                 printf ("<!-- Record %d offset %ld (0x%lx) -->\n",
 289                         num, off, off);
 290             }
 291             len = atoi_n(buf, 5);
 292             if (len < 25 || len > 100000)
 293             {
 294                 long off = ftell(inf) - 5;
 295                 printf("Bad Length %ld read at offset %ld (%lx)\n",
 296                        (long)len, (long) off, (long) off);
 297                 break;
 298             }
 299             rlen = len - 5;
 300             r = fread (buf + 5, 1, rlen, inf);
 301             if (r < rlen)
 302                 break;
 303             while (buf[len-1] != ISO2709_RS)
 304             {
 305                 if (len > sizeof(buf)-2)
 306                     break;
 307                 r = fread (buf + len, 1, 1, inf);
 308                 if (r != 1)
 309                     break;
 310                 len++;
 311             }
 312             if (split_fname)
 313             {
 314                 char fname[256];
 315                 const char *mode = 0;
 316                 FILE *sf;
 317                 if ((marc_no % split_chunk) == 0)
 318                 {
 319                     mode = "wb";
 320                     split_file_no++;
 321                 }
 322                 else
 323                     mode = "ab";
 324                 sprintf(fname, "%.200s%07d", split_fname, split_file_no);
 325                 sf = fopen(fname, mode);
 326                 if (!sf)
 327                 {
 328                     fprintf(stderr, "Could not open %s\n", fname);
 329                     split_fname = 0;
 330                 }
 331                 else
 332                 {
 333                     if (fwrite(buf, 1, len, sf) != len)
 334                     {
 335                         fprintf(stderr, "Could write content to %s\n",
 336                                 fname);
 337                         split_fname = 0;
 338                     }
 339                     fclose(sf);
 340                 }
 341             }
 342             len_result = rlen;
 343             r = yaz_marc_decode_buf(mt, buf, -1, &result, &len_result);
 344             if (r > 0 && result && len_result)
 345             {
 346                 if (fwrite(result, len_result, 1, stdout) != 1)
 347                 {
 348                     fprintf(stderr, "Write to stdout failed\n");
 349                     break;
 350                 }
 351             }
 352             if (r > 0 && cfile)
 353             {
 354                 char *p = buf;
 355                 size_t i;
 356                 if (marc_no)
 357                     fprintf (cfile, ",");
 358                 fprintf (cfile, "\n");
 359                 for (i = 0; i < r; i++)
 360                 {
 361                     if ((i & 15) == 0)
 362                         fprintf (cfile, "  \"");
 363                     fprintf (cfile, "\\x%02X", p[i] & 255);
 364
 365                     if (i < r - 1 && (i & 15) == 15)
 366                         fprintf (cfile, "\"\n");
 367
 368                 }
 369                 fprintf (cfile, "\"\n");
 370             }
 371             num++;
 372             if (verbose)
 373                 printf("\n");
 374         }
 375         if (cfile)
 376             fprintf (cfile, "};\n");
 377         fclose(inf);
 378     }
 379     {
 380         WRBUF wrbuf = wrbuf_alloc();
 381         yaz_marc_write_trailer(mt, wrbuf);
 382         fputs(wrbuf_cstr(wrbuf), stdout);
 383         wrbuf_destroy(wrbuf);
 384     }
 385     if (cd)
 386         yaz_iconv_close(cd);
 387     yaz_marc_destroy(mt);
 388 }
 389
 390 int main (int argc, char **argv)
 391 {
 392     int r;
 393     int print_offset = 0;
 394     char *arg;
 395     int verbose = 0;
 396     int no = 0;
 397     int output_format = YAZ_MARC_LINE;
 398     FILE *cfile = 0;
 399     char *from = 0, *to = 0;
 400     int input_format = YAZ_MARC_ISO2709;
 401     int split_chunk = 1;
 402     const char *split_fname = 0;
 403     const char *leader_spec = 0;
 404     int write_using_libxml2 = 0;
 405
 406 #if HAVE_LOCALE_H
 407     setlocale(LC_CTYPE, "");
 408 #endif
 409 #if HAVE_LANGINFO_H
 410 #ifdef CODESET
 411     to = nl_langinfo(CODESET);
 412 #endif
 413 #endif
 414
 415     prog = *argv;
 416     while ((r = options("i:o:C:npc:xOeXIf:t:s:l:Vv", argv, argc, &arg)) != -2)
 417     {
 418         no++;
 419         switch (r)
 420         {
 421         case 'i':
 422             input_format = yaz_marc_decode_formatstr(arg);
 423             if (input_format == -1)
 424             {
 425                 fprintf(stderr, "%s: bad input format: %s\n", prog, arg);
 426                 exit(1);
 427             }
 428 #if YAZ_HAVE_XML2
 429 #else
 430             if (input_format == YAZ_MARC_MARCXML
 431                 || input_format == YAZ_MARC_XCHANGE)
 432             {
 433                 fprintf(stderr, "%s: Libxml2 support not enabled\n", prog);
 434                 exit(3);
 435             }
 436 #endif
 437             break;
 438         case 'o':
 439             /* dirty hack so we can make Libxml2 do the writing ..
 440                rather than WRBUF */
 441             if (strlen(arg) > 4 && strncmp(arg, "xml,", 4) == 0)
 442             {
 443                 arg = arg + 4;
 444                 write_using_libxml2 = 1;
 445             }
 446             output_format = yaz_marc_decode_formatstr(arg);
 447             if (output_format == -1)
 448             {
 449                 fprintf(stderr, "%s: bad output format: %s\n", prog, arg);
 450                 exit(1);
 451             }
 452             break;
 453         case 'l':
 454             leader_spec = arg;
 455             break;
 456         case 'f':
 457             from = arg;
 458             break;
 459         case 't':
 460             to = arg;
 461             break;
 462         case 'c':
 463             if (cfile)
 464                 fclose (cfile);
 465             cfile = fopen(arg, "w");
 466             break;
 467         case 'x':
 468             fprintf(stderr, "%s: -x no longer supported. "
 469                     "Use -i marcxml instead\n", prog);
 470             exit(1);
 471             break;
 472         case 'O':
 473             fprintf(stderr, "%s: OAI MARC no longer supported."
 474                     " Use MARCXML instead.\n", prog);
 475             exit(1);
 476             break;
 477         case 'e':
 478             fprintf(stderr, "%s: -e no longer supported. "
 479                     "Use -o marcxchange instead\n", prog);
 480             exit(1);
 481             break;
 482         case 'X':
 483             fprintf(stderr, "%s: -X no longer supported. "
 484                     "Use -o marcxml instead\n", prog);
 485             exit(1);
 486             break;
 487         case 'I':
 488             fprintf(stderr, "%s: -I no longer supported. "
 489                     "Use -o marc instead\n", prog);
 490             exit(1);
 491             break;
 492         case 'n':
 493             output_format = YAZ_MARC_CHECK;
 494             break;
 495         case 'p':
 496             print_offset = 1;
 497             break;
 498         case 's':
 499             split_fname = arg;
 500             break;
 501         case 'C':
 502             split_chunk = atoi(arg);
 503             break;
 504         case 0:
 505             dump(arg, from, to, input_format, output_format,
 506                  write_using_libxml2,
 507                  print_offset, split_fname, split_chunk,
 508                  verbose, cfile, leader_spec);
 509             break;
 510         case 'v':
 511             verbose++;
 512             break;
 513         case 'V':
 514             show_version();
 515             break;
 516         default:
 517             usage(prog);
 518             exit(1);
 519         }
 520     }
 521     if (cfile)
 522         fclose (cfile);
 523     if (!no)
 524     {
 525         usage(prog);
 526         exit (1);
 527     }
 528     exit (0);
 529 }
 530 /*
 531  * Local variables:
 532  * c-basic-offset: 4
 533  * c-file-style: "Stroustrup"
 534  * indent-tabs-mode: nil
 535  * End:
 536  * vim: shiftwidth=4 tabstop=8 expandtab
 537  */
 538