533714d25e69d5230e9a6f8674a482ea2f7e050e
[yaz-moved-to-github.git] / util / marcdump.c
1 /* This file is part of the YAZ toolkit.
2  * Copyright (C) 1995-2010 Index Data
3  * See the file LICENSE for details.
4  */
5
6 #define _FILE_OFFSET_BITS 64
7
8 #if HAVE_CONFIG_H
9 #include <config.h>
10 #endif
11
12 #if YAZ_HAVE_XML2
13 #include <libxml/parser.h>
14 #include <libxml/tree.h>
15 #include <libxml/xpath.h>
16 #include <libxml/xpathInternals.h>
17
18 /* Libxml2 version < 2.6.15. xmlreader not reliable/present */
19 #if LIBXML_VERSION < 20615
20 #define USE_XMLREADER 0
21 #else
22 #define USE_XMLREADER 1
23 #endif
24
25 #if USE_XMLREADER
26 #include <libxml/xmlreader.h>
27 #endif
28
29 #endif
30
31 #include <stdio.h>
32 #include <stdlib.h>
33 #include <string.h>
34 #include <errno.h>
35 #include <assert.h>
36
37 #if HAVE_LOCALE_H
38 #include <locale.h>
39 #endif
40 #if HAVE_LANGINFO_H
41 #include <langinfo.h>
42 #endif
43
44 #include <yaz/marcdisp.h>
45 #include <yaz/yaz-util.h>
46 #include <yaz/xmalloc.h>
47 #include <yaz/options.h>
48
49 #ifndef SEEK_SET
50 #define SEEK_SET 0
51 #endif
52 #ifndef SEEK_END
53 #define SEEK_END 2
54 #endif
55
56
57 static char *prog;
58
59 static void usage(const char *prog)
60 {
61     fprintf (stderr, "Usage: %s [-i format] [-o format] [-f from] [-t to] "
62              "[-l pos=value] [-c cfile] [-s prefix] [-C size] [-n] "
63              "[-p] [-v] [-V] file...\n",
64              prog);
65
66
67 static void show_version(void)
68 {
69     char vstr[20], sha1_str[41];
70
71     yaz_version(vstr, sha1_str);
72     printf("YAZ version: %s %s\n", YAZ_VERSION, YAZ_VERSION_SHA1);
73     if (strcmp(sha1_str, YAZ_VERSION_SHA1))
74         printf("YAZ DLL/SO: %s %s\n", vstr, sha1_str);
75     exit(0);
76 }
77
78 static int getbyte_stream(void *client_data)
79 {
80     FILE *f = (FILE*) client_data;
81
82     int c = fgetc(f);
83     if (c == EOF)
84         return 0;
85     return c;
86 }
87
88 static void ungetbyte_stream(int c, void *client_data)
89 {
90     FILE *f = (FILE*) client_data;
91
92     if (c == 0)
93         c = EOF;
94     ungetc(c, f);
95 }
96
97 static void marcdump_read_line(yaz_marc_t mt, const char *fname)
98 {
99     FILE *inf = fopen(fname, "rb");
100     if (!inf)
101     {
102         fprintf (stderr, "%s: cannot open %s:%s\n",
103                  prog, fname, strerror (errno));
104         exit(1);
105     }
106     
107     while (yaz_marc_read_line(mt, getbyte_stream,
108                               ungetbyte_stream, inf) == 0)
109     {
110         WRBUF wrbuf = wrbuf_alloc();
111         yaz_marc_write_mode(mt, wrbuf);
112         fputs(wrbuf_cstr(wrbuf), stdout);
113         wrbuf_destroy(wrbuf);
114     }
115     fclose(inf);
116 }
117
118 #if YAZ_HAVE_XML2
119 static void marcdump_read_xml(yaz_marc_t mt, const char *fname)
120 {
121     WRBUF wrbuf = wrbuf_alloc();
122 #if USE_XMLREADER
123     xmlTextReaderPtr reader = xmlReaderForFile(fname, 0 /* encoding */,
124                                                0 /* options */);
125
126     if (reader)
127     {
128         int ret;
129         while ((ret = xmlTextReaderRead(reader)) == 1)
130         {
131             int type = xmlTextReaderNodeType(reader);
132             if (type == XML_READER_TYPE_ELEMENT)
133             {
134                 const char *name = (const char *) 
135                     xmlTextReaderLocalName(reader);
136                 if (!strcmp(name, "record"))
137                 {
138                     xmlNodePtr ptr = xmlTextReaderExpand(reader);
139         
140                     int r = yaz_marc_read_xml(mt, ptr);
141                     if (r)
142                         fprintf(stderr, "yaz_marc_read_xml failed\n");
143                     else
144                     {
145                         int write_rc = yaz_marc_write_mode(mt, wrbuf);
146                                                 if (write_rc)
147                                                         yaz_log(YLOG_WARN, "yaz_marc_write_mode: write error: %d", write_rc);
148                         
149                         fputs(wrbuf_cstr(wrbuf), stdout);
150                         wrbuf_rewind(wrbuf);
151                     }
152                 }
153             }
154         }
155     }
156 #else
157     xmlDocPtr doc = xmlParseFile(fname);
158     if (doc)
159     {
160         xmlNodePtr ptr = xmlDocGetRootElement(doc);
161         for (; ptr; ptr = ptr->next)
162         {
163             if (ptr->type == XML_ELEMENT_NODE)
164             {
165                 if (!strcmp((const char *) ptr->name, "collection"))
166                 {
167                     ptr = ptr->children;
168                     continue;
169                 }
170                 if (!strcmp((const char *) ptr->name, "record"))
171                 {
172                     int r = yaz_marc_read_xml(mt, ptr);
173                     if (r)
174                         fprintf(stderr, "yaz_marc_read_xml failed\n");
175                     else
176                     {
177                         yaz_marc_write_mode(mt, wrbuf);
178                         
179                         fputs(wrbuf_cstr(wrbuf), stdout);
180                         wrbuf_rewind(wrbuf);
181                     }
182                 }
183             }
184         }
185         xmlFreeDoc(doc);
186     }
187 #endif
188     fputs(wrbuf_cstr(wrbuf), stdout);
189     wrbuf_destroy(wrbuf);
190 }
191 #endif
192
193 static void dump(const char *fname, const char *from, const char *to,
194                  int input_format, int output_format,
195                  int write_using_libxml2,
196                  int print_offset, const char *split_fname, int split_chunk,
197                  int verbose, FILE *cfile, const char *leader_spec)
198 {
199     yaz_marc_t mt = yaz_marc_create();
200     yaz_iconv_t cd = 0;
201
202     if (yaz_marc_leader_spec(mt, leader_spec))
203     {
204         fprintf(stderr, "bad leader spec: %s\n", leader_spec);
205         yaz_marc_destroy(mt);
206         exit(2);
207     }
208     if (from && to)
209     {
210         cd = yaz_iconv_open(to, from);
211         if (!cd)
212         {
213             fprintf(stderr, "conversion from %s to %s "
214                     "unsupported\n", from, to);
215             yaz_marc_destroy(mt);
216             exit(2);
217         }
218         yaz_marc_iconv(mt, cd);
219     }
220     yaz_marc_enable_collection(mt);
221     yaz_marc_set_read_format(mt, input_format);
222     yaz_marc_set_write_format(mt, output_format);
223     yaz_marc_write_using_libxml2(mt, write_using_libxml2);
224     yaz_marc_write_turbo_format(mt, output_format == YAZ_MARC_TMARCXML);
225     yaz_marc_debug(mt, verbose);
226
227     if (input_format == YAZ_MARC_MARCXML || input_format == YAZ_MARC_TMARCXML || input_format == YAZ_MARC_XCHANGE)
228     {
229 #if YAZ_HAVE_XML2
230         marcdump_read_xml(mt, fname);
231 #endif
232     }
233     else if (input_format == YAZ_MARC_LINE)
234     {
235         marcdump_read_line(mt, fname);
236     }
237     else if (input_format == YAZ_MARC_ISO2709)
238     {
239         FILE *inf = fopen(fname, "rb");
240         int num = 1;
241         int marc_no = 0;
242         int split_file_no = -1;
243         if (!inf)
244         {
245             fprintf (stderr, "%s: cannot open %s:%s\n",
246                      prog, fname, strerror (errno));
247             exit(1);
248         }
249         if (cfile)
250             fprintf (cfile, "char *marc_records[] = {\n");
251         for(;; marc_no++)
252         {
253             const char *result = 0;
254             size_t len;
255             size_t rlen;
256             size_t len_result;
257             size_t r;
258             char buf[100001];
259             
260             r = fread (buf, 1, 5, inf);
261             if (r < 5)
262             {
263                 if (r && print_offset && verbose)
264                     printf ("<!-- Extra %ld bytes at end of file -->\n",
265                             (long) r);
266                 break;
267             }
268             while (*buf < '0' || *buf > '9')
269             {
270                 int i;
271                 long off = ftell(inf) - 5;
272                 if (verbose || print_offset)
273                     printf("<!-- Skipping bad byte %d (0x%02X) at offset "
274                            "%ld (0x%lx) -->\n", 
275                            *buf & 0xff, *buf & 0xff,
276                            off, off);
277                 for (i = 0; i<4; i++)
278                     buf[i] = buf[i+1];
279                 r = fread(buf+4, 1, 1, inf);
280                 if (r < 1)
281                     break;
282             }
283             if (r < 1)
284             {
285                 if (verbose || print_offset)
286                     printf ("<!-- End of file with data -->\n");
287                 break;
288             }
289             if (print_offset)
290             {
291                 long off = ftell(inf) - 5;
292                 printf ("<!-- Record %d offset %ld (0x%lx) -->\n",
293                         num, off, off);
294             }
295             len = atoi_n(buf, 5);
296             if (len < 25 || len > 100000)
297             {
298                 long off = ftell(inf) - 5;
299                 printf("Bad Length %ld read at offset %ld (%lx)\n",
300                        (long)len, (long) off, (long) off);
301                 break;
302             }
303             rlen = len - 5;
304             r = fread (buf + 5, 1, rlen, inf);
305             if (r < rlen)
306                 break;
307             while (buf[len-1] != ISO2709_RS)
308             {
309                 if (len > sizeof(buf)-2)
310                     break;
311                 r = fread (buf + len, 1, 1, inf);
312                 if (r != 1)
313                     break;
314                 len++;
315             }
316             if (split_fname)
317             {
318                 char fname[256];
319                 const char *mode = 0;
320                 FILE *sf;
321                 if ((marc_no % split_chunk) == 0)
322                 {
323                     mode = "wb";
324                     split_file_no++;
325                 }
326                 else
327                     mode = "ab";
328                 sprintf(fname, "%.200s%07d", split_fname, split_file_no);
329                 sf = fopen(fname, mode);
330                 if (!sf)
331                 {
332                     fprintf(stderr, "Could not open %s\n", fname);
333                     split_fname = 0;
334                 }
335                 else
336                 {
337                     if (fwrite(buf, 1, len, sf) != len)
338                     {
339                         fprintf(stderr, "Could write content to %s\n",
340                                 fname);
341                         split_fname = 0;
342                     }
343                     fclose(sf);
344                 }
345             }
346             len_result = rlen;
347             r = yaz_marc_decode_buf(mt, buf, -1, &result, &len_result);
348             if (r > 0 && result && len_result)
349             {
350                 if (fwrite(result, len_result, 1, stdout) != 1)
351                 {
352                     fprintf(stderr, "Write to stdout failed\n");
353                     break;
354                 }
355             }
356             if (r > 0 && cfile)
357             {
358                 char *p = buf;
359                 size_t i;
360                 if (marc_no)
361                     fprintf (cfile, ",");
362                 fprintf (cfile, "\n");
363                 for (i = 0; i < r; i++)
364                 {
365                     if ((i & 15) == 0)
366                         fprintf (cfile, "  \"");
367                     fprintf (cfile, "\\x%02X", p[i] & 255);
368                     
369                     if (i < r - 1 && (i & 15) == 15)
370                         fprintf (cfile, "\"\n");
371                     
372                 }
373                 fprintf (cfile, "\"\n");
374             }
375             num++;
376             if (verbose)
377                 printf("\n");
378         }
379         if (cfile)
380             fprintf (cfile, "};\n");
381         fclose(inf);
382     }
383     {
384         WRBUF wrbuf = wrbuf_alloc();
385         yaz_marc_write_trailer(mt, wrbuf);
386         fputs(wrbuf_cstr(wrbuf), stdout);
387         wrbuf_destroy(wrbuf);
388     }
389     if (cd)
390         yaz_iconv_close(cd);
391     yaz_marc_destroy(mt);
392 }
393
394 int main (int argc, char **argv)
395 {
396     int r;
397     int print_offset = 0;
398     char *arg;
399     int verbose = 0;
400     int no = 0;
401     int output_format = YAZ_MARC_LINE;
402     FILE *cfile = 0;
403     char *from = 0, *to = 0;
404     int input_format = YAZ_MARC_ISO2709;
405     int split_chunk = 1;
406     const char *split_fname = 0;
407     const char *leader_spec = 0;
408     int write_using_libxml2 = 0;
409
410 #if HAVE_LOCALE_H
411     setlocale(LC_CTYPE, "");
412 #endif
413 #if HAVE_LANGINFO_H
414 #ifdef CODESET
415     to = nl_langinfo(CODESET);
416 #endif
417 #endif
418
419     prog = *argv;
420     while ((r = options("i:o:C:npc:xOeXIf:t:s:l:Vv", argv, argc, &arg)) != -2)
421     {
422         no++;
423         switch (r)
424         {
425         case 'i':
426             input_format = yaz_marc_decode_formatstr(arg);
427             if (input_format == -1)
428             {
429                 fprintf(stderr, "%s: bad input format: %s\n", prog, arg);
430                 exit(1);
431             }
432 #if YAZ_HAVE_XML2
433 #else
434             if (input_format == YAZ_MARC_MARCXML 
435                 || input_format == YAZ_MARC_XCHANGE)
436             {
437                 fprintf(stderr, "%s: Libxml2 support not enabled\n", prog);
438                 exit(3);
439             }
440 #endif
441             break;
442         case 'o':
443             /* dirty hack so we can make Libxml2 do the writing ..
444                rather than WRBUF */
445             if (strlen(arg) > 4 && strncmp(arg, "xml,", 4) == 0)
446             {
447                 arg = arg + 4;
448                 write_using_libxml2 = 1;
449             }
450             output_format = yaz_marc_decode_formatstr(arg);
451             if (output_format == -1)
452             {
453                 fprintf(stderr, "%s: bad output format: %s\n", prog, arg);
454                 exit(1);
455             }
456             break;
457         case 'l':
458             leader_spec = arg;
459             break;
460         case 'f':
461             from = arg;
462             break;
463         case 't':
464             to = arg;
465             break;
466         case 'c':
467             if (cfile)
468                 fclose (cfile);
469             cfile = fopen(arg, "w");
470             break;
471         case 'x':
472             fprintf(stderr, "%s: -x no longer supported. "
473                     "Use -i marcxml instead\n", prog);
474             exit(1);
475             break;
476         case 'O':
477             fprintf(stderr, "%s: OAI MARC no longer supported."
478                     " Use MARCXML instead.\n", prog);
479             exit(1);
480             break;
481         case 'e':
482             fprintf(stderr, "%s: -e no longer supported. "
483                     "Use -o marcxchange instead\n", prog);
484             exit(1);
485             break;
486         case 'X':
487             fprintf(stderr, "%s: -X no longer supported. "
488                     "Use -o marcxml instead\n", prog);
489             exit(1);
490             break;
491         case 'I':
492             fprintf(stderr, "%s: -I no longer supported. "
493                     "Use -o marc instead\n", prog);
494             exit(1);
495             break;
496         case 'n':
497             output_format = YAZ_MARC_CHECK;
498             break;
499         case 'p':
500             print_offset = 1;
501             break;
502         case 's':
503             split_fname = arg;
504             break;
505         case 'C':
506             split_chunk = atoi(arg);
507             break;
508         case 0:
509             dump(arg, from, to, input_format, output_format,
510                  write_using_libxml2,
511                  print_offset, split_fname, split_chunk,
512                  verbose, cfile, leader_spec);
513             break;
514         case 'v':
515             verbose++;
516             break;
517         case 'V': 
518             show_version();
519             break;
520         default:
521             usage(prog);
522             exit(1);
523         }
524     }
525     if (cfile)
526         fclose (cfile);
527     if (!no)
528     {
529         usage(prog);
530         exit (1);
531     }
532     exit (0);
533 }
534 /*
535  * Local variables:
536  * c-basic-offset: 4
537  * c-file-style: "Stroustrup"
538  * indent-tabs-mode: nil
539  * End:
540  * vim: shiftwidth=4 tabstop=8 expandtab
541  */
542