e0491c446625e068d0c9ea3f20db96df5bc47a3b
[yaz-moved-to-github.git] / util / marcdump.c
1 /* This file is part of the YAZ toolkit.
2  * Copyright (C) 1995-2013 Index Data
3  * See the file LICENSE for details.
4  */
5
6 #define _FILE_OFFSET_BITS 64
7
8 #if HAVE_CONFIG_H
9 #include <config.h>
10 #endif
11
12 #if YAZ_HAVE_XML2
13 #include <libxml/parser.h>
14 #include <libxml/tree.h>
15 #include <libxml/xpath.h>
16 #include <libxml/xpathInternals.h>
17
18 /* Libxml2 version < 2.6.15. xmlreader not reliable/present */
19 #if LIBXML_VERSION < 20615
20 #define USE_XMLREADER 0
21 #else
22 #define USE_XMLREADER 1
23 #endif
24
25 #if USE_XMLREADER
26 #include <libxml/xmlreader.h>
27 #endif
28
29 #endif
30
31 #include <stdio.h>
32 #include <stdlib.h>
33 #include <string.h>
34 #include <errno.h>
35 #include <assert.h>
36
37 #if HAVE_LOCALE_H
38 #include <locale.h>
39 #endif
40 #if HAVE_LANGINFO_H
41 #include <langinfo.h>
42 #endif
43
44 #include <yaz/marcdisp.h>
45 #include <yaz/yaz-util.h>
46 #include <yaz/xmalloc.h>
47 #include <yaz/options.h>
48
49 #ifndef SEEK_SET
50 #define SEEK_SET 0
51 #endif
52 #ifndef SEEK_END
53 #define SEEK_END 2
54 #endif
55
56
57 static char *prog;
58
59 static int no_errors = 0;
60
61 static void usage(const char *prog)
62 {
63     fprintf(stderr, "Usage: %s [-i format] [-o format] [-f from] [-t to] "
64             "[-l pos=value] [-c cfile] [-s prefix] [-C size] [-n] "
65             "[-p] [-v] [-V] file...\n",
66             prog);
67 }
68
69 static void show_version(void)
70 {
71     char vstr[20], sha1_str[41];
72
73     yaz_version(vstr, sha1_str);
74     printf("YAZ version: %s %s\n", YAZ_VERSION, YAZ_VERSION_SHA1);
75     if (strcmp(sha1_str, YAZ_VERSION_SHA1))
76         printf("YAZ DLL/SO: %s %s\n", vstr, sha1_str);
77     exit(0);
78 }
79
80 static int getbyte_stream(void *client_data)
81 {
82     FILE *f = (FILE*) client_data;
83
84     int c = fgetc(f);
85     if (c == EOF)
86         return 0;
87     return c;
88 }
89
90 static void ungetbyte_stream(int c, void *client_data)
91 {
92     FILE *f = (FILE*) client_data;
93
94     if (c == 0)
95         c = EOF;
96     ungetc(c, f);
97 }
98
99 static void marcdump_read_line(yaz_marc_t mt, const char *fname)
100 {
101     FILE *inf = fopen(fname, "rb");
102     if (!inf)
103     {
104         fprintf(stderr, "%s: cannot open %s:%s\n",
105                 prog, fname, strerror(errno));
106         exit(1);
107     }
108
109     while (yaz_marc_read_line(mt, getbyte_stream,
110                               ungetbyte_stream, inf) == 0)
111     {
112         WRBUF wrbuf = wrbuf_alloc();
113         yaz_marc_write_mode(mt, wrbuf);
114         fputs(wrbuf_cstr(wrbuf), stdout);
115         wrbuf_destroy(wrbuf);
116     }
117     fclose(inf);
118 }
119
120 #if YAZ_HAVE_XML2
121 static void marcdump_read_xml(yaz_marc_t mt, const char *fname)
122 {
123     WRBUF wrbuf = wrbuf_alloc();
124 #if USE_XMLREADER
125     xmlTextReaderPtr reader = xmlReaderForFile(fname, 0 /* encoding */,
126                                                0 /* options */);
127
128     if (reader)
129     {
130         int ret;
131         while ((ret = xmlTextReaderRead(reader)) == 1)
132         {
133             int type = xmlTextReaderNodeType(reader);
134             if (type == XML_READER_TYPE_ELEMENT)
135             {
136                 const char *name = (const char *)
137                     xmlTextReaderLocalName(reader);
138                 if (!strcmp(name, "record") || !strcmp(name, "r"))
139                 {
140                     xmlNodePtr ptr = xmlTextReaderExpand(reader);
141
142                     int r = yaz_marc_read_xml(mt, ptr);
143                     if (r)
144                     {
145                         no_errors++;
146                         fprintf(stderr, "yaz_marc_read_xml failed\n");
147                     }
148                     else
149                     {
150                         int write_rc = yaz_marc_write_mode(mt, wrbuf);
151                         if (write_rc)
152                         {
153                             yaz_log(YLOG_WARN, "yaz_marc_write_mode: "
154                                     "write error: %d", write_rc);
155                             no_errors++;
156                         }
157                         fputs(wrbuf_cstr(wrbuf), stdout);
158                         wrbuf_rewind(wrbuf);
159                     }
160                 }
161             }
162         }
163     }
164 #else
165     xmlDocPtr doc = xmlParseFile(fname);
166     if (doc)
167     {
168         xmlNodePtr ptr = xmlDocGetRootElement(doc);
169         for (; ptr; ptr = ptr->next)
170         {
171             if (ptr->type == XML_ELEMENT_NODE)
172             {
173                 if (!strcmp((const char *) ptr->name, "collection"))
174                 {
175                     ptr = ptr->children;
176                     continue;
177                 }
178                 if (!strcmp((const char *) ptr->name, "record") ||
179                     !strcmp((const char *) ptr->name, "r"))
180                 {
181                     int r = yaz_marc_read_xml(mt, ptr);
182                     if (r)
183                     {
184                         no_errors++;
185                         fprintf(stderr, "yaz_marc_read_xml failed\n");
186                     }
187                     else
188                     {
189                         yaz_marc_write_mode(mt, wrbuf);
190
191                         fputs(wrbuf_cstr(wrbuf), stdout);
192                         wrbuf_rewind(wrbuf);
193                     }
194                 }
195             }
196         }
197         xmlFreeDoc(doc);
198     }
199 #endif
200     fputs(wrbuf_cstr(wrbuf), stdout);
201     wrbuf_destroy(wrbuf);
202 }
203 #endif
204
205 static void dump(const char *fname, const char *from, const char *to,
206                  int input_format, int output_format,
207                  int write_using_libxml2,
208                  int print_offset, const char *split_fname, int split_chunk,
209                  int verbose, FILE *cfile, const char *leader_spec)
210 {
211     yaz_marc_t mt = yaz_marc_create();
212     yaz_iconv_t cd = 0;
213
214     if (yaz_marc_leader_spec(mt, leader_spec))
215     {
216         fprintf(stderr, "bad leader spec: %s\n", leader_spec);
217         yaz_marc_destroy(mt);
218         exit(2);
219     }
220     if (from && to)
221     {
222         cd = yaz_iconv_open(to, from);
223         if (!cd)
224         {
225             fprintf(stderr, "conversion from %s to %s "
226                     "unsupported\n", from, to);
227             yaz_marc_destroy(mt);
228             exit(2);
229         }
230         yaz_marc_iconv(mt, cd);
231     }
232     yaz_marc_enable_collection(mt);
233     yaz_marc_xml(mt, output_format);
234     yaz_marc_write_using_libxml2(mt, write_using_libxml2);
235     yaz_marc_debug(mt, verbose);
236
237     if (input_format == YAZ_MARC_MARCXML || input_format == YAZ_MARC_TURBOMARC || input_format == YAZ_MARC_XCHANGE)
238     {
239 #if YAZ_HAVE_XML2
240         marcdump_read_xml(mt, fname);
241 #endif
242     }
243     else if (input_format == YAZ_MARC_LINE)
244     {
245         marcdump_read_line(mt, fname);
246     }
247     else if (input_format == YAZ_MARC_ISO2709)
248     {
249         FILE *inf = fopen(fname, "rb");
250         int num = 1;
251         int marc_no = 0;
252         int split_file_no = -1;
253         if (!inf)
254         {
255             fprintf(stderr, "%s: cannot open %s:%s\n",
256                     prog, fname, strerror(errno));
257             exit(1);
258         }
259         if (cfile)
260             fprintf(cfile, "char *marc_records[] = {\n");
261         for(;; marc_no++)
262         {
263             const char *result = 0;
264             size_t len;
265             size_t rlen;
266             size_t len_result;
267             size_t r;
268             char buf[100001];
269
270             r = fread(buf, 1, 5, inf);
271             if (r < 5)
272             {
273                 if (r == 0) /* normal EOF, all good */
274                     break;
275                 if (print_offset && verbose)
276                 {
277                     printf("<!-- Extra %ld bytes at end of file -->\n",
278                            (long) r);
279                 }
280                 break;
281             }
282             while (*buf < '0' || *buf > '9')
283             {
284                 int i;
285                 long off = ftell(inf) - 5;
286                 printf("<!-- Skipping bad byte %d (0x%02X) at offset "
287                        "%ld (0x%lx) -->\n",
288                        *buf & 0xff, *buf & 0xff,
289                        off, off);
290                 for (i = 0; i<4; i++)
291                     buf[i] = buf[i+1];
292                 r = fread(buf+4, 1, 1, inf);
293                 no_errors++;
294                 if (r < 1)
295                     break;
296             }
297             if (r < 1)
298             {
299                 if (verbose || print_offset)
300                     printf("<!-- End of file with data -->\n");
301                 break;
302             }
303             if (print_offset)
304             {
305                 long off = ftell(inf) - 5;
306                 printf("<!-- Record %d offset %ld (0x%lx) -->\n",
307                        num, off, off);
308             }
309             len = atoi_n(buf, 5);
310             if (len < 25 || len > 100000)
311             {
312                 long off = ftell(inf) - 5;
313                 printf("<!-- Bad Length %ld read at offset %ld (%lx) -->\n",
314                        (long)len, (long) off, (long) off);
315                 no_errors++;
316                 break;
317             }
318             rlen = len - 5;
319             r = fread(buf + 5, 1, rlen, inf);
320             if (r < rlen)
321             {
322                 long off = ftell(inf);
323                 printf("<!-- Premature EOF at offset %ld (%lx) -->\n",
324                        (long) off, (long) off);
325                 no_errors++;
326                 break;
327             }
328             while (buf[len-1] != ISO2709_RS)
329             {
330                 if (len > sizeof(buf)-2)
331                 {
332                     r = 0;
333                     break;
334                 }
335                 r = fread(buf + len, 1, 1, inf);
336                 if (r != 1)
337                     break;
338                 len++;
339             }
340             if (r < 1)
341             {
342                 printf("<!-- EOF while searching for RS -->\n");
343                 no_errors++;
344                 break;
345             }
346             if (split_fname)
347             {
348                 char fname[256];
349                 const char *mode = 0;
350                 FILE *sf;
351                 if ((marc_no % split_chunk) == 0)
352                 {
353                     mode = "wb";
354                     split_file_no++;
355                 }
356                 else
357                     mode = "ab";
358                 sprintf(fname, "%.200s%07d", split_fname, split_file_no);
359                 sf = fopen(fname, mode);
360                 if (!sf)
361                 {
362                     fprintf(stderr, "Could not open %s\n", fname);
363                     split_fname = 0;
364                 }
365                 else
366                 {
367                     if (fwrite(buf, 1, len, sf) != len)
368                     {
369                         fprintf(stderr, "Could write content to %s\n",
370                                 fname);
371                         split_fname = 0;
372                         no_errors++;
373                     }
374                     fclose(sf);
375                 }
376             }
377             len_result = rlen;
378             r = yaz_marc_decode_buf(mt, buf, -1, &result, &len_result);
379             if (r == -1)
380                 no_errors++;
381             if (r > 0 && result && len_result)
382             {
383                 if (fwrite(result, len_result, 1, stdout) != 1)
384                 {
385                     fprintf(stderr, "Write to stdout failed\n");
386                     no_errors++;
387                     break;
388                 }
389             }
390             if (r > 0 && cfile)
391             {
392                 char *p = buf;
393                 size_t i;
394                 if (marc_no)
395                     fprintf(cfile, ",");
396                 fprintf(cfile, "\n");
397                 for (i = 0; i < r; i++)
398                 {
399                     if ((i & 15) == 0)
400                         fprintf(cfile, "  \"");
401                     fprintf(cfile, "\\x%02X", p[i] & 255);
402
403                     if (i < r - 1 && (i & 15) == 15)
404                         fprintf(cfile, "\"\n");
405
406                 }
407                 fprintf(cfile, "\"\n");
408             }
409             num++;
410             if (verbose)
411                 printf("\n");
412         }
413         if (cfile)
414             fprintf(cfile, "};\n");
415         fclose(inf);
416     }
417     {
418         WRBUF wrbuf = wrbuf_alloc();
419         yaz_marc_write_trailer(mt, wrbuf);
420         fputs(wrbuf_cstr(wrbuf), stdout);
421         wrbuf_destroy(wrbuf);
422     }
423     if (cd)
424         yaz_iconv_close(cd);
425     yaz_marc_destroy(mt);
426 }
427
428 int main (int argc, char **argv)
429 {
430     int r;
431     int print_offset = 0;
432     char *arg;
433     int verbose = 0;
434     int no = 0;
435     int output_format = YAZ_MARC_LINE;
436     FILE *cfile = 0;
437     char *from = 0, *to = 0;
438     int input_format = YAZ_MARC_ISO2709;
439     int split_chunk = 1;
440     const char *split_fname = 0;
441     const char *leader_spec = 0;
442     int write_using_libxml2 = 0;
443
444 #if HAVE_LOCALE_H
445     setlocale(LC_CTYPE, "");
446 #endif
447 #if HAVE_LANGINFO_H
448 #ifdef CODESET
449     to = nl_langinfo(CODESET);
450 #endif
451 #endif
452
453     prog = *argv;
454     while ((r = options("i:o:C:npc:xOeXIf:t:s:l:Vv", argv, argc, &arg)) != -2)
455     {
456         no++;
457         switch (r)
458         {
459         case 'i':
460             input_format = yaz_marc_decode_formatstr(arg);
461             if (input_format == -1)
462             {
463                 fprintf(stderr, "%s: bad input format: %s\n", prog, arg);
464                 exit(1);
465             }
466 #if YAZ_HAVE_XML2
467 #else
468             if (input_format == YAZ_MARC_MARCXML
469                 || input_format == YAZ_MARC_XCHANGE)
470             {
471                 fprintf(stderr, "%s: Libxml2 support not enabled\n", prog);
472                 exit(3);
473             }
474 #endif
475             break;
476         case 'o':
477             /* dirty hack so we can make Libxml2 do the writing ..
478                rather than WRBUF */
479             if (strlen(arg) > 4 && strncmp(arg, "xml,", 4) == 0)
480             {
481                 /* Only supported for Libxml2 2.6.0 or later */
482 #if LIBXML_VERSION >= 20600
483                 arg = arg + 4;
484                 write_using_libxml2 = 1;
485 #else
486                 fprintf(stderr, "%s: output using Libxml2 unsupported\n", prog);
487                 exit(4);
488 #endif
489             }
490             output_format = yaz_marc_decode_formatstr(arg);
491             if (output_format == -1)
492             {
493                 fprintf(stderr, "%s: bad output format: %s\n", prog, arg);
494                 exit(1);
495             }
496             break;
497         case 'l':
498             leader_spec = arg;
499             break;
500         case 'f':
501             from = arg;
502             break;
503         case 't':
504             to = arg;
505             break;
506         case 'c':
507             if (cfile)
508                 fclose(cfile);
509             cfile = fopen(arg, "w");
510             break;
511         case 'x':
512             fprintf(stderr, "%s: -x no longer supported. "
513                     "Use -i marcxml instead\n", prog);
514             exit(1);
515             break;
516         case 'O':
517             fprintf(stderr, "%s: OAI MARC no longer supported."
518                     " Use MARCXML instead.\n", prog);
519             exit(1);
520             break;
521         case 'e':
522             fprintf(stderr, "%s: -e no longer supported. "
523                     "Use -o marcxchange instead\n", prog);
524             exit(1);
525             break;
526         case 'X':
527             fprintf(stderr, "%s: -X no longer supported. "
528                     "Use -o marcxml instead\n", prog);
529             exit(1);
530             break;
531         case 'I':
532             fprintf(stderr, "%s: -I no longer supported. "
533                     "Use -o marc instead\n", prog);
534             exit(1);
535             break;
536         case 'n':
537             output_format = YAZ_MARC_CHECK;
538             break;
539         case 'p':
540             print_offset = 1;
541             break;
542         case 's':
543             split_fname = arg;
544             break;
545         case 'C':
546             split_chunk = atoi(arg);
547             break;
548         case 0:
549             dump(arg, from, to, input_format, output_format,
550                  write_using_libxml2,
551                  print_offset, split_fname, split_chunk,
552                  verbose, cfile, leader_spec);
553             break;
554         case 'v':
555             verbose++;
556             break;
557         case 'V':
558             show_version();
559             break;
560         default:
561             usage(prog);
562             exit(1);
563         }
564     }
565     if (cfile)
566         fclose(cfile);
567     if (!no)
568     {
569         usage(prog);
570         exit(1);
571     }
572     if (no_errors)
573         exit(5);
574     exit(0);
575 }
576 /*
577  * Local variables:
578  * c-basic-offset: 4
579  * c-file-style: "Stroustrup"
580  * indent-tabs-mode: nil
581  * End:
582  * vim: shiftwidth=4 tabstop=8 expandtab
583  */
584