43ff281eab5495c20f0e8e183154232bae6e5fa6
[yaz-moved-to-github.git] / util / marcdump.c
1 /* This file is part of the YAZ toolkit.
2  * Copyright (C) 1995-2009 Index Data
3  * See the file LICENSE for details.
4  */
5
6 #define _FILE_OFFSET_BITS 64
7
8 #if HAVE_CONFIG_H
9 #include <config.h>
10 #endif
11
12 #if YAZ_HAVE_XML2
13 #include <libxml/parser.h>
14 #include <libxml/tree.h>
15 #include <libxml/xpath.h>
16 #include <libxml/xpathInternals.h>
17
18 /* Libxml2 version < 2.6.15. xmlreader not reliable/present */
19 #if LIBXML_VERSION < 20615
20 #define USE_XMLREADER 0
21 #else
22 #define USE_XMLREADER 1
23 #endif
24
25 #if USE_XMLREADER
26 #include <libxml/xmlreader.h>
27 #endif
28
29 #endif
30
31 #include <stdio.h>
32 #include <stdlib.h>
33 #include <string.h>
34 #include <errno.h>
35 #include <assert.h>
36
37 #if HAVE_LOCALE_H
38 #include <locale.h>
39 #endif
40 #if HAVE_LANGINFO_H
41 #include <langinfo.h>
42 #endif
43
44 #include <yaz/marcdisp.h>
45 #include <yaz/yaz-util.h>
46 #include <yaz/xmalloc.h>
47 #include <yaz/options.h>
48
49 #ifndef SEEK_SET
50 #define SEEK_SET 0
51 #endif
52 #ifndef SEEK_END
53 #define SEEK_END 2
54 #endif
55
56
57 static char *prog;
58
59 static void usage(const char *prog)
60 {
61     fprintf (stderr, "Usage: %s [-i format] [-o format] [-f from] [-t to] "
62              "[-l pos=value] [-c cfile] [-s prefix] [-C size] [-n] "
63              "[-p] [-v] [-V] file...\n",
64              prog);
65
66
67 static void show_version(void)
68 {
69     char vstr[20], sha1_str[41];
70
71     yaz_version(vstr, sha1_str);
72     printf("YAZ version: %s %s\n", YAZ_VERSION, YAZ_VERSION_SHA1);
73     if (strcmp(sha1_str, YAZ_VERSION_SHA1))
74         printf("YAZ DLL/SO: %s %s\n", vstr, sha1_str);
75     exit(0);
76 }
77
78 static int getbyte_stream(void *client_data)
79 {
80     FILE *f = (FILE*) client_data;
81
82     int c = fgetc(f);
83     if (c == EOF)
84         return 0;
85     return c;
86 }
87
88 static void ungetbyte_stream(int c, void *client_data)
89 {
90     FILE *f = (FILE*) client_data;
91
92     if (c == 0)
93         c = EOF;
94     ungetc(c, f);
95 }
96
97 static void marcdump_read_line(yaz_marc_t mt, const char *fname)
98 {
99     FILE *inf = fopen(fname, "rb");
100     if (!inf)
101     {
102         fprintf (stderr, "%s: cannot open %s:%s\n",
103                  prog, fname, strerror (errno));
104         exit(1);
105     }
106     
107     while (yaz_marc_read_line(mt, getbyte_stream,
108                               ungetbyte_stream, inf) == 0)
109     {
110         WRBUF wrbuf = wrbuf_alloc();
111         yaz_marc_write_mode(mt, wrbuf);
112         fputs(wrbuf_cstr(wrbuf), stdout);
113         wrbuf_destroy(wrbuf);
114     }
115     fclose(inf);
116 }
117
118 #if YAZ_HAVE_XML2
119 static void marcdump_read_xml(yaz_marc_t mt, const char *fname)
120 {
121     WRBUF wrbuf = wrbuf_alloc();
122 #if USE_XMLREADER
123     xmlTextReaderPtr reader = xmlReaderForFile(fname, 0 /* encoding */,
124                                                0 /* options */);
125
126     if (reader)
127     {
128         int ret;
129         while ((ret = xmlTextReaderRead(reader)) == 1)
130         {
131             int type = xmlTextReaderNodeType(reader);
132             if (type == XML_READER_TYPE_ELEMENT)
133             {
134                 const char *name = (const char *) 
135                     xmlTextReaderLocalName(reader);
136                 if (!strcmp(name, "record"))
137                 {
138                     xmlNodePtr ptr = xmlTextReaderExpand(reader);
139         
140                     int r = yaz_marc_read_xml(mt, ptr);
141                     if (r)
142                         fprintf(stderr, "yaz_marc_read_xml failed\n");
143                     else
144                     {
145                         yaz_marc_write_mode(mt, wrbuf);
146                         
147                         fputs(wrbuf_cstr(wrbuf), stdout);
148                         wrbuf_rewind(wrbuf);
149                     }
150                 }
151             }
152         }
153     }
154 #else
155     xmlDocPtr doc = xmlParseFile(fname);
156     if (doc)
157     {
158         xmlNodePtr ptr = xmlDocGetRootElement(doc);
159         for (; ptr; ptr = ptr->next)
160         {
161             if (ptr->type == XML_ELEMENT_NODE)
162             {
163                 if (!strcmp((const char *) ptr->name, "collection"))
164                 {
165                     ptr = ptr->children;
166                     continue;
167                 }
168                 if (!strcmp((const char *) ptr->name, "record"))
169                 {
170                     int r = yaz_marc_read_xml(mt, ptr);
171                     if (r)
172                         fprintf(stderr, "yaz_marc_read_xml failed\n");
173                     else
174                     {
175                         yaz_marc_write_mode(mt, wrbuf);
176                         
177                         fputs(wrbuf_cstr(wrbuf), stdout);
178                         wrbuf_rewind(wrbuf);
179                     }
180                 }
181             }
182         }
183         xmlFreeDoc(doc);
184     }
185 #endif
186     fputs(wrbuf_cstr(wrbuf), stdout);
187     wrbuf_destroy(wrbuf);
188 }
189 #endif
190
191 static void dump(const char *fname, const char *from, const char *to,
192                  int input_format, int output_format,
193                  int write_using_libxml2,
194                  int print_offset, const char *split_fname, int split_chunk,
195                  int verbose, FILE *cfile, const char *leader_spec)
196 {
197     yaz_marc_t mt = yaz_marc_create();
198     yaz_iconv_t cd = 0;
199
200     if (yaz_marc_leader_spec(mt, leader_spec))
201     {
202         fprintf(stderr, "bad leader spec: %s\n", leader_spec);
203         yaz_marc_destroy(mt);
204         exit(2);
205     }
206     if (from && to)
207     {
208         cd = yaz_iconv_open(to, from);
209         if (!cd)
210         {
211             fprintf(stderr, "conversion from %s to %s "
212                     "unsupported\n", from, to);
213             yaz_marc_destroy(mt);
214             exit(2);
215         }
216         yaz_marc_iconv(mt, cd);
217     }
218     yaz_marc_xml(mt, output_format);
219     yaz_marc_enable_collection(mt);
220     yaz_marc_write_using_libxml2(mt, write_using_libxml2);
221     yaz_marc_debug(mt, verbose);
222
223     if (input_format == YAZ_MARC_MARCXML || input_format == YAZ_MARC_XCHANGE)
224     {
225 #if YAZ_HAVE_XML2
226         marcdump_read_xml(mt, fname);
227 #endif
228     }
229     else if (input_format == YAZ_MARC_LINE)
230     {
231         marcdump_read_line(mt, fname);
232     }
233     else if (input_format == YAZ_MARC_ISO2709)
234     {
235         FILE *inf = fopen(fname, "rb");
236         int num = 1;
237         int marc_no = 0;
238         int split_file_no = -1;
239         if (!inf)
240         {
241             fprintf (stderr, "%s: cannot open %s:%s\n",
242                      prog, fname, strerror (errno));
243             exit(1);
244         }
245         if (cfile)
246             fprintf (cfile, "char *marc_records[] = {\n");
247         for(;; marc_no++)
248         {
249             const char *result = 0;
250             size_t len;
251             size_t rlen;
252             size_t len_result;
253             size_t r;
254             char buf[100001];
255             
256             r = fread (buf, 1, 5, inf);
257             if (r < 5)
258             {
259                 if (r && print_offset && verbose)
260                     printf ("<!-- Extra %ld bytes at end of file -->\n",
261                             (long) r);
262                 break;
263             }
264             while (*buf < '0' || *buf > '9')
265             {
266                 int i;
267                 long off = ftell(inf) - 5;
268                 if (verbose || print_offset)
269                     printf("<!-- Skipping bad byte %d (0x%02X) at offset "
270                            "%ld (0x%lx) -->\n", 
271                            *buf & 0xff, *buf & 0xff,
272                            off, off);
273                 for (i = 0; i<4; i++)
274                     buf[i] = buf[i+1];
275                 r = fread(buf+4, 1, 1, inf);
276                 if (r < 1)
277                     break;
278             }
279             if (r < 1)
280             {
281                 if (verbose || print_offset)
282                     printf ("<!-- End of file with data -->\n");
283                 break;
284             }
285             if (print_offset)
286             {
287                 long off = ftell(inf) - 5;
288                 printf ("<!-- Record %d offset %ld (0x%lx) -->\n",
289                         num, off, off);
290             }
291             len = atoi_n(buf, 5);
292             if (len < 25 || len > 100000)
293             {
294                 long off = ftell(inf) - 5;
295                 printf("Bad Length %ld read at offset %ld (%lx)\n",
296                        (long)len, (long) off, (long) off);
297                 break;
298             }
299             rlen = len - 5;
300             r = fread (buf + 5, 1, rlen, inf);
301             if (r < rlen)
302                 break;
303             while (buf[len-1] != ISO2709_RS)
304             {
305                 if (len > sizeof(buf)-2)
306                     break;
307                 r = fread (buf + len, 1, 1, inf);
308                 if (r != 1)
309                     break;
310                 len++;
311             }
312             if (split_fname)
313             {
314                 char fname[256];
315                 const char *mode = 0;
316                 FILE *sf;
317                 if ((marc_no % split_chunk) == 0)
318                 {
319                     mode = "wb";
320                     split_file_no++;
321                 }
322                 else
323                     mode = "ab";
324                 sprintf(fname, "%.200s%07d", split_fname, split_file_no);
325                 sf = fopen(fname, mode);
326                 if (!sf)
327                 {
328                     fprintf(stderr, "Could not open %s\n", fname);
329                     split_fname = 0;
330                 }
331                 else
332                 {
333                     if (fwrite(buf, 1, len, sf) != len)
334                     {
335                         fprintf(stderr, "Could write content to %s\n",
336                                 fname);
337                         split_fname = 0;
338                     }
339                     fclose(sf);
340                 }
341             }
342             len_result = rlen;
343             r = yaz_marc_decode_buf(mt, buf, -1, &result, &len_result);
344             if (r > 0 && result)
345             {
346                 if (fwrite(result, len_result, 1, stdout) != 1)
347                 {
348                     fprintf(stderr, "Write to stdout failed\n");
349                     break;
350                 }
351             }
352             if (r > 0 && cfile)
353             {
354                 char *p = buf;
355                 size_t i;
356                 if (marc_no)
357                     fprintf (cfile, ",");
358                 fprintf (cfile, "\n");
359                 for (i = 0; i < r; i++)
360                 {
361                     if ((i & 15) == 0)
362                         fprintf (cfile, "  \"");
363                     fprintf (cfile, "\\x%02X", p[i] & 255);
364                     
365                     if (i < r - 1 && (i & 15) == 15)
366                         fprintf (cfile, "\"\n");
367                     
368                 }
369                 fprintf (cfile, "\"\n");
370             }
371             num++;
372             if (verbose)
373                 printf("\n");
374         }
375         if (cfile)
376             fprintf (cfile, "};\n");
377         fclose(inf);
378     }
379     {
380         WRBUF wrbuf = wrbuf_alloc();
381         yaz_marc_write_trailer(mt, wrbuf);
382         fputs(wrbuf_cstr(wrbuf), stdout);
383         wrbuf_destroy(wrbuf);
384     }
385     if (cd)
386         yaz_iconv_close(cd);
387     yaz_marc_destroy(mt);
388 }
389
390 int main (int argc, char **argv)
391 {
392     int r;
393     int print_offset = 0;
394     char *arg;
395     int verbose = 0;
396     int no = 0;
397     int output_format = YAZ_MARC_LINE;
398     FILE *cfile = 0;
399     char *from = 0, *to = 0;
400     int input_format = YAZ_MARC_ISO2709;
401     int split_chunk = 1;
402     const char *split_fname = 0;
403     const char *leader_spec = 0;
404     int write_using_libxml2 = 0;
405
406 #if HAVE_LOCALE_H
407     setlocale(LC_CTYPE, "");
408 #endif
409 #if HAVE_LANGINFO_H
410 #ifdef CODESET
411     to = nl_langinfo(CODESET);
412 #endif
413 #endif
414
415     prog = *argv;
416     while ((r = options("i:o:C:npc:xOeXIf:t:s:l:Vv", argv, argc, &arg)) != -2)
417     {
418         no++;
419         switch (r)
420         {
421         case 'i':
422             input_format = yaz_marc_decode_formatstr(arg);
423             if (input_format == -1)
424             {
425                 fprintf(stderr, "%s: bad input format: %s\n", prog, arg);
426                 exit(1);
427             }
428 #if YAZ_HAVE_XML2
429 #else
430             if (input_format == YAZ_MARC_MARCXML 
431                 || input_format == YAZ_MARC_XCHANGE)
432             {
433                 fprintf(stderr, "%s: Libxml2 support not enabled\n", prog);
434                 exit(3);
435             }
436 #endif
437             break;
438         case 'o':
439             /* dirty hack so we can make Libxml2 do the writing ..
440                rather than WRBUF */
441             if (strlen(arg) > 4 && strncmp(arg, "xml,", 4) == 0)
442             {
443                 arg = arg + 4;
444                 write_using_libxml2 = 1;
445             }
446             output_format = yaz_marc_decode_formatstr(arg);
447             if (output_format == -1)
448             {
449                 fprintf(stderr, "%s: bad output format: %s\n", prog, arg);
450                 exit(1);
451             }
452             break;
453         case 'l':
454             leader_spec = arg;
455             break;
456         case 'f':
457             from = arg;
458             break;
459         case 't':
460             to = arg;
461             break;
462         case 'c':
463             if (cfile)
464                 fclose (cfile);
465             cfile = fopen(arg, "w");
466             break;
467         case 'x':
468             fprintf(stderr, "%s: -x no longer supported. "
469                     "Use -i marcxml instead\n", prog);
470             exit(1);
471             break;
472         case 'O':
473             fprintf(stderr, "%s: OAI MARC no longer supported."
474                     " Use MARCXML instead.\n", prog);
475             exit(1);
476             break;
477         case 'e':
478             fprintf(stderr, "%s: -e no longer supported. "
479                     "Use -o marcxchange instead\n", prog);
480             exit(1);
481             break;
482         case 'X':
483             fprintf(stderr, "%s: -X no longer supported. "
484                     "Use -o marcxml instead\n", prog);
485             exit(1);
486             break;
487         case 'I':
488             fprintf(stderr, "%s: -I no longer supported. "
489                     "Use -o marc instead\n", prog);
490             exit(1);
491             break;
492         case 'n':
493             output_format = YAZ_MARC_CHECK;
494             break;
495         case 'p':
496             print_offset = 1;
497             break;
498         case 's':
499             split_fname = arg;
500             break;
501         case 'C':
502             split_chunk = atoi(arg);
503             break;
504         case 0:
505             dump(arg, from, to, input_format, output_format,
506                  write_using_libxml2,
507                  print_offset, split_fname, split_chunk,
508                  verbose, cfile, leader_spec);
509             break;
510         case 'v':
511             verbose++;
512             break;
513         case 'V': 
514             show_version();
515             break;
516         default:
517             usage(prog);
518             exit(1);
519         }
520     }
521     if (cfile)
522         fclose (cfile);
523     if (!no)
524     {
525         usage(prog);
526         exit (1);
527     }
528     exit (0);
529 }
530 /*
531  * Local variables:
532  * c-basic-offset: 4
533  * indent-tabs-mode: nil
534  * End:
535  * vim: shiftwidth=4 tabstop=8 expandtab
536  */
537