MARC-8: allow all characters 0x01-0x20 YAZ-650

[yaz-moved-to-github.git] / src / tokenizer.c

diff --git a/src/tokenizer.c b/src/tokenizer.c
index 1af96f2..993ef91 100644 (file)
--- a/src/tokenizer.c
+++ b/src/tokenizer.c
@@ -1,17 +1,18 @@
 /* This file is part of the YAZ toolkit.
- * Copyright (C) 1995-2009 Index Data
+ * Copyright (C) 1995-2013 Index Data
  * See the file LICENSE for details.
  */
-
 /**
  * \file tokenizer.c
  * \brief Simple tokenizer system.
  */
+#if HAVE_CONFIG_H
+#include <config.h>
+#endif
 
 #include <assert.h>
 #include <stdio.h>
 #include <string.h>
-#include <ctype.h>
 #include <yaz/log.h>
 #include <yaz/wrbuf.h>
 #include <yaz/tokenizer.h>
@@ -20,7 +21,7 @@ struct yaz_tok_parse {
     int unget_byte;
     WRBUF wr_string;
     int look;
-    
+
     yaz_tok_cfg_t cfg;
     yaz_tok_get_byte_t get_byte_func;
     void *get_byte_data;
@@ -117,7 +118,7 @@ yaz_tok_parse_t yaz_tok_parse_create(yaz_tok_cfg_t t,
     tp->wr_string = wrbuf_alloc();
     return tp;
 }
-                                           
+
 
 void yaz_tok_parse_destroy(yaz_tok_parse_t tp)
 {
@@ -135,7 +136,7 @@ int yaz_tok_move(yaz_tok_parse_t tp)
     /* skip white space */
     while (ch && strchr(t->white_space, ch))
         ch = get_byte(tp);
-    if (!ch) 
+    if (!ch)
         ch = YAZ_TOK_EOF;
     else if (strchr(t->comment, ch))
         ch = YAZ_TOK_EOF;
@@ -178,6 +179,7 @@ const char *yaz_tok_parse_string(yaz_tok_parse_t tp)
 /*
  * Local variables:
  * c-basic-offset: 4
+ * c-file-style: "Stroustrup"
  * indent-tabs-mode: nil
  * End:
  * vim: shiftwidth=4 tabstop=8 expandtab