/* This file is part of the YAZ toolkit.
- * Copyright (C) 1995-2009 Index Data
+ * Copyright (C) Index Data
* See the file LICENSE for details.
*/
-
/**
* \file tokenizer.c
* \brief Simple tokenizer system.
*/
+#if HAVE_CONFIG_H
+#include <config.h>
+#endif
#include <assert.h>
#include <stdio.h>
#include <string.h>
-#include <ctype.h>
#include <yaz/log.h>
#include <yaz/wrbuf.h>
#include <yaz/tokenizer.h>
int unget_byte;
WRBUF wr_string;
int look;
-
+
yaz_tok_cfg_t cfg;
yaz_tok_get_byte_t get_byte_func;
void *get_byte_data;
tp->wr_string = wrbuf_alloc();
return tp;
}
-
+
void yaz_tok_parse_destroy(yaz_tok_parse_t tp)
{
/* skip white space */
while (ch && strchr(t->white_space, ch))
ch = get_byte(tp);
- if (!ch)
+ if (!ch)
ch = YAZ_TOK_EOF;
else if (strchr(t->comment, ch))
ch = YAZ_TOK_EOF;