X-Git-Url: http://git.indexdata.com/?p=yaz-moved-to-github.git;a=blobdiff_plain;f=src%2Ftokenizer.c;h=0289771df5ae73fe5db2743f3b89851c5a1c1473;hp=5f783be815e10f4da6630ea97d45e4107205d939;hb=5242cb5a8634bfa38b9333ff7f903e718ac6e292;hpb=4f3bcae93d51a26709c12b51261c3d95af610cb2 diff --git a/src/tokenizer.c b/src/tokenizer.c index 5f783be..0289771 100644 --- a/src/tokenizer.c +++ b/src/tokenizer.c @@ -1,5 +1,5 @@ /* This file is part of the YAZ toolkit. - * Copyright (C) 1995-2011 Index Data + * Copyright (C) 1995-2012 Index Data * See the file LICENSE for details. */ /** @@ -21,7 +21,7 @@ struct yaz_tok_parse { int unget_byte; WRBUF wr_string; int look; - + yaz_tok_cfg_t cfg; yaz_tok_get_byte_t get_byte_func; void *get_byte_data; @@ -118,7 +118,7 @@ yaz_tok_parse_t yaz_tok_parse_create(yaz_tok_cfg_t t, tp->wr_string = wrbuf_alloc(); return tp; } - + void yaz_tok_parse_destroy(yaz_tok_parse_t tp) { @@ -136,7 +136,7 @@ int yaz_tok_move(yaz_tok_parse_t tp) /* skip white space */ while (ch && strchr(t->white_space, ch)) ch = get_byte(tp); - if (!ch) + if (!ch) ch = YAZ_TOK_EOF; else if (strchr(t->comment, ch)) ch = YAZ_TOK_EOF;