X-Git-Url: http://git.indexdata.com/?p=yaz-moved-to-github.git;a=blobdiff_plain;f=src%2Ftokenizer.c;h=db73fe9716e22db175fcb327d3053518151c68b1;hp=197a3237d8839464b70275b3a2f3880d7830f3fe;hb=c1f23597ea64485e2362d658f3653211202cc6a8;hpb=77c5a4fca8b516fd39b8ba213daed17a465a6b2a diff --git a/src/tokenizer.c b/src/tokenizer.c index 197a323..db73fe9 100644 --- a/src/tokenizer.c +++ b/src/tokenizer.c @@ -1,5 +1,5 @@ /* This file is part of the YAZ toolkit. - * Copyright (C) 1995-2012 Index Data + * Copyright (C) Index Data * See the file LICENSE for details. */ /** @@ -21,7 +21,7 @@ struct yaz_tok_parse { int unget_byte; WRBUF wr_string; int look; - + yaz_tok_cfg_t cfg; yaz_tok_get_byte_t get_byte_func; void *get_byte_data; @@ -118,7 +118,7 @@ yaz_tok_parse_t yaz_tok_parse_create(yaz_tok_cfg_t t, tp->wr_string = wrbuf_alloc(); return tp; } - + void yaz_tok_parse_destroy(yaz_tok_parse_t tp) { @@ -136,7 +136,7 @@ int yaz_tok_move(yaz_tok_parse_t tp) /* skip white space */ while (ch && strchr(t->white_space, ch)) ch = get_byte(tp); - if (!ch) + if (!ch) ch = YAZ_TOK_EOF; else if (strchr(t->comment, ch)) ch = YAZ_TOK_EOF;