X-Git-Url: http://git.indexdata.com/?p=yaz-moved-to-github.git;a=blobdiff_plain;f=src%2Ftokenizer.c;h=db73fe9716e22db175fcb327d3053518151c68b1;hp=5f783be815e10f4da6630ea97d45e4107205d939;hb=08cbe0ceed43df90c48a5863de2f1b72cca67cb2;hpb=4f3bcae93d51a26709c12b51261c3d95af610cb2 diff --git a/src/tokenizer.c b/src/tokenizer.c index 5f783be..db73fe9 100644 --- a/src/tokenizer.c +++ b/src/tokenizer.c @@ -1,5 +1,5 @@ /* This file is part of the YAZ toolkit. - * Copyright (C) 1995-2011 Index Data + * Copyright (C) Index Data * See the file LICENSE for details. */ /** @@ -21,7 +21,7 @@ struct yaz_tok_parse { int unget_byte; WRBUF wr_string; int look; - + yaz_tok_cfg_t cfg; yaz_tok_get_byte_t get_byte_func; void *get_byte_data; @@ -118,7 +118,7 @@ yaz_tok_parse_t yaz_tok_parse_create(yaz_tok_cfg_t t, tp->wr_string = wrbuf_alloc(); return tp; } - + void yaz_tok_parse_destroy(yaz_tok_parse_t tp) { @@ -136,7 +136,7 @@ int yaz_tok_move(yaz_tok_parse_t tp) /* skip white space */ while (ch && strchr(t->white_space, ch)) ch = get_byte(tp); - if (!ch) + if (!ch) ch = YAZ_TOK_EOF; else if (strchr(t->comment, ch)) ch = YAZ_TOK_EOF;