X-Git-Url: http://git.indexdata.com/?p=yaz-moved-to-github.git;a=blobdiff_plain;f=src%2Ftokenizer.c;h=db73fe9716e22db175fcb327d3053518151c68b1;hp=e13465ff11f6ec97bc5a5d77e4f2334fc9ff786c;hb=45fb4747b80f37bfa44738e1071c094f18660ae9;hpb=aa60dbcd982a4dbd52870120ef34b91a74d63274

diff --git a/src/tokenizer.c b/src/tokenizer.c
index e13465f..db73fe9 100644
--- a/src/tokenizer.c
+++ b/src/tokenizer.c
@@ -1,19 +1,18 @@
-/*
- * Copyright (C) 1995-2007, Index Data ApS
+/* This file is part of the YAZ toolkit.
+ * Copyright (C) Index Data
  * See the file LICENSE for details.
- *
- * $Id: tokenizer.c,v 1.2 2007-04-27 10:09:45 adam Exp $
  */
-
 /**
  * \file tokenizer.c
- * \brief Implements attribute match of CCL RPN nodes
+ * \brief Simple tokenizer system.
  */
+#if HAVE_CONFIG_H
+#include <config.h>
+#endif
 
 #include <assert.h>
 #include <stdio.h>
 #include <string.h>
-#include <ctype.h>
 #include <yaz/log.h>
 #include <yaz/wrbuf.h>
 #include <yaz/tokenizer.h>
@@ -22,7 +21,7 @@ struct yaz_tok_parse {
     int unget_byte;
     WRBUF wr_string;
     int look;
-    
+
     yaz_tok_cfg_t cfg;
     yaz_tok_get_byte_t get_byte_func;
     void *get_byte_data;
@@ -30,6 +29,7 @@ struct yaz_tok_parse {
 
 struct yaz_tok_cfg {
     int ref_count;
+    char *comment;
     char *white_space;
     char *single_tokens;
     char *quote_tokens_begin;
@@ -44,11 +44,12 @@ void yaz_tok_cfg_single_tokens(yaz_tok_cfg_t t, const char *simple)
 
 yaz_tok_cfg_t yaz_tok_cfg_create(void)
 {
-    yaz_tok_cfg_t t = xmalloc(sizeof(*t));
+    yaz_tok_cfg_t t = (yaz_tok_cfg_t) xmalloc(sizeof(*t));
     t->white_space = xstrdup(" \t\r\n");
     t->single_tokens = xstrdup("");
     t->quote_tokens_begin = xstrdup("\"");
     t->quote_tokens_end = xstrdup("\"");
+    t->comment = xstrdup("#");
     t->ref_count = 1;
     return t;
 }
@@ -62,6 +63,7 @@ void yaz_tok_cfg_destroy(yaz_tok_cfg_t t)
         xfree(t->single_tokens);
         xfree(t->quote_tokens_begin);
         xfree(t->quote_tokens_end);
+        xfree(t->comment);
         xfree(t);
     }
 }
@@ -103,7 +105,7 @@ yaz_tok_parse_t yaz_tok_parse_create(yaz_tok_cfg_t t,
                                      yaz_tok_get_byte_t h,
                                      void *vp)
 {
-    yaz_tok_parse_t tp = xmalloc(sizeof(*tp));
+    yaz_tok_parse_t tp = (yaz_tok_parse_t) xmalloc(sizeof(*tp));
 
     tp->cfg = t;
     tp->cfg->ref_count++;
@@ -116,7 +118,7 @@ yaz_tok_parse_t yaz_tok_parse_create(yaz_tok_cfg_t t,
     tp->wr_string = wrbuf_alloc();
     return tp;
 }
-                                           
+
 
 void yaz_tok_parse_destroy(yaz_tok_parse_t tp)
 {
@@ -134,10 +136,10 @@ int yaz_tok_move(yaz_tok_parse_t tp)
     /* skip white space */
     while (ch && strchr(t->white_space, ch))
         ch = get_byte(tp);
-    if (!ch) 
-    {
+    if (!ch)
+        ch = YAZ_TOK_EOF;
+    else if (strchr(t->comment, ch))
         ch = YAZ_TOK_EOF;
-    }
     else if ((cp = strchr(t->single_tokens, ch)))
         ch = *cp;  /* single token match */
     else if ((cp = strchr(t->quote_tokens_begin, ch)))
@@ -156,7 +158,8 @@ int yaz_tok_move(yaz_tok_parse_t tp)
     {  /* unquoted string */
         wrbuf_rewind(tp->wr_string);
         while (ch && !strchr(t->white_space, ch)
-               && !strchr(t->single_tokens, ch))
+               && !strchr(t->single_tokens, ch)
+               && !strchr(t->comment, ch))
         {
             wrbuf_putc(tp->wr_string, ch);
             ch = get_byte(tp);
@@ -176,6 +179,7 @@ const char *yaz_tok_parse_string(yaz_tok_parse_t tp)
 /*
  * Local variables:
  * c-basic-offset: 4
+ * c-file-style: "Stroustrup"
  * indent-tabs-mode: nil
  * End:
  * vim: shiftwidth=4 tabstop=8 expandtab