X-Git-Url: http://git.indexdata.com/?p=yaz-moved-to-github.git;a=blobdiff_plain;f=src%2Ftokenizer.c;h=cf76c892f0c074fafff532336f231cf2a6f29d48;hp=e13465ff11f6ec97bc5a5d77e4f2334fc9ff786c;hb=5e1889250160764822014ce89ee19dc84e5ce310;hpb=aa60dbcd982a4dbd52870120ef34b91a74d63274

diff --git a/src/tokenizer.c b/src/tokenizer.c
index e13465f..cf76c89 100644
--- a/src/tokenizer.c
+++ b/src/tokenizer.c
@@ -1,13 +1,11 @@
-/*
- * Copyright (C) 1995-2007, Index Data ApS
+/* This file is part of the YAZ toolkit.
+ * Copyright (C) 1995-2010 Index Data
  * See the file LICENSE for details.
- *
- * $Id: tokenizer.c,v 1.2 2007-04-27 10:09:45 adam Exp $
  */
 
 /**
  * \file tokenizer.c
- * \brief Implements attribute match of CCL RPN nodes
+ * \brief Simple tokenizer system.
  */
 
 #include <assert.h>
@@ -30,6 +28,7 @@ struct yaz_tok_parse {
 
 struct yaz_tok_cfg {
     int ref_count;
+    char *comment;
     char *white_space;
     char *single_tokens;
     char *quote_tokens_begin;
@@ -44,11 +43,12 @@ void yaz_tok_cfg_single_tokens(yaz_tok_cfg_t t, const char *simple)
 
 yaz_tok_cfg_t yaz_tok_cfg_create(void)
 {
-    yaz_tok_cfg_t t = xmalloc(sizeof(*t));
+    yaz_tok_cfg_t t = (yaz_tok_cfg_t) xmalloc(sizeof(*t));
     t->white_space = xstrdup(" \t\r\n");
     t->single_tokens = xstrdup("");
     t->quote_tokens_begin = xstrdup("\"");
     t->quote_tokens_end = xstrdup("\"");
+    t->comment = xstrdup("#");
     t->ref_count = 1;
     return t;
 }
@@ -62,6 +62,7 @@ void yaz_tok_cfg_destroy(yaz_tok_cfg_t t)
         xfree(t->single_tokens);
         xfree(t->quote_tokens_begin);
         xfree(t->quote_tokens_end);
+        xfree(t->comment);
         xfree(t);
     }
 }
@@ -103,7 +104,7 @@ yaz_tok_parse_t yaz_tok_parse_create(yaz_tok_cfg_t t,
                                      yaz_tok_get_byte_t h,
                                      void *vp)
 {
-    yaz_tok_parse_t tp = xmalloc(sizeof(*tp));
+    yaz_tok_parse_t tp = (yaz_tok_parse_t) xmalloc(sizeof(*tp));
 
     tp->cfg = t;
     tp->cfg->ref_count++;
@@ -135,9 +136,9 @@ int yaz_tok_move(yaz_tok_parse_t tp)
     while (ch && strchr(t->white_space, ch))
         ch = get_byte(tp);
     if (!ch) 
-    {
         ch = YAZ_TOK_EOF;
-    }
+    else if (strchr(t->comment, ch))
+        ch = YAZ_TOK_EOF;
     else if ((cp = strchr(t->single_tokens, ch)))
         ch = *cp;  /* single token match */
     else if ((cp = strchr(t->quote_tokens_begin, ch)))
@@ -156,7 +157,8 @@ int yaz_tok_move(yaz_tok_parse_t tp)
     {  /* unquoted string */
         wrbuf_rewind(tp->wr_string);
         while (ch && !strchr(t->white_space, ch)
-               && !strchr(t->single_tokens, ch))
+               && !strchr(t->single_tokens, ch)
+               && !strchr(t->comment, ch))
         {
             wrbuf_putc(tp->wr_string, ch);
             ch = get_byte(tp);
@@ -176,6 +178,7 @@ const char *yaz_tok_parse_string(yaz_tok_parse_t tp)
 /*
  * Local variables:
  * c-basic-offset: 4
+ * c-file-style: "Stroustrup"
  * indent-tabs-mode: nil
  * End:
  * vim: shiftwidth=4 tabstop=8 expandtab