X-Git-Url: http://git.indexdata.com/?a=blobdiff_plain;f=src%2Fccltoken.c;h=d7b0d5f37d83eec9952a100493fb7db7cc27ac68;hb=9cc8f4a4d0fd7f1e4bd6c9d5c85ef2dccaab5743;hp=11273b36a56d9d40975f2245769bfaeee8e8f459;hpb=dd7de5f04a0de631f98ad9c4774f2111a85388e5;p=yaz-moved-to-github.git

diff --git a/src/ccltoken.c b/src/ccltoken.c
index 11273b3..d7b0d5f 100644
--- a/src/ccltoken.c
+++ b/src/ccltoken.c
@@ -41,10 +41,14 @@
  * USE OR PERFORMANCE OF THIS SOFTWARE.
  *
  */
+/** 
+ * \file ccltoken.c
+ * \brief Implements CCL lexical analyzer (scanner)
+ */
 /* CCL - lexical analysis
  * Europagate, 1995
  *
- * $Id: ccltoken.c,v 1.4 2004-10-02 13:28:26 adam Exp $
+ * $Id: ccltoken.c,v 1.7 2005-04-15 21:47:56 adam Exp $
  *
  * Old Europagate Log:
  *
@@ -133,68 +137,6 @@ static int token_cmp (CCL_parser cclp, const char *kw, struct ccl_token *token)
 }
 
 /*
- * ccl_token_simple: tokenize CCL raw tokens
- */
-struct ccl_token *ccl_token_simple (const char *command)
-{
-    const char *cp = command;
-    struct ccl_token *first = NULL;
-    struct ccl_token *last = NULL;
-
-    while (1)
-    {
-	while (*cp && strchr (" \t\r\n", *cp))
-	{
-	    cp++;
-	    continue;
-	}
-	if (!first)
-	{
-	    first = last = (struct ccl_token *)xmalloc (sizeof (*first));
-	    ccl_assert (first);
-	    last->prev = NULL;
-	}
-	else
-	{
-	    last->next = (struct ccl_token *)xmalloc (sizeof(*first));
-	    ccl_assert (last->next);
-	    last->next->prev = last;
-	    last = last->next;
-	}
-	last->next = NULL;
-	last->name = cp;
-	last->len = 1;
-	switch (*cp++)
-	{
-        case '\0':
-            last->kind = CCL_TOK_EOL;
-            return first;
-	case '\"':
-	    last->kind = CCL_TOK_TERM;
-	    last->name = cp;
-	    last->len = 0;
-	    while (*cp && *cp != '\"')
-	    {
-		cp++;
-		++ last->len;
-	    }
-	    if (*cp == '\"')
-		cp++;
-	    break;
-	default:
-	    while (*cp && !strchr (" \t\n\r", *cp))
-	    {
-		cp++;
-		++ last->len;
-	    }
-            last->kind = CCL_TOK_TERM;
-	}
-    }
-    return first;
-}
-
-
-/*
  * ccl_tokenize: tokenize CCL command string.
  * return: CCL token list.
  */
@@ -207,11 +149,9 @@ struct ccl_token *ccl_parser_tokenize (CCL_parser cclp, const char *command)
 
     while (1)
     {
+	const unsigned char *cp0 = cp;
 	while (*cp && strchr (" \t\r\n", *cp))
-	{
 	    cp++;
-	    continue;
-	}
 	if (!first)
 	{
 	    first = last = (struct ccl_token *)xmalloc (sizeof (*first));
@@ -225,6 +165,8 @@ struct ccl_token *ccl_parser_tokenize (CCL_parser cclp, const char *command)
 	    last->next->prev = last;
 	    last = last->next;
 	}
+	last->ws_prefix_buf = cp0;
+	last->ws_prefix_len = cp - cp0;
 	last->next = NULL;
 	last->name = (const char *) cp;
 	last->len = 1;
@@ -330,6 +272,8 @@ struct ccl_token *ccl_token_add (struct ccl_token *at)
     n->kind = CCL_TOK_TERM;
     n->name = 0;
     n->len = 0;
+    n->ws_prefix_buf = 0;
+    n->ws_prefix_len = 0;
     return n;
 }