rpn2cql: deal with truncation=102 (regexp-1)

[yaz-moved-to-github.git] / src / ccltoken.c

diff --git a/src/ccltoken.c b/src/ccltoken.c
index b784295..6c74226 100644 (file)
--- a/src/ccltoken.c
+++ b/src/ccltoken.c
@@ -1,16 +1,18 @@
 /* This file is part of the YAZ toolkit.
- * Copyright (C) 1995-2010 Index Data
+ * Copyright (C) 1995-2011 Index Data
  * See the file LICENSE for details.
  */
 /** 
  * \file ccltoken.c
  * \brief Implements CCL lexical analyzer (scanner)
  */
+#if HAVE_CONFIG_H
+#include <config.h>
+#endif
 
 #include <string.h>
 #include <stdlib.h>
-#include <ctype.h>
-
+#include <yaz/yaz-iconv.h>
 #include "cclp.h"
 
 /*
@@ -79,7 +81,6 @@ struct ccl_token *ccl_parser_tokenize(CCL_parser cclp, const char *command)
             last->next->prev = last;
             last = last->next;
         }
-        last->left_trunc = last->right_trunc = 0;
         last->ws_prefix_buf = (const char *) cp0;
         last->ws_prefix_len = cp - cp0;
         last->next = NULL;
@@ -102,7 +103,7 @@ struct ccl_token *ccl_parser_tokenize(CCL_parser cclp, const char *command)
         case '%':
         case '!':
             last->kind = CCL_TOK_PROX;
-            while (isdigit(*cp))
+            while (yaz_isdigit(*cp))
             {
                 ++ last->len;
                 cp++;
@@ -125,11 +126,6 @@ struct ccl_token *ccl_parser_tokenize(CCL_parser cclp, const char *command)
         default:
             --cp;
             --last->len;
-            if (*cp == '?')
-            {
-                last->left_trunc = 1;
-                cp++;
-            }
             if (*cp == '"')
             {
                 cp++;
@@ -147,7 +143,7 @@ struct ccl_token *ccl_parser_tokenize(CCL_parser cclp, const char *command)
             {
                 last->kind = CCL_TOK_TERM;
                 last->name = (const char *) cp;
-                while (*cp && !strchr("(),%!><=? \t\n\r", *cp))
+                while (*cp && !strchr("(),%!><= \t\n\r", *cp))
                 {
                     ++ last->len;
                     cp++;
@@ -177,11 +173,6 @@ struct ccl_token *ccl_parser_tokenize(CCL_parser cclp, const char *command)
                 if (token_cmp(cclp, aliases, last))
                     last->kind = CCL_TOK_SET;
             }
-            if (*cp == '?')
-            {
-                last->right_trunc = 1;
-                cp++;
-            }
         }
     }
     return first;
@@ -198,7 +189,6 @@ struct ccl_token *ccl_token_add(struct ccl_token *at)
         n->next->prev = n;
 
     n->kind = CCL_TOK_TERM;
-    n->left_trunc = n->right_trunc = 0;
     n->name = 0;
     n->len = 0;
     n->ws_prefix_buf = 0;