Rewrite lexer to comply with CQL spec

Add test more tests

diff --git a/src/main/java/org/z3950/zing/cql/CQLLexer.java b/src/main/java/org/z3950/zing/cql/CQLLexer.java
index 293b45a..2247fd0 100644 (file)
--- a/src/main/java/org/z3950/zing/cql/CQLLexer.java
+++ b/src/main/java/org/z3950/zing/cql/CQLLexer.java
@@ -1,247 +1,40 @@
-
+/*
+ * Copyright (c) 1995-2014, Index Datassss
+ * All rights reserved.
+ * See the file LICENSE for details.
+ */

 package org.z3950.zing.cql;
 package org.z3950.zing.cql;

-import java.io.InputStream;
-import java.io.Reader;
-import java.io.StreamTokenizer;
-import java.io.StringReader;
-
-
-// This is a semi-trivial subclass for java.io.StreamTokenizer that:
-//     * Has a halfDecentPushBack() method that actually works
-//     * Includes a render() method
-//     * Knows about the multi-character tokens "<=", ">=" and "<>"
-//     * Recognises a set of keywords as tokens in their own right
-//     * Includes some primitive debugging-output facilities
-// It's used only by CQLParser.
-//
-class CQLLexer extends StreamTokenizer {
-    // New publicly visible token-types
-    public final static int TT_LE        = 1000;       // The "<=" relation
-    public final static int TT_GE        = 1001;       // The ">=" relation
-    public final static int TT_NE        = 1002;       // The "<>" relation
-    public final static int TT_EQEQ      = 1003;       // The "==" relation
-    public final static int TT_AND       = 1004;       // The "and" boolean
-    public final static int TT_OR        = 1005;       // The "or" boolean
-    public final static int TT_NOT       = 1006;       // The "not" boolean
-    public final static int TT_PROX      = 1007;       // The "prox" boolean
-    public final static int TT_SORTBY     = 1008;      // The "sortby" operator
-
-    // Support for keywords.  It would be nice to compile this linear
-    // list into a Hashtable, but it's hard to store ints as hash
-    // values, and next to impossible to use them as hash keys.  So
-    // we'll just scan the (very short) list every time we need to do
-    // a lookup.
-    private class Keyword {
-       int token;
-       String keyword;
-       Keyword(int token, String keyword) {
-           this.token = token;
-           this.keyword = keyword;
-       }
-    }
-    // This should logically be static, but Java won't allow it  :-P
-    private Keyword[] keywords = {
-       new Keyword(TT_AND, "and"),
-       new Keyword(TT_OR,  "or"),
-       new Keyword(TT_NOT, "not"),
-       new Keyword(TT_PROX, "prox"),
-       new Keyword(TT_SORTBY, "sortby"),
-    };
-
-    // For halfDecentPushBack() and the code at the top of nextToken()
-    private static int TT_UNDEFINED = -1000;
-    private int saved_ttype = TT_UNDEFINED;
-    private double saved_nval;
-    private String saved_sval;
-
-    // Controls debugging output
-    private static boolean DEBUG;
-
-    CQLLexer(String cql, boolean lexdebug) {
-        this(new StringReader(cql), lexdebug);
-    }
-    
-    CQLLexer(Reader cql, boolean lexdebug) {
-       super(cql);
-       wordChars('!', '@');    // ASCII-dependency!
-       wordChars('[', '`');    // ASCII-dependency!
-       quoteChar('"');
-       ordinaryChar('=');
-       ordinaryChar('<');
-       ordinaryChar('>');
-       ordinaryChar('/');
-       ordinaryChar('(');
-       ordinaryChar(')');
-        ordinaryChar('.');
-        wordChars('.', '.');
-       wordChars('\'', '\''); // prevent this from introducing strings
-       //parseNumbers();
-       ordinaryChar('-');
-       wordChars('-', '-');
-       ordinaryChars('0', '9');
-       wordChars('0', '9');
-       DEBUG = lexdebug;
-    }
-
-    private static void debug(String str) {
-       if (DEBUG)
-           System.err.println("LEXDEBUG: " + str);
-    }
-
-    // I don't honestly understand why we need this, but the
-    // documentation for java.io.StreamTokenizer.pushBack() is pretty
-    // vague about its semantics, and it seems to me that they could
-    // be summed up as "it doesn't work".  This version has the very
-    // clear semantics "pretend I didn't call nextToken() just then".
-    //
-    private void halfDecentPushBack() {
-       saved_ttype = ttype;
-       saved_nval = nval;
-       saved_sval = sval;
-    }
-
-    @Override
-    public int nextToken() throws java.io.IOException {
-       if (saved_ttype != TT_UNDEFINED) {
-           ttype = saved_ttype;
-           nval = saved_nval;
-           sval = saved_sval;
-           saved_ttype = TT_UNDEFINED;
-           debug("using saved ttype=" + ttype + ", " +
-                 "nval=" + nval + ", sval='" + sval + "'");
-           return ttype;
-       }
-
-       underlyingNextToken();
-       if (ttype == '<') {
-           debug("token starts with '<' ...");
-           underlyingNextToken();
-           if (ttype == '=') {
-               debug("token continues with '=' - it's '<='");
-               ttype = TT_LE;
-           } else if (ttype == '>') {
-               debug("token continues with '>' - it's '<>'");
-               ttype = TT_NE;
-           } else {
-               debug("next token is " + render() + " (pushed back)");
-               halfDecentPushBack();
-               ttype = '<';
-               debug("AFTER: ttype is now " + ttype + " - " + render());
-           }
-       } else if (ttype == '>') {
-           debug("token starts with '>' ...");
-           underlyingNextToken();
-           if (ttype == '=') {
-               debug("token continues with '=' - it's '>='");
-               ttype = TT_GE;
-           } else {
-               debug("next token is " + render() + " (pushed back)");
-               halfDecentPushBack();
-               ttype = '>';
-               debug("AFTER: ttype is now " + ttype + " - " + render());
-           }
-       } else if (ttype == '=') {
-           debug("token starts with '=' ...");
-           underlyingNextToken();
-           if (ttype == '=') {
-               debug("token continues with '=' - it's '=='");
-               ttype = TT_EQEQ;
-           } else {
-               debug("next token is " + render() + " (pushed back)");
-               halfDecentPushBack();
-               ttype = '=';
-               debug("AFTER: ttype is now " + ttype + " - " + render());
-           }
-       }
-
-       debug("done nextToken(): ttype=" + ttype + ", " +
-             "nval=" + nval + ", " + "sval='" + sval + "'" +
-             " (" + render() + ")");
-
-       return ttype;
-    }
-
-    // It's important to do keyword recognition here at the lowest
-    // level, otherwise when one of these words follows "<" or ">"
-    // (which can be the beginning of multi-character tokens) it gets
-    // pushed back as a string, and its keywordiness is not
-    // recognised.
-    //
-    public int underlyingNextToken() throws java.io.IOException {
-       super.nextToken();
-       if (ttype == TT_WORD)
-           for (int i = 0; i < keywords.length; i++)
-               if (sval.equalsIgnoreCase(keywords[i].keyword))
-                   ttype = keywords[i].token;
-
-       return ttype;
-    }
-
-    // Simpler interface for the usual case: current token with quoting
-    String render() {
-       return render(ttype, true);
-    }
-
-    String render(int token, boolean quoteChars) {
-       if (token == TT_EOF) {
-           return "EOF";
-       } else if (token == TT_NUMBER) {
-           if ((double) nval == (int) nval) {
-               return new Integer((int) nval).toString();
-           } else {
-               return new Double((double) nval).toString();
-           }
-       } else if (token == TT_WORD) {
-           return "word: " + sval;
-       } else if (token == '"') {
-           return "string: \"" + sval + "\"";
-       } else if (token == TT_LE) {
-           return "<=";
-       } else if (token == TT_GE) {
-           return ">=";
-       } else if (token == TT_NE) {
-           return "<>";
-       } else if (token == TT_EQEQ) {
-           return "==";
-       }
-
-       // Check whether its associated with one of the keywords
-       for (int i = 0; i < keywords.length; i++)
-           if (token == keywords[i].token)
-               return keywords[i].keyword;
-
-       // Otherwise it must be a single character, such as '(' or '/'.
-       String res = String.valueOf((char) token);
-       if (quoteChars) res = "'" + res + "'";
-        return res;
-    }
-
-    public static void main(String[] args) throws Exception {
-       if (args.length > 1) {
-           System.err.println("Usage: CQLLexer [<CQL-query>]");
-           System.err.println("If unspecified, query is read from stdin");
-           System.exit(1);
-       }
-
-       String cql;
-       if (args.length == 1) {
-           cql = args[0];
-       } else {
-           byte[] bytes = new byte[10000];
-           try {
-               // Read in the whole of standard input in one go
-               int nbytes = System.in.read(bytes);
-           } catch (java.io.IOException ex) {
-               System.err.println("Can't read query: " + ex.getMessage());
-               System.exit(2);
-           }
-           cql = new String(bytes);
-       }

 
 

-       CQLLexer lexer = new CQLLexer(cql, true);
-       int token;
-       while ((token = lexer.nextToken()) != TT_EOF) {
-           // Nothing to do: debug() statements render tokens for us
-       }
-    }
+/**
+ *
+ * @author jakub
+ */
+public interface CQLLexer {
+  
+  public static final int TT_EOF  = -1;
+  public static final int TT_WORD = -3;
+  public static final int TT_NOTHING = -4;
+  
+  public final static int TT_LE     = 1000;    // The "<=" relation
+  public final static int TT_GE     = 1001;    // The ">=" relation
+  public final static int TT_NE     = 1002;    // The "<>" relation
+  public final static int TT_EQEQ   = 1003;    // The "==" relation
+  public final static int TT_AND    = 1004;    // The "and" boolean
+  public final static int TT_OR     = 1005;    // The "or" boolean
+  public final static int TT_NOT    = 1006;    // The "not" boolean
+  public final static int TT_PROX   = 1007;    // The "prox" boolean
+  public final static int TT_SORTBY = 1008;    // The "sortby" operator
+  
+  public void move();
+  
+  public String value();
+  
+  public int what();
+  
+  public String render();
+  
+  public String render(int what, boolean quote);
+  
+  public int pos();
+  

 }
 }

diff --git a/src/main/java/org/z3950/zing/cql/CQLLexerSimple.java b/src/main/java/org/z3950/zing/cql/CQLLexerSimple.java
new file mode 100644 (file)
index 0000000..80931cc
--- /dev/null
+++ b/src/main/java/org/z3950/zing/cql/CQLLexerSimple.java
@@ -0,0 +1,173 @@
+/*
+ * Copyright (c) 1995-2014, Index Datassss
+ * All rights reserved.
+ * See the file LICENSE for details.
+ */
+package org.z3950.zing.cql;
+
+/**
+ *
+ * @author jakub
+ */
+public class CQLLexerSimple implements CQLLexer {
+  private String qs;
+  private int qi;
+  private int ql;
+  private int what = TT_NOTHING;
+  private String val;
+  private String lval;
+  private StringBuilder buf = new StringBuilder();
+
+  public CQLLexerSimple(String cql, boolean debug) {
+    qs = cql;
+    ql = cql.length();
+  }
+
+  @Override
+  public void move() {
+    //eat whitespace
+    while (qi < ql && strchr(" \t\r\n", qs.charAt(qi)))
+      qi++;
+    //eof
+    if (qi == ql) {
+      what = TT_EOF;
+      return;
+    }
+    //current char
+    char c = qs.charAt(qi);
+    //separators
+    if (strchr("()/", c)) {
+      what = c;
+      qi++;
+    //comparitor
+    } else if (strchr("<>=", c)) {
+      what = c;
+      qi++;
+      //two-char comparitor
+      if (qi < ql) {
+        char d = qs.charAt(qi);
+        String comp = String.valueOf((char) c) + String.valueOf((char) d);
+        if (comp.equals("==")) {
+          what = TT_EQEQ;
+          qi++;
+        }
+        else if (comp.equals("<=")) {
+          what = TT_LE;
+          qi++;
+        }
+        else if (comp.equals(">=")) {
+          what = TT_GE;
+          qi++;
+        }
+        else if (comp.equals("<>")) {
+          what = TT_NE;
+          qi++;
+        }
+      }
+    //quoted string
+    } else if (strchr("\"", c)) { //no single-quotes
+      what = '"';
+      //remember quote char
+      char mark = c;
+      qi++;
+      boolean escaped = false;
+      buf.setLength(0); //reset buffer
+      while (qi < ql) {
+        if (!escaped && qs.charAt(qi) == mark) //terminator
+          break;
+        if (escaped && strchr("*?^\\", qs.charAt(qi))) //no escaping for d-quote
+          buf.append("\\");
+        if (!escaped && qs.charAt(qi) == '\\') { //escape-char
+          escaped = true;
+          qi++;
+          continue;
+        }
+        escaped = false; //reset escape
+        buf.append(qs.charAt(qi));
+        qi++;
+      }
+      val = buf.toString();
+      lval = val.toLowerCase();
+      if (qi < ql)
+        qi++;
+      else //unterminated
+        what = TT_EOF; //notify error
+      //unquoted string
+    } else {
+      what = TT_WORD;
+      buf.setLength(0); //reset buffer
+      while (qi < ql
+        && !strchr("()/<>= \t\r\n", qs.charAt(qi))) {
+        buf.append(qs.charAt(qi));
+        qi++;
+      }
+      val = buf.toString();
+      lval = val.toLowerCase();
+      if (lval.equals("or")) what = TT_OR;
+      else if (lval.equals("and")) what = TT_AND;
+      else if (lval.equals("not")) what = TT_NOT;
+      else if (lval.equals("prox")) what = TT_PROX;
+      else if (lval.equals("sortby")) what = TT_SORTBY;
+    }
+  }
+
+  private boolean strchr(String s, char ch) {
+    return s.indexOf(ch) >= 0;
+  }
+
+  @Override
+  public String value() {
+    return val;
+  }
+
+  @Override
+  public int what() {
+    return what;
+  }
+
+  @Override
+  public String render() {
+    return render(what, true);
+  }
+
+  @Override
+  public String render(int token, boolean quoteChars) {
+    switch (token) {
+      case TT_EOF:
+        return "EOF";
+      case TT_WORD:
+        return "word: " + val;
+      case '"':
+        return "string: \"" + val + "\"";
+      case TT_LE:
+        return "<=";
+      case TT_GE:
+        return ">=";
+      case TT_NE:
+        return "<>";
+      case TT_EQEQ:
+        return "==";
+      case TT_AND:
+        return "and";
+      case TT_NOT:
+        return "not";
+      case TT_OR:
+        return "or";
+      case TT_PROX:
+        return "prox";
+      case TT_SORTBY:
+        return "sortby";
+      default:
+        //a single character, such as '(' or '/' or relation
+        String res = String.valueOf((char) token);
+        if (quoteChars)
+          res = "'" + res + "'";
+        return res;
+    }
+  }
+
+  @Override
+  public int pos() {
+    return qi;
+  }
+}

diff --git a/src/main/java/org/z3950/zing/cql/CQLParser.java b/src/main/java/org/z3950/zing/cql/CQLParser.java
index 72a39a3..c373f09 100644 (file)
--- a/src/main/java/org/z3950/zing/cql/CQLParser.java
+++ b/src/main/java/org/z3950/zing/cql/CQLParser.java
@@ -21,7 +21,6 @@ import java.util.Set;
  */
 public class CQLParser {
     private CQLLexer lexer;
  */
 public class CQLParser {
     private CQLLexer lexer;

-    private PositionAwareReader par; //active reader with position

     private final int compat;  // When false, implement CQL 1.2
     private final Set<String> customRelations = new HashSet<String>();
     
     private final int compat;  // When false, implement CQL 1.2
     private final Set<String> customRelations = new HashSet<String>();
     

@@ -94,26 +93,6 @@ public class CQLParser {
     public boolean unregisterCustomRelation(String relation) {
       return customRelations.remove(relation);
     }
     public boolean unregisterCustomRelation(String relation) {
       return customRelations.remove(relation);
     }

-    
-    /**
-     * Compiles a CQL query.
-     * <P>
-     * The resulting parse tree may be further processed by hand (see
-     * the individual node-types' documentation for details on the
-     * data structure) or, more often, simply rendered out in the
-     * desired form using one of the back-ends.  <TT>toCQL()</TT>
-     * returns a decompiled CQL query equivalent to the one that was
-     * compiled in the first place; <TT>toXCQL()</TT> returns an
-     * XML snippet representing the query; and <TT>toPQF()</TT>
-     * returns the query rendered in Index Data's Prefix Query
-     * Format.
-     *
-     * @param cql      The query
-     * @return         A CQLNode object which is the root of a parse
-     * tree representing the query.  */
-    public CQLNode parse(String cql) throws CQLParseException, IOException {
-        return parse(new StringReader(cql));
-    }

 
     /**
      * Compiles a CQL query.
 
     /**
      * Compiles a CQL query.

@@ -131,18 +110,17 @@ public class CQLParser {
      * @param cql      The query
      * @return         A CQLNode object which is the root of a parse
      * tree representing the query.  */
      * @param cql      The query
      * @return         A CQLNode object which is the root of a parse
      * tree representing the query.  */

-    public CQLNode parse(Reader cql)
+    public CQLNode parse(String cql)

        throws CQLParseException, IOException {
        throws CQLParseException, IOException {

-        par = new PositionAwareReader(cql);
-       lexer = new CQLLexer(par, LEXDEBUG);
+       lexer = new CQLLexerSimple(cql, LEXDEBUG);

 
 

-       lexer.nextToken();
+       lexer.move();

        debug("about to parseQuery()");
        CQLNode root = parseTopLevelPrefixes("cql.serverChoice",
                new CQLRelation(compat == V1POINT2 ? "=" : "scr"));
        debug("about to parseQuery()");
        CQLNode root = parseTopLevelPrefixes("cql.serverChoice",
                new CQLRelation(compat == V1POINT2 ? "=" : "scr"));

-       if (lexer.ttype != CQLLexer.TT_EOF)
+       if (lexer.what() != CQLLexer.TT_EOF)

            throw new CQLParseException("junk after end: " + lexer.render(), 
            throw new CQLParseException("junk after end: " + lexer.render(), 

-              par.getPosition());
+              lexer.pos());

 
        return root;
     }
 
        return root;
     }

@@ -151,25 +129,25 @@ public class CQLParser {
        throws CQLParseException, IOException {
        debug("top-level prefix mapping");
 
        throws CQLParseException, IOException {
        debug("top-level prefix mapping");
 

-       if (lexer.ttype == '>') {
+       if (lexer.what() == '>') {

            return parsePrefix(index, relation, true);
        }
 
        CQLNode node = parseQuery(index, relation);
        if ((compat == V1POINT2 || compat == V1POINT1SORT) &&
            return parsePrefix(index, relation, true);
        }
 
        CQLNode node = parseQuery(index, relation);
        if ((compat == V1POINT2 || compat == V1POINT1SORT) &&

-           lexer.ttype == CQLLexer.TT_SORTBY) {
-           match(lexer.ttype);
+           lexer.what() == CQLLexer.TT_SORTBY) {
+           match(lexer.what());

            debug("sortspec");
 
            CQLSortNode sortnode = new CQLSortNode(node);
            debug("sortspec");
 
            CQLSortNode sortnode = new CQLSortNode(node);

-           while (lexer.ttype != CQLLexer.TT_EOF) {
+           while (lexer.what() != CQLLexer.TT_EOF) {

                String sortindex = matchSymbol("sort index");
                ModifierSet ms = gatherModifiers(sortindex);
                sortnode.addSortIndex(ms);
            }
 
            if (sortnode.keys.size() == 0) {
                String sortindex = matchSymbol("sort index");
                ModifierSet ms = gatherModifiers(sortindex);
                sortnode.addSortIndex(ms);
            }
 
            if (sortnode.keys.size() == 0) {

-               throw new CQLParseException("no sort keys", par.getPosition());
+               throw new CQLParseException("no sort keys", lexer.pos());

            }
 
            node = sortnode;
            }
 
            node = sortnode;

@@ -183,15 +161,15 @@ public class CQLParser {
        debug("in parseQuery()");
 
        CQLNode term = parseTerm(index, relation);
        debug("in parseQuery()");
 
        CQLNode term = parseTerm(index, relation);

-       while (lexer.ttype != CQLLexer.TT_EOF &&
-              lexer.ttype != ')' &&
-              lexer.ttype != CQLLexer.TT_SORTBY) {
-           if (lexer.ttype == CQLLexer.TT_AND ||
-               lexer.ttype == CQLLexer.TT_OR ||
-               lexer.ttype == CQLLexer.TT_NOT ||
-               lexer.ttype == CQLLexer.TT_PROX) {
-               int type = lexer.ttype;
-               String val = lexer.sval;
+       while (lexer.what() != CQLLexer.TT_EOF &&
+              lexer.what() != ')' &&
+              lexer.what() != CQLLexer.TT_SORTBY) {
+           if (lexer.what() == CQLLexer.TT_AND ||
+               lexer.what() == CQLLexer.TT_OR ||
+               lexer.what() == CQLLexer.TT_NOT ||
+               lexer.what() == CQLLexer.TT_PROX) {
+               int type = lexer.what();
+               String val = lexer.value();

                match(type);
                ModifierSet ms = gatherModifiers(val);
                CQLNode term2 = parseTerm(index, relation);
                match(type);
                ModifierSet ms = gatherModifiers(val);
                CQLNode term2 = parseTerm(index, relation);

@@ -201,7 +179,7 @@ public class CQLParser {
                                                 new CQLProxNode(term, term2, ms));
            } else {
                throw new CQLParseException("expected boolean, got " +
                                                 new CQLProxNode(term, term2, ms));
            } else {
                throw new CQLParseException("expected boolean, got " +

-                                           lexer.render(), par.getPosition());
+                                           lexer.render(), lexer.pos());

            }
        }
 
            }
        }
 

@@ -214,21 +192,21 @@ public class CQLParser {
        debug("in gatherModifiers()");
 
        ModifierSet ms = new ModifierSet(base);
        debug("in gatherModifiers()");
 
        ModifierSet ms = new ModifierSet(base);

-       while (lexer.ttype == '/') {
+       while (lexer.what() == '/') {

            match('/');
            match('/');

-           if (lexer.ttype != CQLLexer.TT_WORD)
+           if (lexer.what() != CQLLexer.TT_WORD)

                throw new CQLParseException("expected modifier, "
                                            + "got " + lexer.render(), 
                throw new CQLParseException("expected modifier, "
                                            + "got " + lexer.render(), 

-                  par.getPosition());
-           String type = lexer.sval.toLowerCase();
-           match(lexer.ttype);
+                  lexer.pos());
+           String type = lexer.value().toLowerCase();
+           match(lexer.what());

            if (!isSymbolicRelation()) {
                // It's a simple modifier consisting of type only
                ms.addModifier(type);
            } else {
                // It's a complex modifier of the form type=value
            if (!isSymbolicRelation()) {
                // It's a simple modifier consisting of type only
                ms.addModifier(type);
            } else {
                // It's a complex modifier of the form type=value

-               String comparision = lexer.render(lexer.ttype, false);
-               match(lexer.ttype);
+               String comparision = lexer.render(lexer.what(), false);
+               match(lexer.what());

                String value = matchSymbol("modifier value");
                ms.addModifier(type, comparision, value);
            }
                String value = matchSymbol("modifier value");
                ms.addModifier(type, comparision, value);
            }

@@ -243,20 +221,20 @@ public class CQLParser {
 
        String word;
        while (true) {
 
        String word;
        while (true) {

-           if (lexer.ttype == '(') {
+           if (lexer.what() == '(') {

                debug("parenthesised term");
                match('(');
                CQLNode expr = parseQuery(index, relation);
                match(')');
                return expr;
                debug("parenthesised term");
                match('(');
                CQLNode expr = parseQuery(index, relation);
                match(')');
                return expr;

-           } else if (lexer.ttype == '>') {
+           } else if (lexer.what() == '>') {

                return parsePrefix(index, relation, false);
            }
 
            debug("non-parenthesised term");
            word = matchSymbol("index or term");
                return parsePrefix(index, relation, false);
            }
 
            debug("non-parenthesised term");
            word = matchSymbol("index or term");

-            while (lexer.ttype == CQLLexer.TT_WORD && !isRelation()) {
-              word = word + " " + lexer.sval;
+            while (lexer.what() == CQLLexer.TT_WORD && !isRelation()) {
+              word = word + " " + lexer.value();

               match(CQLLexer.TT_WORD);
             }
 
               match(CQLLexer.TT_WORD);
             }
 

@@ -264,10 +242,10 @@ public class CQLParser {
                break;
 
            index = word;
                break;
 
            index = word;

-           String relstr = (lexer.ttype == CQLLexer.TT_WORD ?
-                            lexer.sval : lexer.render(lexer.ttype, false));
+           String relstr = (lexer.what() == CQLLexer.TT_WORD ?
+                            lexer.value() : lexer.render(lexer.what(), false));

            relation = new CQLRelation(relstr);
            relation = new CQLRelation(relstr);

-           match(lexer.ttype);
+           match(lexer.what());

            ModifierSet ms = gatherModifiers(relstr);
            relation.ms = ms;
            debug("index='" + index + ", " +
            ModifierSet ms = gatherModifiers(relstr);
            relation.ms = ms;
            debug("index='" + index + ", " +

@@ -287,7 +265,7 @@ public class CQLParser {
        match('>');
        String name = null;
        String identifier = matchSymbol("prefix-name");
        match('>');
        String name = null;
        String identifier = matchSymbol("prefix-name");

-       if (lexer.ttype == '=') {
+       if (lexer.what() == '=') {

            match('=');
            name = identifier;
            identifier = matchSymbol("prefix-identifer");
            match('=');
            name = identifier;
            identifier = matchSymbol("prefix-identifer");

@@ -300,75 +278,71 @@ public class CQLParser {
     }
 
     private boolean isRelation() {
     }
 
     private boolean isRelation() {

-       debug("isRelation: checking ttype=" + lexer.ttype +
+       debug("isRelation: checking what()=" + lexer.what() +

              " (" + lexer.render() + ")");
              " (" + lexer.render() + ")");

-        if (lexer.ttype == CQLLexer.TT_WORD &&
-            (lexer.sval.indexOf('.') >= 0 ||
-             lexer.sval.equals("any") ||
-             lexer.sval.equals("all") ||
-             lexer.sval.equals("within") ||
-             lexer.sval.equals("encloses") ||
-             (lexer.sval.equals("exact") && compat != V1POINT2) ||
-             (lexer.sval.equals("scr") && compat != V1POINT2) ||
-             (lexer.sval.equals("adj") && compat == V1POINT2) ||
-             customRelations.contains(lexer.sval)))
+        if (lexer.what() == CQLLexer.TT_WORD &&
+            (lexer.value().indexOf('.') >= 0 ||
+             lexer.value().equals("any") ||
+             lexer.value().equals("all") ||
+             lexer.value().equals("within") ||
+             lexer.value().equals("encloses") ||
+             (lexer.value().equals("exact") && compat != V1POINT2) ||
+             (lexer.value().equals("scr") && compat != V1POINT2) ||
+             (lexer.value().equals("adj") && compat == V1POINT2) ||
+             customRelations.contains(lexer.value())))

           return true;
 
         return isSymbolicRelation();
     }
 
     private boolean isSymbolicRelation() {
           return true;
 
         return isSymbolicRelation();
     }
 
     private boolean isSymbolicRelation() {

-       debug("isSymbolicRelation: checking ttype=" + lexer.ttype +
+       debug("isSymbolicRelation: checking what()=" + lexer.what() +

              " (" + lexer.render() + ")");
              " (" + lexer.render() + ")");

-       return (lexer.ttype == '<' ||
-               lexer.ttype == '>' ||
-               lexer.ttype == '=' ||
-               lexer.ttype == CQLLexer.TT_LE ||
-               lexer.ttype == CQLLexer.TT_GE ||
-               lexer.ttype == CQLLexer.TT_NE ||
-               lexer.ttype == CQLLexer.TT_EQEQ);
+       return (lexer.what() == '<' ||
+               lexer.what() == '>' ||
+               lexer.what() == '=' ||
+               lexer.what() == CQLLexer.TT_LE ||
+               lexer.what() == CQLLexer.TT_GE ||
+               lexer.what() == CQLLexer.TT_NE ||
+               lexer.what() == CQLLexer.TT_EQEQ);

     }
 
     private void match(int token)
        throws CQLParseException, IOException {
        debug("in match(" + lexer.render(token, true) + ")");
     }
 
     private void match(int token)
        throws CQLParseException, IOException {
        debug("in match(" + lexer.render(token, true) + ")");

-       if (lexer.ttype != token)
+       if (lexer.what() != token)

            throw new CQLParseException("expected " +
                                        lexer.render(token, true) +
                                        ", " + "got " + lexer.render(), 
            throw new CQLParseException("expected " +
                                        lexer.render(token, true) +
                                        ", " + "got " + lexer.render(), 

-              par.getPosition());
-       int tmp = lexer.nextToken();
-       debug("match() got token=" + lexer.ttype + ", " +
-             "nval=" + lexer.nval + ", sval='" + lexer.sval + "'" +
-             " (tmp=" + tmp + ")");
+              lexer.pos());
+       lexer.move();
+       debug("match() got token=" + lexer.what() + ", value()='" + lexer.value() + "'");

     }
 
     private String matchSymbol(String expected)
        throws CQLParseException, IOException {
 
        debug("in matchSymbol()");
     }
 
     private String matchSymbol(String expected)
        throws CQLParseException, IOException {
 
        debug("in matchSymbol()");

-       if (lexer.ttype == CQLLexer.TT_WORD ||
-           lexer.ttype == CQLLexer.TT_NUMBER ||
-           lexer.ttype == '"' ||
+       if (lexer.what() == CQLLexer.TT_WORD ||
+           lexer.what() == '"' ||

            // The following is a complete list of keywords.  Because
            // they're listed here, they can be used unquoted as
            // indexes, terms, prefix names and prefix identifiers.
            // ### Instead, we should ask the lexer whether what we
            // have is a keyword, and let the knowledge reside there.
             (allowKeywordTerms &&
            // The following is a complete list of keywords.  Because
            // they're listed here, they can be used unquoted as
            // indexes, terms, prefix names and prefix identifiers.
            // ### Instead, we should ask the lexer whether what we
            // have is a keyword, and let the knowledge reside there.
             (allowKeywordTerms &&

-           lexer.ttype == CQLLexer.TT_AND ||
-           lexer.ttype == CQLLexer.TT_OR ||
-           lexer.ttype == CQLLexer.TT_NOT ||
-           lexer.ttype == CQLLexer.TT_PROX ||
-           lexer.ttype == CQLLexer.TT_SORTBY)) {
-           String symbol = (lexer.ttype == CQLLexer.TT_NUMBER) ?
-               lexer.render() : lexer.sval;
-           match(lexer.ttype);
+           lexer.what() == CQLLexer.TT_AND ||
+           lexer.what() == CQLLexer.TT_OR ||
+           lexer.what() == CQLLexer.TT_NOT ||
+           lexer.what() == CQLLexer.TT_PROX ||
+           lexer.what() == CQLLexer.TT_SORTBY)) {
+           String symbol = lexer.value();
+           match(lexer.what());

            return symbol;
        }
 
        throw new CQLParseException("expected " + expected + ", " +
            return symbol;
        }
 
        throw new CQLParseException("expected " + expected + ", " +

-                                   "got " + lexer.render(), par.getPosition());
+                                   "got " + lexer.render(), lexer.pos());

     }
 
 
     }
 
 

diff --git a/src/main/java/org/z3950/zing/cql/PositionAwareReader.java b/src/main/java/org/z3950/zing/cql/PositionAwareReader.java
deleted file mode 100644 (file)
index b3294c0..0000000
--- a/src/main/java/org/z3950/zing/cql/PositionAwareReader.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Copyright (c) 1995-2012, Index Data
- * All rights reserved.
- * See the file LICENSE for details.
- */
-package org.z3950.zing.cql;
-
-import java.io.IOException;
-import java.io.Reader;
-import java.nio.CharBuffer;
-
-/**
- * Reader proxy to count how many characters has been read so far.
- * @author jakub
- */
-public class PositionAwareReader extends Reader {
-  protected Reader reader;
-  protected int pos = -1;
-  
-  public PositionAwareReader(Reader reader) {
-    this.reader = reader;
-  }
-
-  /*
-   * Position of the last read character or -1 if either reading from an empty 
-   * stream or no 'read' has been invoked for this reader.
-   */
-  public int getPosition() {
-    return pos;
-  }
-
-  @Override
-  public void mark(int readAheadLimit) throws IOException {
-    reader.mark(readAheadLimit);
-  }
-
-  @Override
-  public boolean markSupported() {
-    return reader.markSupported();
-  }
-
-  @Override
-  public int read() throws IOException {
-    int c = reader.read();
-    if (c != -1) pos++;
-    return c;
-  }
-
-  @Override
-  public int read(char[] cbuf) throws IOException {
-    int c = reader.read(cbuf);
-    if (c != -1) pos+=c;
-    return c;
-  }
-
-  @Override
-  public int read(CharBuffer target) throws IOException {
-    int c = reader.read(target);
-    if (c != -1) pos+=c;
-    return c;
-  }
-  
-  @Override
-  public int read(char[] cbuf, int off, int len) throws IOException {
-    int c = reader.read(cbuf, off, len);
-    if (c != -1) pos+=c;
-    return c;
-  }
-
-  @Override
-  public boolean ready() throws IOException {
-    return reader.ready();
-  }
-
-  @Override
-  public long skip(long n) throws IOException {
-    return reader.skip(n);
-  }
-  
-  @Override
-  public void close() throws IOException {
-    reader.close();
-  }
-
-  @Override
-  public void reset() throws IOException {
-    reader.reset();
-  }
-  
-  //override object methods, to be on the safe-side
-
-  @Override
-  public boolean equals(Object obj) {
-    return reader.equals(obj);
-  }
-
-  @Override
-  public String toString() {
-    return reader.toString();
-  }
-
-  @Override
-  public int hashCode() {
-    return reader.hashCode();
-  }
-  
-}

diff --git a/src/test/resources/regression/12/01.cql b/src/test/resources/regression/12/01.cql
new file mode 100644 (file)
index 0000000..d8ec47d
--- /dev/null
+++ b/src/test/resources/regression/12/01.cql
@@ -0,0 +1 @@
+"te\rm\*\?\^"

diff --git a/src/test/resources/regression/12/01.xcql b/src/test/resources/regression/12/01.xcql
new file mode 100644 (file)
index 0000000..c9fc4b6
--- /dev/null
+++ b/src/test/resources/regression/12/01.xcql
@@ -0,0 +1,7 @@
+<searchClause>
+  <index>cql.serverChoice</index>
+  <relation>
+    <value>=</value>
+  </relation>
+  <term>term\*\?\^</term>
+</searchClause>

diff --git a/src/test/resources/regression/12/name b/src/test/resources/regression/12/name
new file mode 100644 (file)
index 0000000..09d34d3
--- /dev/null
+++ b/src/test/resources/regression/12/name
@@ -0,0 +1 @@
+wildcards