From 277aa21b21ccafbb392f58ba30390040bdb47c94 Mon Sep 17 00:00:00 2001 From: Jakub Skoczen Date: Fri, 14 Mar 2014 17:19:54 +0100 Subject: [PATCH] Rewrite lexer to comply with CQL spec Add test more tests --- src/main/java/org/z3950/zing/cql/CQLLexer.java | 281 +++----------------- .../java/org/z3950/zing/cql/CQLLexerSimple.java | 173 ++++++++++++ src/main/java/org/z3950/zing/cql/CQLParser.java | 162 +++++------ .../org/z3950/zing/cql/PositionAwareReader.java | 107 -------- src/test/resources/regression/12/01.cql | 1 + src/test/resources/regression/12/01.xcql | 7 + src/test/resources/regression/12/name | 1 + 7 files changed, 287 insertions(+), 445 deletions(-) create mode 100644 src/main/java/org/z3950/zing/cql/CQLLexerSimple.java delete mode 100644 src/main/java/org/z3950/zing/cql/PositionAwareReader.java create mode 100644 src/test/resources/regression/12/01.cql create mode 100644 src/test/resources/regression/12/01.xcql create mode 100644 src/test/resources/regression/12/name diff --git a/src/main/java/org/z3950/zing/cql/CQLLexer.java b/src/main/java/org/z3950/zing/cql/CQLLexer.java index 293b45a..2247fd0 100644 --- a/src/main/java/org/z3950/zing/cql/CQLLexer.java +++ b/src/main/java/org/z3950/zing/cql/CQLLexer.java @@ -1,247 +1,40 @@ - +/* + * Copyright (c) 1995-2014, Index Datassss + * All rights reserved. + * See the file LICENSE for details. + */ package org.z3950.zing.cql; -import java.io.InputStream; -import java.io.Reader; -import java.io.StreamTokenizer; -import java.io.StringReader; - - -// This is a semi-trivial subclass for java.io.StreamTokenizer that: -// * Has a halfDecentPushBack() method that actually works -// * Includes a render() method -// * Knows about the multi-character tokens "<=", ">=" and "<>" -// * Recognises a set of keywords as tokens in their own right -// * Includes some primitive debugging-output facilities -// It's used only by CQLParser. -// -class CQLLexer extends StreamTokenizer { - // New publicly visible token-types - public final static int TT_LE = 1000; // The "<=" relation - public final static int TT_GE = 1001; // The ">=" relation - public final static int TT_NE = 1002; // The "<>" relation - public final static int TT_EQEQ = 1003; // The "==" relation - public final static int TT_AND = 1004; // The "and" boolean - public final static int TT_OR = 1005; // The "or" boolean - public final static int TT_NOT = 1006; // The "not" boolean - public final static int TT_PROX = 1007; // The "prox" boolean - public final static int TT_SORTBY = 1008; // The "sortby" operator - - // Support for keywords. It would be nice to compile this linear - // list into a Hashtable, but it's hard to store ints as hash - // values, and next to impossible to use them as hash keys. So - // we'll just scan the (very short) list every time we need to do - // a lookup. - private class Keyword { - int token; - String keyword; - Keyword(int token, String keyword) { - this.token = token; - this.keyword = keyword; - } - } - // This should logically be static, but Java won't allow it :-P - private Keyword[] keywords = { - new Keyword(TT_AND, "and"), - new Keyword(TT_OR, "or"), - new Keyword(TT_NOT, "not"), - new Keyword(TT_PROX, "prox"), - new Keyword(TT_SORTBY, "sortby"), - }; - - // For halfDecentPushBack() and the code at the top of nextToken() - private static int TT_UNDEFINED = -1000; - private int saved_ttype = TT_UNDEFINED; - private double saved_nval; - private String saved_sval; - - // Controls debugging output - private static boolean DEBUG; - - CQLLexer(String cql, boolean lexdebug) { - this(new StringReader(cql), lexdebug); - } - - CQLLexer(Reader cql, boolean lexdebug) { - super(cql); - wordChars('!', '@'); // ASCII-dependency! - wordChars('[', '`'); // ASCII-dependency! - quoteChar('"'); - ordinaryChar('='); - ordinaryChar('<'); - ordinaryChar('>'); - ordinaryChar('/'); - ordinaryChar('('); - ordinaryChar(')'); - ordinaryChar('.'); - wordChars('.', '.'); - wordChars('\'', '\''); // prevent this from introducing strings - //parseNumbers(); - ordinaryChar('-'); - wordChars('-', '-'); - ordinaryChars('0', '9'); - wordChars('0', '9'); - DEBUG = lexdebug; - } - - private static void debug(String str) { - if (DEBUG) - System.err.println("LEXDEBUG: " + str); - } - - // I don't honestly understand why we need this, but the - // documentation for java.io.StreamTokenizer.pushBack() is pretty - // vague about its semantics, and it seems to me that they could - // be summed up as "it doesn't work". This version has the very - // clear semantics "pretend I didn't call nextToken() just then". - // - private void halfDecentPushBack() { - saved_ttype = ttype; - saved_nval = nval; - saved_sval = sval; - } - - @Override - public int nextToken() throws java.io.IOException { - if (saved_ttype != TT_UNDEFINED) { - ttype = saved_ttype; - nval = saved_nval; - sval = saved_sval; - saved_ttype = TT_UNDEFINED; - debug("using saved ttype=" + ttype + ", " + - "nval=" + nval + ", sval='" + sval + "'"); - return ttype; - } - - underlyingNextToken(); - if (ttype == '<') { - debug("token starts with '<' ..."); - underlyingNextToken(); - if (ttype == '=') { - debug("token continues with '=' - it's '<='"); - ttype = TT_LE; - } else if (ttype == '>') { - debug("token continues with '>' - it's '<>'"); - ttype = TT_NE; - } else { - debug("next token is " + render() + " (pushed back)"); - halfDecentPushBack(); - ttype = '<'; - debug("AFTER: ttype is now " + ttype + " - " + render()); - } - } else if (ttype == '>') { - debug("token starts with '>' ..."); - underlyingNextToken(); - if (ttype == '=') { - debug("token continues with '=' - it's '>='"); - ttype = TT_GE; - } else { - debug("next token is " + render() + " (pushed back)"); - halfDecentPushBack(); - ttype = '>'; - debug("AFTER: ttype is now " + ttype + " - " + render()); - } - } else if (ttype == '=') { - debug("token starts with '=' ..."); - underlyingNextToken(); - if (ttype == '=') { - debug("token continues with '=' - it's '=='"); - ttype = TT_EQEQ; - } else { - debug("next token is " + render() + " (pushed back)"); - halfDecentPushBack(); - ttype = '='; - debug("AFTER: ttype is now " + ttype + " - " + render()); - } - } - - debug("done nextToken(): ttype=" + ttype + ", " + - "nval=" + nval + ", " + "sval='" + sval + "'" + - " (" + render() + ")"); - - return ttype; - } - - // It's important to do keyword recognition here at the lowest - // level, otherwise when one of these words follows "<" or ">" - // (which can be the beginning of multi-character tokens) it gets - // pushed back as a string, and its keywordiness is not - // recognised. - // - public int underlyingNextToken() throws java.io.IOException { - super.nextToken(); - if (ttype == TT_WORD) - for (int i = 0; i < keywords.length; i++) - if (sval.equalsIgnoreCase(keywords[i].keyword)) - ttype = keywords[i].token; - - return ttype; - } - - // Simpler interface for the usual case: current token with quoting - String render() { - return render(ttype, true); - } - - String render(int token, boolean quoteChars) { - if (token == TT_EOF) { - return "EOF"; - } else if (token == TT_NUMBER) { - if ((double) nval == (int) nval) { - return new Integer((int) nval).toString(); - } else { - return new Double((double) nval).toString(); - } - } else if (token == TT_WORD) { - return "word: " + sval; - } else if (token == '"') { - return "string: \"" + sval + "\""; - } else if (token == TT_LE) { - return "<="; - } else if (token == TT_GE) { - return ">="; - } else if (token == TT_NE) { - return "<>"; - } else if (token == TT_EQEQ) { - return "=="; - } - - // Check whether its associated with one of the keywords - for (int i = 0; i < keywords.length; i++) - if (token == keywords[i].token) - return keywords[i].keyword; - - // Otherwise it must be a single character, such as '(' or '/'. - String res = String.valueOf((char) token); - if (quoteChars) res = "'" + res + "'"; - return res; - } - - public static void main(String[] args) throws Exception { - if (args.length > 1) { - System.err.println("Usage: CQLLexer []"); - System.err.println("If unspecified, query is read from stdin"); - System.exit(1); - } - - String cql; - if (args.length == 1) { - cql = args[0]; - } else { - byte[] bytes = new byte[10000]; - try { - // Read in the whole of standard input in one go - int nbytes = System.in.read(bytes); - } catch (java.io.IOException ex) { - System.err.println("Can't read query: " + ex.getMessage()); - System.exit(2); - } - cql = new String(bytes); - } - CQLLexer lexer = new CQLLexer(cql, true); - int token; - while ((token = lexer.nextToken()) != TT_EOF) { - // Nothing to do: debug() statements render tokens for us - } - } +/** + * + * @author jakub + */ +public interface CQLLexer { + + public static final int TT_EOF = -1; + public static final int TT_WORD = -3; + public static final int TT_NOTHING = -4; + + public final static int TT_LE = 1000; // The "<=" relation + public final static int TT_GE = 1001; // The ">=" relation + public final static int TT_NE = 1002; // The "<>" relation + public final static int TT_EQEQ = 1003; // The "==" relation + public final static int TT_AND = 1004; // The "and" boolean + public final static int TT_OR = 1005; // The "or" boolean + public final static int TT_NOT = 1006; // The "not" boolean + public final static int TT_PROX = 1007; // The "prox" boolean + public final static int TT_SORTBY = 1008; // The "sortby" operator + + public void move(); + + public String value(); + + public int what(); + + public String render(); + + public String render(int what, boolean quote); + + public int pos(); + } diff --git a/src/main/java/org/z3950/zing/cql/CQLLexerSimple.java b/src/main/java/org/z3950/zing/cql/CQLLexerSimple.java new file mode 100644 index 0000000..80931cc --- /dev/null +++ b/src/main/java/org/z3950/zing/cql/CQLLexerSimple.java @@ -0,0 +1,173 @@ +/* + * Copyright (c) 1995-2014, Index Datassss + * All rights reserved. + * See the file LICENSE for details. + */ +package org.z3950.zing.cql; + +/** + * + * @author jakub + */ +public class CQLLexerSimple implements CQLLexer { + private String qs; + private int qi; + private int ql; + private int what = TT_NOTHING; + private String val; + private String lval; + private StringBuilder buf = new StringBuilder(); + + public CQLLexerSimple(String cql, boolean debug) { + qs = cql; + ql = cql.length(); + } + + @Override + public void move() { + //eat whitespace + while (qi < ql && strchr(" \t\r\n", qs.charAt(qi))) + qi++; + //eof + if (qi == ql) { + what = TT_EOF; + return; + } + //current char + char c = qs.charAt(qi); + //separators + if (strchr("()/", c)) { + what = c; + qi++; + //comparitor + } else if (strchr("<>=", c)) { + what = c; + qi++; + //two-char comparitor + if (qi < ql) { + char d = qs.charAt(qi); + String comp = String.valueOf((char) c) + String.valueOf((char) d); + if (comp.equals("==")) { + what = TT_EQEQ; + qi++; + } + else if (comp.equals("<=")) { + what = TT_LE; + qi++; + } + else if (comp.equals(">=")) { + what = TT_GE; + qi++; + } + else if (comp.equals("<>")) { + what = TT_NE; + qi++; + } + } + //quoted string + } else if (strchr("\"", c)) { //no single-quotes + what = '"'; + //remember quote char + char mark = c; + qi++; + boolean escaped = false; + buf.setLength(0); //reset buffer + while (qi < ql) { + if (!escaped && qs.charAt(qi) == mark) //terminator + break; + if (escaped && strchr("*?^\\", qs.charAt(qi))) //no escaping for d-quote + buf.append("\\"); + if (!escaped && qs.charAt(qi) == '\\') { //escape-char + escaped = true; + qi++; + continue; + } + escaped = false; //reset escape + buf.append(qs.charAt(qi)); + qi++; + } + val = buf.toString(); + lval = val.toLowerCase(); + if (qi < ql) + qi++; + else //unterminated + what = TT_EOF; //notify error + //unquoted string + } else { + what = TT_WORD; + buf.setLength(0); //reset buffer + while (qi < ql + && !strchr("()/<>= \t\r\n", qs.charAt(qi))) { + buf.append(qs.charAt(qi)); + qi++; + } + val = buf.toString(); + lval = val.toLowerCase(); + if (lval.equals("or")) what = TT_OR; + else if (lval.equals("and")) what = TT_AND; + else if (lval.equals("not")) what = TT_NOT; + else if (lval.equals("prox")) what = TT_PROX; + else if (lval.equals("sortby")) what = TT_SORTBY; + } + } + + private boolean strchr(String s, char ch) { + return s.indexOf(ch) >= 0; + } + + @Override + public String value() { + return val; + } + + @Override + public int what() { + return what; + } + + @Override + public String render() { + return render(what, true); + } + + @Override + public String render(int token, boolean quoteChars) { + switch (token) { + case TT_EOF: + return "EOF"; + case TT_WORD: + return "word: " + val; + case '"': + return "string: \"" + val + "\""; + case TT_LE: + return "<="; + case TT_GE: + return ">="; + case TT_NE: + return "<>"; + case TT_EQEQ: + return "=="; + case TT_AND: + return "and"; + case TT_NOT: + return "not"; + case TT_OR: + return "or"; + case TT_PROX: + return "prox"; + case TT_SORTBY: + return "sortby"; + default: + //a single character, such as '(' or '/' or relation + String res = String.valueOf((char) token); + if (quoteChars) + res = "'" + res + "'"; + return res; + } + } + + @Override + public int pos() { + return qi; + } +} diff --git a/src/main/java/org/z3950/zing/cql/CQLParser.java b/src/main/java/org/z3950/zing/cql/CQLParser.java index 72a39a3..c373f09 100644 --- a/src/main/java/org/z3950/zing/cql/CQLParser.java +++ b/src/main/java/org/z3950/zing/cql/CQLParser.java @@ -21,7 +21,6 @@ import java.util.Set; */ public class CQLParser { private CQLLexer lexer; - private PositionAwareReader par; //active reader with position private final int compat; // When false, implement CQL 1.2 private final Set customRelations = new HashSet(); @@ -94,26 +93,6 @@ public class CQLParser { public boolean unregisterCustomRelation(String relation) { return customRelations.remove(relation); } - - /** - * Compiles a CQL query. - *

- * The resulting parse tree may be further processed by hand (see - * the individual node-types' documentation for details on the - * data structure) or, more often, simply rendered out in the - * desired form using one of the back-ends. toCQL() - * returns a decompiled CQL query equivalent to the one that was - * compiled in the first place; toXCQL() returns an - * XML snippet representing the query; and toPQF() - * returns the query rendered in Index Data's Prefix Query - * Format. - * - * @param cql The query - * @return A CQLNode object which is the root of a parse - * tree representing the query. */ - public CQLNode parse(String cql) throws CQLParseException, IOException { - return parse(new StringReader(cql)); - } /** * Compiles a CQL query. @@ -131,18 +110,17 @@ public class CQLParser { * @param cql The query * @return A CQLNode object which is the root of a parse * tree representing the query. */ - public CQLNode parse(Reader cql) + public CQLNode parse(String cql) throws CQLParseException, IOException { - par = new PositionAwareReader(cql); - lexer = new CQLLexer(par, LEXDEBUG); + lexer = new CQLLexerSimple(cql, LEXDEBUG); - lexer.nextToken(); + lexer.move(); debug("about to parseQuery()"); CQLNode root = parseTopLevelPrefixes("cql.serverChoice", new CQLRelation(compat == V1POINT2 ? "=" : "scr")); - if (lexer.ttype != CQLLexer.TT_EOF) + if (lexer.what() != CQLLexer.TT_EOF) throw new CQLParseException("junk after end: " + lexer.render(), - par.getPosition()); + lexer.pos()); return root; } @@ -151,25 +129,25 @@ public class CQLParser { throws CQLParseException, IOException { debug("top-level prefix mapping"); - if (lexer.ttype == '>') { + if (lexer.what() == '>') { return parsePrefix(index, relation, true); } CQLNode node = parseQuery(index, relation); if ((compat == V1POINT2 || compat == V1POINT1SORT) && - lexer.ttype == CQLLexer.TT_SORTBY) { - match(lexer.ttype); + lexer.what() == CQLLexer.TT_SORTBY) { + match(lexer.what()); debug("sortspec"); CQLSortNode sortnode = new CQLSortNode(node); - while (lexer.ttype != CQLLexer.TT_EOF) { + while (lexer.what() != CQLLexer.TT_EOF) { String sortindex = matchSymbol("sort index"); ModifierSet ms = gatherModifiers(sortindex); sortnode.addSortIndex(ms); } if (sortnode.keys.size() == 0) { - throw new CQLParseException("no sort keys", par.getPosition()); + throw new CQLParseException("no sort keys", lexer.pos()); } node = sortnode; @@ -183,15 +161,15 @@ public class CQLParser { debug("in parseQuery()"); CQLNode term = parseTerm(index, relation); - while (lexer.ttype != CQLLexer.TT_EOF && - lexer.ttype != ')' && - lexer.ttype != CQLLexer.TT_SORTBY) { - if (lexer.ttype == CQLLexer.TT_AND || - lexer.ttype == CQLLexer.TT_OR || - lexer.ttype == CQLLexer.TT_NOT || - lexer.ttype == CQLLexer.TT_PROX) { - int type = lexer.ttype; - String val = lexer.sval; + while (lexer.what() != CQLLexer.TT_EOF && + lexer.what() != ')' && + lexer.what() != CQLLexer.TT_SORTBY) { + if (lexer.what() == CQLLexer.TT_AND || + lexer.what() == CQLLexer.TT_OR || + lexer.what() == CQLLexer.TT_NOT || + lexer.what() == CQLLexer.TT_PROX) { + int type = lexer.what(); + String val = lexer.value(); match(type); ModifierSet ms = gatherModifiers(val); CQLNode term2 = parseTerm(index, relation); @@ -201,7 +179,7 @@ public class CQLParser { new CQLProxNode(term, term2, ms)); } else { throw new CQLParseException("expected boolean, got " + - lexer.render(), par.getPosition()); + lexer.render(), lexer.pos()); } } @@ -214,21 +192,21 @@ public class CQLParser { debug("in gatherModifiers()"); ModifierSet ms = new ModifierSet(base); - while (lexer.ttype == '/') { + while (lexer.what() == '/') { match('/'); - if (lexer.ttype != CQLLexer.TT_WORD) + if (lexer.what() != CQLLexer.TT_WORD) throw new CQLParseException("expected modifier, " + "got " + lexer.render(), - par.getPosition()); - String type = lexer.sval.toLowerCase(); - match(lexer.ttype); + lexer.pos()); + String type = lexer.value().toLowerCase(); + match(lexer.what()); if (!isSymbolicRelation()) { // It's a simple modifier consisting of type only ms.addModifier(type); } else { // It's a complex modifier of the form type=value - String comparision = lexer.render(lexer.ttype, false); - match(lexer.ttype); + String comparision = lexer.render(lexer.what(), false); + match(lexer.what()); String value = matchSymbol("modifier value"); ms.addModifier(type, comparision, value); } @@ -243,20 +221,20 @@ public class CQLParser { String word; while (true) { - if (lexer.ttype == '(') { + if (lexer.what() == '(') { debug("parenthesised term"); match('('); CQLNode expr = parseQuery(index, relation); match(')'); return expr; - } else if (lexer.ttype == '>') { + } else if (lexer.what() == '>') { return parsePrefix(index, relation, false); } debug("non-parenthesised term"); word = matchSymbol("index or term"); - while (lexer.ttype == CQLLexer.TT_WORD && !isRelation()) { - word = word + " " + lexer.sval; + while (lexer.what() == CQLLexer.TT_WORD && !isRelation()) { + word = word + " " + lexer.value(); match(CQLLexer.TT_WORD); } @@ -264,10 +242,10 @@ public class CQLParser { break; index = word; - String relstr = (lexer.ttype == CQLLexer.TT_WORD ? - lexer.sval : lexer.render(lexer.ttype, false)); + String relstr = (lexer.what() == CQLLexer.TT_WORD ? + lexer.value() : lexer.render(lexer.what(), false)); relation = new CQLRelation(relstr); - match(lexer.ttype); + match(lexer.what()); ModifierSet ms = gatherModifiers(relstr); relation.ms = ms; debug("index='" + index + ", " + @@ -287,7 +265,7 @@ public class CQLParser { match('>'); String name = null; String identifier = matchSymbol("prefix-name"); - if (lexer.ttype == '=') { + if (lexer.what() == '=') { match('='); name = identifier; identifier = matchSymbol("prefix-identifer"); @@ -300,75 +278,71 @@ public class CQLParser { } private boolean isRelation() { - debug("isRelation: checking ttype=" + lexer.ttype + + debug("isRelation: checking what()=" + lexer.what() + " (" + lexer.render() + ")"); - if (lexer.ttype == CQLLexer.TT_WORD && - (lexer.sval.indexOf('.') >= 0 || - lexer.sval.equals("any") || - lexer.sval.equals("all") || - lexer.sval.equals("within") || - lexer.sval.equals("encloses") || - (lexer.sval.equals("exact") && compat != V1POINT2) || - (lexer.sval.equals("scr") && compat != V1POINT2) || - (lexer.sval.equals("adj") && compat == V1POINT2) || - customRelations.contains(lexer.sval))) + if (lexer.what() == CQLLexer.TT_WORD && + (lexer.value().indexOf('.') >= 0 || + lexer.value().equals("any") || + lexer.value().equals("all") || + lexer.value().equals("within") || + lexer.value().equals("encloses") || + (lexer.value().equals("exact") && compat != V1POINT2) || + (lexer.value().equals("scr") && compat != V1POINT2) || + (lexer.value().equals("adj") && compat == V1POINT2) || + customRelations.contains(lexer.value()))) return true; return isSymbolicRelation(); } private boolean isSymbolicRelation() { - debug("isSymbolicRelation: checking ttype=" + lexer.ttype + + debug("isSymbolicRelation: checking what()=" + lexer.what() + " (" + lexer.render() + ")"); - return (lexer.ttype == '<' || - lexer.ttype == '>' || - lexer.ttype == '=' || - lexer.ttype == CQLLexer.TT_LE || - lexer.ttype == CQLLexer.TT_GE || - lexer.ttype == CQLLexer.TT_NE || - lexer.ttype == CQLLexer.TT_EQEQ); + return (lexer.what() == '<' || + lexer.what() == '>' || + lexer.what() == '=' || + lexer.what() == CQLLexer.TT_LE || + lexer.what() == CQLLexer.TT_GE || + lexer.what() == CQLLexer.TT_NE || + lexer.what() == CQLLexer.TT_EQEQ); } private void match(int token) throws CQLParseException, IOException { debug("in match(" + lexer.render(token, true) + ")"); - if (lexer.ttype != token) + if (lexer.what() != token) throw new CQLParseException("expected " + lexer.render(token, true) + ", " + "got " + lexer.render(), - par.getPosition()); - int tmp = lexer.nextToken(); - debug("match() got token=" + lexer.ttype + ", " + - "nval=" + lexer.nval + ", sval='" + lexer.sval + "'" + - " (tmp=" + tmp + ")"); + lexer.pos()); + lexer.move(); + debug("match() got token=" + lexer.what() + ", value()='" + lexer.value() + "'"); } private String matchSymbol(String expected) throws CQLParseException, IOException { debug("in matchSymbol()"); - if (lexer.ttype == CQLLexer.TT_WORD || - lexer.ttype == CQLLexer.TT_NUMBER || - lexer.ttype == '"' || + if (lexer.what() == CQLLexer.TT_WORD || + lexer.what() == '"' || // The following is a complete list of keywords. Because // they're listed here, they can be used unquoted as // indexes, terms, prefix names and prefix identifiers. // ### Instead, we should ask the lexer whether what we // have is a keyword, and let the knowledge reside there. (allowKeywordTerms && - lexer.ttype == CQLLexer.TT_AND || - lexer.ttype == CQLLexer.TT_OR || - lexer.ttype == CQLLexer.TT_NOT || - lexer.ttype == CQLLexer.TT_PROX || - lexer.ttype == CQLLexer.TT_SORTBY)) { - String symbol = (lexer.ttype == CQLLexer.TT_NUMBER) ? - lexer.render() : lexer.sval; - match(lexer.ttype); + lexer.what() == CQLLexer.TT_AND || + lexer.what() == CQLLexer.TT_OR || + lexer.what() == CQLLexer.TT_NOT || + lexer.what() == CQLLexer.TT_PROX || + lexer.what() == CQLLexer.TT_SORTBY)) { + String symbol = lexer.value(); + match(lexer.what()); return symbol; } throw new CQLParseException("expected " + expected + ", " + - "got " + lexer.render(), par.getPosition()); + "got " + lexer.render(), lexer.pos()); } diff --git a/src/main/java/org/z3950/zing/cql/PositionAwareReader.java b/src/main/java/org/z3950/zing/cql/PositionAwareReader.java deleted file mode 100644 index b3294c0..0000000 --- a/src/main/java/org/z3950/zing/cql/PositionAwareReader.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright (c) 1995-2012, Index Data - * All rights reserved. - * See the file LICENSE for details. - */ -package org.z3950.zing.cql; - -import java.io.IOException; -import java.io.Reader; -import java.nio.CharBuffer; - -/** - * Reader proxy to count how many characters has been read so far. - * @author jakub - */ -public class PositionAwareReader extends Reader { - protected Reader reader; - protected int pos = -1; - - public PositionAwareReader(Reader reader) { - this.reader = reader; - } - - /* - * Position of the last read character or -1 if either reading from an empty - * stream or no 'read' has been invoked for this reader. - */ - public int getPosition() { - return pos; - } - - @Override - public void mark(int readAheadLimit) throws IOException { - reader.mark(readAheadLimit); - } - - @Override - public boolean markSupported() { - return reader.markSupported(); - } - - @Override - public int read() throws IOException { - int c = reader.read(); - if (c != -1) pos++; - return c; - } - - @Override - public int read(char[] cbuf) throws IOException { - int c = reader.read(cbuf); - if (c != -1) pos+=c; - return c; - } - - @Override - public int read(CharBuffer target) throws IOException { - int c = reader.read(target); - if (c != -1) pos+=c; - return c; - } - - @Override - public int read(char[] cbuf, int off, int len) throws IOException { - int c = reader.read(cbuf, off, len); - if (c != -1) pos+=c; - return c; - } - - @Override - public boolean ready() throws IOException { - return reader.ready(); - } - - @Override - public long skip(long n) throws IOException { - return reader.skip(n); - } - - @Override - public void close() throws IOException { - reader.close(); - } - - @Override - public void reset() throws IOException { - reader.reset(); - } - - //override object methods, to be on the safe-side - - @Override - public boolean equals(Object obj) { - return reader.equals(obj); - } - - @Override - public String toString() { - return reader.toString(); - } - - @Override - public int hashCode() { - return reader.hashCode(); - } - -} diff --git a/src/test/resources/regression/12/01.cql b/src/test/resources/regression/12/01.cql new file mode 100644 index 0000000..d8ec47d --- /dev/null +++ b/src/test/resources/regression/12/01.cql @@ -0,0 +1 @@ +"te\rm\*\?\^" diff --git a/src/test/resources/regression/12/01.xcql b/src/test/resources/regression/12/01.xcql new file mode 100644 index 0000000..c9fc4b6 --- /dev/null +++ b/src/test/resources/regression/12/01.xcql @@ -0,0 +1,7 @@ + + cql.serverChoice + + = + + term\*\?\^ + diff --git a/src/test/resources/regression/12/name b/src/test/resources/regression/12/name new file mode 100644 index 0000000..09d34d3 --- /dev/null +++ b/src/test/resources/regression/12/name @@ -0,0 +1 @@ +wildcards -- 1.7.10.4