Author: shawkins Date: 2012-09-30 08:57:02 -0400 (Sun, 30 Sep 2012) New Revision: 4491 Modified: trunk/engine/src/main/java/org/teiid/query/parser/QueryParser.java trunk/engine/src/main/java/org/teiid/query/parser/TeiidSQLParserTokenManager.java trunk/engine/src/test/java/org/teiid/query/parser/TestParser.java Log: TEIID-2233 better handling for token errors Modified: trunk/engine/src/main/java/org/teiid/query/parser/QueryParser.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/parser/QueryParser.java 2012-09-29 19:14:01 UTC (rev 4490) +++ trunk/engine/src/main/java/org/teiid/query/parser/QueryParser.java 2012-09-30 12:57:02 UTC (rev 4491) @@ -22,6 +22,8 @@ package org.teiid.query.parser; +import static org.teiid.query.parser.TeiidSQLParserTokenManager.*; + import java.io.Reader; import java.io.StringReader; import java.util.ArrayList; @@ -219,7 +221,14 @@ return pe.getMessage(); } StringBuilder sb = encountered(pe, 1); - sb.append(pe.getMessage()); + if (pe.currentToken.kind == INVALID_TOKEN) { + sb.append(QueryPlugin.Util.getString("QueryParser.lexicalError", pe.currentToken.image)); //$NON-NLS-1$ + } else if (pe.currentToken.next != null && pe.currentToken.next.kind == -1) { + sb.append(QueryPlugin.Util.getString("QueryParser.lexicalError", pe.currentToken.next.image)); //$NON-NLS-1$ + } + if (pe.getMessage() != null) { + sb.append(pe.getMessage()); + } return sb.toString(); } StringBuffer expected = new StringBuffer(); @@ -236,7 +245,7 @@ }); int maxSize = expectedTokenSequences[0].length; StringBuilder retval = encountered(pe, maxSize); - if (currentToken.next.kind == -1) { + if (currentToken.next.kind == INVALID_TOKEN) { retval.append(QueryPlugin.Util.getString("QueryParser.lexicalError", currentToken.next.image)); //$NON-NLS-1$ return retval.toString(); } @@ -288,22 +297,35 @@ } else { addTokenSequence(1, retval, currentToken); } - if (currentToken.next.kind == -1) { - maxSize = 1; - } - retval.append(" [*]"); //$NON-NLS-1$ - Token last = addTokenSequence(maxSize, retval, currentToken.next); - if (last.kind != 0) { - retval.append("[*]"); //$NON-NLS-1$ - if (last.next == null) { - this.parser.getNextToken(); + if (currentToken.next != null) { + boolean space = true; + if (currentToken.next.kind == INVALID_TOKEN) { + maxSize = 1; + space = currentToken.endColumn + 1 != currentToken.next.beginColumn; } - if (last.next != null) { + if (space) { retval.append(" "); //$NON-NLS-1$ - addTokenSequence(1, retval, last.next); } + retval.append("[*]"); //$NON-NLS-1$ + Token last = addTokenSequence(maxSize, retval, currentToken.next); + if (last.kind != 0) { + retval.append("[*]"); //$NON-NLS-1$ + if (last.next == null) { + this.parser.getNextToken(); + } + if (last.next != null) { + if (!space) { + space = last.endColumn + 1 != last.next.beginColumn; + } + if (space) { + retval.append(" "); //$NON-NLS-1$ + } + addTokenSequence(1, retval, last.next); + } + } + currentToken = currentToken.next; } - retval.append("\" at line ").append(currentToken.next.beginLine).append(", column ").append(currentToken.next.beginColumn); //$NON-NLS-1$ //$NON-NLS-2$ + retval.append("\" at line ").append(currentToken.beginLine).append(", column ").append(currentToken.beginColumn); //$NON-NLS-1$ //$NON-NLS-2$ retval.append(".").append(pe.eol); //$NON-NLS-1$ return retval; } Modified: trunk/engine/src/main/java/org/teiid/query/parser/TeiidSQLParserTokenManager.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/parser/TeiidSQLParserTokenManager.java 2012-09-29 19:14:01 UTC (rev 4490) +++ trunk/engine/src/main/java/org/teiid/query/parser/TeiidSQLParserTokenManager.java 2012-09-30 12:57:02 UTC (rev 4491) @@ -29,6 +29,7 @@ */ class TeiidSQLParserTokenManager extends SQLParserTokenManager { + static final int INVALID_TOKEN = -1; int tokenCount = 0; Token head; @@ -57,12 +58,12 @@ return t; } catch (TokenMgrError err) { Token t = new Token(); - t.kind = -1; + t.kind = INVALID_TOKEN; t.beginColumn = this.input_stream.getBeginColumn(); t.beginLine = this.input_stream.getBeginLine(); - t.endColumn = t.beginColumn+1; + t.endColumn = t.beginColumn; t.endLine = t.beginLine; - t.image = new String(this.input_stream.buffer, this.input_stream.bufpos+1, 1); + t.image = this.input_stream.GetImage().substring(0, 1); try { //mark the char a consumed this.input_stream.readChar(); Modified: trunk/engine/src/test/java/org/teiid/query/parser/TestParser.java =================================================================== --- trunk/engine/src/test/java/org/teiid/query/parser/TestParser.java 2012-09-29 19:14:01 UTC (rev 4490) +++ trunk/engine/src/test/java/org/teiid/query/parser/TestParser.java 2012-09-30 12:57:02 UTC (rev 4491) @@ -2696,8 +2696,16 @@ /** SELECT xx.yy%.a from xx.yy */ @Test public void testFailsWildcardInSelect(){ - helpException("SELECT xx.yy%.a from xx.yy", "TEIID31100 Parsing error: Encountered \"SELECT xx.yy [*]%[*] .\" at line 1, column 13.\nLexical error. Character is not a valid token: % "); //$NON-NLS-1$ + helpException("SELECT xx.yy%.a from xx.yy", "TEIID31100 Parsing error: Encountered \"SELECT xx.yy[*]%[*].\" at line 1, column 13.\nLexical error. Character is not a valid token: % "); //$NON-NLS-1$ } + + @Test public void testFailsWildcardInSelect1(){ + helpException("SELECT % from xx.yy", "TEIID31100 Parsing error: Encountered \"SELECT [*]%[*] from\" at line 1, column 8.\nLexical error. Character is not a valid token: % "); //$NON-NLS-1$ + } + + @Test public void testInvalidToken(){ + helpException("%", ""); + } /** SELECT a or b from g */ @Test public void testOrInSelect(){