Author: blafond Date: 2009-11-19 18:07:26 -0500 (Thu, 19 Nov 2009) New Revision: 1331 Added: branches/ddl_sequencer/dna-common/src/test/java/org/jboss/dna/common/text/PositionTest.java Modified: branches/ddl_sequencer/dna-common/src/main/java/org/jboss/dna/common/CommonI18n.java branches/ddl_sequencer/dna-common/src/main/java/org/jboss/dna/common/text/Position.java branches/ddl_sequencer/dna-common/src/main/java/org/jboss/dna/common/text/TokenStream.java branches/ddl_sequencer/dna-common/src/main/resources/org/jboss/dna/common/CommonI18n.properties branches/ddl_sequencer/dna-common/src/test/java/org/jboss/dna/common/text/TokenStreamTest.java Log: Added starting Index "in content" to Position object. Changed integer Token Stream constants to bit-mask values. Modified: branches/ddl_sequencer/dna-common/src/main/java/org/jboss/dna/common/CommonI18n.java =================================================================== --- branches/ddl_sequencer/dna-common/src/main/java/org/jboss/dna/common/CommonI18n.java 2009-11-18 19:42:04 UTC (rev 1330) +++ branches/ddl_sequencer/dna-common/src/main/java/org/jboss/dna/common/CommonI18n.java 2009-11-19 23:07:26 UTC (rev 1331) @@ -104,6 +104,7 @@ public static I18n expectingValidIntegerAtLineAndColumn; public static I18n expectingValidLongAtLineAndColumn; public static I18n expectingValidBooleanAtLineAndColumn; + public static I18n endPositionMustBeGreaterThanStartingPosition; static { try { Modified: branches/ddl_sequencer/dna-common/src/main/java/org/jboss/dna/common/text/Position.java =================================================================== --- branches/ddl_sequencer/dna-common/src/main/java/org/jboss/dna/common/text/Position.java 2009-11-18 19:42:04 UTC (rev 1330) +++ branches/ddl_sequencer/dna-common/src/main/java/org/jboss/dna/common/text/Position.java 2009-11-19 23:07:26 UTC (rev 1331) @@ -7,14 +7,37 @@ */ @Immutable public final class Position { + /** + * The position is used when there is no content. + */ + public final static Position EMPTY_CONTENT_POSITION = new Position(-1, 1, 0); + private final int line; private final int column; + private final int indexInContent; - public Position( int line, + public Position( int indexInContent, + int line, int column ) { + this.indexInContent = indexInContent < 0 ? -1:indexInContent; this.line = line; this.column = column; + + assert this.indexInContent >= -1; + assert this.line > 0; + assert this.column >= 0; + // make sure that negative index means an EMPTY_CONTENT_POSITION + assert this.indexInContent < 0 ? this.line == 1 && this.column == 0: true; } + + /** + * Get the 0-based index of this position in the content character array. + * + * @return the index; never negative except for the first position in an empty content. + */ + public int getIndexInContent() { + return indexInContent; + } /** * Get the 1-based column number of the character. @@ -41,7 +64,7 @@ */ @Override public int hashCode() { - return line; + return indexInContent; } /** @@ -51,6 +74,28 @@ */ @Override public String toString() { - return "" + line + ':' + column; + return "" + indexInContent + ':' + line + ':' + column; } + + /** + * Return a new position that is the addition of this position and that supplied. + * + * @param position the position to add to this object; may not be null + * @return the combined position + */ + public Position add(Position position) { + if( this.getIndexInContent() < 0 ) { + return position.getIndexInContent() < 0 ? EMPTY_CONTENT_POSITION:position; + } + + if( position.getIndexInContent() < 0 ) { + return this; + } + + int index = this.getIndexInContent() + position.getIndexInContent(); + int line = position.getLine() + this.getLine() - 1; + int column = this.getLine() == 1 ? this.getColumn() + position.getColumn() : this.getColumn(); + + return new Position(index, line, column); + } } Modified: branches/ddl_sequencer/dna-common/src/main/java/org/jboss/dna/common/text/TokenStream.java =================================================================== --- branches/ddl_sequencer/dna-common/src/main/java/org/jboss/dna/common/text/TokenStream.java 2009-11-18 19:42:04 UTC (rev 1330) +++ branches/ddl_sequencer/dna-common/src/main/java/org/jboss/dna/common/text/TokenStream.java 2009-11-19 23:07:26 UTC (rev 1331) @@ -384,6 +384,20 @@ private final boolean caseSensitive; private final Tokenizer tokenizer; private List<Token> tokens; + /** + * This class navigates the Token objects using this iterator. However, because it very often needs to access the + * "current token" in the "consume(...)" and "canConsume(...)" and "matches(...)" methods, the class caches a "current token" + * and makes this iterator point to the 2nd token. + * + * <pre> + * T1 T2 T3 T4 T5 + * &circ; &circ; &circ; + * | | | + * | | +- The position of the tokenIterator, where tokenIterator.hasNext() will return T3 + * | +---- The token referenced by currentToken + * +-------- The logical position of the TokenStream object, where the "consume()" would return T2 + * </pre> + */ private ListIterator<Token> tokenIterator; private Token currentToken; private boolean completed; @@ -412,7 +426,7 @@ TokenFactory tokenFactory = caseSensitive ? new CaseSensitiveTokenFactory() : new CaseInsensitiveTokenFactory(); CharacterStream characterStream = new CharacterArrayStream(inputContent); tokenizer.tokenize(characterStream, tokenFactory); - this.tokens = tokenFactory.getTokens(); + this.tokens = initializeTokens(tokenFactory.getTokens()); } // Create the iterator ... @@ -420,6 +434,28 @@ moveToNextToken(); return this; } + + /** + * Method to allow subclasses to preprocess the set of tokens and return the correct tokens to use. + * The default behavior is to simply return the supplied tokens. + * + * @param tokens + * @return list of tokens. + */ + protected List<Token> initializeTokens( List<Token> tokens) { + return tokens; + } + + /** + * Method to allow tokens to be re-used from the start without re-tokenizing content. + */ + public void rewind() { + // recreate the iterator ... + tokenIterator = this.tokens.listIterator(); + completed = false; + currentToken = null; + moveToNextToken(); + } /** * Get the position of the previous token. @@ -526,7 +562,7 @@ protected void throwNoMoreContent() throws ParsingException { String msg = CommonI18n.noMoreContent.text(); - Position pos = tokens.isEmpty() ? new Position(1, 0) : tokens.get(tokens.size() - 1).position(); + Position pos = tokens.isEmpty() ? new Position(-1, 1, 0) : tokens.get(tokens.size() - 1).position(); throw new ParsingException(pos, msg); } @@ -954,7 +990,7 @@ * @throws IllegalStateException if this method was called before the stream was {@link #start() started} */ public boolean matches( int expectedType ) throws IllegalStateException { - return !completed && (expectedType == ANY_TYPE || currentToken().type() == expectedType); + return !completed && currentToken().matches(expectedType); } /** @@ -1047,7 +1083,7 @@ ListIterator<Token> iter = tokens.listIterator(tokenIterator.previousIndex()); if (!iter.hasNext()) return false; Token token = iter.next(); - if (currentExpectedType != ANY_TYPE || currentToken().type() != currentExpectedType) return false; + if (currentExpectedType != ANY_TYPE && currentToken().type() != currentExpectedType) return false; for (int nextExpectedType : expectedTypeForNextTokens) { if (!iter.hasNext()) return false; token = iter.next(); @@ -1075,7 +1111,7 @@ if (!iter.hasNext()) return false; token = iter.next(); if (nextExpectedType == ANY_TYPE) continue; - if (token.type() != nextExpectedType) return false; + if (!token.matches(nextExpectedType)) return false; } return true; } @@ -1233,6 +1269,29 @@ } /** + * Gets the content string starting at the first position (inclusive) and continuing up to the end position (exclusive). + * + * @param starting the position marking the beginning of the desired content string. + * @param end the position located directly after the returned content string; can be null, which means end of content + * @return the content string; never null + */ + public String getContentBetween(Position starting, Position end) { + CheckArg.isNotNull(starting, "starting"); + + int startIndex = starting.getIndexInContent(); + int endIndex = inputString.length(); + if( end != null ) { + endIndex = end.getIndexInContent(); + } + + if( startIndex >= endIndex ) { + throw new IllegalArgumentException(CommonI18n.endPositionMustBeGreaterThanStartingPosition.text(startIndex, endIndex)); + } + + return inputString.substring(startIndex, endIndex); + } + + /** * Get the previous token. This does not modify the state. * * @return the previous token; never null @@ -1249,11 +1308,10 @@ } throw new IllegalStateException(CommonI18n.startMethodMustBeCalledBeforeConsumingOrMatching.text()); } - if (!tokenIterator.hasPrevious()) { + if (tokenIterator.previousIndex() == 0 ) { throw new NoSuchElementException(CommonI18n.noMoreContent.text()); } - ListIterator<Token> temp = tokens.listIterator(tokenIterator.previousIndex()); - return temp.next(); + return tokens.get(tokenIterator.previousIndex()-1); } String generateFragment() { @@ -1336,10 +1394,11 @@ /** * Get the position for the last character returned from {@link #next()}. + * @param startIndex * * @return the position of the last character returned; never null */ - Position position(); + Position position(int startIndex); /** * Determine if the next character on the sream is a {@link Character#isWhitespace(char) whitespace character}. This @@ -1495,6 +1554,14 @@ * @return true if the token's value matches the supplied character value, or false otherwise */ boolean matches( char expected ); + + /** + * Determine if the token matches the supplied type. + * + * @param expectedType the expected integer type + * @return true if the token's value matches the supplied integer type, or false otherwise + */ + boolean matches( int expectedType ); /** * Get the type of the token. @@ -1530,6 +1597,14 @@ * @return the position; never null */ Position position(); + + /** + * Bitmask ORed with existing type value. + * + * @param typeMask + * @return copy of Token with new type + */ + Token withType( int typeMask); } /** @@ -1553,6 +1628,17 @@ } /** + * {@inheritDoc} + * + * @see org.jboss.dna.common.text.TokenStream.Token#withType(int) + */ + @Override + public Token withType(int typeMask) { + int type = this.type | typeMask; + return new CaseSensitiveToken(startIndex, endIndex, type, position); + } + + /** * {@inheritDoc} * * @see org.jboss.dna.common.text.TokenStream.Token#type() @@ -1607,6 +1693,16 @@ } /** + * {@inheritDoc} + * + * @see org.jboss.dna.common.text.TokenStream.Token#matches(int) + */ + @Override + public final boolean matches(int expectedType) { + return expectedType == ANY_TYPE || (currentToken().type() & expectedType) == expectedType; + } + + /** * {@inheritDoc} * * @see org.jboss.dna.common.text.TokenStream.Token#value() @@ -1657,6 +1753,17 @@ protected String matchString() { return inputUppercased; } + + /** + * {@inheritDoc} + * + * @see org.jboss.dna.common.text.TokenStream.Token#withType(int) + */ + @Override + public Token withType(int typeMask) { + int type = this.type() | typeMask; + return new CaseInsensitiveToken(startIndex(), endIndex(), type, position()); + } } protected abstract class TokenFactory implements Tokens { @@ -1755,11 +1862,13 @@ /** * {@inheritDoc} + * @param startIndex + * @return the position of the token. never null * - * @see org.jboss.dna.common.text.TokenStream.CharacterStream#position() + * @see org.jboss.dna.common.text.TokenStream.CharacterStream#position(int) */ - public Position position() { - return new Position(lineNumber, columnNumber); + public Position position(int startIndex) { + return new Position(startIndex, lineNumber, columnNumber); } /** @@ -1776,10 +1885,10 @@ if (result == '\r') { nextCharMayBeLineFeed = true; ++lineNumber; - columnNumber = 1; + columnNumber = 0; } else if (result == '\n') { if (!nextCharMayBeLineFeed) ++lineNumber; - columnNumber = 1; + columnNumber = 0; } else if (nextCharMayBeLineFeed) { nextCharMayBeLineFeed = false; } @@ -1922,22 +2031,22 @@ /** * The {@link Token#type() token type} for tokens that consist of an individual '.' character. */ - public static final int DECIMAL = 3; + public static final int DECIMAL = 4; /** * The {@link Token#type() token type} for tokens that consist of all the characters within single-quotes. Single quote * characters are included if they are preceded (escaped) by a '\' character. */ - public static final int SINGLE_QUOTED_STRING = 4; + public static final int SINGLE_QUOTED_STRING = 8; /** * The {@link Token#type() token type} for tokens that consist of all the characters within double-quotes. Double quote * characters are included if they are preceded (escaped) by a '\' character. */ - public static final int DOUBLE_QUOTED_STRING = 5; + public static final int DOUBLE_QUOTED_STRING = 16; /** * The {@link Token#type() token type} for tokens that consist of all the characters between "/*" and "&#42;/" or between * "//" and the next line terminator (e.g., '\n', '\r' or "\r\n"). */ - public static final int COMMENT = 6; + public static final int COMMENT = 32; private final boolean useComments; @@ -1981,14 +2090,14 @@ case '|': case '=': case ':': - tokens.addToken(input.position(), input.index(), input.index() + 1, SYMBOL); + tokens.addToken(input.position(input.index()), input.index(), input.index() + 1, SYMBOL); break; case '.': - tokens.addToken(input.position(), input.index(), input.index() + 1, DECIMAL); + tokens.addToken(input.position(input.index()), input.index(), input.index() + 1, DECIMAL); break; case '\"': int startIndex = input.index(); - Position startingPosition = input.position(); + Position startingPosition = input.position(startIndex); boolean foundClosingQuote = false; while (input.hasNext()) { c = input.next(); @@ -2005,11 +2114,11 @@ throw new ParsingException(startingPosition, msg); } int endIndex = input.index() + 1; // beyond last character read - tokens.addToken(input.position(), startIndex, endIndex, DOUBLE_QUOTED_STRING); + tokens.addToken(startingPosition, startIndex, endIndex, DOUBLE_QUOTED_STRING); break; case '\'': startIndex = input.index(); - startingPosition = input.position(); + startingPosition = input.position(startIndex); foundClosingQuote = false; while (input.hasNext()) { c = input.next(); @@ -2026,10 +2135,11 @@ throw new ParsingException(startingPosition, msg); } endIndex = input.index() + 1; // beyond last character read - tokens.addToken(input.position(), startIndex, endIndex, SINGLE_QUOTED_STRING); + tokens.addToken(startingPosition, startIndex, endIndex, SINGLE_QUOTED_STRING); break; case '/': startIndex = input.index(); + startingPosition = input.position(startIndex); if (input.isNext('/')) { // End-of-line comment ... boolean foundLineTerminator = false; @@ -2044,7 +2154,7 @@ if (!foundLineTerminator) ++endIndex; // must point beyond last char if (c == '\r' && input.isNext('\n')) input.next(); if (useComments) { - tokens.addToken(input.position(), startIndex, endIndex, COMMENT); + tokens.addToken(startingPosition, startIndex, endIndex, COMMENT); } } else if (input.isNext('*')) { // Multi-line comment ... @@ -2055,21 +2165,22 @@ if (input.hasNext()) input.next(); // consume the '/' if (useComments) { endIndex = input.index() + 1; // the token will include the '/' and '*' characters - tokens.addToken(input.position(), startIndex, endIndex, COMMENT); + tokens.addToken(startingPosition, startIndex, endIndex, COMMENT); } } else { // just a regular slash ... - tokens.addToken(input.position(), startIndex, startIndex + 1, SYMBOL); + tokens.addToken(startingPosition, startIndex, startIndex + 1, SYMBOL); } break; default: startIndex = input.index(); + startingPosition = input.position(startIndex); // Read until another whitespace/symbol/decimal/slash is found while (input.hasNext() && !(input.isNextWhitespace() || input.isNextAnyOf("/.-(){}*,;+%?$[]!<>|=:"))) { c = input.next(); } endIndex = input.index() + 1; // beyond last character that was included - tokens.addToken(input.position(), startIndex, endIndex, WORD); + tokens.addToken(startingPosition, startIndex, endIndex, WORD); } } } Modified: branches/ddl_sequencer/dna-common/src/main/resources/org/jboss/dna/common/CommonI18n.properties =================================================================== --- branches/ddl_sequencer/dna-common/src/main/resources/org/jboss/dna/common/CommonI18n.properties 2009-11-18 19:42:04 UTC (rev 1330) +++ branches/ddl_sequencer/dna-common/src/main/resources/org/jboss/dna/common/CommonI18n.properties 2009-11-19 23:07:26 UTC (rev 1331) @@ -93,3 +93,4 @@ expectingValidIntegerAtLineAndColumn = Expecting a valid integer value but found '{0}' at line {1}, column {2} expectingValidLongAtLineAndColumn = Expecting a valid long value but found '{0}' at line {1}, column {2} expectingValidBooleanAtLineAndColumn = Expecting a valid boolean value but found '{0}' at line {1}, column {2} +endPositionMustBeGreaterThanStartingPosition = End position {1} must be greater than starting position {0} Added: branches/ddl_sequencer/dna-common/src/test/java/org/jboss/dna/common/text/PositionTest.java =================================================================== --- branches/ddl_sequencer/dna-common/src/test/java/org/jboss/dna/common/text/PositionTest.java (rev 0) +++ branches/ddl_sequencer/dna-common/src/test/java/org/jboss/dna/common/text/PositionTest.java 2009-11-19 23:07:26 UTC (rev 1331) @@ -0,0 +1,85 @@ +/* + * JBoss DNA (http://www.jboss.org/dna) + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * See the AUTHORS.txt file in the distribution for a full listing of + * individual contributors. + * + * JBoss DNA is free software. Unless otherwise indicated, all code in JBoss DNA + * is licensed to you under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of + * the License, or (at your option) any later version. + * + * JBoss DNA is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this software; if not, write to the Free + * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA + * 02110-1301 USA, or see the FSF site: http://www.fsf.org. + */ +package org.jboss.dna.common.text; + +import static org.junit.Assert.assertThat; + +import static org.hamcrest.core.Is.is; + +import org.junit.Test; + + +public class PositionTest { + + private int combinedIndex(int firstIndex, int secondIndex) { + Position first = new Position(firstIndex, 1, 0); + Position second = new Position(secondIndex, 1, 0); + + int firstPlusSecond = first.add(second).getIndexInContent(); + int secondPlusFirst = second.add(first).getIndexInContent(); + + assertThat(firstPlusSecond, is(secondPlusFirst)); + + return firstPlusSecond; + } + + @Test + public void shouldAddNoContentPositionToValidPosition() { + // -1 to >=0 + assertThat(combinedIndex(-1, 0), is(0)); + assertThat(combinedIndex(-1, 1), is(1)); + assertThat(combinedIndex(-1, 10), is(10)); + } + + @Test + public void shouldAddValidPositionToNoContentPosition() { + // >= 0 to -1 + assertThat(combinedIndex(0, -1), is(0)); + assertThat(combinedIndex(1, -1), is(1)); + assertThat(combinedIndex(10, -1), is(10)); + } + + @Test + public void shouldAddValidPositionToValidPosition() { + // positive to positive + assertThat(combinedIndex(1, 1), is(2)); + assertThat(combinedIndex(10, 1), is(11)); + assertThat(combinedIndex(1, 10), is(11)); + assertThat(combinedIndex(10, 10), is(20)); + } + + @Test + public void shouldAddStartingPositionToStartingPosition() { + // 0 to 0 + assertThat(combinedIndex(0, 0), is(0)); + } + + @Test + public void shouldAddNoContentPositionToNoContentPosition() { + // -1 to -1 + assertThat(combinedIndex(-1, -1), is(-1)); + assertThat(combinedIndex(-10, -1), is(-1)); + assertThat(combinedIndex(-1, -10), is(-1)); + } +} Property changes on: branches/ddl_sequencer/dna-common/src/test/java/org/jboss/dna/common/text/PositionTest.java ___________________________________________________________________ Name: svn:mime-type + text/plain Modified: branches/ddl_sequencer/dna-common/src/test/java/org/jboss/dna/common/text/TokenStreamTest.java =================================================================== --- branches/ddl_sequencer/dna-common/src/test/java/org/jboss/dna/common/text/TokenStreamTest.java 2009-11-18 19:42:04 UTC (rev 1330) +++ branches/ddl_sequencer/dna-common/src/test/java/org/jboss/dna/common/text/TokenStreamTest.java 2009-11-19 23:07:26 UTC (rev 1331) @@ -25,7 +25,9 @@ import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertThat; + import java.util.Arrays; + import org.jboss.dna.common.text.TokenStream.BasicTokenizer; import org.jboss.dna.common.text.TokenStream.Tokenizer; import org.junit.Before; @@ -336,6 +338,19 @@ assertThat(tokens.canConsume("FROM", "THIS", "TABLE"), is(true)); assertThat(tokens.hasNext(), is(false)); } + + @Test + public void shouldReturnTrueFromMatchIfAllTypeValuesMatch() { + makeCaseInsensitive(); + assertThat(tokens.matches(BasicTokenizer.WORD, BasicTokenizer.WORD), is(true)); + } + + @Test + public void shouldReturnFalseFromMatchIfAllTypeValuesDoNotMatch() { + makeCaseInsensitive(); + assertThat(tokens.matches(BasicTokenizer.WORD, BasicTokenizer.DECIMAL), is(false)); + assertThat(tokens.matches(BasicTokenizer.DECIMAL, BasicTokenizer.WORD), is(false)); + } @Test public void shouldConsumeMultipleTokensWithAnyValueConstant() { @@ -397,4 +412,51 @@ assertThat(tokens.canConsume(TokenStream.ANY_VALUE), is(false)); assertThat(tokens.canConsume(BasicTokenizer.SYMBOL), is(false)); } + + @Test + public void shouldFindNextPositionStartIndex() { + makeCaseInsensitive(); + // "Select all columns from this table"; + tokens.consume(); + // Next position should be line 1, column 8 + assertThat(tokens.nextPosition().getIndexInContent(), is(7)); + assertThat(tokens.nextPosition().getColumn(), is(8)); + assertThat(tokens.nextPosition().getLine(), is(1)); + } + + @Test + public void shouldFindPreviousPositionStartIndex() { + makeCaseInsensitive(); + // "Select all columns from this table"; + tokens.consume(); + tokens.consume(); + // previous position should be line 1, column 8 + assertThat(tokens.previousPosition().getIndexInContent(), is(7)); + assertThat(tokens.previousPosition().getColumn(), is(8)); + assertThat(tokens.previousPosition().getLine(), is(1)); + } + + @Test + public void shouldParseMultiLineString() { + makeCaseInsensitive(); + String content = "ALTER DATABASE \n" + + "DO SOMETHING; \n" + + "ALTER DATABASE \n" + + " SET DEFAULT BIGFILE TABLESPACE;"; + tokens = new TokenStream(content, tokenizer, true); + tokens.start(); + + tokens.consume(); // LINE + tokens.consume(); // ONE + tokens.consume(); // DO + tokens.consume(); // SOMETHING + tokens.consume(); // ; + + assertThat(tokens.nextPosition().getIndexInContent(), is(31)); + assertThat(tokens.nextPosition().getColumn(), is(1)); + tokens.consume(); // ALTER + assertThat(tokens.nextPosition().getIndexInContent(), is(37)); + assertThat(tokens.nextPosition().getColumn(), is(7)); + + } }