5:07 p.m.
Author: blafond
Date: 2009-11-19 18:07:26 -0500 (Thu, 19 Nov 2009)
New Revision: 1331
Added:
branches/ddl_sequencer/dna-common/src/test/java/org/jboss/dna/common/text/PositionTest.java
Modified:
branches/ddl_sequencer/dna-common/src/main/java/org/jboss/dna/common/CommonI18n.java
branches/ddl_sequencer/dna-common/src/main/java/org/jboss/dna/common/text/Position.java
branches/ddl_sequencer/dna-common/src/main/java/org/jboss/dna/common/text/TokenStream.java
branches/ddl_sequencer/dna-common/src/main/resources/org/jboss/dna/common/CommonI18n.properties
branches/ddl_sequencer/dna-common/src/test/java/org/jboss/dna/common/text/TokenStreamTest.java
Log:
Added starting Index "in content" to Position object. Changed integer Token
Stream constants to bit-mask values.
Modified:
branches/ddl_sequencer/dna-common/src/main/java/org/jboss/dna/common/CommonI18n.java
===================================================================
---
branches/ddl_sequencer/dna-common/src/main/java/org/jboss/dna/common/CommonI18n.java 2009-11-18
19:42:04 UTC (rev 1330)
+++
branches/ddl_sequencer/dna-common/src/main/java/org/jboss/dna/common/CommonI18n.java 2009-11-19
23:07:26 UTC (rev 1331)
@@ -104,6 +104,7 @@
public static I18n expectingValidIntegerAtLineAndColumn;
public static I18n expectingValidLongAtLineAndColumn;
public static I18n expectingValidBooleanAtLineAndColumn;
+ public static I18n endPositionMustBeGreaterThanStartingPosition;
static {
try {
Modified:
branches/ddl_sequencer/dna-common/src/main/java/org/jboss/dna/common/text/Position.java
===================================================================
---
branches/ddl_sequencer/dna-common/src/main/java/org/jboss/dna/common/text/Position.java 2009-11-18
19:42:04 UTC (rev 1330)
+++
branches/ddl_sequencer/dna-common/src/main/java/org/jboss/dna/common/text/Position.java 2009-11-19
23:07:26 UTC (rev 1331)
@@ -7,14 +7,37 @@
*/
@Immutable
public final class Position {
+ /**
+ * The position is used when there is no content.
+ */
+ public final static Position EMPTY_CONTENT_POSITION = new Position(-1, 1, 0);
+
private final int line;
private final int column;
+ private final int indexInContent;
- public Position( int line,
+ public Position( int indexInContent,
+ int line,
int column ) {
+ this.indexInContent = indexInContent < 0 ? -1:indexInContent;
this.line = line;
this.column = column;
+
+ assert this.indexInContent >= -1;
+ assert this.line > 0;
+ assert this.column >= 0;
+ // make sure that negative index means an EMPTY_CONTENT_POSITION
+ assert this.indexInContent < 0 ? this.line == 1 && this.column == 0:
true;
}
+
+ /**
+ * Get the 0-based index of this position in the content character array.
+ *
+ * @return the index; never negative except for the first position in an empty
content.
+ */
+ public int getIndexInContent() {
+ return indexInContent;
+ }
/**
* Get the 1-based column number of the character.
@@ -41,7 +64,7 @@
*/
@Override
public int hashCode() {
- return line;
+ return indexInContent;
}
/**
@@ -51,6 +74,28 @@
*/
@Override
public String toString() {
- return "" + line + ':' + column;
+ return "" + indexInContent + ':' + line + ':' +
column;
}
+
+ /**
+ * Return a new position that is the addition of this position and that supplied.
+ *
+ * @param position the position to add to this object; may not be null
+ * @return the combined position
+ */
+ public Position add(Position position) {
+ if( this.getIndexInContent() < 0 ) {
+ return position.getIndexInContent() < 0 ? EMPTY_CONTENT_POSITION:position;
+ }
+
+ if( position.getIndexInContent() < 0 ) {
+ return this;
+ }
+
+ int index = this.getIndexInContent() + position.getIndexInContent();
+ int line = position.getLine() + this.getLine() - 1;
+ int column = this.getLine() == 1 ? this.getColumn() + position.getColumn() :
this.getColumn();
+
+ return new Position(index, line, column);
+ }
}
Modified:
branches/ddl_sequencer/dna-common/src/main/java/org/jboss/dna/common/text/TokenStream.java
===================================================================
---
branches/ddl_sequencer/dna-common/src/main/java/org/jboss/dna/common/text/TokenStream.java 2009-11-18
19:42:04 UTC (rev 1330)
+++
branches/ddl_sequencer/dna-common/src/main/java/org/jboss/dna/common/text/TokenStream.java 2009-11-19
23:07:26 UTC (rev 1331)
@@ -384,6 +384,20 @@
private final boolean caseSensitive;
private final Tokenizer tokenizer;
private List<Token> tokens;
+ /**
+ * This class navigates the Token objects using this iterator. However, because it
very often needs to access the
+ * "current token" in the "consume(...)" and
"canConsume(...)" and "matches(...)" methods, the class caches a
"current token"
+ * and makes this iterator point to the 2nd token.
+ *
+ * <pre>
+ * T1 T2 T3 T4 T5
+ * ˆ ˆ ˆ
+ * | | |
+ * | | +- The position of the tokenIterator, where tokenIterator.hasNext()
will return T3
+ * | +---- The token referenced by currentToken
+ * +-------- The logical position of the TokenStream object, where the
"consume()" would return T2
+ * </pre>
+ */
private ListIterator<Token> tokenIterator;
private Token currentToken;
private boolean completed;
@@ -412,7 +426,7 @@
TokenFactory tokenFactory = caseSensitive ? new CaseSensitiveTokenFactory() :
new CaseInsensitiveTokenFactory();
CharacterStream characterStream = new CharacterArrayStream(inputContent);
tokenizer.tokenize(characterStream, tokenFactory);
- this.tokens = tokenFactory.getTokens();
+ this.tokens = initializeTokens(tokenFactory.getTokens());
}
// Create the iterator ...
@@ -420,6 +434,28 @@
moveToNextToken();
return this;
}
+
+ /**
+ * Method to allow subclasses to preprocess the set of tokens and return the correct
tokens to use.
+ * The default behavior is to simply return the supplied tokens.
+ *
+ * @param tokens
+ * @return list of tokens.
+ */
+ protected List<Token> initializeTokens( List<Token> tokens) {
+ return tokens;
+ }
+
+ /**
+ * Method to allow tokens to be re-used from the start without re-tokenizing
content.
+ */
+ public void rewind() {
+ // recreate the iterator ...
+ tokenIterator = this.tokens.listIterator();
+ completed = false;
+ currentToken = null;
+ moveToNextToken();
+ }
/**
* Get the position of the previous token.
@@ -526,7 +562,7 @@
protected void throwNoMoreContent() throws ParsingException {
String msg = CommonI18n.noMoreContent.text();
- Position pos = tokens.isEmpty() ? new Position(1, 0) : tokens.get(tokens.size() -
1).position();
+ Position pos = tokens.isEmpty() ? new Position(-1, 1, 0) :
tokens.get(tokens.size() - 1).position();
throw new ParsingException(pos, msg);
}
@@ -954,7 +990,7 @@
* @throws IllegalStateException if this method was called before the stream was
{@link #start() started}
*/
public boolean matches( int expectedType ) throws IllegalStateException {
- return !completed && (expectedType == ANY_TYPE || currentToken().type()
== expectedType);
+ return !completed && currentToken().matches(expectedType);
}
/**
@@ -1047,7 +1083,7 @@
ListIterator<Token> iter =
tokens.listIterator(tokenIterator.previousIndex());
if (!iter.hasNext()) return false;
Token token = iter.next();
- if (currentExpectedType != ANY_TYPE || currentToken().type() !=
currentExpectedType) return false;
+ if (currentExpectedType != ANY_TYPE && currentToken().type() !=
currentExpectedType) return false;
for (int nextExpectedType : expectedTypeForNextTokens) {
if (!iter.hasNext()) return false;
token = iter.next();
@@ -1075,7 +1111,7 @@
if (!iter.hasNext()) return false;
token = iter.next();
if (nextExpectedType == ANY_TYPE) continue;
- if (token.type() != nextExpectedType) return false;
+ if (!token.matches(nextExpectedType)) return false;
}
return true;
}
@@ -1233,6 +1269,29 @@
}
/**
+ * Gets the content string starting at the first position (inclusive) and continuing
up to the end position (exclusive).
+ *
+ * @param starting the position marking the beginning of the desired content string.
+ * @param end the position located directly after the returned content string; can
be null, which means end of content
+ * @return the content string; never null
+ */
+ public String getContentBetween(Position starting, Position end) {
+ CheckArg.isNotNull(starting, "starting");
+
+ int startIndex = starting.getIndexInContent();
+ int endIndex = inputString.length();
+ if( end != null ) {
+ endIndex = end.getIndexInContent();
+ }
+
+ if( startIndex >= endIndex ) {
+ throw new
IllegalArgumentException(CommonI18n.endPositionMustBeGreaterThanStartingPosition.text(startIndex,
endIndex));
+ }
+
+ return inputString.substring(startIndex, endIndex);
+ }
+
+ /**
* Get the previous token. This does not modify the state.
*
* @return the previous token; never null
@@ -1249,11 +1308,10 @@
}
throw new
IllegalStateException(CommonI18n.startMethodMustBeCalledBeforeConsumingOrMatching.text());
}
- if (!tokenIterator.hasPrevious()) {
+ if (tokenIterator.previousIndex() == 0 ) {
throw new NoSuchElementException(CommonI18n.noMoreContent.text());
}
- ListIterator<Token> temp =
tokens.listIterator(tokenIterator.previousIndex());
- return temp.next();
+ return tokens.get(tokenIterator.previousIndex()-1);
}
String generateFragment() {
@@ -1336,10 +1394,11 @@
/**
* Get the position for the last character returned from {@link #next()}.
+ * @param startIndex
*
* @return the position of the last character returned; never null
*/
- Position position();
+ Position position(int startIndex);
/**
* Determine if the next character on the sream is a {@link
Character#isWhitespace(char) whitespace character}. This
@@ -1495,6 +1554,14 @@
* @return true if the token's value matches the supplied character value, or
false otherwise
*/
boolean matches( char expected );
+
+ /**
+ * Determine if the token matches the supplied type.
+ *
+ * @param expectedType the expected integer type
+ * @return true if the token's value matches the supplied integer type, or
false otherwise
+ */
+ boolean matches( int expectedType );
/**
* Get the type of the token.
@@ -1530,6 +1597,14 @@
* @return the position; never null
*/
Position position();
+
+ /**
+ * Bitmask ORed with existing type value.
+ *
+ * @param typeMask
+ * @return copy of Token with new type
+ */
+ Token withType( int typeMask);
}
/**
@@ -1553,6 +1628,17 @@
}
/**
+ * {@inheritDoc}
+ *
+ * @see org.jboss.dna.common.text.TokenStream.Token#withType(int)
+ */
+ @Override
+ public Token withType(int typeMask) {
+ int type = this.type | typeMask;
+ return new CaseSensitiveToken(startIndex, endIndex, type, position);
+ }
+
+ /**
* {@inheritDoc}
*
* @see org.jboss.dna.common.text.TokenStream.Token#type()
@@ -1607,6 +1693,16 @@
}
/**
+ * {@inheritDoc}
+ *
+ * @see org.jboss.dna.common.text.TokenStream.Token#matches(int)
+ */
+ @Override
+ public final boolean matches(int expectedType) {
+ return expectedType == ANY_TYPE || (currentToken().type() & expectedType) ==
expectedType;
+ }
+
+ /**
* {@inheritDoc}
*
* @see org.jboss.dna.common.text.TokenStream.Token#value()
@@ -1657,6 +1753,17 @@
protected String matchString() {
return inputUppercased;
}
+
+ /**
+ * {@inheritDoc}
+ *
+ * @see org.jboss.dna.common.text.TokenStream.Token#withType(int)
+ */
+ @Override
+ public Token withType(int typeMask) {
+ int type = this.type() | typeMask;
+ return new CaseInsensitiveToken(startIndex(), endIndex(), type, position());
+ }
}
protected abstract class TokenFactory implements Tokens {
@@ -1755,11 +1862,13 @@
/**
* {@inheritDoc}
+ * @param startIndex
+ * @return the position of the token. never null
*
- * @see org.jboss.dna.common.text.TokenStream.CharacterStream#position()
+ * @see org.jboss.dna.common.text.TokenStream.CharacterStream#position(int)
*/
- public Position position() {
- return new Position(lineNumber, columnNumber);
+ public Position position(int startIndex) {
+ return new Position(startIndex, lineNumber, columnNumber);
}
/**
@@ -1776,10 +1885,10 @@
if (result == '\r') {
nextCharMayBeLineFeed = true;
++lineNumber;
- columnNumber = 1;
+ columnNumber = 0;
} else if (result == '\n') {
if (!nextCharMayBeLineFeed) ++lineNumber;
- columnNumber = 1;
+ columnNumber = 0;
} else if (nextCharMayBeLineFeed) {
nextCharMayBeLineFeed = false;
}
@@ -1922,22 +2031,22 @@
/**
* The {@link Token#type() token type} for tokens that consist of an individual
'.' character.
*/
- public static final int DECIMAL = 3;
+ public static final int DECIMAL = 4;
/**
* The {@link Token#type() token type} for tokens that consist of all the
characters within single-quotes. Single quote
* characters are included if they are preceded (escaped) by a '\'
character.
*/
- public static final int SINGLE_QUOTED_STRING = 4;
+ public static final int SINGLE_QUOTED_STRING = 8;
/**
* The {@link Token#type() token type} for tokens that consist of all the
characters within double-quotes. Double quote
* characters are included if they are preceded (escaped) by a '\'
character.
*/
- public static final int DOUBLE_QUOTED_STRING = 5;
+ public static final int DOUBLE_QUOTED_STRING = 16;
/**
* The {@link Token#type() token type} for tokens that consist of all the
characters between "/*" and "*/" or between
* "//" and the next line terminator (e.g., '\n', '\r'
or "\r\n").
*/
- public static final int COMMENT = 6;
+ public static final int COMMENT = 32;
private final boolean useComments;
@@ -1981,14 +2090,14 @@
case '|':
case '=':
case ':':
- tokens.addToken(input.position(), input.index(), input.index() +
1, SYMBOL);
+ tokens.addToken(input.position(input.index()), input.index(),
input.index() + 1, SYMBOL);
break;
case '.':
- tokens.addToken(input.position(), input.index(), input.index() +
1, DECIMAL);
+ tokens.addToken(input.position(input.index()), input.index(),
input.index() + 1, DECIMAL);
break;
case '\"':
int startIndex = input.index();
- Position startingPosition = input.position();
+ Position startingPosition = input.position(startIndex);
boolean foundClosingQuote = false;
while (input.hasNext()) {
c = input.next();
@@ -2005,11 +2114,11 @@
throw new ParsingException(startingPosition, msg);
}
int endIndex = input.index() + 1; // beyond last character read
- tokens.addToken(input.position(), startIndex, endIndex,
DOUBLE_QUOTED_STRING);
+ tokens.addToken(startingPosition, startIndex, endIndex,
DOUBLE_QUOTED_STRING);
break;
case '\'':
startIndex = input.index();
- startingPosition = input.position();
+ startingPosition = input.position(startIndex);
foundClosingQuote = false;
while (input.hasNext()) {
c = input.next();
@@ -2026,10 +2135,11 @@
throw new ParsingException(startingPosition, msg);
}
endIndex = input.index() + 1; // beyond last character read
- tokens.addToken(input.position(), startIndex, endIndex,
SINGLE_QUOTED_STRING);
+ tokens.addToken(startingPosition, startIndex, endIndex,
SINGLE_QUOTED_STRING);
break;
case '/':
startIndex = input.index();
+ startingPosition = input.position(startIndex);
if (input.isNext('/')) {
// End-of-line comment ...
boolean foundLineTerminator = false;
@@ -2044,7 +2154,7 @@
if (!foundLineTerminator) ++endIndex; // must point beyond
last char
if (c == '\r' && input.isNext('\n'))
input.next();
if (useComments) {
- tokens.addToken(input.position(), startIndex, endIndex,
COMMENT);
+ tokens.addToken(startingPosition, startIndex, endIndex,
COMMENT);
}
} else if (input.isNext('*')) {
// Multi-line comment ...
@@ -2055,21 +2165,22 @@
if (input.hasNext()) input.next(); // consume the
'/'
if (useComments) {
endIndex = input.index() + 1; // the token will include
the '/' and '*' characters
- tokens.addToken(input.position(), startIndex, endIndex,
COMMENT);
+ tokens.addToken(startingPosition, startIndex, endIndex,
COMMENT);
}
} else {
// just a regular slash ...
- tokens.addToken(input.position(), startIndex, startIndex + 1,
SYMBOL);
+ tokens.addToken(startingPosition, startIndex, startIndex + 1,
SYMBOL);
}
break;
default:
startIndex = input.index();
+ startingPosition = input.position(startIndex);
// Read until another whitespace/symbol/decimal/slash is found
while (input.hasNext() && !(input.isNextWhitespace() ||
input.isNextAnyOf("/.-(){}*,;+%?$[]!<>|=:"))) {
c = input.next();
}
endIndex = input.index() + 1; // beyond last character that was
included
- tokens.addToken(input.position(), startIndex, endIndex, WORD);
+ tokens.addToken(startingPosition, startIndex, endIndex, WORD);
}
}
}
Modified:
branches/ddl_sequencer/dna-common/src/main/resources/org/jboss/dna/common/CommonI18n.properties
===================================================================
---
branches/ddl_sequencer/dna-common/src/main/resources/org/jboss/dna/common/CommonI18n.properties 2009-11-18
19:42:04 UTC (rev 1330)
+++
branches/ddl_sequencer/dna-common/src/main/resources/org/jboss/dna/common/CommonI18n.properties 2009-11-19
23:07:26 UTC (rev 1331)
@@ -93,3 +93,4 @@
expectingValidIntegerAtLineAndColumn = Expecting a valid integer value but found
'{0}' at line {1}, column {2}
expectingValidLongAtLineAndColumn = Expecting a valid long value but found '{0}'
at line {1}, column {2}
expectingValidBooleanAtLineAndColumn = Expecting a valid boolean value but found
'{0}' at line {1}, column {2}
+endPositionMustBeGreaterThanStartingPosition = End position {1} must be greater than
starting position {0}
Added:
branches/ddl_sequencer/dna-common/src/test/java/org/jboss/dna/common/text/PositionTest.java
===================================================================
---
branches/ddl_sequencer/dna-common/src/test/java/org/jboss/dna/common/text/PositionTest.java
(rev 0)
+++
branches/ddl_sequencer/dna-common/src/test/java/org/jboss/dna/common/text/PositionTest.java 2009-11-19
23:07:26 UTC (rev 1331)
@@ -0,0 +1,85 @@
+/*
+ * JBoss DNA (http://www.jboss.org/dna)
+ * See the COPYRIGHT.txt file distributed with this work for information
+ * regarding copyright ownership. Some portions may be licensed
+ * to Red Hat, Inc. under one or more contributor license agreements.
+ * See the AUTHORS.txt file in the distribution for a full listing of
+ * individual contributors.
+ *
+ * JBoss DNA is free software. Unless otherwise indicated, all code in JBoss DNA
+ * is licensed to you under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * JBoss DNA is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this software; if not, write to the Free
+ * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
+ */
+package org.jboss.dna.common.text;
+
+import static org.junit.Assert.assertThat;
+
+import static org.hamcrest.core.Is.is;
+
+import org.junit.Test;
+
+
+public class PositionTest {
+
+ private int combinedIndex(int firstIndex, int secondIndex) {
+ Position first = new Position(firstIndex, 1, 0);
+ Position second = new Position(secondIndex, 1, 0);
+
+ int firstPlusSecond = first.add(second).getIndexInContent();
+ int secondPlusFirst = second.add(first).getIndexInContent();
+
+ assertThat(firstPlusSecond, is(secondPlusFirst));
+
+ return firstPlusSecond;
+ }
+
+ @Test
+ public void shouldAddNoContentPositionToValidPosition() {
+ // -1 to >=0
+ assertThat(combinedIndex(-1, 0), is(0));
+ assertThat(combinedIndex(-1, 1), is(1));
+ assertThat(combinedIndex(-1, 10), is(10));
+ }
+
+ @Test
+ public void shouldAddValidPositionToNoContentPosition() {
+ // >= 0 to -1
+ assertThat(combinedIndex(0, -1), is(0));
+ assertThat(combinedIndex(1, -1), is(1));
+ assertThat(combinedIndex(10, -1), is(10));
+ }
+
+ @Test
+ public void shouldAddValidPositionToValidPosition() {
+ // positive to positive
+ assertThat(combinedIndex(1, 1), is(2));
+ assertThat(combinedIndex(10, 1), is(11));
+ assertThat(combinedIndex(1, 10), is(11));
+ assertThat(combinedIndex(10, 10), is(20));
+ }
+
+ @Test
+ public void shouldAddStartingPositionToStartingPosition() {
+ // 0 to 0
+ assertThat(combinedIndex(0, 0), is(0));
+ }
+
+ @Test
+ public void shouldAddNoContentPositionToNoContentPosition() {
+ // -1 to -1
+ assertThat(combinedIndex(-1, -1), is(-1));
+ assertThat(combinedIndex(-10, -1), is(-1));
+ assertThat(combinedIndex(-1, -10), is(-1));
+ }
+}
Property changes on:
branches/ddl_sequencer/dna-common/src/test/java/org/jboss/dna/common/text/PositionTest.java
___________________________________________________________________
Name: svn:mime-type
+ text/plain
Modified:
branches/ddl_sequencer/dna-common/src/test/java/org/jboss/dna/common/text/TokenStreamTest.java
===================================================================
---
branches/ddl_sequencer/dna-common/src/test/java/org/jboss/dna/common/text/TokenStreamTest.java 2009-11-18
19:42:04 UTC (rev 1330)
+++
branches/ddl_sequencer/dna-common/src/test/java/org/jboss/dna/common/text/TokenStreamTest.java 2009-11-19
23:07:26 UTC (rev 1331)
@@ -25,7 +25,9 @@
import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertThat;
+
import java.util.Arrays;
+
import org.jboss.dna.common.text.TokenStream.BasicTokenizer;
import org.jboss.dna.common.text.TokenStream.Tokenizer;
import org.junit.Before;
@@ -336,6 +338,19 @@
assertThat(tokens.canConsume("FROM", "THIS",
"TABLE"), is(true));
assertThat(tokens.hasNext(), is(false));
}
+
+ @Test
+ public void shouldReturnTrueFromMatchIfAllTypeValuesMatch() {
+ makeCaseInsensitive();
+ assertThat(tokens.matches(BasicTokenizer.WORD, BasicTokenizer.WORD), is(true));
+ }
+
+ @Test
+ public void shouldReturnFalseFromMatchIfAllTypeValuesDoNotMatch() {
+ makeCaseInsensitive();
+ assertThat(tokens.matches(BasicTokenizer.WORD, BasicTokenizer.DECIMAL),
is(false));
+ assertThat(tokens.matches(BasicTokenizer.DECIMAL, BasicTokenizer.WORD),
is(false));
+ }
@Test
public void shouldConsumeMultipleTokensWithAnyValueConstant() {
@@ -397,4 +412,51 @@
assertThat(tokens.canConsume(TokenStream.ANY_VALUE), is(false));
assertThat(tokens.canConsume(BasicTokenizer.SYMBOL), is(false));
}
+
+ @Test
+ public void shouldFindNextPositionStartIndex() {
+ makeCaseInsensitive();
+ // "Select all columns from this table";
+ tokens.consume();
+ // Next position should be line 1, column 8
+ assertThat(tokens.nextPosition().getIndexInContent(), is(7));
+ assertThat(tokens.nextPosition().getColumn(), is(8));
+ assertThat(tokens.nextPosition().getLine(), is(1));
+ }
+
+ @Test
+ public void shouldFindPreviousPositionStartIndex() {
+ makeCaseInsensitive();
+ // "Select all columns from this table";
+ tokens.consume();
+ tokens.consume();
+ // previous position should be line 1, column 8
+ assertThat(tokens.previousPosition().getIndexInContent(), is(7));
+ assertThat(tokens.previousPosition().getColumn(), is(8));
+ assertThat(tokens.previousPosition().getLine(), is(1));
+ }
+
+ @Test
+ public void shouldParseMultiLineString() {
+ makeCaseInsensitive();
+ String content = "ALTER DATABASE \n"
+ + "DO SOMETHING; \n"
+ + "ALTER DATABASE \n"
+ + " SET DEFAULT BIGFILE TABLESPACE;";
+ tokens = new TokenStream(content, tokenizer, true);
+ tokens.start();
+
+ tokens.consume(); // LINE
+ tokens.consume(); // ONE
+ tokens.consume(); // DO
+ tokens.consume(); // SOMETHING
+ tokens.consume(); // ;
+
+ assertThat(tokens.nextPosition().getIndexInContent(), is(31));
+ assertThat(tokens.nextPosition().getColumn(), is(1));
+ tokens.consume(); // ALTER
+ assertThat(tokens.nextPosition().getIndexInContent(), is(37));
+ assertThat(tokens.nextPosition().getColumn(), is(7));
+
+ }
}