3:17 p.m.
Author: blafond
Date: 2009-11-24 16:17:39 -0500 (Tue, 24 Nov 2009)
New Revision: 1341
Added:
trunk/dna-common/src/test/java/org/jboss/dna/common/text/PositionTest.java
Modified:
trunk/dna-cnd/src/main/java/org/jboss/dna/cnd/CndTokenizer.java
trunk/dna-common/src/main/java/org/jboss/dna/common/CommonI18n.java
trunk/dna-common/src/main/java/org/jboss/dna/common/text/Position.java
trunk/dna-common/src/main/java/org/jboss/dna/common/text/TokenStream.java
trunk/dna-common/src/main/resources/org/jboss/dna/common/CommonI18n.properties
trunk/dna-common/src/test/java/org/jboss/dna/common/text/TokenStreamTest.java
trunk/dna-graph/src/main/java/org/jboss/dna/graph/query/parse/SqlQueryParser.java
trunk/dna-graph/src/test/java/org/jboss/dna/graph/query/parse/SqlQueryParserTest.java
trunk/dna-jcr/src/main/java/org/jboss/dna/jcr/xpath/XPathParser.java
Log:
DNA-555 added "index in content" ability to token stream and position class.
Allows extracting and managing string segments from tokenized content.
Modified: trunk/dna-cnd/src/main/java/org/jboss/dna/cnd/CndTokenizer.java
===================================================================
--- trunk/dna-cnd/src/main/java/org/jboss/dna/cnd/CndTokenizer.java 2009-11-24 17:46:37
UTC (rev 1340)
+++ trunk/dna-cnd/src/main/java/org/jboss/dna/cnd/CndTokenizer.java 2009-11-24 21:17:39
UTC (rev 1341)
@@ -82,7 +82,7 @@
case '(':
case ')':
case ',':
- tokens.addToken(input.position(), input.index(), input.index() + 1,
SYMBOL);
+ tokens.addToken(input.position(input.index()), input.index(),
input.index() + 1, SYMBOL);
break;
// case '.':
// tokens.addToken(input.position(), input.index(), input.index() + 1,
DECIMAL);
@@ -90,7 +90,7 @@
case '{':
// Vendor extension, meant to be excluded
int startIndex = input.index();
- Position startingPosition = input.position();
+ Position startingPosition = input.position(startIndex);
boolean foundClosingBrace = false;
String vendorName = "";
while (input.hasNext()) {
@@ -114,7 +114,7 @@
break;
case '\"':
startIndex = input.index();
- startingPosition = input.position();
+ startingPosition = input.position(startIndex);
boolean foundClosingQuote = false;
while (input.hasNext()) {
c = input.next();
@@ -135,7 +135,7 @@
break;
case '\'':
startIndex = input.index();
- startingPosition = input.position();
+ startingPosition = input.position(startIndex);
foundClosingQuote = false;
while (input.hasNext()) {
c = input.next();
@@ -156,7 +156,7 @@
break;
case '/':
startIndex = input.index();
- startingPosition = input.position();
+ startingPosition = input.position(startIndex);
if (input.isNext('/')) {
// End-of-line comment ...
boolean foundLineTerminator = false;
@@ -211,7 +211,7 @@
// since these do not appear in other rules above, they will result
in one-character tokens.
//
startIndex = input.index();
- startingPosition = input.position();
+ startingPosition = input.position(startIndex);
// Read as long as there is a valid XML character ...
while (input.hasNext() && !(input.isNextWhitespace() ||
input.isNextAnyOf("[]<>=-+(),\"'/{*|"))) {
c = input.next();
Modified: trunk/dna-common/src/main/java/org/jboss/dna/common/CommonI18n.java
===================================================================
--- trunk/dna-common/src/main/java/org/jboss/dna/common/CommonI18n.java 2009-11-24
17:46:37 UTC (rev 1340)
+++ trunk/dna-common/src/main/java/org/jboss/dna/common/CommonI18n.java 2009-11-24
21:17:39 UTC (rev 1341)
@@ -104,6 +104,7 @@
public static I18n expectingValidIntegerAtLineAndColumn;
public static I18n expectingValidLongAtLineAndColumn;
public static I18n expectingValidBooleanAtLineAndColumn;
+ public static I18n endPositionMustBeGreaterThanStartingPosition;
static {
try {
Modified: trunk/dna-common/src/main/java/org/jboss/dna/common/text/Position.java
===================================================================
--- trunk/dna-common/src/main/java/org/jboss/dna/common/text/Position.java 2009-11-24
17:46:37 UTC (rev 1340)
+++ trunk/dna-common/src/main/java/org/jboss/dna/common/text/Position.java 2009-11-24
21:17:39 UTC (rev 1341)
@@ -7,14 +7,37 @@
*/
@Immutable
public final class Position {
+ /**
+ * The position is used when there is no content.
+ */
+ public final static Position EMPTY_CONTENT_POSITION = new Position(-1, 1, 0);
+
private final int line;
private final int column;
+ private final int indexInContent;
- public Position( int line,
+ public Position( int indexInContent,
+ int line,
int column ) {
+ this.indexInContent = indexInContent < 0 ? -1:indexInContent;
this.line = line;
this.column = column;
+
+ assert this.indexInContent >= -1;
+ assert this.line > 0;
+ assert this.column >= 0;
+ // make sure that negative index means an EMPTY_CONTENT_POSITION
+ assert this.indexInContent < 0 ? this.line == 1 && this.column == 0:
true;
}
+
+ /**
+ * Get the 0-based index of this position in the content character array.
+ *
+ * @return the index; never negative except for the first position in an empty
content.
+ */
+ public int getIndexInContent() {
+ return indexInContent;
+ }
/**
* Get the 1-based column number of the character.
@@ -41,7 +64,7 @@
*/
@Override
public int hashCode() {
- return line;
+ return indexInContent;
}
/**
@@ -51,6 +74,28 @@
*/
@Override
public String toString() {
- return "" + line + ':' + column;
+ return "" + indexInContent + ':' + line + ':' +
column;
}
+
+ /**
+ * Return a new position that is the addition of this position and that supplied.
+ *
+ * @param position the position to add to this object; may not be null
+ * @return the combined position
+ */
+ public Position add(Position position) {
+ if( this.getIndexInContent() < 0 ) {
+ return position.getIndexInContent() < 0 ? EMPTY_CONTENT_POSITION:position;
+ }
+
+ if( position.getIndexInContent() < 0 ) {
+ return this;
+ }
+
+ int index = this.getIndexInContent() + position.getIndexInContent();
+ int line = position.getLine() + this.getLine() - 1;
+ int column = this.getLine() == 1 ? this.getColumn() + position.getColumn() :
this.getColumn();
+
+ return new Position(index, line, column);
+ }
}
Modified: trunk/dna-common/src/main/java/org/jboss/dna/common/text/TokenStream.java
===================================================================
--- trunk/dna-common/src/main/java/org/jboss/dna/common/text/TokenStream.java 2009-11-24
17:46:37 UTC (rev 1340)
+++ trunk/dna-common/src/main/java/org/jboss/dna/common/text/TokenStream.java 2009-11-24
21:17:39 UTC (rev 1341)
@@ -384,6 +384,20 @@
private final boolean caseSensitive;
private final Tokenizer tokenizer;
private List<Token> tokens;
+ /**
+ * This class navigates the Token objects using this iterator. However, because it
very often needs to access the
+ * "current token" in the "consume(...)" and
"canConsume(...)" and "matches(...)" methods, the class caches a
"current token"
+ * and makes this iterator point to the 2nd token.
+ *
+ * <pre>
+ * T1 T2 T3 T4 T5
+ * ˆ ˆ ˆ
+ * | | |
+ * | | +- The position of the tokenIterator, where tokenIterator.hasNext()
will return T3
+ * | +---- The token referenced by currentToken
+ * +-------- The logical position of the TokenStream object, where the
"consume()" would return T2
+ * </pre>
+ */
private ListIterator<Token> tokenIterator;
private Token currentToken;
private boolean completed;
@@ -412,7 +426,7 @@
TokenFactory tokenFactory = caseSensitive ? new CaseSensitiveTokenFactory() :
new CaseInsensitiveTokenFactory();
CharacterStream characterStream = new CharacterArrayStream(inputContent);
tokenizer.tokenize(characterStream, tokenFactory);
- this.tokens = tokenFactory.getTokens();
+ this.tokens = initializeTokens(tokenFactory.getTokens());
}
// Create the iterator ...
@@ -422,6 +436,28 @@
}
/**
+ * Method to allow subclasses to preprocess the set of tokens and return the correct
tokens to use. The default behavior is to
+ * simply return the supplied tokens.
+ *
+ * @param tokens
+ * @return list of tokens.
+ */
+ protected List<Token> initializeTokens( List<Token> tokens ) {
+ return tokens;
+ }
+
+ /**
+ * Method to allow tokens to be re-used from the start without re-tokenizing
content.
+ */
+ public void rewind() {
+ // recreate the iterator ...
+ tokenIterator = this.tokens.listIterator();
+ completed = false;
+ currentToken = null;
+ moveToNextToken();
+ }
+
+ /**
* Get the position of the previous token.
*
* @return the previous token's position; never null
@@ -437,7 +473,7 @@
*
* @return the current token's position; never null
* @throws IllegalStateException if this method was called before the stream was
{@link #start() started}
- * @throws NoSuchElementException if there is no next token
+ * @throws NoSuchElementException if there is no previous token
*/
public Position nextPosition() {
return currentToken().position();
@@ -526,7 +562,7 @@
protected void throwNoMoreContent() throws ParsingException {
String msg = CommonI18n.noMoreContent.text();
- Position pos = tokens.isEmpty() ? new Position(1, 0) : tokens.get(tokens.size() -
1).position();
+ Position pos = tokens.isEmpty() ? new Position(-1, 1, 0) :
tokens.get(tokens.size() - 1).position();
throw new ParsingException(pos, msg);
}
@@ -725,9 +761,12 @@
*
* <pre>
*
- * if ( tokens.matches(currentExpected,expectedForNextTokens) ) {
tokens.consume(currentExpected,expectedForNextTokens); }
+ * if (tokens.matches(currentExpected, expectedForNextTokens)) {
+ * tokens.consume(currentExpected, expectedForNextTokens);
+ * }
*
* </pre>
+ *
* </p>
* <p>
* The {@link #ANY_VALUE ANY_VALUE} constant can be used in the expected values as a
wildcard.
@@ -770,9 +809,12 @@
*
* <pre>
*
- * if ( tokens.matches(currentExpected,expectedForNextTokens) ) {
tokens.consume(currentExpected,expectedForNextTokens); }
+ * if (tokens.matches(currentExpected, expectedForNextTokens)) {
+ * tokens.consume(currentExpected, expectedForNextTokens);
+ * }
*
* </pre>
+ *
* </p>
* <p>
* The {@link #ANY_VALUE ANY_VALUE} constant can be used in the expected values as a
wildcard.
@@ -811,9 +853,12 @@
*
* <pre>
*
- * if ( tokens.matches(currentExpected,expectedForNextTokens) ) {
tokens.consume(currentExpected,expectedForNextTokens); }
+ * if (tokens.matches(currentExpected, expectedForNextTokens)) {
+ * tokens.consume(currentExpected, expectedForNextTokens);
+ * }
*
* </pre>
+ *
* </p>
* <p>
* The {@link #ANY_VALUE ANY_VALUE} constant can be used in the expected values as a
wildcard.
@@ -954,7 +999,7 @@
* @throws IllegalStateException if this method was called before the stream was
{@link #start() started}
*/
public boolean matches( int expectedType ) throws IllegalStateException {
- return !completed && (expectedType == ANY_TYPE || currentToken().type()
== expectedType);
+ return !completed && currentToken().matches(expectedType);
}
/**
@@ -1075,7 +1120,7 @@
if (!iter.hasNext()) return false;
token = iter.next();
if (nextExpectedType == ANY_TYPE) continue;
- if (token.type() != nextExpectedType) return false;
+ if (!token.matches(nextExpectedType)) return false;
}
return true;
}
@@ -1233,6 +1278,30 @@
}
/**
+ * Gets the content string starting at the first position (inclusive) and continuing
up to the end position (exclusive).
+ *
+ * @param starting the position marking the beginning of the desired content string.
+ * @param end the position located directly after the returned content string; can be
null, which means end of content
+ * @return the content string; never null
+ */
+ public String getContentBetween( Position starting,
+ Position end ) {
+ CheckArg.isNotNull(starting, "starting");
+
+ int startIndex = starting.getIndexInContent();
+ int endIndex = inputString.length();
+ if (end != null) {
+ endIndex = end.getIndexInContent();
+ }
+
+ if (startIndex >= endIndex) {
+ throw new
IllegalArgumentException(CommonI18n.endPositionMustBeGreaterThanStartingPosition.text(startIndex,
endIndex));
+ }
+
+ return inputString.substring(startIndex, endIndex);
+ }
+
+ /**
* Get the previous token. This does not modify the state.
*
* @return the previous token; never null
@@ -1249,11 +1318,10 @@
}
throw new
IllegalStateException(CommonI18n.startMethodMustBeCalledBeforeConsumingOrMatching.text());
}
- if (!tokenIterator.hasPrevious()) {
+ if (tokenIterator.previousIndex() == 0) {
throw new NoSuchElementException(CommonI18n.noMoreContent.text());
}
- ListIterator<Token> temp =
tokens.listIterator(tokenIterator.previousIndex());
- return temp.next();
+ return tokens.get(tokenIterator.previousIndex() - 1);
}
String generateFragment() {
@@ -1337,9 +1405,10 @@
/**
* Get the position for the last character returned from {@link #next()}.
*
+ * @param startIndex
* @return the position of the last character returned; never null
*/
- Position position();
+ Position position( int startIndex );
/**
* Determine if the next character on the sream is a {@link
Character#isWhitespace(char) whitespace character}. This
@@ -1513,6 +1582,14 @@
boolean matches( char expected );
/**
+ * Determine if the token matches the supplied type.
+ *
+ * @param expectedType the expected integer type
+ * @return true if the token's value matches the supplied integer type, or
false otherwise
+ */
+ boolean matches( int expectedType );
+
+ /**
* Get the type of the token.
*
* @return the token's type
@@ -1546,6 +1623,14 @@
* @return the position; never null
*/
Position position();
+
+ /**
+ * Bitmask ORed with existing type value.
+ *
+ * @param typeMask
+ * @return copy of Token with new type
+ */
+ Token withType( int typeMask );
}
/**
@@ -1571,6 +1656,16 @@
/**
* {@inheritDoc}
*
+ * @see org.jboss.dna.common.text.TokenStream.Token#withType(int)
+ */
+ public Token withType( int typeMask ) {
+ int type = this.type | typeMask;
+ return new CaseSensitiveToken(startIndex, endIndex, type, position);
+ }
+
+ /**
+ * {@inheritDoc}
+ *
* @see org.jboss.dna.common.text.TokenStream.Token#type()
*/
public final int type() {
@@ -1625,6 +1720,15 @@
/**
* {@inheritDoc}
*
+ * @see org.jboss.dna.common.text.TokenStream.Token#matches(int)
+ */
+ public final boolean matches( int expectedType ) {
+ return expectedType == ANY_TYPE || (currentToken().type() & expectedType)
== expectedType;
+ }
+
+ /**
+ * {@inheritDoc}
+ *
* @see org.jboss.dna.common.text.TokenStream.Token#value()
*/
public final String value() {
@@ -1673,6 +1777,17 @@
protected String matchString() {
return inputUppercased;
}
+
+ /**
+ * {@inheritDoc}
+ *
+ * @see org.jboss.dna.common.text.TokenStream.Token#withType(int)
+ */
+ @Override
+ public Token withType( int typeMask ) {
+ int type = this.type() | typeMask;
+ return new CaseInsensitiveToken(startIndex(), endIndex(), type, position());
+ }
}
protected abstract class TokenFactory implements Tokens {
@@ -1772,10 +1887,12 @@
/**
* {@inheritDoc}
*
- * @see org.jboss.dna.common.text.TokenStream.CharacterStream#position()
+ * @param startIndex
+ * @return the position of the token. never null
+ * @see org.jboss.dna.common.text.TokenStream.CharacterStream#position(int)
*/
- public Position position() {
- return new Position(lineNumber, columnNumber);
+ public Position position( int startIndex ) {
+ return new Position(startIndex, lineNumber, columnNumber);
}
/**
@@ -1792,10 +1909,10 @@
if (result == '\r') {
nextCharMayBeLineFeed = true;
++lineNumber;
- columnNumber = 1;
+ columnNumber = 0;
} else if (result == '\n') {
if (!nextCharMayBeLineFeed) ++lineNumber;
- columnNumber = 1;
+ columnNumber = 0;
} else if (nextCharMayBeLineFeed) {
nextCharMayBeLineFeed = false;
}
@@ -1958,22 +2075,22 @@
/**
* The {@link Token#type() token type} for tokens that consist of an individual
'.' character.
*/
- public static final int DECIMAL = 3;
+ public static final int DECIMAL = 4;
/**
* The {@link Token#type() token type} for tokens that consist of all the
characters within single-quotes. Single quote
* characters are included if they are preceded (escaped) by a '\'
character.
*/
- public static final int SINGLE_QUOTED_STRING = 4;
+ public static final int SINGLE_QUOTED_STRING = 8;
/**
* The {@link Token#type() token type} for tokens that consist of all the
characters within double-quotes. Double quote
* characters are included if they are preceded (escaped) by a '\'
character.
*/
- public static final int DOUBLE_QUOTED_STRING = 5;
+ public static final int DOUBLE_QUOTED_STRING = 16;
/**
* The {@link Token#type() token type} for tokens that consist of all the
characters between "/*" and "*/" or between
* "//" and the next line terminator (e.g., '\n', '\r'
or "\r\n").
*/
- public static final int COMMENT = 6;
+ public static final int COMMENT = 32;
private final boolean useComments;
@@ -2017,14 +2134,14 @@
case '|':
case '=':
case ':':
- tokens.addToken(input.position(), input.index(), input.index() +
1, SYMBOL);
+ tokens.addToken(input.position(input.index()), input.index(),
input.index() + 1, SYMBOL);
break;
case '.':
- tokens.addToken(input.position(), input.index(), input.index() +
1, DECIMAL);
+ tokens.addToken(input.position(input.index()), input.index(),
input.index() + 1, DECIMAL);
break;
case '\"':
int startIndex = input.index();
- Position startingPosition = input.position();
+ Position startingPosition = input.position(startIndex);
boolean foundClosingQuote = false;
while (input.hasNext()) {
c = input.next();
@@ -2041,11 +2158,11 @@
throw new ParsingException(startingPosition, msg);
}
int endIndex = input.index() + 1; // beyond last character read
- tokens.addToken(input.position(), startIndex, endIndex,
DOUBLE_QUOTED_STRING);
+ tokens.addToken(startingPosition, startIndex, endIndex,
DOUBLE_QUOTED_STRING);
break;
case '\'':
startIndex = input.index();
- startingPosition = input.position();
+ startingPosition = input.position(startIndex);
foundClosingQuote = false;
while (input.hasNext()) {
c = input.next();
@@ -2062,10 +2179,11 @@
throw new ParsingException(startingPosition, msg);
}
endIndex = input.index() + 1; // beyond last character read
- tokens.addToken(input.position(), startIndex, endIndex,
SINGLE_QUOTED_STRING);
+ tokens.addToken(startingPosition, startIndex, endIndex,
SINGLE_QUOTED_STRING);
break;
case '/':
startIndex = input.index();
+ startingPosition = input.position(startIndex);
if (input.isNext('/')) {
// End-of-line comment ...
boolean foundLineTerminator = false;
@@ -2080,7 +2198,7 @@
if (!foundLineTerminator) ++endIndex; // must point beyond
last char
if (c == '\r' && input.isNext('\n'))
input.next();
if (useComments) {
- tokens.addToken(input.position(), startIndex, endIndex,
COMMENT);
+ tokens.addToken(startingPosition, startIndex, endIndex,
COMMENT);
}
} else if (input.isNext('*')) {
// Multi-line comment ...
@@ -2091,21 +2209,22 @@
if (input.hasNext()) input.next(); // consume the
'/'
if (useComments) {
endIndex = input.index() + 1; // the token will include
the '/' and '*' characters
- tokens.addToken(input.position(), startIndex, endIndex,
COMMENT);
+ tokens.addToken(startingPosition, startIndex, endIndex,
COMMENT);
}
} else {
// just a regular slash ...
- tokens.addToken(input.position(), startIndex, startIndex + 1,
SYMBOL);
+ tokens.addToken(startingPosition, startIndex, startIndex + 1,
SYMBOL);
}
break;
default:
startIndex = input.index();
+ startingPosition = input.position(startIndex);
// Read until another whitespace/symbol/decimal/slash is found
while (input.hasNext() && !(input.isNextWhitespace() ||
input.isNextAnyOf("/.-(){}*,;+%?$[]!<>|=:"))) {
c = input.next();
}
endIndex = input.index() + 1; // beyond last character that was
included
- tokens.addToken(input.position(), startIndex, endIndex, WORD);
+ tokens.addToken(startingPosition, startIndex, endIndex, WORD);
}
}
}
Modified: trunk/dna-common/src/main/resources/org/jboss/dna/common/CommonI18n.properties
===================================================================
---
trunk/dna-common/src/main/resources/org/jboss/dna/common/CommonI18n.properties 2009-11-24
17:46:37 UTC (rev 1340)
+++
trunk/dna-common/src/main/resources/org/jboss/dna/common/CommonI18n.properties 2009-11-24
21:17:39 UTC (rev 1341)
@@ -93,3 +93,4 @@
expectingValidIntegerAtLineAndColumn = Expecting a valid integer value but found
'{0}' at line {1}, column {2}
expectingValidLongAtLineAndColumn = Expecting a valid long value but found '{0}'
at line {1}, column {2}
expectingValidBooleanAtLineAndColumn = Expecting a valid boolean value but found
'{0}' at line {1}, column {2}
+endPositionMustBeGreaterThanStartingPosition = End position {1} must be greater than
starting position {0}
Added: trunk/dna-common/src/test/java/org/jboss/dna/common/text/PositionTest.java
===================================================================
--- trunk/dna-common/src/test/java/org/jboss/dna/common/text/PositionTest.java
(rev 0)
+++ trunk/dna-common/src/test/java/org/jboss/dna/common/text/PositionTest.java 2009-11-24
21:17:39 UTC (rev 1341)
@@ -0,0 +1,85 @@
+/*
+ * JBoss DNA (http://www.jboss.org/dna)
+ * See the COPYRIGHT.txt file distributed with this work for information
+ * regarding copyright ownership. Some portions may be licensed
+ * to Red Hat, Inc. under one or more contributor license agreements.
+ * See the AUTHORS.txt file in the distribution for a full listing of
+ * individual contributors.
+ *
+ * JBoss DNA is free software. Unless otherwise indicated, all code in JBoss DNA
+ * is licensed to you under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * JBoss DNA is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this software; if not, write to the Free
+ * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
+ */
+package org.jboss.dna.common.text;
+
+import static org.junit.Assert.assertThat;
+
+import static org.hamcrest.core.Is.is;
+
+import org.junit.Test;
+
+
+public class PositionTest {
+
+ private int combinedIndex(int firstIndex, int secondIndex) {
+ Position first = new Position(firstIndex, 1, 0);
+ Position second = new Position(secondIndex, 1, 0);
+
+ int firstPlusSecond = first.add(second).getIndexInContent();
+ int secondPlusFirst = second.add(first).getIndexInContent();
+
+ assertThat(firstPlusSecond, is(secondPlusFirst));
+
+ return firstPlusSecond;
+ }
+
+ @Test
+ public void shouldAddNoContentPositionToValidPosition() {
+ // -1 to >=0
+ assertThat(combinedIndex(-1, 0), is(0));
+ assertThat(combinedIndex(-1, 1), is(1));
+ assertThat(combinedIndex(-1, 10), is(10));
+ }
+
+ @Test
+ public void shouldAddValidPositionToNoContentPosition() {
+ // >= 0 to -1
+ assertThat(combinedIndex(0, -1), is(0));
+ assertThat(combinedIndex(1, -1), is(1));
+ assertThat(combinedIndex(10, -1), is(10));
+ }
+
+ @Test
+ public void shouldAddValidPositionToValidPosition() {
+ // positive to positive
+ assertThat(combinedIndex(1, 1), is(2));
+ assertThat(combinedIndex(10, 1), is(11));
+ assertThat(combinedIndex(1, 10), is(11));
+ assertThat(combinedIndex(10, 10), is(20));
+ }
+
+ @Test
+ public void shouldAddStartingPositionToStartingPosition() {
+ // 0 to 0
+ assertThat(combinedIndex(0, 0), is(0));
+ }
+
+ @Test
+ public void shouldAddNoContentPositionToNoContentPosition() {
+ // -1 to -1
+ assertThat(combinedIndex(-1, -1), is(-1));
+ assertThat(combinedIndex(-10, -1), is(-1));
+ assertThat(combinedIndex(-1, -10), is(-1));
+ }
+}
Property changes on:
trunk/dna-common/src/test/java/org/jboss/dna/common/text/PositionTest.java
___________________________________________________________________
Name: svn:mime-type
+ text/plain
Modified: trunk/dna-common/src/test/java/org/jboss/dna/common/text/TokenStreamTest.java
===================================================================
---
trunk/dna-common/src/test/java/org/jboss/dna/common/text/TokenStreamTest.java 2009-11-24
17:46:37 UTC (rev 1340)
+++
trunk/dna-common/src/test/java/org/jboss/dna/common/text/TokenStreamTest.java 2009-11-24
21:17:39 UTC (rev 1341)
@@ -336,6 +336,19 @@
assertThat(tokens.canConsume("FROM", "THIS",
"TABLE"), is(true));
assertThat(tokens.hasNext(), is(false));
}
+
+ @Test
+ public void shouldReturnTrueFromMatchIfAllTypeValuesMatch() {
+ makeCaseInsensitive();
+ assertThat(tokens.matches(BasicTokenizer.WORD, BasicTokenizer.WORD), is(true));
+ }
+
+ @Test
+ public void shouldReturnFalseFromMatchIfAllTypeValuesDoNotMatch() {
+ makeCaseInsensitive();
+ assertThat(tokens.matches(BasicTokenizer.WORD, BasicTokenizer.DECIMAL),
is(false));
+ assertThat(tokens.matches(BasicTokenizer.DECIMAL, BasicTokenizer.WORD),
is(false));
+ }
@Test
public void shouldConsumeMultipleTokensWithAnyValueConstant() {
@@ -397,4 +410,51 @@
assertThat(tokens.canConsume(TokenStream.ANY_VALUE), is(false));
assertThat(tokens.canConsume(BasicTokenizer.SYMBOL), is(false));
}
+
+ @Test
+ public void shouldFindNextPositionStartIndex() {
+ makeCaseInsensitive();
+ // "Select all columns from this table";
+ tokens.consume();
+ // Next position should be line 1, column 8
+ assertThat(tokens.nextPosition().getIndexInContent(), is(7));
+ assertThat(tokens.nextPosition().getColumn(), is(8));
+ assertThat(tokens.nextPosition().getLine(), is(1));
+ }
+
+ @Test
+ public void shouldFindPreviousPositionStartIndex() {
+ makeCaseInsensitive();
+ // "Select all columns from this table";
+ tokens.consume();
+ tokens.consume();
+ // previous position should be line 1, column 8
+ assertThat(tokens.previousPosition().getIndexInContent(), is(7));
+ assertThat(tokens.previousPosition().getColumn(), is(8));
+ assertThat(tokens.previousPosition().getLine(), is(1));
+ }
+
+ @Test
+ public void shouldParseMultiLineString() {
+ makeCaseInsensitive();
+ String content = "ALTER DATABASE \n"
+ + "DO SOMETHING; \n"
+ + "ALTER DATABASE \n"
+ + " SET DEFAULT BIGFILE TABLESPACE;";
+ tokens = new TokenStream(content, tokenizer, true);
+ tokens.start();
+
+ tokens.consume(); // LINE
+ tokens.consume(); // ONE
+ tokens.consume(); // DO
+ tokens.consume(); // SOMETHING
+ tokens.consume(); // ;
+
+ assertThat(tokens.nextPosition().getIndexInContent(), is(31));
+ assertThat(tokens.nextPosition().getColumn(), is(1));
+ tokens.consume(); // ALTER
+ assertThat(tokens.nextPosition().getIndexInContent(), is(37));
+ assertThat(tokens.nextPosition().getColumn(), is(7));
+
+ }
}
Modified:
trunk/dna-graph/src/main/java/org/jboss/dna/graph/query/parse/SqlQueryParser.java
===================================================================
---
trunk/dna-graph/src/main/java/org/jboss/dna/graph/query/parse/SqlQueryParser.java 2009-11-24
17:46:37 UTC (rev 1340)
+++
trunk/dna-graph/src/main/java/org/jboss/dna/graph/query/parse/SqlQueryParser.java 2009-11-24
21:17:39 UTC (rev 1341)
@@ -799,15 +799,12 @@
}
protected Term parseFullTextSearchExpression( String expression,
- Position position ) {
+ Position startOfExpression ) {
try {
return new FullTextSearchParser().parse(expression);
} catch (ParsingException e) {
// Convert the position in the exception into a position in the query.
- Position exprPos = e.getPosition();
- int line = position.getLine() + exprPos.getLine() - 1;
- int column = exprPos.getLine() == 1 ? exprPos.getColumn() +
position.getColumn() : exprPos.getColumn();
- Position queryPos = new Position(line, column);
+ Position queryPos = startOfExpression.add(e.getPosition());
throw new ParsingException(queryPos, e.getMessage());
}
}
@@ -1264,14 +1261,14 @@
case '|':
case '=':
case ':':
- tokens.addToken(input.position(), input.index(), input.index() +
1, SYMBOL);
+ tokens.addToken(input.position(input.index()), input.index(),
input.index() + 1, SYMBOL);
break;
case '\'':
case '[':
case '\"':
int startIndex = input.index();
char closingChar = c == '[' ? ']' : c;
- Position pos = input.position();
+ Position pos = input.position(startIndex);
boolean foundClosingQuote = false;
while (input.hasNext()) {
c = input.next();
@@ -1296,7 +1293,7 @@
break;
case '-':
startIndex = input.index();
- pos = input.position();
+ pos = input.position(input.index());
if (input.isNext('-')) {
// End-of-line comment ...
boolean foundLineTerminator = false;
@@ -1314,13 +1311,13 @@
tokens.addToken(pos, startIndex, endIndex, COMMENT);
}
} else {
- tokens.addToken(input.position(), input.index(),
input.index() + 1, SYMBOL);
+ tokens.addToken(input.position(input.index()), input.index(),
input.index() + 1, SYMBOL);
break;
}
break;
case '/':
startIndex = input.index();
- pos = input.position();
+ pos = input.position(input.index());
if (input.isNext('*')) {
// Multi-line comment ...
while (input.hasNext() && !input.isNext('*',
'/')) {
@@ -1333,13 +1330,13 @@
tokens.addToken(pos, startIndex, endIndex, COMMENT);
}
} else {
- tokens.addToken(input.position(), input.index(),
input.index() + 1, SYMBOL);
+ tokens.addToken(input.position(input.index()), input.index(),
input.index() + 1, SYMBOL);
break;
}
break;
default:
startIndex = input.index();
- pos = input.position();
+ pos = input.position(input.index());
// Read as long as there is a valid XML character ...
int tokenType = (Character.isLetterOrDigit(c) || c ==
'_') ? WORD : OTHER;
while (input.isNextLetterOrDigit() || input.isNext('_'))
{
Modified:
trunk/dna-graph/src/test/java/org/jboss/dna/graph/query/parse/SqlQueryParserTest.java
===================================================================
---
trunk/dna-graph/src/test/java/org/jboss/dna/graph/query/parse/SqlQueryParserTest.java 2009-11-24
17:46:37 UTC (rev 1340)
+++
trunk/dna-graph/src/test/java/org/jboss/dna/graph/query/parse/SqlQueryParserTest.java 2009-11-24
21:17:39 UTC (rev 1341)
@@ -789,8 +789,8 @@
@Test
public void shouldParseFullTextSearchExpressionFromStringWithValidExpression() {
- Position pos = new Position(100, 13);
- Term result = parser.parseFullTextSearchExpression("term1 term2 OR -term3 OR
-term4 OR term5", pos);
+ Position pos = new Position(500, 100, 13);
+ FullTextSearch.Term result = parser.parseFullTextSearchExpression("term1
term2 OR -term3 OR -term4 OR term5", pos);
assertThat(result, is(notNullValue()));
assertThat(result, is(instanceOf(Disjunction.class)));
Disjunction disjunction = (Disjunction)result;
@@ -808,11 +808,12 @@
@Test
public void shouldConvertPositionWhenUnableToParseFullTextSearchExpression() {
try {
- parser.parseFullTextSearchExpression("", new Position(100, 13));
+ parser.parseFullTextSearchExpression("", new Position(500, 100,
13));
fail("Should have thrown an exception");
} catch (ParsingException e) {
assertThat(e.getPosition().getLine(), is(100));
assertThat(e.getPosition().getColumn(), is(13));
+ assertThat(e.getPosition().getIndexInContent(), is(500));
}
}
Modified: trunk/dna-jcr/src/main/java/org/jboss/dna/jcr/xpath/XPathParser.java
===================================================================
--- trunk/dna-jcr/src/main/java/org/jboss/dna/jcr/xpath/XPathParser.java 2009-11-24
17:46:37 UTC (rev 1340)
+++ trunk/dna-jcr/src/main/java/org/jboss/dna/jcr/xpath/XPathParser.java 2009-11-24
21:17:39 UTC (rev 1341)
@@ -751,13 +751,13 @@
case '\\':
case '#':
case '@':
- tokens.addToken(input.position(), input.index(), input.index() +
1, SYMBOL);
+ tokens.addToken(input.position(input.index()), input.index(),
input.index() + 1, SYMBOL);
break;
case '\'':
case '\"':
int startIndex = input.index();
char closingChar = c;
- Position pos = input.position();
+ Position pos = input.position(startIndex);
boolean foundClosingQuote = false;
while (input.hasNext()) {
c = input.next();
@@ -782,7 +782,7 @@
startIndex = input.index();
if (input.isNext(':')) {
// This is a comment ...
- pos = input.position();
+ pos = input.position(startIndex);
while (input.hasNext() && !input.isNext(':',
')')) {
c = input.next();
}
@@ -793,13 +793,13 @@
tokens.addToken(pos, startIndex, endIndex, COMMENT);
}
} else {
- tokens.addToken(input.position(), input.index(),
input.index() + 1, SYMBOL);
+ tokens.addToken(input.position(startIndex), input.index(),
input.index() + 1, SYMBOL);
break;
}
break;
default:
startIndex = input.index();
- pos = input.position();
+ pos = input.position(startIndex);
// Read as long as there is a valid XML character ...
int tokenType = (XmlCharacters.isValidNcNameStart(c)) ? NAME :
OTHER;
while (input.isNextValidXmlNcNameCharacter()) {