6:59 a.m.
Author: blafond
Date: 2009-11-25 07:59:39 -0500 (Wed, 25 Nov 2009)
New Revision: 1343
Modified:
branches/ddl_sequencer/dna-common/src/main/java/org/jboss/dna/common/text/TokenStream.java
Log:
DNA-555 Fixed problem in token positions having the wrong column number.
Modified:
branches/ddl_sequencer/dna-common/src/main/java/org/jboss/dna/common/text/TokenStream.java
===================================================================
---
branches/ddl_sequencer/dna-common/src/main/java/org/jboss/dna/common/text/TokenStream.java 2009-11-25
00:59:30 UTC (rev 1342)
+++
branches/ddl_sequencer/dna-common/src/main/java/org/jboss/dna/common/text/TokenStream.java 2009-11-25
12:59:39 UTC (rev 1343)
@@ -395,7 +395,7 @@
* | | |
* | | +- The position of the tokenIterator, where tokenIterator.hasNext()
will return T3
* | +---- The token referenced by currentToken
- * +-------- The logical position of the TokenStream object, where the
"consume()" would return T2
+ * +-------- The logical position of the TokenStream object, where the
"consume()" would return T2
* </pre>
*/
private ListIterator<Token> tokenIterator;
@@ -434,18 +434,18 @@
moveToNextToken();
return this;
}
-
+
/**
- * Method to allow subclasses to preprocess the set of tokens and return the correct
tokens to use.
- * The default behavior is to simply return the supplied tokens.
+ * Method to allow subclasses to preprocess the set of tokens and return the correct
tokens to use. The default behavior is to
+ * simply return the supplied tokens.
*
* @param tokens
* @return list of tokens.
*/
- protected List<Token> initializeTokens( List<Token> tokens) {
- return tokens;
+ protected List<Token> initializeTokens( List<Token> tokens ) {
+ return tokens;
}
-
+
/**
* Method to allow tokens to be re-used from the start without re-tokenizing
content.
*/
@@ -761,9 +761,12 @@
*
* <pre>
*
- * if ( tokens.matches(currentExpected,expectedForNextTokens) ) {
tokens.consume(currentExpected,expectedForNextTokens); }
+ * if (tokens.matches(currentExpected, expectedForNextTokens)) {
+ * tokens.consume(currentExpected, expectedForNextTokens);
+ * }
*
* </pre>
+ *
* </p>
* <p>
* The {@link #ANY_VALUE ANY_VALUE} constant can be used in the expected values as a
wildcard.
@@ -806,9 +809,12 @@
*
* <pre>
*
- * if ( tokens.matches(currentExpected,expectedForNextTokens) ) {
tokens.consume(currentExpected,expectedForNextTokens); }
+ * if (tokens.matches(currentExpected, expectedForNextTokens)) {
+ * tokens.consume(currentExpected, expectedForNextTokens);
+ * }
*
* </pre>
+ *
* </p>
* <p>
* The {@link #ANY_VALUE ANY_VALUE} constant can be used in the expected values as a
wildcard.
@@ -847,9 +853,12 @@
*
* <pre>
*
- * if ( tokens.matches(currentExpected,expectedForNextTokens) ) {
tokens.consume(currentExpected,expectedForNextTokens); }
+ * if (tokens.matches(currentExpected, expectedForNextTokens)) {
+ * tokens.consume(currentExpected, expectedForNextTokens);
+ * }
*
* </pre>
+ *
* </p>
* <p>
* The {@link #ANY_VALUE ANY_VALUE} constant can be used in the expected values as a
wildcard.
@@ -1272,25 +1281,26 @@
* Gets the content string starting at the first position (inclusive) and continuing
up to the end position (exclusive).
*
* @param starting the position marking the beginning of the desired content string.
- * @param end the position located directly after the returned content string; can
be null, which means end of content
+ * @param end the position located directly after the returned content string; can be
null, which means end of content
* @return the content string; never null
*/
- public String getContentBetween(Position starting, Position end) {
- CheckArg.isNotNull(starting, "starting");
-
- int startIndex = starting.getIndexInContent();
- int endIndex = inputString.length();
- if( end != null ) {
- endIndex = end.getIndexInContent();
- }
-
- if( startIndex >= endIndex ) {
- throw new
IllegalArgumentException(CommonI18n.endPositionMustBeGreaterThanStartingPosition.text(startIndex,
endIndex));
- }
-
- return inputString.substring(startIndex, endIndex);
+ public String getContentBetween( Position starting,
+ Position end ) {
+ CheckArg.isNotNull(starting, "starting");
+
+ int startIndex = starting.getIndexInContent();
+ int endIndex = inputString.length();
+ if (end != null) {
+ endIndex = end.getIndexInContent();
+ }
+
+ if (startIndex >= endIndex) {
+ throw new
IllegalArgumentException(CommonI18n.endPositionMustBeGreaterThanStartingPosition.text(startIndex,
endIndex));
+ }
+
+ return inputString.substring(startIndex, endIndex);
}
-
+
/**
* Get the previous token. This does not modify the state.
*
@@ -1308,10 +1318,10 @@
}
throw new
IllegalStateException(CommonI18n.startMethodMustBeCalledBeforeConsumingOrMatching.text());
}
- if (tokenIterator.previousIndex() == 0 ) {
+ if (tokenIterator.previousIndex() == 0) {
throw new NoSuchElementException(CommonI18n.noMoreContent.text());
}
- return tokens.get(tokenIterator.previousIndex()-1);
+ return tokens.get(tokenIterator.previousIndex() - 1);
}
String generateFragment() {
@@ -1394,11 +1404,11 @@
/**
* Get the position for the last character returned from {@link #next()}.
- * @param startIndex
*
+ * @param startIndex
* @return the position of the last character returned; never null
*/
- Position position(int startIndex);
+ Position position( int startIndex );
/**
* Determine if the next character on the sream is a {@link
Character#isWhitespace(char) whitespace character}. This
@@ -1498,7 +1508,8 @@
* Create a single- or multi-character token with the characters in the range
given by the starting and ending index in
* the character stream. The character at the ending index is
<i>not</i> included in the token (as this is standard
* practice when using 0-based indexes). The token type is set to 0, meaning this
is equivalent to calling <code>
- * addToken(startIndex,endIndex,0)</code> .
+ * addToken(startIndex,endIndex,0)</code>
+ * .
*
* @param position the position (line and column numbers) of this new token; may
not be null
* @param startIndex the index of the first character to appear in the token;
must be a valid index in the stream
@@ -1554,7 +1565,7 @@
* @return true if the token's value matches the supplied character value, or
false otherwise
*/
boolean matches( char expected );
-
+
/**
* Determine if the token matches the supplied type.
*
@@ -1597,14 +1608,14 @@
* @return the position; never null
*/
Position position();
-
+
/**
* Bitmask ORed with existing type value.
*
* @param typeMask
* @return copy of Token with new type
*/
- Token withType( int typeMask);
+ Token withType( int typeMask );
}
/**
@@ -1628,17 +1639,16 @@
}
/**
- * {@inheritDoc}
- *
- * @see org.jboss.dna.common.text.TokenStream.Token#withType(int)
- */
- @Override
- public Token withType(int typeMask) {
- int type = this.type | typeMask;
- return new CaseSensitiveToken(startIndex, endIndex, type, position);
- }
+ * {@inheritDoc}
+ *
+ * @see org.jboss.dna.common.text.TokenStream.Token#withType(int)
+ */
+ public Token withType( int typeMask ) {
+ int type = this.type | typeMask;
+ return new CaseSensitiveToken(startIndex, endIndex, type, position);
+ }
- /**
+ /**
* {@inheritDoc}
*
* @see org.jboss.dna.common.text.TokenStream.Token#type()
@@ -1693,16 +1703,15 @@
}
/**
- * {@inheritDoc}
- *
- * @see org.jboss.dna.common.text.TokenStream.Token#matches(int)
- */
- @Override
- public final boolean matches(int expectedType) {
- return expectedType == ANY_TYPE || (currentToken().type() & expectedType) ==
expectedType;
- }
+ * {@inheritDoc}
+ *
+ * @see org.jboss.dna.common.text.TokenStream.Token#matches(int)
+ */
+ public final boolean matches( int expectedType ) {
+ return expectedType == ANY_TYPE || (currentToken().type() & expectedType)
== expectedType;
+ }
- /**
+ /**
* {@inheritDoc}
*
* @see org.jboss.dna.common.text.TokenStream.Token#value()
@@ -1753,17 +1762,17 @@
protected String matchString() {
return inputUppercased;
}
-
+
/**
- * {@inheritDoc}
- *
- * @see org.jboss.dna.common.text.TokenStream.Token#withType(int)
- */
- @Override
- public Token withType(int typeMask) {
- int type = this.type() | typeMask;
- return new CaseInsensitiveToken(startIndex(), endIndex(), type, position());
- }
+ * {@inheritDoc}
+ *
+ * @see org.jboss.dna.common.text.TokenStream.Token#withType(int)
+ */
+ @Override
+ public Token withType( int typeMask ) {
+ int type = this.type() | typeMask;
+ return new CaseInsensitiveToken(startIndex(), endIndex(), type, position());
+ }
}
protected abstract class TokenFactory implements Tokens {
@@ -1862,12 +1871,12 @@
/**
* {@inheritDoc}
- * @param startIndex
+ *
+ * @param startIndex
* @return the position of the token. never null
- *
* @see org.jboss.dna.common.text.TokenStream.CharacterStream#position(int)
*/
- public Position position(int startIndex) {
+ public Position position( int startIndex ) {
return new Position(startIndex, lineNumber, columnNumber);
}