Author: blafond Date: 2009-11-25 07:59:39 -0500 (Wed, 25 Nov 2009) New Revision: 1343 Modified: branches/ddl_sequencer/dna-common/src/main/java/org/jboss/dna/common/text/TokenStream.java Log: DNA-555 Fixed problem in token positions having the wrong column number. Modified: branches/ddl_sequencer/dna-common/src/main/java/org/jboss/dna/common/text/TokenStream.java =================================================================== --- branches/ddl_sequencer/dna-common/src/main/java/org/jboss/dna/common/text/TokenStream.java 2009-11-25 00:59:30 UTC (rev 1342) +++ branches/ddl_sequencer/dna-common/src/main/java/org/jboss/dna/common/text/TokenStream.java 2009-11-25 12:59:39 UTC (rev 1343) @@ -395,7 +395,7 @@ * | | | * | | +- The position of the tokenIterator, where tokenIterator.hasNext() will return T3 * | +---- The token referenced by currentToken - * +-------- The logical position of the TokenStream object, where the "consume()" would return T2 + * +-------- The logical position of the TokenStream object, where the &quot;consume()&quot; would return T2 * </pre> */ private ListIterator<Token> tokenIterator; @@ -434,18 +434,18 @@ moveToNextToken(); return this; } - + /** - * Method to allow subclasses to preprocess the set of tokens and return the correct tokens to use. - * The default behavior is to simply return the supplied tokens. + * Method to allow subclasses to preprocess the set of tokens and return the correct tokens to use. The default behavior is to + * simply return the supplied tokens. * * @param tokens * @return list of tokens. */ - protected List<Token> initializeTokens( List<Token> tokens) { - return tokens; + protected List<Token> initializeTokens( List<Token> tokens ) { + return tokens; } - + /** * Method to allow tokens to be re-used from the start without re-tokenizing content. */ @@ -761,9 +761,12 @@ * * <pre> * - * if ( tokens.matches(currentExpected,expectedForNextTokens) ) { tokens.consume(currentExpected,expectedForNextTokens); } + * if (tokens.matches(currentExpected, expectedForNextTokens)) { + * tokens.consume(currentExpected, expectedForNextTokens); + * } * * </pre> + * * </p> * <p> * The {@link #ANY_VALUE ANY_VALUE} constant can be used in the expected values as a wildcard. @@ -806,9 +809,12 @@ * * <pre> * - * if ( tokens.matches(currentExpected,expectedForNextTokens) ) { tokens.consume(currentExpected,expectedForNextTokens); } + * if (tokens.matches(currentExpected, expectedForNextTokens)) { + * tokens.consume(currentExpected, expectedForNextTokens); + * } * * </pre> + * * </p> * <p> * The {@link #ANY_VALUE ANY_VALUE} constant can be used in the expected values as a wildcard. @@ -847,9 +853,12 @@ * * <pre> * - * if ( tokens.matches(currentExpected,expectedForNextTokens) ) { tokens.consume(currentExpected,expectedForNextTokens); } + * if (tokens.matches(currentExpected, expectedForNextTokens)) { + * tokens.consume(currentExpected, expectedForNextTokens); + * } * * </pre> + * * </p> * <p> * The {@link #ANY_VALUE ANY_VALUE} constant can be used in the expected values as a wildcard. @@ -1272,25 +1281,26 @@ * Gets the content string starting at the first position (inclusive) and continuing up to the end position (exclusive). * * @param starting the position marking the beginning of the desired content string. - * @param end the position located directly after the returned content string; can be null, which means end of content + * @param end the position located directly after the returned content string; can be null, which means end of content * @return the content string; never null */ - public String getContentBetween(Position starting, Position end) { - CheckArg.isNotNull(starting, "starting"); - - int startIndex = starting.getIndexInContent(); - int endIndex = inputString.length(); - if( end != null ) { - endIndex = end.getIndexInContent(); - } - - if( startIndex >= endIndex ) { - throw new IllegalArgumentException(CommonI18n.endPositionMustBeGreaterThanStartingPosition.text(startIndex, endIndex)); - } - - return inputString.substring(startIndex, endIndex); + public String getContentBetween( Position starting, + Position end ) { + CheckArg.isNotNull(starting, "starting"); + + int startIndex = starting.getIndexInContent(); + int endIndex = inputString.length(); + if (end != null) { + endIndex = end.getIndexInContent(); + } + + if (startIndex >= endIndex) { + throw new IllegalArgumentException(CommonI18n.endPositionMustBeGreaterThanStartingPosition.text(startIndex, endIndex)); + } + + return inputString.substring(startIndex, endIndex); } - + /** * Get the previous token. This does not modify the state. * @@ -1308,10 +1318,10 @@ } throw new IllegalStateException(CommonI18n.startMethodMustBeCalledBeforeConsumingOrMatching.text()); } - if (tokenIterator.previousIndex() == 0 ) { + if (tokenIterator.previousIndex() == 0) { throw new NoSuchElementException(CommonI18n.noMoreContent.text()); } - return tokens.get(tokenIterator.previousIndex()-1); + return tokens.get(tokenIterator.previousIndex() - 1); } String generateFragment() { @@ -1394,11 +1404,11 @@ /** * Get the position for the last character returned from {@link #next()}. - * @param startIndex * + * @param startIndex * @return the position of the last character returned; never null */ - Position position(int startIndex); + Position position( int startIndex ); /** * Determine if the next character on the sream is a {@link Character#isWhitespace(char) whitespace character}. This @@ -1498,7 +1508,8 @@ * Create a single- or multi-character token with the characters in the range given by the starting and ending index in * the character stream. The character at the ending index is <i>not</i> included in the token (as this is standard * practice when using 0-based indexes). The token type is set to 0, meaning this is equivalent to calling <code> - * addToken(startIndex,endIndex,0)</code> . + * addToken(startIndex,endIndex,0)</code> + * . * * @param position the position (line and column numbers) of this new token; may not be null * @param startIndex the index of the first character to appear in the token; must be a valid index in the stream @@ -1554,7 +1565,7 @@ * @return true if the token's value matches the supplied character value, or false otherwise */ boolean matches( char expected ); - + /** * Determine if the token matches the supplied type. * @@ -1597,14 +1608,14 @@ * @return the position; never null */ Position position(); - + /** * Bitmask ORed with existing type value. * * @param typeMask * @return copy of Token with new type */ - Token withType( int typeMask); + Token withType( int typeMask ); } /** @@ -1628,17 +1639,16 @@ } /** - * {@inheritDoc} - * - * @see org.jboss.dna.common.text.TokenStream.Token#withType(int) - */ - @Override - public Token withType(int typeMask) { - int type = this.type | typeMask; - return new CaseSensitiveToken(startIndex, endIndex, type, position); - } + * {@inheritDoc} + * + * @see org.jboss.dna.common.text.TokenStream.Token#withType(int) + */ + public Token withType( int typeMask ) { + int type = this.type | typeMask; + return new CaseSensitiveToken(startIndex, endIndex, type, position); + } - /** + /** * {@inheritDoc} * * @see org.jboss.dna.common.text.TokenStream.Token#type() @@ -1693,16 +1703,15 @@ } /** - * {@inheritDoc} - * - * @see org.jboss.dna.common.text.TokenStream.Token#matches(int) - */ - @Override - public final boolean matches(int expectedType) { - return expectedType == ANY_TYPE || (currentToken().type() & expectedType) == expectedType; - } + * {@inheritDoc} + * + * @see org.jboss.dna.common.text.TokenStream.Token#matches(int) + */ + public final boolean matches( int expectedType ) { + return expectedType == ANY_TYPE || (currentToken().type() & expectedType) == expectedType; + } - /** + /** * {@inheritDoc} * * @see org.jboss.dna.common.text.TokenStream.Token#value() @@ -1753,17 +1762,17 @@ protected String matchString() { return inputUppercased; } - + /** - * {@inheritDoc} - * - * @see org.jboss.dna.common.text.TokenStream.Token#withType(int) - */ - @Override - public Token withType(int typeMask) { - int type = this.type() | typeMask; - return new CaseInsensitiveToken(startIndex(), endIndex(), type, position()); - } + * {@inheritDoc} + * + * @see org.jboss.dna.common.text.TokenStream.Token#withType(int) + */ + @Override + public Token withType( int typeMask ) { + int type = this.type() | typeMask; + return new CaseInsensitiveToken(startIndex(), endIndex(), type, position()); + } } protected abstract class TokenFactory implements Tokens { @@ -1862,12 +1871,12 @@ /** * {@inheritDoc} - * @param startIndex + * + * @param startIndex * @return the position of the token. never null - * * @see org.jboss.dna.common.text.TokenStream.CharacterStream#position(int) */ - public Position position(int startIndex) { + public Position position( int startIndex ) { return new Position(startIndex, lineNumber, columnNumber); }