Author: shawkins Date: 2011-02-18 10:19:32 -0500 (Fri, 18 Feb 2011) New Revision: 2918 Modified: trunk/engine/src/main/java/org/teiid/query/processor/relational/TextTableNode.java trunk/engine/src/test/java/org/teiid/query/processor/TestTextTable.java Log: TEIID-1475 refining previous approach Modified: trunk/engine/src/main/java/org/teiid/query/processor/relational/TextTableNode.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/processor/relational/TextTableNode.java 2011-02-17 18:55:02 UTC (rev 2917) +++ trunk/engine/src/main/java/org/teiid/query/processor/relational/TextTableNode.java 2011-02-18 15:19:32 UTC (rev 2918) @@ -204,58 +204,59 @@ return null; } StringBuilder sb = new StringBuilder(exact ? maxLength : (maxLength >> 4)); + while (true) { + char c = readChar(); + if (c == '\n') { + if (sb.length() == 0) { + if (eof) { + return null; + } + continue; //skip empty lines + } + if (exact && sb.length() < lineWidth) { + throw new TeiidProcessingException(QueryPlugin.Util.getString("TextTableNode.invalid_width", sb.length(), lineWidth, textLine, systemId)); //$NON-NLS-1$ + } + return sb.toString(); + } + sb.append(c); + if (sb.length() > maxLength) { + if (exact) { + sb.deleteCharAt(sb.length() - 1); + //we're not forcing them to fully specify the line, so just drop the rest + //TODO: there should be a max read length + while (readChar() != '\n') { + + } + return sb.toString(); + } + throw new TeiidProcessingException(QueryPlugin.Util.getString("TextTableNode.line_too_long", textLine+1, systemId, maxLength)); //$NON-NLS-1$ + } + } + } + + private char readChar() throws TeiidProcessingException { try { - while (true) { - char c = readChar(); + int c = reader.read(); + if (cr) { if (c == '\n') { - if (sb.length() == 0) { - if (eof) { - return null; - } - continue; //skip empty lines - } - if (exact && sb.length() < lineWidth) { - throw new TeiidProcessingException(QueryPlugin.Util.getString("TextTableNode.invalid_width", sb.length(), lineWidth, textLine, systemId)); //$NON-NLS-1$ - } - return sb.toString(); - } - sb.append(c); - if (sb.length() > maxLength) { - if (exact) { - //we're not forcing them to fully specify the line, so just drop the rest - //TODO: there should be a max read length - while ((c = readChar()) != '\n') { - - } - return sb.toString(); - } - throw new TeiidProcessingException(QueryPlugin.Util.getString("TextTableNode.line_too_long", textLine+1, systemId, maxLength)); //$NON-NLS-1$ - } - } + c = reader.read(); + } + cr = false; + } + switch (c) { + case '\r': + cr = true; + case -1: + eof = true; + case '\n': + textLine++; + return '\n'; + } + return (char)c; } catch (IOException e) { throw new TeiidProcessingException(e); } } - - private char readChar() throws IOException { - int c = reader.read(); - if (cr) { - if (c == '\n') { - c = reader.read(); - } - cr = false; - } - switch (c) { - case '\r': - cr = true; - case -1: - eof = true; - case '\n': - textLine++; - return '\n'; - } - return (char)c; - } private void initReader() throws ExpressionEvaluationException, BlockedException, TeiidComponentException, TeiidProcessingException { @@ -291,14 +292,17 @@ } while (textLine < skip) { boolean isHeader = textLine == header; - //if we don't need a header, then we could just scan, but for now we'll enforce a max length - String line = readLine(Math.min(lineWidth, DataTypeManager.MAX_STRING_LENGTH), false); - if (line == null) { //just return an empty batch - reset(); - return; - } if (isHeader) { + String line = readLine(lineWidth, false); + if (line == null) { //just return an empty batch + reset(); + return; + } processHeader(parseLine(line)); + } else { + while (readChar() != '\n') { + + } } } } Modified: trunk/engine/src/test/java/org/teiid/query/processor/TestTextTable.java =================================================================== --- trunk/engine/src/test/java/org/teiid/query/processor/TestTextTable.java 2011-02-17 18:55:02 UTC (rev 2917) +++ trunk/engine/src/test/java/org/teiid/query/processor/TestTextTable.java 2011-02-18 15:19:32 UTC (rev 2918) @@ -63,10 +63,10 @@ } @Test public void testTextTableFixed() throws Exception { - String sql = "select count(*) from texttable(? COLUMNS compkey string width 78, CDM_ID string width 14, CURRENCY string width 9, \"START\" string width 31, MATURITY string width 38, AMOUNT double width 13, RECORDSOURCE string width 13, SUMMIT_ID string width 25, RATE double width 21, SPREAD double width 9, DESK string width 13) x"; //$NON-NLS-1$ + String sql = "select max(compkey), max(cdm_id), max(currency), max(\"start\"), max(maturity), max(amount), count(*) from texttable(? COLUMNS compkey string width 76, CDM_ID string width 14, CURRENCY string width 9, \"START\" string width 31, MATURITY string width 31, AMOUNT double width 21, RECORDSOURCE string width 13, SUMMIT_ID string width 15, RATE double width 20, SPREAD double width 20, DESK string width 14) x"; //$NON-NLS-1$ List[] expected = new List[] { - Arrays.asList(52), + Arrays.asList("000369USD05/20/200405/20/2007", "000369", "USD", "12/18/2000", "12/19/2005", 6.7209685146E8, 52), }; FakeDataManager dataManager = new FakeDataManager(); @@ -75,6 +75,19 @@ processPreparedStatement(sql, expected, dataManager, new DefaultCapabilitiesFinder(), FakeMetadataFactory.example1Cached(), Arrays.asList(clobFromFile("text/cdm_dos.txt"))); } + @Test public void testTextTableFixedPartial() throws Exception { + String sql = "select max(length(compkey)) from texttable(? COLUMNS compkey string width 76) x"; //$NON-NLS-1$ + + List[] expected = new List[] { + Arrays.asList(30), + }; + + FakeDataManager dataManager = new FakeDataManager(); + sampleData1(dataManager); + + processPreparedStatement(sql, expected, dataManager, new DefaultCapabilitiesFinder(), FakeMetadataFactory.example1Cached(), Arrays.asList(clobFromFile("text/cdm_dos.txt"))); + } + @Test public void testNamedMultilineHeader() throws Exception { String sql = "SELECT * from texttable(? COLUMNS Col3Head string HEADER) x";