[teiid-commits] teiid SVN: r2915 - in trunk/engine/src: test/java/org/teiid/query/processor and 1 other directory.
teiid-commits at lists.jboss.org
teiid-commits at lists.jboss.org
Thu Feb 17 12:32:44 EST 2011
Author: shawkins
Date: 2011-02-17 12:32:44 -0500 (Thu, 17 Feb 2011)
New Revision: 2915
Modified:
trunk/engine/src/main/java/org/teiid/query/processor/relational/TextTableNode.java
trunk/engine/src/test/java/org/teiid/query/processor/TestTextTable.java
Log:
TEIID-1475 added checks to ensure that we do pull the file into memory on invalid input
Modified: trunk/engine/src/main/java/org/teiid/query/processor/relational/TextTableNode.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/query/processor/relational/TextTableNode.java 2011-02-17 15:12:44 UTC (rev 2914)
+++ trunk/engine/src/main/java/org/teiid/query/processor/relational/TextTableNode.java 2011-02-17 17:32:44 UTC (rev 2915)
@@ -51,8 +51,7 @@
/**
* Handles text file processing.
*
- * TODO: unix style escape handling \t \n, etc.
- * TODO: parse using something more memory safe than strings
+ * TODO: unix style escape handling \t \n, etc. - see also the unescape function
* TODO: allow for escaping with fixed parsing
* TODO: allow for fixed parsing without new lines
* TODO: allow for a configurable line terminator
@@ -75,6 +74,9 @@
private int textLine = 0;
private Map<String, Integer> nameIndexes;
private String systemId;
+
+ private boolean cr;
+ private boolean eof;
public TextTableNode(int nodeID) {
super(nodeID);
@@ -110,6 +112,7 @@
noQuote = table.isEscape();
quote = table.getQuote();
}
+ lineWidth = table.getColumns().size() * DataTypeManager.MAX_STRING_LENGTH;
}
Map elementMap = createLookupMap(table.getProjectedSymbols());
this.projectionIndexes = getProjectionIndexes(elementMap, getElements());
@@ -133,6 +136,8 @@
}
this.nameIndexes = null;
this.textLine = 0;
+ this.cr = false;
+ this.eof = false;
}
public void setTable(TextTable table) {
@@ -161,7 +166,7 @@
}
while (!isBatchFull()) {
- String line = readLine();
+ String line = readLine(lineWidth, table.isFixedWidth());
if (line == null) {
terminateBatches();
@@ -194,22 +199,63 @@
return pullBatch();
}
- private String readLine() throws TeiidProcessingException {
- while (true) {
- try {
- String line = reader.readLine();
- if (line != null) {
- textLine++;
- if (line.length() == 0) {
- continue;
+ private String readLine(int maxLength, boolean exact) throws TeiidProcessingException {
+ if (eof) {
+ return null;
+ }
+ StringBuilder sb = new StringBuilder(exact ? maxLength : (maxLength >> 4));
+ try {
+ while (true) {
+ char c = readChar();
+ if (c == '\n') {
+ if (sb.length() == 0) {
+ if (eof) {
+ return null;
+ }
+ continue; //skip empty lines
}
- }
- return line;
- } catch (IOException e) {
- throw new TeiidProcessingException(e);
+ if (exact && sb.length() < lineWidth) {
+ throw new TeiidProcessingException(QueryPlugin.Util.getString("TextTableNode.invalid_width", sb.length(), lineWidth, textLine, systemId)); //$NON-NLS-1$
+ }
+ return sb.toString();
+ }
+ sb.append(c);
+ if (sb.length() > maxLength) {
+ if (exact) {
+ //we're not forcing them to fully specify the line, so just drop the rest
+ //TODO: there should be a max read length
+ while ((c = readChar()) != '\n') {
+
+ }
+ return sb.toString();
+ }
+ throw new TeiidProcessingException(QueryPlugin.Util.getString("TextTableNode.line_too_long", textLine+1, systemId, maxLength)); //$NON-NLS-1$
+ }
}
+ } catch (IOException e) {
+ throw new TeiidProcessingException(e);
}
}
+
+ private char readChar() throws IOException {
+ int c = reader.read();
+ if (cr) {
+ if (c == '\n') {
+ c = reader.read();
+ }
+ cr = false;
+ }
+ switch (c) {
+ case '\r':
+ cr = true;
+ case -1:
+ eof = true;
+ case '\n':
+ textLine++;
+ return '\n';
+ }
+ return (char)c;
+ }
private void initReader() throws ExpressionEvaluationException,
BlockedException, TeiidComponentException, TeiidProcessingException {
@@ -245,7 +291,8 @@
}
while (textLine < skip) {
boolean isHeader = textLine == header;
- String line = readLine();
+ //if we don't need a header, then we could just scan, but for now we'll enforce a max length
+ String line = readLine(Math.min(lineWidth, DataTypeManager.MAX_STRING_LENGTH), false);
if (line == null) { //just return an empty batch
reset();
return;
@@ -289,9 +336,13 @@
while (true) {
if (line == null) {
if (escaped) {
- builder.append('\n'); //allow for escaped new lines
+ //allow for escaped new lines
+ if (cr) {
+ builder.append('\r');
+ }
+ builder.append('\n');
escaped = false;
- line = readLine();
+ line = readLine(lineWidth, false);
continue;
}
if (!qualified) {
@@ -299,7 +350,7 @@
addValue(result, wasQualified, builder.toString());
return result;
}
- line = readLine();
+ line = readLine(lineWidth, false);
if (line == null) {
throw new TeiidProcessingException(QueryPlugin.Util.getString("TextTableNode.unclosed", systemId)); //$NON-NLS-1$
}
@@ -368,11 +419,7 @@
result.add(val);
}
- private List<String> parseFixedWidth(String line)
- throws TeiidProcessingException {
- if (line.length() < lineWidth) {
- throw new TeiidProcessingException(QueryPlugin.Util.getString("TextTableNode.invalid_width", line.length(), lineWidth, textLine, systemId)); //$NON-NLS-1$
- }
+ private List<String> parseFixedWidth(String line) {
ArrayList<String> result = new ArrayList<String>();
int beginIndex = 0;
for (TextColumn col : table.getColumns()) {
Modified: trunk/engine/src/test/java/org/teiid/query/processor/TestTextTable.java
===================================================================
--- trunk/engine/src/test/java/org/teiid/query/processor/TestTextTable.java 2011-02-17 15:12:44 UTC (rev 2914)
+++ trunk/engine/src/test/java/org/teiid/query/processor/TestTextTable.java 2011-02-17 17:32:44 UTC (rev 2915)
@@ -22,14 +22,15 @@
package org.teiid.query.processor;
-import static org.teiid.query.optimizer.TestOptimizer.getTypicalCapabilities;
-import static org.teiid.query.optimizer.TestOptimizer.helpPlan;
+import static org.teiid.query.optimizer.TestOptimizer.*;
import static org.teiid.query.processor.TestProcessor.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import javax.sql.rowset.serial.SerialClob;
+
import org.junit.Test;
import org.teiid.core.TeiidProcessingException;
import org.teiid.core.types.ClobImpl;
@@ -62,7 +63,7 @@
}
@Test public void testTextTableFixed() throws Exception {
- String sql = "select count(*) from texttable(? COLUMNS compkey string width 77, CDM_ID string width 14, CURRENCY string width 9, \"START\" string width 31, MATURITY string width 38, AMOUNT double width 13, RECORDSOURCE string width 13, SUMMIT_ID string width 25, RATE double width 21, SPREAD double width 9, DESK string width 13) x"; //$NON-NLS-1$
+ String sql = "select count(*) from texttable(? COLUMNS compkey string width 78, CDM_ID string width 14, CURRENCY string width 9, \"START\" string width 31, MATURITY string width 38, AMOUNT double width 13, RECORDSOURCE string width 13, SUMMIT_ID string width 25, RATE double width 21, SPREAD double width 9, DESK string width 13) x"; //$NON-NLS-1$
List[] expected = new List[] {
Arrays.asList(52),
@@ -255,5 +256,15 @@
helpProcess(plan, hdm, expected);
}
+
+ @Test(expected=TeiidProcessingException.class) public void testTextTableInvalidData() throws Exception {
+ String sql = "select count(*) from texttable(? COLUMNS PARTNAME string) x"; //$NON-NLS-1$
+
+ FakeDataManager dataManager = new FakeDataManager();
+ sampleData1(dataManager);
+
+ char[] data = new char[5000];
+ processPreparedStatement(sql, null, dataManager, new DefaultCapabilitiesFinder(), FakeMetadataFactory.example1Cached(), Arrays.asList(new ClobType(new SerialClob(data))));
+ }
}
More information about the teiid-commits
mailing list