teiid SVN: r4555 - in branches/7.7.x: engine/src/main/java/org/teiid/common/buffer/impl and 2 other directories.
by teiid-commits＠lists.jboss.org
Author: jolee
Date: 2013-02-27 15:22:37 -0500 (Wed, 27 Feb 2013)
New Revision: 4555
Modified:
branches/7.7.x/common-core/src/main/java/org/teiid/core/types/DataTypeManager.java
branches/7.7.x/engine/src/main/java/org/teiid/common/buffer/impl/BufferManagerImpl.java
branches/7.7.x/engine/src/main/java/org/teiid/common/buffer/impl/SizeUtility.java
branches/7.7.x/engine/src/main/java/org/teiid/dqp/internal/process/DQPCore.java
branches/7.7.x/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java
branches/7.7.x/engine/src/test/java/org/teiid/dqp/internal/process/TestDQPCore.java
Log:
TEIID-2410: issues with output buffer blocking
Modified: branches/7.7.x/common-core/src/main/java/org/teiid/core/types/DataTypeManager.java
===================================================================
--- branches/7.7.x/common-core/src/main/java/org/teiid/core/types/DataTypeManager.java 2013-02-27 17:16:02 UTC (rev 4554)
+++ branches/7.7.x/common-core/src/main/java/org/teiid/core/types/DataTypeManager.java 2013-02-27 20:22:37 UTC (rev 4555)
@@ -174,8 +174,16 @@
};
public static final int MAX_STRING_LENGTH = 4000;
- public static final int MAX_LOB_MEMORY_BYTES = 1 << 13;
+ public static final int MAX_LOB_MEMORY_BYTES = Math.max(nextPowOf2(2*MAX_STRING_LENGTH), 1<<13);
+ public static int nextPowOf2(int val) {
+ int result = 1;
+ while (result < val) {
+ result <<= 1;
+ }
+ return result;
+ }
+
public static final class DataTypeAliases {
public static final String VARCHAR = "varchar"; //$NON-NLS-1$
public static final String TINYINT = "tinyint"; //$NON-NLS-1$
Modified: branches/7.7.x/engine/src/main/java/org/teiid/common/buffer/impl/BufferManagerImpl.java
===================================================================
--- branches/7.7.x/engine/src/main/java/org/teiid/common/buffer/impl/BufferManagerImpl.java 2013-02-27 17:16:02 UTC (rev 4554)
+++ branches/7.7.x/engine/src/main/java/org/teiid/common/buffer/impl/BufferManagerImpl.java 2013-02-27 20:22:37 UTC (rev 4555)
@@ -114,7 +114,7 @@
if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.TRACE)) {
LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, "Asynch eviction run", impl.reserveBatchBytes.get(), impl.maxReserveBytes.get(), impl.activeBatchBytes.get()); //$NON-NLS-1$
}
- impl.doEvictions(0, false);
+ impl.doEvictions(0, !agingOut);
if (!agingOut) {
try {
Thread.sleep(100); //we don't want to evict too fast, because the processing threads are more than capable of evicting
@@ -691,13 +691,13 @@
}
return;
}
- long maxToFree = Math.max(maxProcessingBytes>>1, reserveBatch>>3);
+ long maxToFree = Math.min(maxProcessingBytes, (activeBatch - reserveBatch)<<1);
doEvictions(maxToFree, true);
}
void doEvictions(long maxToFree, boolean checkActiveBatch) {
int freed = 0;
- while (freed <= maxToFree && (!checkActiveBatch || activeBatchBytes.get() > reserveBatchBytes.get() * .8)) {
+ while (freed <= maxToFree && (!checkActiveBatch || (maxToFree == 0 && activeBatchBytes.get() > reserveBatchBytes.get() * .7) || (maxToFree > 0 && activeBatchBytes.get() > reserveBatchBytes.get() * .8))) {
CacheEntry ce = evictionQueue.firstEntry(true);
if (ce == null) {
break;
@@ -715,9 +715,14 @@
} finally {
synchronized (ce) {
if (evicted && memoryEntries.remove(ce.getId()) != null) {
- freed += ce.getSizeEstimate();
+ if (maxToFree > 0) {
+ freed += ce.getSizeEstimate();
+ }
activeBatchBytes.addAndGet(-ce.getSizeEstimate());
evictionQueue.remove(ce); //ensures that an intervening get will still be cleaned
+ if (!checkActiveBatch) {
+ break;
+ }
}
}
}
Modified: branches/7.7.x/engine/src/main/java/org/teiid/common/buffer/impl/SizeUtility.java
===================================================================
--- branches/7.7.x/engine/src/main/java/org/teiid/common/buffer/impl/SizeUtility.java 2013-02-27 17:16:02 UTC (rev 4554)
+++ branches/7.7.x/engine/src/main/java/org/teiid/common/buffer/impl/SizeUtility.java 2013-02-27 20:22:37 UTC (rev 4555)
@@ -47,7 +47,7 @@
private static Map<Class<?>, int[]> SIZE_ESTIMATES = new HashMap<Class<?>, int[]>(128);
private static Set<Class<?>> VARIABLE_SIZE_TYPES = new HashSet<Class<?>>();
static {
- SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.STRING, new int[] {100, 256});
+ SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.STRING, new int[] {100, Math.max(100, DataTypeManager.nextPowOf2(DataTypeManager.MAX_STRING_LENGTH/16))});
SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.DATE, new int[] {20, 28});
SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.TIME, new int[] {20, 28});
SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.TIMESTAMP, new int[] {20, 28});
Modified: branches/7.7.x/engine/src/main/java/org/teiid/dqp/internal/process/DQPCore.java
===================================================================
--- branches/7.7.x/engine/src/main/java/org/teiid/dqp/internal/process/DQPCore.java 2013-02-27 17:16:02 UTC (rev 4554)
+++ branches/7.7.x/engine/src/main/java/org/teiid/dqp/internal/process/DQPCore.java 2013-02-27 20:22:37 UTC (rev 4555)
@@ -431,15 +431,18 @@
}
}
- public boolean hasWaitingPlans(RequestWorkItem item) {
- synchronized (waitingPlans) {
- if (!waitingPlans.isEmpty()) {
- return true;
- }
- this.bufferFullPlans.add(item);
- }
- return false;
- }
+ public boolean blockOnOutputBuffer(RequestWorkItem item) {
+ synchronized (waitingPlans) {
+ if (!waitingPlans.isEmpty()) {
+ return false;
+ }
+ if (item.useCallingThread || item.getDqpWorkContext().getSession().isEmbedded()) {
+ return false;
+ }
+ this.bufferFullPlans.add(item);
+ }
+ return true;
+ }
void removeRequest(final RequestWorkItem workItem) {
finishProcessing(workItem);
Modified: branches/7.7.x/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java
===================================================================
--- branches/7.7.x/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java 2013-02-27 17:16:02 UTC (rev 4554)
+++ branches/7.7.x/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java 2013-02-27 20:22:37 UTC (rev 4555)
@@ -550,11 +550,12 @@
super.flushBatchDirect(batch, add);
if (!add && !processor.hasFinalBuffer()) {
resultsBuffer.setRowCount(batch.getEndRow());
- } else if (!processor.hasFinalBuffer() //restrict the buffer size for forward only results
+ } else if (isForwardOnly() && add
+ && !processor.hasFinalBuffer() //restrict the buffer size for forward only results
&& !batch.getTerminationFlag()
&& transactionState != TransactionState.ACTIVE
- && this.getTupleBuffer().getManagedRowCount() >= OUTPUT_BUFFER_MAX_BATCHES * this.getTupleBuffer().getBatchSize()) {
- if (!dqpCore.hasWaitingPlans(RequestWorkItem.this)) {
+ && resultsBuffer.getManagedRowCount() >= OUTPUT_BUFFER_MAX_BATCHES * resultsBuffer.getBatchSize()) {
+ if (dqpCore.blockOnOutputBuffer(RequestWorkItem.this)) {
//requestMore will trigger more processing
throw BlockedException.block(requestID, "Blocking due to full results TupleBuffer", //$NON-NLS-1$
this.getTupleBuffer().getId(), "rows", this.getTupleBuffer().getManagedRowCount(), "batch size", this.getTupleBuffer().getBatchSize()); //$NON-NLS-1$ //$NON-NLS-2$
Modified: branches/7.7.x/engine/src/test/java/org/teiid/dqp/internal/process/TestDQPCore.java
===================================================================
--- branches/7.7.x/engine/src/test/java/org/teiid/dqp/internal/process/TestDQPCore.java 2013-02-27 17:16:02 UTC (rev 4554)
+++ branches/7.7.x/engine/src/test/java/org/teiid/dqp/internal/process/TestDQPCore.java 2013-02-27 20:22:37 UTC (rev 4555)
@@ -345,8 +345,30 @@
break;
}
}
+
+ //insensitive should not block
+ reqMsg.setCursorType(ResultSet.TYPE_SCROLL_INSENSITIVE);
+
+ message = core.executeRequest(reqMsg.getExecutionId(), reqMsg);
+ rm = message.get(500000, TimeUnit.MILLISECONDS);
+ assertNull(rm.getException());
+
+ assertEquals(rowsPerBatch, rm.getResultsList().size());
+ item = core.getRequestWorkItem(DQPWorkContext.getWorkContext().getRequestID(reqMsg.getExecutionId()));
+
+ message = core.processCursorRequest(reqMsg.getExecutionId(), 9, rowsPerBatch);
+ rm = message.get(500000, TimeUnit.MILLISECONDS);
+ assertNull(rm.getException());
+ assertEquals(rowsPerBatch, rm.getResultsList().size());
+ //ensure that we are idle
+ for (int i = 0; i < 10 && item.getThreadState() != ThreadState.IDLE; i++) {
+ Thread.sleep(100);
+ }
+ assertEquals(ThreadState.IDLE, item.getThreadState());
+ assertEquals(item.resultsBuffer.getManagedRowCount(), 400); //should have the full results
}
+
@Test public void testBufferReuse() throws Exception {
//the sql should return 100 rows
String sql = "SELECT A.IntKey FROM BQT1.SmallA as A, BQT1.SmallA as B ORDER BY A.IntKey"; //$NON-NLS-1$
12 years, 8 months
teiid SVN: r4554 - in branches/7.7.x: connectors/translator-jdbc/src/test/java/org/teiid/translator/jdbc/sqlserver and 4 other directories.
by teiid-commits＠lists.jboss.org
Author: jolee
Date: 2013-02-27 12:16:02 -0500 (Wed, 27 Feb 2013)
New Revision: 4554
Modified:
branches/7.7.x/api/src/main/java/org/teiid/language/visitor/SQLStringVisitor.java
branches/7.7.x/connectors/translator-jdbc/src/test/java/org/teiid/translator/jdbc/sqlserver/TestSqlServerConversionVisitor.java
branches/7.7.x/engine/src/main/java/org/teiid/dqp/internal/datamgr/LanguageBridgeFactory.java
branches/7.7.x/engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java
branches/7.7.x/engine/src/main/java/org/teiid/query/sql/visitor/SQLStringVisitor.java
branches/7.7.x/engine/src/test/java/org/teiid/query/processor/TestWithClauseProcessing.java
Log:
TEIID-2392: Pushdown common table query invalid
Modified: branches/7.7.x/api/src/main/java/org/teiid/language/visitor/SQLStringVisitor.java
===================================================================
--- branches/7.7.x/api/src/main/java/org/teiid/language/visitor/SQLStringVisitor.java 2013-02-27 16:33:26 UTC (rev 4553)
+++ branches/7.7.x/api/src/main/java/org/teiid/language/visitor/SQLStringVisitor.java 2013-02-27 17:16:02 UTC (rev 4554)
@@ -98,6 +98,7 @@
protected StringBuilder buffer = new StringBuilder();
private boolean appendedSourceComment;
+ private boolean shortNameOnly = false;
/**
* Gets the name of a group or element from the RuntimeMetadata
@@ -276,7 +277,7 @@
}
public void visit(ColumnReference obj) {
- buffer.append(getElementName(obj, true));
+ buffer.append(getElementName(obj, !shortNameOnly));
}
private String getElementName(ColumnReference obj, boolean qualify) {
@@ -520,14 +521,9 @@
append(obj.getTable());
buffer.append(Tokens.SPACE).append(Tokens.LPAREN);
- int elementCount = obj.getColumns().size();
- for (int i = 0; i < elementCount; i++) {
- buffer.append(getElementName(obj.getColumns().get(i), false));
- if (i < elementCount - 1) {
- buffer.append(Tokens.COMMA);
- buffer.append(Tokens.SPACE);
- }
- }
+ this.shortNameOnly = true;
+ append(obj.getColumns());
+ this.shortNameOnly = false;
buffer.append(Tokens.RPAREN);
buffer.append(Tokens.SPACE);
@@ -953,7 +949,9 @@
buffer.append(Tokens.SPACE);
if (obj.getColumns() != null) {
buffer.append(Tokens.LPAREN);
+ shortNameOnly = true;
append(obj.getColumns());
+ shortNameOnly = false;
buffer.append(Tokens.RPAREN);
buffer.append(Tokens.SPACE);
}
Modified: branches/7.7.x/connectors/translator-jdbc/src/test/java/org/teiid/translator/jdbc/sqlserver/TestSqlServerConversionVisitor.java
===================================================================
--- branches/7.7.x/connectors/translator-jdbc/src/test/java/org/teiid/translator/jdbc/sqlserver/TestSqlServerConversionVisitor.java 2013-02-27 16:33:26 UTC (rev 4553)
+++ branches/7.7.x/connectors/translator-jdbc/src/test/java/org/teiid/translator/jdbc/sqlserver/TestSqlServerConversionVisitor.java 2013-02-27 17:16:02 UTC (rev 4554)
@@ -203,7 +203,7 @@
@Test public void testWith() throws Exception {
String input = "with x as (select intkey from bqt1.smalla) select intkey from x limit 100"; //$NON-NLS-1$
- String output = "WITH x AS (SELECT SmallA.IntKey FROM SmallA) SELECT TOP 100 g_0.intkey AS c_0 FROM x g_0"; //$NON-NLS-1$
+ String output = "WITH x (IntKey) AS (SELECT SmallA.IntKey FROM SmallA) SELECT TOP 100 g_0.intkey AS c_0 FROM x g_0"; //$NON-NLS-1$
CommandBuilder commandBuilder = new CommandBuilder(RealMetadataFactory.exampleBQTCached());
Command obj = commandBuilder.getCommand(input, true, true);
Modified: branches/7.7.x/engine/src/main/java/org/teiid/dqp/internal/datamgr/LanguageBridgeFactory.java
===================================================================
--- branches/7.7.x/engine/src/main/java/org/teiid/dqp/internal/datamgr/LanguageBridgeFactory.java 2013-02-27 16:33:26 UTC (rev 4553)
+++ branches/7.7.x/engine/src/main/java/org/teiid/dqp/internal/datamgr/LanguageBridgeFactory.java 2013-02-27 17:16:02 UTC (rev 4554)
@@ -36,13 +36,13 @@
import org.teiid.core.TeiidException;
import org.teiid.core.TeiidRuntimeException;
import org.teiid.language.*;
-import org.teiid.language.DerivedColumn;
-import org.teiid.language.Select;
-import org.teiid.language.WindowSpecification;
import org.teiid.language.Argument.Direction;
import org.teiid.language.Comparison.Operator;
import org.teiid.language.SortSpecification.Ordering;
import org.teiid.language.SubqueryComparison.Quantifier;
+import org.teiid.language.DerivedColumn;
+import org.teiid.language.Select;
+import org.teiid.language.WindowSpecification;
import org.teiid.metadata.Procedure;
import org.teiid.metadata.ProcedureParameter;
import org.teiid.metadata.FunctionMethod.PushDown;
@@ -176,6 +176,7 @@
for (ElementSymbol es: withQueryCommand.getColumns()) {
translatedElements.add(translate(es));
}
+ item.setColumns(translatedElements);
}
item.setSubquery(translate(withQueryCommand.getCommand()));
items.add(item);
Modified: branches/7.7.x/engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java
===================================================================
--- branches/7.7.x/engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java 2013-02-27 16:33:26 UTC (rev 4553)
+++ branches/7.7.x/engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java 2013-02-27 17:16:02 UTC (rev 4554)
@@ -206,6 +206,7 @@
if (queryCommand == null || CriteriaCapabilityValidatorVisitor.validateCommandPushdown(modelID, metadata, capFinder, aNode) == null) {
supportsWithPushdown = false;
} else {
+ //TODO: do this prior to alias generation, or rerun alias generation here
queryCommand.setWith(pushDownWith);
}
} else {
Modified: branches/7.7.x/engine/src/main/java/org/teiid/query/sql/visitor/SQLStringVisitor.java
===================================================================
--- branches/7.7.x/engine/src/main/java/org/teiid/query/sql/visitor/SQLStringVisitor.java 2013-02-27 16:33:26 UTC (rev 4553)
+++ branches/7.7.x/engine/src/main/java/org/teiid/query/sql/visitor/SQLStringVisitor.java 2013-02-27 17:16:02 UTC (rev 4554)
@@ -64,6 +64,7 @@
private static final String END_HINT = "*/"; //$NON-NLS-1$
private static final char ID_ESCAPE_CHAR = '\"';
protected StringBuilder parts = new StringBuilder();
+ private boolean shortNameOnly = false;
/**
* Helper to quickly get the parser string for an object using the visitor.
@@ -680,7 +681,9 @@
append(SPACE);
if (obj.getColumns() != null && !obj.getColumns().isEmpty()) {
append(Tokens.LPAREN);
+ shortNameOnly = true;
registerNodes(obj.getColumns(), 0);
+ shortNameOnly = false;
append(Tokens.RPAREN);
append(SPACE);
}
@@ -1179,7 +1182,7 @@
}
public void visit( ElementSymbol obj ) {
- if (obj.getDisplayMode().equals(ElementSymbol.DisplayMode.SHORT_OUTPUT_NAME)) {
+ if (obj.getDisplayMode().equals(ElementSymbol.DisplayMode.SHORT_OUTPUT_NAME) || shortNameOnly) {
outputShortName(obj);
return;
}
Modified: branches/7.7.x/engine/src/test/java/org/teiid/query/processor/TestWithClauseProcessing.java
===================================================================
--- branches/7.7.x/engine/src/test/java/org/teiid/query/processor/TestWithClauseProcessing.java 2013-02-27 16:33:26 UTC (rev 4553)
+++ branches/7.7.x/engine/src/test/java/org/teiid/query/processor/TestWithClauseProcessing.java 2013-02-27 17:16:02 UTC (rev 4554)
@@ -76,6 +76,28 @@
helpProcess(plan, dataManager, expected);
}
+ @Test public void testWithPushdown1() throws TeiidException {
+ FakeCapabilitiesFinder capFinder = new FakeCapabilitiesFinder();
+ BasicSourceCapabilities caps = TestOptimizer.getTypicalCapabilities();
+ caps.setCapabilitySupport(Capability.COMMON_TABLE_EXPRESSIONS, true);
+ caps.setCapabilitySupport(Capability.QUERY_FROM_JOIN_SELFJOIN, true);
+ caps.setCapabilitySupport(Capability.QUERY_AGGREGATES_COUNT, true);
+ capFinder.addCapabilities("pm1", caps); //$NON-NLS-1$
+
+ String sql = "with a as (select x, y, z from (select e1 as x, e2 as y, e3 as z from pm1.g1) v) SELECT count(a.x) from a, a z"; //$NON-NLS-1$
+
+ HardcodedDataManager dataManager = new HardcodedDataManager(RealMetadataFactory.example1Cached());
+ List<?>[] expected = new List[] {
+ Arrays.asList("a", 1, Boolean.FALSE),
+ };
+
+ dataManager.addData("WITH a (x, y, z) AS (SELECT g_0.e1, g_0.e2, g_0.e3 FROM g1 AS g_0) SELECT COUNT(g_0.x) FROM a AS g_0, a AS g_1", expected);
+
+ ProcessorPlan plan = TestOptimizer.helpPlan(sql, RealMetadataFactory.example1Cached(), null, capFinder, new String[] {"WITH a (x, y, z) AS (SELECT g_0.e1, g_0.e2, g_0.e3 FROM pm1.g1 AS g_0) SELECT COUNT(g_0.x) FROM a AS g_0, a AS g_1"}, ComparisonMode.EXACT_COMMAND_STRING);
+
+ helpProcess(plan, dataManager, expected);
+ }
+
@Test public void testWithPushdownWithConstants() throws TeiidException {
FakeCapabilitiesFinder capFinder = new FakeCapabilitiesFinder();
BasicSourceCapabilities caps = TestOptimizer.getTypicalCapabilities();
12 years, 8 months
teiid SVN: r4553 - in branches/7.7.x/common-core/src: test/java/org/teiid/core/util and 1 other directory.
by teiid-commits＠lists.jboss.org
Author: jolee
Date: 2013-02-27 11:33:26 -0500 (Wed, 27 Feb 2013)
New Revision: 4553
Modified:
branches/7.7.x/common-core/src/main/java/org/teiid/core/util/InputStreamReader.java
branches/7.7.x/common-core/src/test/java/org/teiid/core/util/TestInputStreamReader.java
Log:
TEIID-2383: InputStreamReader throws MalformedInputException when handling multi-byte characters
Modified: branches/7.7.x/common-core/src/main/java/org/teiid/core/util/InputStreamReader.java
===================================================================
--- branches/7.7.x/common-core/src/main/java/org/teiid/core/util/InputStreamReader.java 2013-02-05 20:34:44 UTC (rev 4552)
+++ branches/7.7.x/common-core/src/main/java/org/teiid/core/util/InputStreamReader.java 2013-02-27 16:33:26 UTC (rev 4553)
@@ -74,6 +74,7 @@
}
while (!done && !cb.hasRemaining()) {
int read = 0;
+ int pos = bb.position();
while ((read = rbc.read(bb)) == 0) {
//blocking read
}
@@ -90,7 +91,7 @@
}
done = true;
}
- if (bb.position() != read) {
+ if (bb.position() != read + pos) {
bb.compact();
} else {
bb.clear();
Modified: branches/7.7.x/common-core/src/test/java/org/teiid/core/util/TestInputStreamReader.java
===================================================================
--- branches/7.7.x/common-core/src/test/java/org/teiid/core/util/TestInputStreamReader.java 2013-02-05 20:34:44 UTC (rev 4552)
+++ branches/7.7.x/common-core/src/test/java/org/teiid/core/util/TestInputStreamReader.java 2013-02-27 16:33:26 UTC (rev 4553)
@@ -7,7 +7,7 @@
import org.junit.Test;
-@SuppressWarnings("nls")
+@SuppressWarnings({"nls","resource"})
public class TestInputStreamReader {
@Test public void testMultiByte() throws Exception {
@@ -15,4 +15,14 @@
assertEquals(80, isr.read());
assertEquals(250, isr.read());
}
+
+ @Test public void testMultiByte1() throws Exception {
+ InputStreamReader isr = new InputStreamReader(new ByteArrayInputStream(new byte[] {(byte)80, (byte)-61, (byte)-70, (byte)-61, (byte)-70, (byte)80, (byte)-61, (byte)-70}), Charset.forName("UTF-8").newDecoder(), 4);
+ int count = 0;
+ while (isr.read() != -1) {
+ count++;
+ }
+ assertEquals(5, count);
+ }
+
}
12 years, 8 months
