[teiid-commits] teiid SVN: r3567 - in trunk: engine/src/main/java/org/teiid/dqp/internal/process and 3 other directories.
teiid-commits at lists.jboss.org
teiid-commits at lists.jboss.org
Tue Oct 18 14:59:58 EDT 2011
Author: shawkins
Date: 2011-10-18 14:59:58 -0400 (Tue, 18 Oct 2011)
New Revision: 3567
Modified:
trunk/build/kits/jboss-container/teiid-releasenotes.html
trunk/engine/src/main/java/org/teiid/dqp/internal/process/AccessInfo.java
trunk/engine/src/main/java/org/teiid/dqp/internal/process/DQPWorkContext.java
trunk/engine/src/main/java/org/teiid/dqp/internal/process/PreparedPlan.java
trunk/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java
trunk/engine/src/main/java/org/teiid/query/metadata/TempMetadataID.java
trunk/engine/src/test/java/org/teiid/dqp/internal/process/TestDQPCore.java
trunk/test-integration/common/src/test/java/org/teiid/jdbc/FakeServer.java
Log:
TEIID-1788 preventing the plan from being reported unnecessarily and preventing prepared plans from expiring needlessly TEIID-1750 updating the release notes
Modified: trunk/build/kits/jboss-container/teiid-releasenotes.html
===================================================================
--- trunk/build/kits/jboss-container/teiid-releasenotes.html 2011-10-18 17:24:41 UTC (rev 3566)
+++ trunk/build/kits/jboss-container/teiid-releasenotes.html 2011-10-18 18:59:58 UTC (rev 3567)
@@ -33,6 +33,7 @@
<LI><B>Buffering Improvements</B> - Added the ability to inline memory based or small lobs and added tracking of the memory held by soft references. Also switched to a LFRU algorithm that significantly reduces writes and read misses with temporary tables.
<LI><B>GSSAPI</B> - both the Teiid JDBC client/server and the ODBC pg backend can now support GSSAPI for single sign-on.
<LI><B>Server-side Query Timeouts</B> - default query timeouts can be configured at both the VDB (via the query-timeout VDB property) and entire server (via the teiid-jboss-beans.xml queryTimeout property).
+ <LI><B>Memory Improvements</B> - buffering was optimized for concurrency and to better handle table querying instead of tuple buffers. Added a memory buffer to better handle file storage. The memory buffer may be optional configured as off-heap for better large memory performance.
</UL>
<h2><a name="Compatibility">Compatibility Issues</a></h2>
Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/process/AccessInfo.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/dqp/internal/process/AccessInfo.java 2011-10-18 17:24:41 UTC (rev 3566)
+++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/AccessInfo.java 2011-10-18 18:59:58 UTC (rev 3567)
@@ -52,7 +52,7 @@
private static final long serialVersionUID = -2608267960584191359L;
private transient Set<Object> objectsAccessed;
-
+ private boolean sensitiveToMetadataChanges = true;
private List<List<String>> externalNames;
private transient long creationTime = System.currentTimeMillis();
@@ -67,6 +67,14 @@
this.creationTime = System.currentTimeMillis();
}
+ public boolean isSensitiveToMetadataChanges() {
+ return sensitiveToMetadataChanges;
+ }
+
+ public void setSensitiveToMetadataChanges(boolean sensitiveToMetadataChanges) {
+ this.sensitiveToMetadataChanges = sensitiveToMetadataChanges;
+ }
+
private static List<List<String>> initExternalList(List<List<String>> externalNames, Set<? extends Object> accessed) {
if (externalNames == null) {
externalNames = new ArrayList<List<String>>(accessed.size());
@@ -153,8 +161,14 @@
}
for (Object o : this.objectsAccessed) {
if (!data) {
- if (o instanceof Modifiable && ((Modifiable)o).getLastModified() - modTime > this.creationTime) {
- return false;
+ if (o instanceof Modifiable) {
+ Modifiable m = (Modifiable)o;
+ if (m.getLastModified() < 0) {
+ return false; //invalid object
+ }
+ if (sensitiveToMetadataChanges && m.getLastModified() - modTime > this.creationTime) {
+ return false;
+ }
}
} else if (o instanceof DataModifiable && ((DataModifiable)o).getLastDataModification() - modTime > this.creationTime) {
return false;
Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/process/DQPWorkContext.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/dqp/internal/process/DQPWorkContext.java 2011-10-18 17:24:41 UTC (rev 3566)
+++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/DQPWorkContext.java 2011-10-18 18:59:58 UTC (rev 3567)
@@ -54,7 +54,9 @@
SEVEN_1("7.1"), //$NON-NLS-1$
SEVEN_2("7.2"), //$NON-NLS-1$
SEVEN_3("7.3"), //$NON-NLS-1$
- SEVEN_4("7.4"); //$NON-NLS-1$
+ SEVEN_4("7.4"), //$NON-NLS-1$
+ SEVEN_5("7.5"), //$NON-NLS-1$
+ SEVEN_6("7.6"); //$NON-NLS-1$
private String string;
Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/process/PreparedPlan.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/dqp/internal/process/PreparedPlan.java 2011-10-18 17:24:41 UTC (rev 3566)
+++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/PreparedPlan.java 2011-10-18 18:59:58 UTC (rev 3567)
@@ -29,6 +29,8 @@
import org.teiid.common.buffer.BufferManager;
import org.teiid.query.analysis.AnalysisRecord;
import org.teiid.query.processor.ProcessorPlan;
+import org.teiid.query.processor.relational.AccessNode;
+import org.teiid.query.processor.relational.RelationalPlan;
import org.teiid.query.sql.lang.Command;
import org.teiid.query.sql.symbol.Reference;
import org.teiid.query.util.CommandContext;
@@ -77,6 +79,13 @@
public void setPlan(ProcessorPlan planValue, CommandContext context){
plan = planValue;
this.accessInfo.populate(context, false);
+ //TODO: expand this logic
+ if (planValue instanceof RelationalPlan) {
+ RelationalPlan rp = (RelationalPlan)planValue;
+ if (rp.getRootNode() instanceof AccessNode) {
+ this.accessInfo.setSensitiveToMetadataChanges(false);
+ }
+ }
}
/**
Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java 2011-10-18 17:24:41 UTC (rev 3566)
+++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java 2011-10-18 18:59:58 UTC (rev 3567)
@@ -52,6 +52,7 @@
import org.teiid.core.types.DataTypeManager;
import org.teiid.dqp.internal.process.DQPCore.CompletionListener;
import org.teiid.dqp.internal.process.DQPCore.FutureWork;
+import org.teiid.dqp.internal.process.DQPWorkContext.Version;
import org.teiid.dqp.internal.process.SessionAwareCache.CacheID;
import org.teiid.dqp.internal.process.ThreadReuseExecutor.PrioritizedRunnable;
import org.teiid.dqp.message.AtomicRequestID;
@@ -543,7 +544,6 @@
resultsBuffer = this.processor.getBufferManager().createTupleBuffer(this.originalCommand.getProjectedSymbols(), this.request.context.getConnectionID(), TupleSourceType.FINAL);
}
analysisRecord = request.analysisRecord;
- analysisRecord.setQueryPlan(processor.getProcessorPlan().getDescriptionProperties());
transactionContext = request.transactionContext;
if (this.transactionContext != null && this.transactionContext.getTransactionType() != Scope.NONE) {
this.transactionState = TransactionState.ACTIVE;
@@ -677,6 +677,7 @@
dataTypes[i] = DataTypeManager.getDataTypeName(symbol.getType());
}
ResultsMessage result = new ResultsMessage(batch, columnNames, dataTypes);
+ result.setClientSerializationVersion((byte)(this.dqpWorkContext.getClientVersion().compareTo(Version.SEVEN_6) >= 0?1:0));
setAnalysisRecords(result);
return result;
}
Modified: trunk/engine/src/main/java/org/teiid/query/metadata/TempMetadataID.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/query/metadata/TempMetadataID.java 2011-10-18 17:24:41 UTC (rev 3566)
+++ trunk/engine/src/main/java/org/teiid/query/metadata/TempMetadataID.java 2011-10-18 18:59:58 UTC (rev 3567)
@@ -62,13 +62,17 @@
List<List<TempMetadataID>> keys;
List<List<TempMetadataID>> indexes;
long lastDataModification;
- long lastModified;
+ long lastModified = System.currentTimeMillis();
int modCount;
public long getLastDataModification() {
return lastDataModification;
}
+ public void removed() {
+ this.lastModified = -1;
+ }
+
public void dataModified(int updateCount) {
if (updateCount == 0) {
return;
Modified: trunk/engine/src/test/java/org/teiid/dqp/internal/process/TestDQPCore.java
===================================================================
--- trunk/engine/src/test/java/org/teiid/dqp/internal/process/TestDQPCore.java 2011-10-18 17:24:41 UTC (rev 3566)
+++ trunk/engine/src/test/java/org/teiid/dqp/internal/process/TestDQPCore.java 2011-10-18 18:59:58 UTC (rev 3567)
@@ -441,22 +441,35 @@
assertNull(rm.getException());
assertEquals(0, rm.getResultsList().size());
}
-
+
@Test public void testPreparedPlanInvalidation() throws Exception {
- String sql = "insert into #temp select * FROM vqt.SmallB"; //$NON-NLS-1$
+ helpTestPlanInvalidation("select * from #temp a, #temp b limit 10");
+
+ assertEquals(2, this.core.getPrepPlanCache().getCacheHitCount());
+ }
+
+ @Test public void testPreparedPlanSimpleNoInvalidation() throws Exception {
+ helpTestPlanInvalidation("select * from #temp");
+
+ assertEquals(3, this.core.getPrepPlanCache().getCacheHitCount());
+ }
+
+ private void helpTestPlanInvalidation(String query) throws InterruptedException,
+ ExecutionException, TimeoutException {
+ String sql = "insert into #temp select * FROM vqt.SmallB"; //$NON-NLS-1$
String userName = "1"; //$NON-NLS-1$
int sessionid = 1; //$NON-NLS-1$
RequestMessage reqMsg = exampleRequestMessage(sql);
ResultsMessage rm = execute(userName, sessionid, reqMsg);
assertEquals(1, rm.getResultsList().size()); //$NON-NLS-1$
- sql = "select * from #temp"; //$NON-NLS-1$
+ sql = query;
reqMsg = exampleRequestMessage(sql);
reqMsg.setStatementType(StatementType.PREPARED);
rm = execute(userName, sessionid, reqMsg);
assertEquals(10, rm.getResultsList().size()); //$NON-NLS-1$
- sql = "select * from #temp"; //$NON-NLS-1$
+ sql = query;
reqMsg = exampleRequestMessage(sql);
reqMsg.setStatementType(StatementType.PREPARED);
rm = execute(userName, sessionid, reqMsg);
@@ -472,7 +485,7 @@
rm = execute(userName, sessionid, reqMsg);
assertEquals(1, rm.getResultsList().size()); //$NON-NLS-1$
- sql = "select * from #temp"; //$NON-NLS-1$
+ sql = query;
reqMsg = exampleRequestMessage(sql);
reqMsg.setStatementType(StatementType.PREPARED);
rm = execute(userName, sessionid, reqMsg);
@@ -480,20 +493,18 @@
assertEquals(2, this.core.getPrepPlanCache().getCacheHitCount());
- //perform a major update, we will purge the plan
+ //perform a major update, it might purge the plan
sql = "delete from #temp"; //$NON-NLS-1$
reqMsg = exampleRequestMessage(sql);
rm = execute(userName, sessionid, reqMsg);
assertEquals(1, rm.getResultsList().size()); //$NON-NLS-1$
- sql = "select * from #temp"; //$NON-NLS-1$
+ sql = query;
reqMsg = exampleRequestMessage(sql);
reqMsg.setStatementType(StatementType.PREPARED);
rm = execute(userName, sessionid, reqMsg);
assertEquals(0, rm.getResultsList().size()); //$NON-NLS-1$
-
- assertEquals(2, this.core.getPrepPlanCache().getCacheHitCount());
- }
+ }
@Test public void testRsCacheInvalidation() throws Exception {
String sql = "select * FROM vqt.SmallB"; //$NON-NLS-1$
Modified: trunk/test-integration/common/src/test/java/org/teiid/jdbc/FakeServer.java
===================================================================
--- trunk/test-integration/common/src/test/java/org/teiid/jdbc/FakeServer.java 2011-10-18 17:24:41 UTC (rev 3566)
+++ trunk/test-integration/common/src/test/java/org/teiid/jdbc/FakeServer.java 2011-10-18 18:59:58 UTC (rev 3567)
@@ -38,6 +38,7 @@
import org.teiid.cache.CacheConfiguration.Policy;
import org.teiid.client.DQP;
import org.teiid.client.security.ILogon;
+import org.teiid.core.util.UnitTestUtil;
import org.teiid.deployers.CompositeVDB;
import org.teiid.deployers.MetadataStoreGroup;
import org.teiid.deployers.UDFMetaData;
@@ -65,6 +66,7 @@
import org.teiid.query.optimizer.capabilities.SourceCapabilities;
import org.teiid.query.tempdata.GlobalTableStore;
import org.teiid.query.tempdata.GlobalTableStoreImpl;
+import org.teiid.services.BufferServiceImpl;
import org.teiid.services.SessionServiceImpl;
import org.teiid.transport.ClientServiceRegistry;
import org.teiid.transport.ClientServiceRegistryImpl;
@@ -91,6 +93,10 @@
}
public FakeServer(DQPConfiguration config) {
+ this(config, false);
+ }
+
+ public FakeServer(DQPConfiguration config, boolean realBufferMangaer) {
this.logon = new LogonImpl(sessionService, null);
this.repo.addListener(new VDBLifeCycleListener() {
@@ -118,7 +124,14 @@
this.repo.start();
this.sessionService.setVDBRepository(repo);
- this.dqp.setBufferService(new FakeBufferService());
+ if (!realBufferMangaer) {
+ this.dqp.setBufferService(new FakeBufferService());
+ } else {
+ BufferServiceImpl bsi = new BufferServiceImpl();
+ bsi.setDiskDirectory(UnitTestUtil.getTestScratchPath());
+ this.dqp.setBufferService(bsi);
+ }
+
this.dqp.setCacheFactory(new DefaultCacheFactory());
this.dqp.setTransactionService(new FakeTransactionService());
@@ -144,6 +157,10 @@
registerClientService(DQP.class, dqp, null);
}
+ public DQPCore getDqp() {
+ return dqp;
+ }
+
public void setConnectorManagerRepository(ConnectorManagerRepository cmr) {
this.cmr = cmr;
}
More information about the teiid-commits
mailing list