Author: shawkins Date: 2011-10-18 14:59:58 -0400 (Tue, 18 Oct 2011) New Revision: 3567 Modified: trunk/build/kits/jboss-container/teiid-releasenotes.html trunk/engine/src/main/java/org/teiid/dqp/internal/process/AccessInfo.java trunk/engine/src/main/java/org/teiid/dqp/internal/process/DQPWorkContext.java trunk/engine/src/main/java/org/teiid/dqp/internal/process/PreparedPlan.java trunk/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java trunk/engine/src/main/java/org/teiid/query/metadata/TempMetadataID.java trunk/engine/src/test/java/org/teiid/dqp/internal/process/TestDQPCore.java trunk/test-integration/common/src/test/java/org/teiid/jdbc/FakeServer.java Log: TEIID-1788 preventing the plan from being reported unnecessarily and preventing prepared plans from expiring needlessly TEIID-1750 updating the release notes Modified: trunk/build/kits/jboss-container/teiid-releasenotes.html =================================================================== --- trunk/build/kits/jboss-container/teiid-releasenotes.html 2011-10-18 17:24:41 UTC (rev 3566) +++ trunk/build/kits/jboss-container/teiid-releasenotes.html 2011-10-18 18:59:58 UTC (rev 3567) @@ -33,6 +33,7 @@ <LI><B>Buffering Improvements</B> - Added the ability to inline memory based or small lobs and added tracking of the memory held by soft references. Also switched to a LFRU algorithm that significantly reduces writes and read misses with temporary tables. <LI><B>GSSAPI</B> - both the Teiid JDBC client/server and the ODBC pg backend can now support GSSAPI for single sign-on. <LI><B>Server-side Query Timeouts</B> - default query timeouts can be configured at both the VDB (via the query-timeout VDB property) and entire server (via the teiid-jboss-beans.xml queryTimeout property). + <LI><B>Memory Improvements</B> - buffering was optimized for concurrency and to better handle table querying instead of tuple buffers. Added a memory buffer to better handle file storage. The memory buffer may be optional configured as off-heap for better large memory performance. </UL> <h2><a name="Compatibility">Compatibility Issues</a></h2> Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/process/AccessInfo.java =================================================================== --- trunk/engine/src/main/java/org/teiid/dqp/internal/process/AccessInfo.java 2011-10-18 17:24:41 UTC (rev 3566) +++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/AccessInfo.java 2011-10-18 18:59:58 UTC (rev 3567) @@ -52,7 +52,7 @@ private static final long serialVersionUID = -2608267960584191359L; private transient Set<Object> objectsAccessed; - + private boolean sensitiveToMetadataChanges = true; private List<List<String>> externalNames; private transient long creationTime = System.currentTimeMillis(); @@ -67,6 +67,14 @@ this.creationTime = System.currentTimeMillis(); } + public boolean isSensitiveToMetadataChanges() { + return sensitiveToMetadataChanges; + } + + public void setSensitiveToMetadataChanges(boolean sensitiveToMetadataChanges) { + this.sensitiveToMetadataChanges = sensitiveToMetadataChanges; + } + private static List<List<String>> initExternalList(List<List<String>> externalNames, Set<? extends Object> accessed) { if (externalNames == null) { externalNames = new ArrayList<List<String>>(accessed.size()); @@ -153,8 +161,14 @@ } for (Object o : this.objectsAccessed) { if (!data) { - if (o instanceof Modifiable && ((Modifiable)o).getLastModified() - modTime > this.creationTime) { - return false; + if (o instanceof Modifiable) { + Modifiable m = (Modifiable)o; + if (m.getLastModified() < 0) { + return false; //invalid object + } + if (sensitiveToMetadataChanges && m.getLastModified() - modTime > this.creationTime) { + return false; + } } } else if (o instanceof DataModifiable && ((DataModifiable)o).getLastDataModification() - modTime > this.creationTime) { return false; Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/process/DQPWorkContext.java =================================================================== --- trunk/engine/src/main/java/org/teiid/dqp/internal/process/DQPWorkContext.java 2011-10-18 17:24:41 UTC (rev 3566) +++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/DQPWorkContext.java 2011-10-18 18:59:58 UTC (rev 3567) @@ -54,7 +54,9 @@ SEVEN_1("7.1"), //$NON-NLS-1$ SEVEN_2("7.2"), //$NON-NLS-1$ SEVEN_3("7.3"), //$NON-NLS-1$ - SEVEN_4("7.4"); //$NON-NLS-1$ + SEVEN_4("7.4"), //$NON-NLS-1$ + SEVEN_5("7.5"), //$NON-NLS-1$ + SEVEN_6("7.6"); //$NON-NLS-1$ private String string; Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/process/PreparedPlan.java =================================================================== --- trunk/engine/src/main/java/org/teiid/dqp/internal/process/PreparedPlan.java 2011-10-18 17:24:41 UTC (rev 3566) +++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/PreparedPlan.java 2011-10-18 18:59:58 UTC (rev 3567) @@ -29,6 +29,8 @@ import org.teiid.common.buffer.BufferManager; import org.teiid.query.analysis.AnalysisRecord; import org.teiid.query.processor.ProcessorPlan; +import org.teiid.query.processor.relational.AccessNode; +import org.teiid.query.processor.relational.RelationalPlan; import org.teiid.query.sql.lang.Command; import org.teiid.query.sql.symbol.Reference; import org.teiid.query.util.CommandContext; @@ -77,6 +79,13 @@ public void setPlan(ProcessorPlan planValue, CommandContext context){ plan = planValue; this.accessInfo.populate(context, false); + //TODO: expand this logic + if (planValue instanceof RelationalPlan) { + RelationalPlan rp = (RelationalPlan)planValue; + if (rp.getRootNode() instanceof AccessNode) { + this.accessInfo.setSensitiveToMetadataChanges(false); + } + } } /** Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java =================================================================== --- trunk/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java 2011-10-18 17:24:41 UTC (rev 3566) +++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java 2011-10-18 18:59:58 UTC (rev 3567) @@ -52,6 +52,7 @@ import org.teiid.core.types.DataTypeManager; import org.teiid.dqp.internal.process.DQPCore.CompletionListener; import org.teiid.dqp.internal.process.DQPCore.FutureWork; +import org.teiid.dqp.internal.process.DQPWorkContext.Version; import org.teiid.dqp.internal.process.SessionAwareCache.CacheID; import org.teiid.dqp.internal.process.ThreadReuseExecutor.PrioritizedRunnable; import org.teiid.dqp.message.AtomicRequestID; @@ -543,7 +544,6 @@ resultsBuffer = this.processor.getBufferManager().createTupleBuffer(this.originalCommand.getProjectedSymbols(), this.request.context.getConnectionID(), TupleSourceType.FINAL); } analysisRecord = request.analysisRecord; - analysisRecord.setQueryPlan(processor.getProcessorPlan().getDescriptionProperties()); transactionContext = request.transactionContext; if (this.transactionContext != null && this.transactionContext.getTransactionType() != Scope.NONE) { this.transactionState = TransactionState.ACTIVE; @@ -677,6 +677,7 @@ dataTypes[i] = DataTypeManager.getDataTypeName(symbol.getType()); } ResultsMessage result = new ResultsMessage(batch, columnNames, dataTypes); + result.setClientSerializationVersion((byte)(this.dqpWorkContext.getClientVersion().compareTo(Version.SEVEN_6) >= 0?1:0)); setAnalysisRecords(result); return result; } Modified: trunk/engine/src/main/java/org/teiid/query/metadata/TempMetadataID.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/metadata/TempMetadataID.java 2011-10-18 17:24:41 UTC (rev 3566) +++ trunk/engine/src/main/java/org/teiid/query/metadata/TempMetadataID.java 2011-10-18 18:59:58 UTC (rev 3567) @@ -62,13 +62,17 @@ List<List<TempMetadataID>> keys; List<List<TempMetadataID>> indexes; long lastDataModification; - long lastModified; + long lastModified = System.currentTimeMillis(); int modCount; public long getLastDataModification() { return lastDataModification; } + public void removed() { + this.lastModified = -1; + } + public void dataModified(int updateCount) { if (updateCount == 0) { return; Modified: trunk/engine/src/test/java/org/teiid/dqp/internal/process/TestDQPCore.java =================================================================== --- trunk/engine/src/test/java/org/teiid/dqp/internal/process/TestDQPCore.java 2011-10-18 17:24:41 UTC (rev 3566) +++ trunk/engine/src/test/java/org/teiid/dqp/internal/process/TestDQPCore.java 2011-10-18 18:59:58 UTC (rev 3567) @@ -441,22 +441,35 @@ assertNull(rm.getException()); assertEquals(0, rm.getResultsList().size()); } - + @Test public void testPreparedPlanInvalidation() throws Exception { - String sql = "insert into #temp select * FROM vqt.SmallB"; //$NON-NLS-1$ + helpTestPlanInvalidation("select * from #temp a, #temp b limit 10"); + + assertEquals(2, this.core.getPrepPlanCache().getCacheHitCount()); + } + + @Test public void testPreparedPlanSimpleNoInvalidation() throws Exception { + helpTestPlanInvalidation("select * from #temp"); + + assertEquals(3, this.core.getPrepPlanCache().getCacheHitCount()); + } + + private void helpTestPlanInvalidation(String query) throws InterruptedException, + ExecutionException, TimeoutException { + String sql = "insert into #temp select * FROM vqt.SmallB"; //$NON-NLS-1$ String userName = "1"; //$NON-NLS-1$ int sessionid = 1; //$NON-NLS-1$ RequestMessage reqMsg = exampleRequestMessage(sql); ResultsMessage rm = execute(userName, sessionid, reqMsg); assertEquals(1, rm.getResultsList().size()); //$NON-NLS-1$ - sql = "select * from #temp"; //$NON-NLS-1$ + sql = query; reqMsg = exampleRequestMessage(sql); reqMsg.setStatementType(StatementType.PREPARED); rm = execute(userName, sessionid, reqMsg); assertEquals(10, rm.getResultsList().size()); //$NON-NLS-1$ - sql = "select * from #temp"; //$NON-NLS-1$ + sql = query; reqMsg = exampleRequestMessage(sql); reqMsg.setStatementType(StatementType.PREPARED); rm = execute(userName, sessionid, reqMsg); @@ -472,7 +485,7 @@ rm = execute(userName, sessionid, reqMsg); assertEquals(1, rm.getResultsList().size()); //$NON-NLS-1$ - sql = "select * from #temp"; //$NON-NLS-1$ + sql = query; reqMsg = exampleRequestMessage(sql); reqMsg.setStatementType(StatementType.PREPARED); rm = execute(userName, sessionid, reqMsg); @@ -480,20 +493,18 @@ assertEquals(2, this.core.getPrepPlanCache().getCacheHitCount()); - //perform a major update, we will purge the plan + //perform a major update, it might purge the plan sql = "delete from #temp"; //$NON-NLS-1$ reqMsg = exampleRequestMessage(sql); rm = execute(userName, sessionid, reqMsg); assertEquals(1, rm.getResultsList().size()); //$NON-NLS-1$ - sql = "select * from #temp"; //$NON-NLS-1$ + sql = query; reqMsg = exampleRequestMessage(sql); reqMsg.setStatementType(StatementType.PREPARED); rm = execute(userName, sessionid, reqMsg); assertEquals(0, rm.getResultsList().size()); //$NON-NLS-1$ - - assertEquals(2, this.core.getPrepPlanCache().getCacheHitCount()); - } + } @Test public void testRsCacheInvalidation() throws Exception { String sql = "select * FROM vqt.SmallB"; //$NON-NLS-1$ Modified: trunk/test-integration/common/src/test/java/org/teiid/jdbc/FakeServer.java =================================================================== --- trunk/test-integration/common/src/test/java/org/teiid/jdbc/FakeServer.java 2011-10-18 17:24:41 UTC (rev 3566) +++ trunk/test-integration/common/src/test/java/org/teiid/jdbc/FakeServer.java 2011-10-18 18:59:58 UTC (rev 3567) @@ -38,6 +38,7 @@ import org.teiid.cache.CacheConfiguration.Policy; import org.teiid.client.DQP; import org.teiid.client.security.ILogon; +import org.teiid.core.util.UnitTestUtil; import org.teiid.deployers.CompositeVDB; import org.teiid.deployers.MetadataStoreGroup; import org.teiid.deployers.UDFMetaData; @@ -65,6 +66,7 @@ import org.teiid.query.optimizer.capabilities.SourceCapabilities; import org.teiid.query.tempdata.GlobalTableStore; import org.teiid.query.tempdata.GlobalTableStoreImpl; +import org.teiid.services.BufferServiceImpl; import org.teiid.services.SessionServiceImpl; import org.teiid.transport.ClientServiceRegistry; import org.teiid.transport.ClientServiceRegistryImpl; @@ -91,6 +93,10 @@ } public FakeServer(DQPConfiguration config) { + this(config, false); + } + + public FakeServer(DQPConfiguration config, boolean realBufferMangaer) { this.logon = new LogonImpl(sessionService, null); this.repo.addListener(new VDBLifeCycleListener() { @@ -118,7 +124,14 @@ this.repo.start(); this.sessionService.setVDBRepository(repo); - this.dqp.setBufferService(new FakeBufferService()); + if (!realBufferMangaer) { + this.dqp.setBufferService(new FakeBufferService()); + } else { + BufferServiceImpl bsi = new BufferServiceImpl(); + bsi.setDiskDirectory(UnitTestUtil.getTestScratchPath()); + this.dqp.setBufferService(bsi); + } + this.dqp.setCacheFactory(new DefaultCacheFactory()); this.dqp.setTransactionService(new FakeTransactionService()); @@ -144,6 +157,10 @@ registerClientService(DQP.class, dqp, null); } + public DQPCore getDqp() { + return dqp; + } + public void setConnectorManagerRepository(ConnectorManagerRepository cmr) { this.cmr = cmr; }