Author: shawkins Date: 2010-08-04 00:01:19 -0400 (Wed, 04 Aug 2010) New Revision: 2412 Modified: trunk/engine/src/main/java/org/teiid/common/buffer/BatchManager.java trunk/engine/src/main/java/org/teiid/common/buffer/SPage.java trunk/engine/src/main/java/org/teiid/common/buffer/TupleBuffer.java trunk/engine/src/main/java/org/teiid/common/buffer/impl/BufferManagerImpl.java trunk/engine/src/main/java/org/teiid/dqp/internal/process/PreparedStatementRequest.java trunk/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java trunk/engine/src/main/java/org/teiid/query/function/FunctionDescriptor.java trunk/engine/src/main/java/org/teiid/query/util/CommandContext.java trunk/engine/src/main/resources/org/teiid/dqp/i18n.properties trunk/engine/src/test/java/org/teiid/common/buffer/TestTupleBuffer.java trunk/engine/src/test/java/org/teiid/dqp/internal/process/TestPreparedPlanCache.java Log: TEIID-168 minor updates to caching logic TEIID-829 changing code tables to use temp tables rather than being a separate cache Modified: trunk/engine/src/main/java/org/teiid/common/buffer/BatchManager.java =================================================================== --- trunk/engine/src/main/java/org/teiid/common/buffer/BatchManager.java 2010-08-04 04:00:09 UTC (rev 2411) +++ trunk/engine/src/main/java/org/teiid/common/buffer/BatchManager.java 2010-08-04 04:01:19 UTC (rev 2412) @@ -34,7 +34,7 @@ } - ManagedBatch createManagedBatch(TupleBatch batch) throws TeiidComponentException; + ManagedBatch createManagedBatch(TupleBatch batch, boolean softCache) throws TeiidComponentException; void remove(); Modified: trunk/engine/src/main/java/org/teiid/common/buffer/SPage.java =================================================================== --- trunk/engine/src/main/java/org/teiid/common/buffer/SPage.java 2010-08-04 04:00:09 UTC (rev 2411) +++ trunk/engine/src/main/java/org/teiid/common/buffer/SPage.java 2010-08-04 04:01:19 UTC (rev 2412) @@ -135,9 +135,9 @@ values.setDataTypes(stree.types); } if (children != null) { - managedBatch = stree.keyManager.createManagedBatch(values); + managedBatch = stree.keyManager.createManagedBatch(values, true); } else { - managedBatch = stree.leafManager.createManagedBatch(values); + managedBatch = stree.leafManager.createManagedBatch(values, stree.preferMemory); } } Modified: trunk/engine/src/main/java/org/teiid/common/buffer/TupleBuffer.java =================================================================== --- trunk/engine/src/main/java/org/teiid/common/buffer/TupleBuffer.java 2010-08-04 04:00:09 UTC (rev 2411) +++ trunk/engine/src/main/java/org/teiid/common/buffer/TupleBuffer.java 2010-08-04 04:01:19 UTC (rev 2412) @@ -164,7 +164,7 @@ writeBatch.setTerminationFlag(true); } writeBatch.setDataTypes(types); - BatchManager.ManagedBatch mbatch = manager.createManagedBatch(writeBatch); + BatchManager.ManagedBatch mbatch = manager.createManagedBatch(writeBatch, false); this.batches.put(writeBatch.getBeginRow(), mbatch); batchBuffer = null; } Modified: trunk/engine/src/main/java/org/teiid/common/buffer/impl/BufferManagerImpl.java =================================================================== --- trunk/engine/src/main/java/org/teiid/common/buffer/impl/BufferManagerImpl.java 2010-08-04 04:00:09 UTC (rev 2411) +++ trunk/engine/src/main/java/org/teiid/common/buffer/impl/BufferManagerImpl.java 2010-08-04 04:01:19 UTC (rev 2412) @@ -29,6 +29,7 @@ import java.io.ObjectOutputStream; import java.io.OutputStream; import java.lang.ref.Reference; +import java.lang.ref.SoftReference; import java.lang.ref.WeakReference; import java.util.ArrayList; import java.util.Collections; @@ -113,9 +114,9 @@ } @Override - public ManagedBatch createManagedBatch(TupleBatch batch) + public ManagedBatch createManagedBatch(TupleBatch batch, boolean softCache) throws TeiidComponentException { - ManagedBatchImpl mbi = new ManagedBatchImpl(batch, this); + ManagedBatchImpl mbi = new ManagedBatchImpl(batch, this, softCache); mbi.addToCache(false); persistBatchReferences(); return mbi; @@ -193,6 +194,7 @@ private final class ManagedBatchImpl implements ManagedBatch { private boolean persistent; + private boolean softCache; private volatile TupleBatch activeBatch; private volatile Reference<TupleBatch> batchReference; private int beginRow; @@ -200,7 +202,8 @@ private long id; private LobManager lobManager; - public ManagedBatchImpl(TupleBatch batch, BatchManagerImpl manager) { + public ManagedBatchImpl(TupleBatch batch, BatchManagerImpl manager, boolean softCache) { + this.softCache = softCache; id = batchAdded.incrementAndGet(); LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, "Add batch to BufferManager", id); //$NON-NLS-1$ this.activeBatch = batch; @@ -331,9 +334,11 @@ } LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, batchManager.id, id, "batch written starting at:", offset); //$NON-NLS-1$ } - this.batchReference = new WeakReference<TupleBatch>(batch); - } else { - assert persistent; + if (softCache) { + this.batchReference = new SoftReference<TupleBatch>(batch); + } else { + this.batchReference = new WeakReference<TupleBatch>(batch); + } } } catch (IOException e) { throw new TeiidComponentException(e); Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/process/PreparedStatementRequest.java =================================================================== --- trunk/engine/src/main/java/org/teiid/dqp/internal/process/PreparedStatementRequest.java 2010-08-04 04:00:09 UTC (rev 2411) +++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/PreparedStatementRequest.java 2010-08-04 04:01:19 UTC (rev 2412) @@ -146,7 +146,7 @@ // Defect 13751: Clone the plan in its current state (i.e. before processing) so that it can be used for later queries prepPlan.setPlan(processPlan.clone()); prepPlan.setAnalysisRecord(analysisRecord); - this.prepPlanCache.put(id, this.context.isSessionFunctionEvaluated(), this.context.isUserFunctionEvaluated(), prepPlan); + this.prepPlanCache.put(id, this.context.getDeterminismLevel(), prepPlan); } } else { LogManager.logTrace(LogConstants.CTX_DQP, new Object[] { "Query exist in cache: ", sqlQuery }); //$NON-NLS-1$ Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java =================================================================== --- trunk/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java 2010-08-04 04:00:09 UTC (rev 2411) +++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java 2010-08-04 04:01:19 UTC (rev 2412) @@ -59,6 +59,7 @@ import org.teiid.logging.CommandLogMessage.Event; import org.teiid.query.analysis.AnalysisRecord; import org.teiid.query.execution.QueryExecPlugin; +import org.teiid.query.function.metadata.FunctionMethod; import org.teiid.query.processor.BatchCollector; import org.teiid.query.processor.QueryProcessor; import org.teiid.query.sql.lang.Command; @@ -362,13 +363,15 @@ doneProducingBatches(); } if (doneProducingBatches && cid != null) { - boolean sessionScope = processor.getContext().isSessionFunctionEvaluated(); - boolean userScope = processor.getContext().isUserFunctionEvaluated(); + int determinismLevel = processor.getContext().getDeterminismLevel(); CachedResults cr = new CachedResults(); cr.setCommand(originalCommand); cr.setAnalysisRecord(analysisRecord); cr.setResults(resultsBuffer); - dqpCore.getRsCache().put(cid, sessionScope, userScope, cr); + if (determinismLevel > FunctionMethod.SESSION_DETERMINISTIC) { + LogManager.logInfo(LogConstants.CTX_DQP, DQPPlugin.Util.getString("RequestWorkItem.cache_nondeterministic", originalCommand)); //$NON-NLS-1$ + } + dqpCore.getRsCache().put(cid, determinismLevel, cr); } add = sendResultsIfNeeded(batch); if (!added) { Modified: trunk/engine/src/main/java/org/teiid/query/function/FunctionDescriptor.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/function/FunctionDescriptor.java 2010-08-04 04:00:09 UTC (rev 2411) +++ trunk/engine/src/main/java/org/teiid/query/function/FunctionDescriptor.java 2010-08-04 04:01:19 UTC (rev 2412) @@ -239,16 +239,11 @@ throw new FunctionExecutionException(ErrorMessageKeys.FUNCTION_0002, QueryPlugin.Util.getString(ErrorMessageKeys.FUNCTION_0002, getName())); } - if (getDeterministic() == FunctionMethod.USER_DETERMINISTIC && values.length > 0 && values[0] instanceof CommandContext) { + if (getDeterministic() >= FunctionMethod.USER_DETERMINISTIC && values.length > 0 && values[0] instanceof CommandContext) { CommandContext cc = (CommandContext)values[0]; - cc.setUserFunctionEvaluated(true); + cc.setDeterminismLevel(getDeterministic()); } - if (getDeterministic() >= FunctionMethod.SESSION_DETERMINISTIC && values.length > 0 && values[0] instanceof CommandContext) { - CommandContext cc = (CommandContext)values[0]; - cc.setSessionFunctionEvaluated(true); - } - // Invoke the method and return the result try { if (method.isVarArgs()) { Modified: trunk/engine/src/main/java/org/teiid/query/util/CommandContext.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/util/CommandContext.java 2010-08-04 04:00:09 UTC (rev 2411) +++ trunk/engine/src/main/java/org/teiid/query/util/CommandContext.java 2010-08-04 04:01:19 UTC (rev 2412) @@ -86,10 +86,8 @@ private QueryProcessor.ProcessorFactory queryProcessorFactory; - private boolean sessionFunctionEvaluated; + private int determinismLevel; - private boolean userFunctionEvaluated; - private Set<String> groups; private long timeSliceEnd = Long.MAX_VALUE; @@ -140,22 +138,14 @@ public CommandContext() { } - public boolean isSessionFunctionEvaluated() { - return globalState.sessionFunctionEvaluated; + public int getDeterminismLevel() { + return globalState.determinismLevel; } - public void setSessionFunctionEvaluated(boolean sessionFunctionEvaluated) { - globalState.sessionFunctionEvaluated = sessionFunctionEvaluated; - } + public void setDeterminismLevel(int level) { + globalState.determinismLevel = Math.max(globalState.determinismLevel, level); + } - public boolean isUserFunctionEvaluated() { - return globalState.userFunctionEvaluated; - } - - public void setUserFunctionEvaluated(boolean userFunctionEvaluated) { - globalState.userFunctionEvaluated = userFunctionEvaluated; - } - /** * @return */ Modified: trunk/engine/src/main/resources/org/teiid/dqp/i18n.properties =================================================================== --- trunk/engine/src/main/resources/org/teiid/dqp/i18n.properties 2010-08-04 04:00:09 UTC (rev 2411) +++ trunk/engine/src/main/resources/org/teiid/dqp/i18n.properties 2010-08-04 04:01:19 UTC (rev 2412) @@ -514,7 +514,6 @@ TransformationMetadata.Invalid_type=Invalid type: {0}. TransformationMetadata.Unable_to_determine_fullname_for_element__1=Unable to determine fullname for element -CachedRequestWorkItem.not_available=Cache result is no longer available. CachedFinder.no_connector_found=No connector with jndi-name {0} found for Model {1} with source name {2} failed_to_get_connection= Failed to get connection for translator {0} failed_to_close_connection=Failed to close the connection for translator {0} @@ -522,5 +521,7 @@ datasource_not_found=Data Source {0} not accessible. failed_to_bind_translator=Failed to bind the translator {0} on the jndi tree failed_to_unbind_translator=Failed to un-bind the translator {0} from the jndi tree. + +RequestWorkItem.cache_nondeterministic=Caching command '{0}'' at a session level, but less deterministic functions were evaluated. not_found_cache=Results not found in cache failed_to_put_in_cache=Failed to put results in the cache \ No newline at end of file Modified: trunk/engine/src/test/java/org/teiid/common/buffer/TestTupleBuffer.java =================================================================== --- trunk/engine/src/test/java/org/teiid/common/buffer/TestTupleBuffer.java 2010-08-04 04:00:09 UTC (rev 2411) +++ trunk/engine/src/test/java/org/teiid/common/buffer/TestTupleBuffer.java 2010-08-04 04:01:19 UTC (rev 2412) @@ -45,7 +45,7 @@ } @Override - public ManagedBatch createManagedBatch(final TupleBatch batch) + public ManagedBatch createManagedBatch(final TupleBatch batch, boolean softCache) throws TeiidComponentException { return new ManagedBatch() { Modified: trunk/engine/src/test/java/org/teiid/dqp/internal/process/TestPreparedPlanCache.java =================================================================== --- trunk/engine/src/test/java/org/teiid/dqp/internal/process/TestPreparedPlanCache.java 2010-08-04 04:00:09 UTC (rev 2411) +++ trunk/engine/src/test/java/org/teiid/dqp/internal/process/TestPreparedPlanCache.java 2010-08-04 04:01:19 UTC (rev 2412) @@ -31,6 +31,7 @@ import org.teiid.api.exception.query.QueryParserException; import org.teiid.dqp.internal.process.SessionAwareCache.CacheID; import org.teiid.query.analysis.AnalysisRecord; +import org.teiid.query.function.metadata.FunctionMethod; import org.teiid.query.parser.ParseInfo; import org.teiid.query.parser.QueryParser; import org.teiid.query.processor.relational.ProjectNode; @@ -61,7 +62,7 @@ //No PreparedPlan at the begining assertNull(cache.get(id)); //create one - cache.put(id, true, false, new PreparedPlan()); + cache.put(id, FunctionMethod.SESSION_DETERMINISTIC, new PreparedPlan()); //should have one now assertNotNull("Unable to get prepared plan from cache", cache.get(id)); //$NON-NLS-1$ } @@ -148,7 +149,7 @@ CacheID id = new CacheID(session, pi, dummy.toString()); PreparedPlan pPlan = new PreparedPlan(); - cache.put(id, true, false, pPlan); + cache.put(id, FunctionMethod.SESSION_DETERMINISTIC, pPlan); pPlan.setCommand(dummy); pPlan.setPlan(new RelationalPlan(new ProjectNode(i))); AnalysisRecord analysisRecord = new AnalysisRecord(true, false);