[teiid-commits] teiid SVN: r3010 - in trunk/engine/src: main/java/org/teiid/query/processor/relational and 4 other directories.

Author: shawkins Date: 2011-03-19 10:32:44 -0400 (Sat, 19 Mar 2011) New Revision: 3010 Modified: trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/CriteriaCapabilityValidatorVisitor.java trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleMergeVirtual.java trunk/engine/src/main/java/org/teiid/query/processor/relational/SubqueryAwareEvaluator.java trunk/engine/src/main/java/org/teiid/query/rewriter/QueryRewriter.java trunk/engine/src/main/java/org/teiid/query/sql/util/VariableContext.java trunk/engine/src/main/java/org/teiid/query/sql/visitor/FunctionCollectorVisitor.java trunk/engine/src/test/java/org/teiid/query/processor/TestProcessor.java Log: TEIID-1497 adding determinism checks and cleaning up correlation logic Modified: trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/CriteriaCapabilityValidatorVisitor.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/CriteriaCapabilityValidatorVisitor.java 2011-03-19 01:18:44 UTC (rev 3009) +++ trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/CriteriaCapabilityValidatorVisitor.java 2011-03-19 14:32:44 UTC (rev 3010) @@ -76,6 +76,7 @@ import org.teiid.query.sql.symbol.XMLSerialize; import org.teiid.query.sql.util.SymbolMap; import org.teiid.query.sql.visitor.EvaluatableVisitor; +import org.teiid.query.sql.visitor.FunctionCollectorVisitor; import org.teiid.query.sql.visitor.GroupCollectorVisitor; @@ -386,7 +387,7 @@ try { if(!this.caps.supportsCapability(Capability.QUERY_SUBQUERIES_SCALAR) || validateSubqueryPushdown(obj, modelID, metadata, capFinder, analysisRecord) == null) { - if (obj.getCommand().getCorrelatedReferences() == null) { + if (obj.getCommand().getCorrelatedReferences() == null && !FunctionCollectorVisitor.isNonDeterministic(obj.getCommand())) { obj.setShouldEvaluate(true); } else { markInvalid(obj.getCommand(), !this.caps.supportsCapability(Capability.QUERY_SUBQUERIES_SCALAR)? Modified: trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleMergeVirtual.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleMergeVirtual.java 2011-03-19 01:18:44 UTC (rev 3009) +++ trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleMergeVirtual.java 2011-03-19 14:32:44 UTC (rev 3010) @@ -32,7 +32,6 @@ import org.teiid.api.exception.query.QueryMetadataException; import org.teiid.api.exception.query.QueryPlannerException; import org.teiid.core.TeiidComponentException; -import org.teiid.metadata.FunctionMethod.Determinism; import org.teiid.query.analysis.AnalysisRecord; import org.teiid.query.metadata.QueryMetadataInterface; import org.teiid.query.optimizer.capabilities.CapabilitiesFinder; @@ -47,7 +46,6 @@ import org.teiid.query.sql.lang.OrderByItem; import org.teiid.query.sql.symbol.ElementSymbol; import org.teiid.query.sql.symbol.Expression; -import org.teiid.query.sql.symbol.Function; import org.teiid.query.sql.symbol.GroupSymbol; import org.teiid.query.sql.symbol.SingleElementSymbol; import org.teiid.query.sql.util.SymbolMap; @@ -355,11 +353,8 @@ return false; } // TEIID-16: We do not want to merge a non-deterministic scalar function - Collection<Function> functions = FunctionCollectorVisitor.getFunctions(symbol, true, true); - for (Function function : functions) { - if ( function.getFunctionDescriptor().getDeterministic() == Determinism.NONDETERMINISTIC) { - return false; - } + if (FunctionCollectorVisitor.isNonDeterministic(symbol)) { + return false; } } Modified: trunk/engine/src/main/java/org/teiid/query/processor/relational/SubqueryAwareEvaluator.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/processor/relational/SubqueryAwareEvaluator.java 2011-03-19 01:18:44 UTC (rev 3009) +++ trunk/engine/src/main/java/org/teiid/query/processor/relational/SubqueryAwareEvaluator.java 2011-03-19 14:32:44 UTC (rev 3010) @@ -30,6 +30,7 @@ import org.teiid.common.buffer.BufferManager; import org.teiid.core.TeiidComponentException; import org.teiid.core.TeiidProcessingException; +import org.teiid.core.types.DataTypeManager; import org.teiid.query.eval.Evaluator; import org.teiid.query.processor.BatchCollector; import org.teiid.query.processor.ProcessorDataManager; @@ -39,8 +40,11 @@ import org.teiid.query.sql.symbol.ContextReference; import org.teiid.query.sql.symbol.ElementSymbol; import org.teiid.query.sql.symbol.Expression; +import org.teiid.query.sql.symbol.ScalarSubquery; +import org.teiid.query.sql.util.SymbolMap; import org.teiid.query.sql.util.ValueIterator; import org.teiid.query.sql.util.VariableContext; +import org.teiid.query.sql.visitor.FunctionCollectorVisitor; import org.teiid.query.util.CommandContext; @@ -54,8 +58,10 @@ QueryProcessor processor; BatchCollector collector; boolean done; - List<?> tuple; ProcessorPlan plan; + boolean nonDeterministic; + List<Object> refValues; + boolean comparable = true; void close() { if (processor == null) { @@ -73,13 +79,13 @@ //processing state private Map<String, SubqueryState> subqueries = new HashMap<String, SubqueryState>(); - + public SubqueryAwareEvaluator(Map elements, ProcessorDataManager dataMgr, CommandContext context, BufferManager manager) { super(elements, dataMgr, context); this.manager = manager; } - + public void reset() { for (SubqueryState subQueryState : subqueries.values()) { subQueryState.plan.reset(); @@ -102,26 +108,48 @@ if (state == null) { state = new SubqueryState(); state.plan = container.getCommand().getProcessorPlan().clone(); + if (container instanceof ScalarSubquery) { + state.nonDeterministic = FunctionCollectorVisitor.isNonDeterministic(container.getCommand()); + } + if (container.getCommand().getCorrelatedReferences() != null) { + for (ElementSymbol es : container.getCommand().getCorrelatedReferences().getKeys()) { + if (DataTypeManager.isNonComparable(DataTypeManager.getDataTypeName(es.getType()))) { + state.comparable = false; + break; + } + } + } this.subqueries.put(key, state); } - if ((tuple == null && state.tuple != null) || (tuple != null && !tuple.equals(state.tuple))) { - if (container.getCommand().getCorrelatedReferences() != null) { - state.close(); + SymbolMap correlatedRefs = container.getCommand().getCorrelatedReferences(); + VariableContext currentContext = null; + boolean shouldClose = state.done && state.nonDeterministic; + if (correlatedRefs != null) { + currentContext = new VariableContext(); + for (Map.Entry<ElementSymbol, Expression> entry : container.getCommand().getCorrelatedReferences().asMap().entrySet()) { + currentContext.setValue(entry.getKey(), evaluate(entry.getValue(), tuple)); } - state.tuple = tuple; + List<Object> refValues = currentContext.getLocalValues(); + if (!refValues.equals(state.refValues)) { + state.refValues = refValues; + shouldClose = true; + } } + if (shouldClose) { + //if (state.done && state.comparable) { + //cache + //} else { + state.close(); + //} + } if (!state.done) { if (state.processor == null) { CommandContext subContext = context.clone(); state.plan.reset(); state.processor = new QueryProcessor(state.plan, subContext, manager, this.dataMgr); - if (container.getCommand().getCorrelatedReferences() != null) { - VariableContext currentContext = new VariableContext(); - for (Map.Entry<ElementSymbol, Expression> entry : container.getCommand().getCorrelatedReferences().asMap().entrySet()) { - currentContext.setValue(entry.getKey(), evaluate(entry.getValue(), tuple)); - } - state.processor.getContext().pushVariableContext(currentContext); - } + if (currentContext != null) { + state.processor.getContext().pushVariableContext(currentContext); + } state.collector = state.processor.createBatchCollector(); } state.done = true; Modified: trunk/engine/src/main/java/org/teiid/query/rewriter/QueryRewriter.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/rewriter/QueryRewriter.java 2011-03-19 01:18:44 UTC (rev 3009) +++ trunk/engine/src/main/java/org/teiid/query/rewriter/QueryRewriter.java 2011-03-19 14:32:44 UTC (rev 3010) @@ -55,7 +55,6 @@ import org.teiid.core.util.Assertion; import org.teiid.core.util.TimestampWithTimezone; import org.teiid.language.SQLConstants.NonReserved; -import org.teiid.metadata.FunctionMethod.Determinism; import org.teiid.query.QueryPlugin; import org.teiid.query.eval.Evaluator; import org.teiid.query.function.FunctionDescriptor; @@ -162,7 +161,6 @@ import org.teiid.query.sql.visitor.EvaluatableVisitor; import org.teiid.query.sql.visitor.ExpressionMappingVisitor; import org.teiid.query.sql.visitor.FunctionCollectorVisitor; -import org.teiid.query.sql.visitor.ValueIteratorProviderCollectorVisitor; import org.teiid.query.sql.visitor.EvaluatableVisitor.EvaluationLevel; import org.teiid.query.util.CommandContext; import org.teiid.query.validator.UpdateValidator.UpdateInfo; @@ -750,11 +748,8 @@ } else if (query.getSelect().isDistinct()) { for (SingleElementSymbol projectSymbol : query.getSelect().getProjectedSymbols()) { Expression ex = SymbolMap.getExpression(projectSymbol); - Collection<Function> functions = FunctionCollectorVisitor.getFunctions(ex, true, false); - for (Function function : functions) { - if ( function.getFunctionDescriptor().getDeterministic() == Determinism.NONDETERMINISTIC) { - return true; - } + if (FunctionCollectorVisitor.isNonDeterministic(ex)) { + return true; } if (!ValueIteratorProviderCollectorVisitor.getValueIteratorProviders(ex).isEmpty()) { return true; Modified: trunk/engine/src/main/java/org/teiid/query/sql/util/VariableContext.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/sql/util/VariableContext.java 2011-03-19 01:18:44 UTC (rev 3009) +++ trunk/engine/src/main/java/org/teiid/query/sql/util/VariableContext.java 2011-03-19 14:32:44 UTC (rev 3010) @@ -22,7 +22,9 @@ package org.teiid.query.sql.util; -import java.util.HashMap; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import org.teiid.core.TeiidComponentException; @@ -33,7 +35,7 @@ public class VariableContext { // map between variables and their values - private Map variableMap = new HashMap(); + private Map variableMap = new LinkedHashMap(); // reference to the parent variable context private VariableContext parentContext; @@ -48,7 +50,6 @@ public VariableContext(boolean delegateSets) { this.delegateSets = delegateSets; - this.variableMap = new HashMap(); } public void setGlobalValue(String variable, Object value) { @@ -185,4 +186,8 @@ this.variableMap.putAll(other.variableMap); } + public List<Object> getLocalValues() { + return new ArrayList<Object>(this.variableMap.values()); + } + } Modified: trunk/engine/src/main/java/org/teiid/query/sql/visitor/FunctionCollectorVisitor.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/sql/visitor/FunctionCollectorVisitor.java 2011-03-19 01:18:44 UTC (rev 3009) +++ trunk/engine/src/main/java/org/teiid/query/sql/visitor/FunctionCollectorVisitor.java 2011-03-19 14:32:44 UTC (rev 3010) @@ -26,6 +26,7 @@ import java.util.Collection; import java.util.HashSet; +import org.teiid.metadata.FunctionMethod.Determinism; import org.teiid.query.QueryPlugin; import org.teiid.query.sql.LanguageObject; import org.teiid.query.sql.LanguageVisitor; @@ -141,4 +142,15 @@ getFunctions(obj, functions, deep); return functions; } + + public static boolean isNonDeterministic(LanguageObject ex) { + Collection<Function> functions = FunctionCollectorVisitor.getFunctions(ex, true, false); + for (Function function : functions) { + if ( function.getFunctionDescriptor().getDeterministic() == Determinism.NONDETERMINISTIC) { + return true; + } + } + return false; + } + } Modified: trunk/engine/src/test/java/org/teiid/query/processor/TestProcessor.java =================================================================== --- trunk/engine/src/test/java/org/teiid/query/processor/TestProcessor.java 2011-03-19 01:18:44 UTC (rev 3009) +++ trunk/engine/src/test/java/org/teiid/query/processor/TestProcessor.java 2011-03-19 14:32:44 UTC (rev 3010) @@ -7540,6 +7540,30 @@ helpProcess(plan, hdm, expected); } + @Test public void testNonDeterministicScalarSubquery() throws Exception { + FakeCapabilitiesFinder capFinder = new FakeCapabilitiesFinder(); + FakeMetadataFacade metadata = FakeMetadataFactory.example1Cached(); + + BasicSourceCapabilities caps = getTypicalCapabilities(); + caps.setCapabilitySupport(Capability.QUERY_SUBQUERIES_SCALAR, false); + caps.setCapabilitySupport(Capability.QUERY_AGGREGATES, true); + caps.setCapabilitySupport(Capability.QUERY_AGGREGATES_MAX, true); + capFinder.addCapabilities("pm1", caps); //$NON-NLS-1$ + + ProcessorPlan plan = helpPlan("select count(distinct x) from (select (select uuid()) as x from pm1.g1) as v", metadata, //$NON-NLS-1 + null, capFinder, + new String[] { "SELECT 1 FROM pm1.g1 AS g_0" }, ComparisonMode.EXACT_COMMAND_STRING); //$NON-NLS-1$ + + HardcodedDataManager hdm = new HardcodedDataManager(); + hdm.addData("SELECT 1 FROM pm1.g1 AS g_0", new List[] {Arrays.asList(1), Arrays.asList(1)}); + hdm.setBlockOnce(true); + List[] expected = new List[] { + Arrays.asList(2), + }; + + helpProcess(plan, hdm, expected); + } + @Test public void testUncorrelatedScalarSubqueryPushdown1() throws Exception { FakeCapabilitiesFinder capFinder = new FakeCapabilitiesFinder(); FakeMetadataFacade metadata = FakeMetadataFactory.example1Cached();