[teiid-commits] teiid SVN: r4562 - in branches/7.7.x/engine/src: main/java/org/teiid/query/processor/relational and 2 other directories.

Author: jolee Date: 2013-05-06 10:30:47 -0400 (Mon, 06 May 2013) New Revision: 4562 Modified: branches/7.7.x/engine/src/main/java/org/teiid/query/optimizer/relational/AliasGenerator.java branches/7.7.x/engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java branches/7.7.x/engine/src/main/java/org/teiid/query/processor/relational/RelationalPlan.java branches/7.7.x/engine/src/main/java/org/teiid/query/tempdata/TempTableDataManager.java branches/7.7.x/engine/src/main/java/org/teiid/query/tempdata/TempTableStore.java branches/7.7.x/engine/src/test/java/org/teiid/query/processor/TestWithClauseProcessing.java Log: TEIID-2478 backport: incomplete with pushdown. (required supporting changes) Modified: branches/7.7.x/engine/src/main/java/org/teiid/query/optimizer/relational/AliasGenerator.java =================================================================== --- branches/7.7.x/engine/src/main/java/org/teiid/query/optimizer/relational/AliasGenerator.java 2013-04-17 18:59:47 UTC (rev 4561) +++ branches/7.7.x/engine/src/main/java/org/teiid/query/optimizer/relational/AliasGenerator.java 2013-05-06 14:30:47 UTC (rev 4562) @@ -75,10 +75,14 @@ public SQLNamingContext(SQLNamingContext parent) { this.parent = parent; } + + public String getElementName(SingleElementSymbol symbol, boolean renameGroup) { + return getElementName(symbol, renameGroup, false); + } - public String getElementName(SingleElementSymbol symbol, boolean renameGroup) { + public String getElementName(SingleElementSymbol symbol, boolean renameGroup, boolean ignoreCurrent) { String name = null; - if (currentSymbols != null) { + if (currentSymbols != null && !ignoreCurrent) { name = currentSymbols.get(symbol); if (name != null) { if (renameGroup && symbol instanceof ElementSymbol) { @@ -255,7 +259,7 @@ private boolean needsAlias(String newAlias, ElementSymbol symbol) { - return !(symbol.getMetadataID() instanceof TempMetadataID) || !newAlias.equalsIgnoreCase(visitor.namingContext.getElementName(symbol, false)); + return !(symbol.getMetadataID() instanceof TempMetadataID) || !newAlias.equalsIgnoreCase(visitor.namingContext.getElementName(symbol, false, true)); } /** Modified: branches/7.7.x/engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java =================================================================== --- branches/7.7.x/engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java 2013-04-17 18:59:47 UTC (rev 4561) +++ branches/7.7.x/engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java 2013-05-06 14:30:47 UTC (rev 4562) @@ -48,10 +48,10 @@ import org.teiid.query.optimizer.capabilities.CapabilitiesFinder; import org.teiid.query.optimizer.capabilities.SourceCapabilities.Capability; import org.teiid.query.optimizer.relational.plantree.NodeConstants; +import org.teiid.query.optimizer.relational.plantree.NodeConstants.Info; import org.teiid.query.optimizer.relational.plantree.NodeEditor; import org.teiid.query.optimizer.relational.plantree.NodeFactory; import org.teiid.query.optimizer.relational.plantree.PlanNode; -import org.teiid.query.optimizer.relational.plantree.NodeConstants.Info; import org.teiid.query.optimizer.relational.rules.CapabilitiesUtil; import org.teiid.query.optimizer.relational.rules.CriteriaCapabilityValidatorVisitor; import org.teiid.query.optimizer.relational.rules.RuleAssignOutputElements; @@ -62,15 +62,16 @@ import org.teiid.query.optimizer.relational.rules.RulePushAggregates; import org.teiid.query.processor.ProcessorPlan; import org.teiid.query.processor.relational.AccessNode; +import org.teiid.query.processor.relational.JoinNode.JoinStrategyType; +import org.teiid.query.processor.relational.RelationalNode; import org.teiid.query.processor.relational.RelationalPlan; -import org.teiid.query.processor.relational.JoinNode.JoinStrategyType; import org.teiid.query.resolver.ProcedureContainerResolver; import org.teiid.query.resolver.QueryResolver; import org.teiid.query.resolver.util.ResolverUtil; import org.teiid.query.rewriter.QueryRewriter; import org.teiid.query.sql.LanguageObject; +import org.teiid.query.sql.LanguageObject.Util; import org.teiid.query.sql.LanguageVisitor; -import org.teiid.query.sql.LanguageObject.Util; import org.teiid.query.sql.lang.*; import org.teiid.query.sql.navigator.PreOrPostOrderNavigator; import org.teiid.query.sql.proc.CreateUpdateProcedureCommand; @@ -81,11 +82,12 @@ import org.teiid.query.sql.visitor.CorrelatedReferenceCollectorVisitor; import org.teiid.query.sql.visitor.ElementCollectorVisitor; import org.teiid.query.sql.visitor.ExpressionMappingVisitor; +import org.teiid.query.sql.visitor.GroupCollectorVisitor; import org.teiid.query.sql.visitor.GroupsUsedByElementsVisitor; import org.teiid.query.sql.visitor.ValueIteratorProviderCollectorVisitor; import org.teiid.query.util.CommandContext; +import org.teiid.query.validator.UpdateValidator.UpdateInfo; import org.teiid.query.validator.ValidationVisitor; -import org.teiid.query.validator.UpdateValidator.UpdateInfo; /** @@ -115,6 +117,13 @@ } }; + private static final Comparator<GroupSymbol> nonCorrelatedComparator = new Comparator<GroupSymbol>() { + @Override + public int compare(GroupSymbol arg0, GroupSymbol arg1) { + return arg0.getNonCorrelationName().compareTo(arg1.getNonCorrelationName()); + } + }; + public ProcessorPlan optimize( Command command) throws @@ -138,40 +147,44 @@ //plan with List<WithQueryCommand> withList = null; - Object modelID = null; - boolean supportsWithPushdown = true; - List<WithQueryCommand> pushDownWith = null; + Map<String, WithQueryCommand> pushdownWith = null; + Set<GroupSymbol> withGroups = null; if (command instanceof QueryCommand) { QueryCommand queryCommand = (QueryCommand)command; - final HashSet<String> names = new HashSet<String>(); - if (queryCommand.getWith() != null) { - withList = queryCommand.getWith(); - for (WithQueryCommand with : queryCommand.getWith()) { - Command subCommand = with.getCommand(); - ProcessorPlan procPlan = QueryOptimizer.optimizePlan(subCommand, metadata, idGenerator, capFinder, analysisRecord, context); + withList = queryCommand.getWith(); + if (withList != null) { + for (WithQueryCommand with : withList) { + QueryCommand subCommand = with.getCommand(); + RelationalPlan procPlan = (RelationalPlan)QueryOptimizer.optimizePlan(subCommand, metadata, idGenerator, capFinder, analysisRecord, context); + RelationalNode root = procPlan.getRootNode(); + Number planCardinality = root.getEstimateNodeCardinality(); + if (planCardinality != null) { + ((TempMetadataID)with.getGroupSymbol().getMetadataID()).setCardinality(planCardinality.intValue()); + } subCommand.setProcessorPlan(procPlan); AccessNode aNode = CriteriaCapabilityValidatorVisitor.getAccessNode(procPlan); - if (aNode != null && supportsWithPushdown) { - modelID = CriteriaCapabilityValidatorVisitor.validateCommandPushdown(modelID, metadata, capFinder, aNode); - } + if (aNode == null) { + continue; + } + Object modelID = CriteriaCapabilityValidatorVisitor.validateCommandPushdown(null, metadata, capFinder, aNode); QueryCommand withCommand = CriteriaCapabilityValidatorVisitor.getQueryCommand(aNode); if (modelID == null || withCommand == null) { - supportsWithPushdown = false; - } else { - if (pushDownWith == null) { - pushDownWith = new ArrayList<WithQueryCommand>(); - } - WithQueryCommand wqc = new WithQueryCommand(with.getGroupSymbol(), with.getColumns(), withCommand); - pushDownWith.add(wqc); + continue; } - names.add(with.getGroupSymbol().getCanonicalName()); + if (!CapabilitiesUtil.supports(Capability.COMMON_TABLE_EXPRESSIONS, modelID, metadata, capFinder)) { + continue; + } + WithQueryCommand wqc = new WithQueryCommand(with.getGroupSymbol(), with.getColumns(), withCommand); + with.getGroupSymbol().setModelMetadataId(modelID); + if (pushdownWith == null) { + pushdownWith = new LinkedHashMap<String, WithQueryCommand>(); + withGroups = new TreeSet<GroupSymbol>(nonCorrelatedComparator); + } + pushdownWith.put(with.getGroupSymbol().getName(), wqc); } - if (modelID != null && supportsWithPushdown) { - supportsWithPushdown = CapabilitiesUtil.supports(Capability.COMMON_TABLE_EXPRESSIONS, modelID, metadata, capFinder); + if (pushdownWith != null) { + addModelIds(command, pushdownWith); } - if (supportsWithPushdown) { - addModelIds(command, modelID, names); - } } } @@ -187,7 +200,7 @@ } // Connect ProcessorPlan to SubqueryContainer (if any) of SELECT or PROJECT nodes - connectSubqueryContainers(plan); //TODO: merge with node creation + connectSubqueryContainers(plan, pushdownWith, withGroups); //TODO: merge with node creation // Set top column information on top node List<SingleElementSymbol> topCols = Util.deepClone(command.getProjectedSymbols(), SingleElementSymbol.class); @@ -199,27 +212,62 @@ plan = executeRules(rules, plan); RelationalPlan result = planToProcessConverter.convert(plan); - if (withList != null && supportsWithPushdown) { - AccessNode aNode = CriteriaCapabilityValidatorVisitor.getAccessNode(result); - if (aNode != null) { - QueryCommand queryCommand = CriteriaCapabilityValidatorVisitor.getQueryCommand(aNode); - if (queryCommand == null || CriteriaCapabilityValidatorVisitor.validateCommandPushdown(modelID, metadata, capFinder, aNode) == null) { - supportsWithPushdown = false; - } else { - //TODO: do this prior to alias generation, or rerun alias generation here - queryCommand.setWith(pushDownWith); - } - } else { - supportsWithPushdown = false; - } + boolean fullPushdown = false; + if (pushdownWith != null) { + AccessNode aNode = CriteriaCapabilityValidatorVisitor.getAccessNode(result); + if (aNode != null) { + QueryCommand queryCommand = CriteriaCapabilityValidatorVisitor.getQueryCommand(aNode); + if (queryCommand != null) { + fullPushdown = true; + } + } + //distribute the appropriate clauses to the pushdowns + assignWithClause(result.getRootNode(), pushdownWith, withGroups); } - if (!supportsWithPushdown) { + if (!fullPushdown) { + //generally any with item associated with a pushdown will not be needed as we're converting to a source query result.setWith(withList); } result.setOutputElements(topCols); result.setSourceHint(sourceHint); return result; } + + private static void assignWithClause(RelationalNode node, Map<String, WithQueryCommand> pushdownWith, Set<GroupSymbol> groups) { + if(node instanceof AccessNode) { + AccessNode accessNode = (AccessNode) node; + Command command = accessNode.getCommand(); + if (command instanceof QueryCommand) { + groups.clear(); + GroupCollectorVisitor.getGroupsIgnoreInlineViews(command, groups); + List<WithQueryCommand> with = new ArrayList<WithQueryCommand>(); + for (GroupSymbol groupSymbol : groups) { + WithQueryCommand clause = pushdownWith.get(groupSymbol.getNonCorrelationName()); + if (clause != null) { + with.add(clause.clone()); + } + } + if (!with.isEmpty()) { + QueryCommand query = (QueryCommand)command; + if (query.getWith() != null) { + //we need to accumulate as a with clause could have been used at a lower scope + query.getWith().addAll(with); + } else { + query.setWith(with); + } + } + } + } + + // Recurse through children + RelationalNode[] children = node.getChildren(); + for(int i=0; i<children.length; i++) { + if (children[i] == null) { + break; + } + assignWithClause(children[i], pushdownWith, groups); + } + } /** * mark all relevant group symbols as being from the modelid @@ -227,14 +275,14 @@ * @param modelID * @param names */ - private void addModelIds(Command command, final Object modelID, - final HashSet<String> names) { + private void addModelIds(Command command, final Map<String, WithQueryCommand> with) { PreOrPostOrderNavigator.doVisit(command, new LanguageVisitor() { @Override public void visit(UnaryFromClause obj) { GroupSymbol group = obj.getGroup(); - if (names.contains(group.getNonCorrelationName().toUpperCase())) { - group.setModelMetadataId(modelID); + WithQueryCommand wqc = with.get(group.getNonCorrelationName()); + if (wqc != null) { + group.setModelMetadataId(wqc.getGroupSymbol().getModelMetadataId()); } } }, PreOrPostOrderNavigator.POST_ORDER, true); @@ -251,7 +299,7 @@ this.context = context; } - private void connectSubqueryContainers(PlanNode plan) throws QueryPlannerException, QueryMetadataException, TeiidComponentException { + private void connectSubqueryContainers(PlanNode plan, Map<String, WithQueryCommand> pushdownWith, Set<GroupSymbol> groups) throws QueryPlannerException, QueryMetadataException, TeiidComponentException { Set<GroupSymbol> groupSymbols = getGroupSymbols(plan); for (PlanNode node : NodeEditor.findAllNodes(plan, NodeConstants.Types.PROJECT | NodeConstants.Types.SELECT | NodeConstants.Types.JOIN | NodeConstants.Types.SOURCE)) { @@ -285,6 +333,20 @@ } } ProcessorPlan procPlan = QueryOptimizer.optimizePlan(subCommand, metadata, idGenerator, capFinder, analysisRecord, context); + if (procPlan instanceof RelationalPlan && pushdownWith != null) { + Map<String, WithQueryCommand> parentPushdownWith = pushdownWith; + if (subCommand instanceof QueryCommand) { + QueryCommand query = (QueryCommand)subCommand; + List<WithQueryCommand> with = query.getWith(); + if (with != null && !with.isEmpty()) { + parentPushdownWith = new HashMap<String, WithQueryCommand>(parentPushdownWith); + for (WithQueryCommand withQueryCommand : with) { + parentPushdownWith.remove(withQueryCommand.getGroupSymbol().getNonCorrelationName()); + } + } + } + assignWithClause(((RelationalPlan) procPlan).getRootNode(), parentPushdownWith, groups); + } container.getCommand().setProcessorPlan(procPlan); setCorrelatedReferences(container, correlatedReferences); } Modified: branches/7.7.x/engine/src/main/java/org/teiid/query/processor/relational/RelationalPlan.java =================================================================== --- branches/7.7.x/engine/src/main/java/org/teiid/query/processor/relational/RelationalPlan.java 2013-04-17 18:59:47 UTC (rev 4561) +++ branches/7.7.x/engine/src/main/java/org/teiid/query/processor/relational/RelationalPlan.java 2013-05-06 14:30:47 UTC (rev 4562) @@ -22,8 +22,8 @@ package org.teiid.query.processor.relational; -import java.util.LinkedList; import java.util.List; +import java.util.TreeMap; import org.teiid.client.plan.PlanNode; import org.teiid.common.buffer.BlockedException; @@ -34,19 +34,16 @@ import org.teiid.core.TeiidProcessingException; import org.teiid.language.SQLConstants; import org.teiid.query.analysis.AnalysisRecord; -import org.teiid.query.metadata.TempMetadataAdapter; -import org.teiid.query.processor.CollectionTupleSource; import org.teiid.query.processor.ProcessorDataManager; import org.teiid.query.processor.ProcessorPlan; import org.teiid.query.processor.QueryProcessor; import org.teiid.query.processor.relational.ProjectIntoNode.Mode; import org.teiid.query.sql.LanguageObject; -import org.teiid.query.sql.lang.Create; -import org.teiid.query.sql.lang.Insert; import org.teiid.query.sql.lang.QueryCommand; import org.teiid.query.sql.lang.SourceHint; import org.teiid.query.sql.lang.WithQueryCommand; import org.teiid.query.tempdata.TempTableStore; +import org.teiid.query.tempdata.TempTableStore.TableProcessor; import org.teiid.query.tempdata.TempTableStore.TransactionMode; import org.teiid.query.util.CommandContext; @@ -59,8 +56,6 @@ private List outputCols; private List<WithQueryCommand> with; - private List<WithQueryCommand> withToProcess; - private QueryProcessor withProcessor; private TempTableStore tempTableStore; private boolean multisourceUpdate; private SourceHint sourceHint; @@ -104,9 +99,6 @@ tempTableStore = new TempTableStore(context.getConnectionID(), TransactionMode.NONE); tempTableStore.setParentTempTableStore(context.getTempTableStore()); context.setTempTableStore(tempTableStore); - for (WithQueryCommand withCommand : this.with) { - withCommand.getCommand().getProcessorPlan().initialize(context, dataMgr, bufferMgr); - } } setContext(context); connectExternal(this.root, context, dataMgr, bufferMgr); @@ -136,34 +128,15 @@ public void open() throws TeiidComponentException, TeiidProcessingException { - if (this.with != null) { - if (withToProcess == null) { - withToProcess = new LinkedList<WithQueryCommand>(with); - } - while (!withToProcess.isEmpty()) { - WithQueryCommand withCommand = withToProcess.get(0); - if (withProcessor == null) { - ProcessorPlan plan = withCommand.getCommand().getProcessorPlan(); - withProcessor = new QueryProcessor(plan, getContext(), this.root.getBufferManager(), this.root.getDataManager()); - Create create = new Create(); - create.setElementSymbolsAsColumns(withCommand.getColumns()); - create.setTable(withCommand.getGroupSymbol()); - this.root.getDataManager().registerRequest(getContext(), create, TempMetadataAdapter.TEMP_MODEL.getID(), null, 0, -1); - } - while (true) { - TupleBatch batch = withProcessor.nextBatch(); - Insert insert = new Insert(withCommand.getGroupSymbol(), withCommand.getColumns(), null); - insert.setTupleSource(new CollectionTupleSource(batch.getTuples().iterator())); - this.root.getDataManager().registerRequest(getContext(), insert, TempMetadataAdapter.TEMP_MODEL.getID(), null, 0, -1); - if (batch.getTerminationFlag()) { - break; - } - } - this.tempTableStore.setUpdatable(withCommand.getGroupSymbol().getCanonicalName(), false); - withToProcess.remove(0); - withProcessor = null; + if (with != null && tempTableStore.getProcessors() == null) { + TreeMap<String, TableProcessor> processors = new TreeMap<String, TableProcessor>(String.CASE_INSENSITIVE_ORDER); + tempTableStore.setProcessors(processors); + for (WithQueryCommand withCommand : this.with) { + ProcessorPlan plan = withCommand.getCommand().getProcessorPlan(); + QueryProcessor withProcessor = new QueryProcessor(plan, getContext(), root.getBufferManager(), root.getDataManager()); + processors.put(withCommand.getGroupSymbol().getName(), new TableProcessor(withProcessor, withCommand.getColumns())); } - } + } this.root.open(); } @@ -178,14 +151,15 @@ public void close() throws TeiidComponentException { - if (this.with != null) { - for (WithQueryCommand withCommand : this.with) { - withCommand.getCommand().getProcessorPlan().close(); + if (this.tempTableStore != null) { + this.tempTableStore.removeTempTables(); + if (this.tempTableStore.getProcessors() != null) { + for (TableProcessor proc : this.tempTableStore.getProcessors().values()) { + proc.getQueryProcessor().closeProcessing(); + } + this.tempTableStore.setProcessors(null); } - if (this.tempTableStore != null) { - this.tempTableStore.removeTempTables(); - } - } + } this.root.close(); } @@ -197,8 +171,6 @@ this.root.reset(); if (this.with != null) { - withToProcess = null; - withProcessor = null; for (WithQueryCommand withCommand : this.with) { withCommand.getCommand().getProcessorPlan().reset(); } Modified: branches/7.7.x/engine/src/main/java/org/teiid/query/tempdata/TempTableDataManager.java =================================================================== --- branches/7.7.x/engine/src/main/java/org/teiid/query/tempdata/TempTableDataManager.java 2013-04-17 18:59:47 UTC (rev 4561) +++ branches/7.7.x/engine/src/main/java/org/teiid/query/tempdata/TempTableDataManager.java 2013-05-06 14:30:47 UTC (rev 4562) @@ -53,8 +53,8 @@ import org.teiid.language.SQLConstants.Reserved; import org.teiid.logging.LogConstants; import org.teiid.logging.LogManager; +import org.teiid.metadata.FunctionMethod.Determinism; import org.teiid.metadata.MetadataRepository; -import org.teiid.metadata.FunctionMethod.Determinism; import org.teiid.query.QueryPlugin; import org.teiid.query.eval.Evaluator; import org.teiid.query.metadata.QueryMetadataInterface; @@ -67,21 +67,7 @@ import org.teiid.query.processor.ProcessorDataManager; import org.teiid.query.processor.QueryProcessor; import org.teiid.query.resolver.util.ResolverUtil; -import org.teiid.query.sql.lang.CacheHint; -import org.teiid.query.sql.lang.Command; -import org.teiid.query.sql.lang.CompareCriteria; -import org.teiid.query.sql.lang.Create; -import org.teiid.query.sql.lang.Criteria; -import org.teiid.query.sql.lang.Delete; -import org.teiid.query.sql.lang.Drop; -import org.teiid.query.sql.lang.Insert; -import org.teiid.query.sql.lang.Option; -import org.teiid.query.sql.lang.ProcedureContainer; -import org.teiid.query.sql.lang.Query; -import org.teiid.query.sql.lang.SPParameter; -import org.teiid.query.sql.lang.StoredProcedure; -import org.teiid.query.sql.lang.UnaryFromClause; -import org.teiid.query.sql.lang.Update; +import org.teiid.query.sql.lang.*; import org.teiid.query.sql.navigator.PostOrderNavigator; import org.teiid.query.sql.symbol.Constant; import org.teiid.query.sql.symbol.ElementSymbol; @@ -354,7 +340,7 @@ } private TupleSource registerQuery(final CommandContext context, - TempTableStore contextStore, Query query) + final TempTableStore contextStore, final Query query) throws TeiidComponentException, QueryMetadataException, TeiidProcessingException, ExpressionEvaluationException, QueryProcessingException { @@ -416,16 +402,36 @@ } table = globalStore.getTempTableStore().getOrCreateTempTable(tableName, query, bufferManager, false, false, context); context.accessedDataObject(group.getMetadataID()); - } else { - table = contextStore.getOrCreateTempTable(tableName, query, bufferManager, true, false, context); - if (context.getDataObjects() != null) { - Object id = RelationalPlanner.getTrackableGroup(group, context.getMetadata()); - if (id != null) { - context.accessedDataObject(group.getMetadataID()); + return table.createTupleSource(query.getProjectedSymbols(), query.getCriteria(), query.getOrderBy()); + } + final GroupSymbol g = group; + //it's not expected for a blocked exception to bubble up from here, so return a tuplesource to perform getOrCreateTempTable + return new TupleSource() { + TupleSource ts = null; + + @Override + public List<?> nextTuple() throws TeiidComponentException, + TeiidProcessingException { + if (ts == null) { + TempTable tt = contextStore.getOrCreateTempTable(tableName, query, bufferManager, true, false, context); + if (context.getDataObjects() != null) { + Object id = RelationalPlanner.getTrackableGroup(g, context.getMetadata()); + if (id != null) { + context.accessedDataObject(g.getMetadataID()); + } + } + ts = tt.createTupleSource(query.getProjectedSymbols(), query.getCriteria(), query.getOrderBy()); } + return ts.nextTuple(); } - } - return table.createTupleSource(query.getProjectedSymbols(), query.getCriteria(), query.getOrderBy()); + + @Override + public void closeSource() { + if (ts != null) { + ts.closeSource(); + } + } + }; } private void loadAsynch(final CommandContext context, Modified: branches/7.7.x/engine/src/main/java/org/teiid/query/tempdata/TempTableStore.java =================================================================== --- branches/7.7.x/engine/src/main/java/org/teiid/query/tempdata/TempTableStore.java 2013-04-17 18:59:47 UTC (rev 4561) +++ branches/7.7.x/engine/src/main/java/org/teiid/query/tempdata/TempTableStore.java 2013-05-06 14:30:47 UTC (rev 4562) @@ -36,6 +36,7 @@ import javax.transaction.Synchronization; import javax.transaction.SystemException; +import org.teiid.api.exception.query.ExpressionEvaluationException; import org.teiid.api.exception.query.QueryProcessingException; import org.teiid.common.buffer.BufferManager; import org.teiid.core.TeiidComponentException; @@ -47,6 +48,8 @@ import org.teiid.query.QueryPlugin; import org.teiid.query.metadata.TempMetadataID; import org.teiid.query.metadata.TempMetadataStore; +import org.teiid.query.processor.BatchIterator; +import org.teiid.query.processor.QueryProcessor; import org.teiid.query.resolver.command.TempTableResolver; import org.teiid.query.sql.lang.Command; import org.teiid.query.sql.lang.Create; @@ -79,6 +82,21 @@ NONE } + public static class TableProcessor { + QueryProcessor queryProcessor; + List<ElementSymbol> columns; + + public TableProcessor(QueryProcessor queryProcessor, + List<ElementSymbol> columns) { + this.queryProcessor = queryProcessor; + this.columns = columns; + } + + public QueryProcessor getQueryProcessor() { + return queryProcessor; + } + } + public class TempTableSynchronization implements Synchronization { private String id; @@ -180,6 +198,8 @@ private String sessionID; private TempTableStore parentTempTableStore; + private Map<String, TableProcessor> processors; + public TempTableStore(String sessionID, TransactionMode transactionMode) { this.sessionID = sessionID; this.transactionMode = transactionMode; @@ -192,6 +212,10 @@ public boolean hasTempTable(String tempTableName) { return tempTables.containsKey(tempTableName); } + + public void setProcessors(Map<String, TableProcessor> plans) { + this.processors = plans; + } TempTable addTempTable(final String tempTableName, Create create, BufferManager buffer, boolean add, CommandContext context) throws TeiidProcessingException { List<ElementSymbol> columns = create.getColumnSymbols(); @@ -284,9 +308,19 @@ return this.tempTables.get(tempTableID); } - TempTable getOrCreateTempTable(String tempTableID, Command command, BufferManager buffer, boolean delegate, boolean forUpdate, CommandContext context) throws TeiidProcessingException{ + public Map<String, TableProcessor> getProcessors() { + return processors; + } + + TempTable getOrCreateTempTable(String tempTableID, Command command, BufferManager buffer, boolean delegate, boolean forUpdate, CommandContext context) throws TeiidProcessingException, TeiidComponentException{ TempTable tempTable = getTempTable(tempTableID, command, buffer, delegate, forUpdate, context); if (tempTable != null) { + if (processors != null) { + TableProcessor withProcessor = processors.get(tempTableID); + if (withProcessor != null) { + buildWithTable(tempTableID, withProcessor, tempTable); + } + } return tempTable; } //allow implicit temp group definition @@ -299,6 +333,18 @@ } } if (columns == null) { + if (processors != null) { + TableProcessor withProcessor = processors.get(tempTableID); + if (withProcessor != null) { + LogManager.logDetail(LogConstants.CTX_DQP, "Creating temporary table for with clause", tempTableID); //$NON-NLS-1$ + Create create = new Create(); + create.setTable(new GroupSymbol(tempTableID)); + create.setElementSymbolsAsColumns(withProcessor.columns); + tempTable = addTempTable(tempTableID, create, buffer, true, context); + buildWithTable(tempTableID, withProcessor, tempTable); + return tempTable; + } + } throw new QueryProcessingException(QueryPlugin.Util.getString("TempTableStore.table_doesnt_exist_error", tempTableID)); //$NON-NLS-1$ } LogManager.logDetail(LogConstants.CTX_DQP, "Creating temporary table", tempTableID); //$NON-NLS-1$ @@ -307,6 +353,15 @@ create.setElementSymbolsAsColumns(columns); return addTempTable(tempTableID, create, buffer, true, context); } + + private void buildWithTable(String tempTableID, + TableProcessor withProcessor, TempTable tempTable) + throws TeiidComponentException, ExpressionEvaluationException, + TeiidProcessingException { + tempTable.insert(new BatchIterator(withProcessor.queryProcessor), withProcessor.columns, false); + tempTable.setUpdatable(false); + processors.remove(tempTableID); + } private TempTable getTempTable(String tempTableID, Command command, BufferManager buffer, boolean delegate, boolean forUpdate, CommandContext context) Modified: branches/7.7.x/engine/src/test/java/org/teiid/query/processor/TestWithClauseProcessing.java =================================================================== --- branches/7.7.x/engine/src/test/java/org/teiid/query/processor/TestWithClauseProcessing.java 2013-04-17 18:59:47 UTC (rev 4561) +++ branches/7.7.x/engine/src/test/java/org/teiid/query/processor/TestWithClauseProcessing.java 2013-05-06 14:30:47 UTC (rev 4562) @@ -28,6 +28,7 @@ }; FakeDataManager dataManager = new FakeDataManager(); + dataManager.setBlockOnce(); sampleData1(dataManager); ProcessorPlan plan = helpGetPlan(helpParse(sql), RealMetadataFactory.example1Cached()); @@ -98,19 +99,98 @@ helpProcess(plan, dataManager, expected); } - @Test public void testWithPushdownWithConstants() throws TeiidException { - FakeCapabilitiesFinder capFinder = new FakeCapabilitiesFinder(); + /** + * This tests both an intervening parent plan construct (count) and a reference to a parent with in a subquery + */ + @Test public void testWithPushdownNotFullyPushed() throws TeiidException { + FakeCapabilitiesFinder capFinder = new FakeCapabilitiesFinder(); BasicSourceCapabilities caps = TestOptimizer.getTypicalCapabilities(); caps.setCapabilitySupport(Capability.COMMON_TABLE_EXPRESSIONS, true); caps.setCapabilitySupport(Capability.QUERY_FROM_JOIN_SELFJOIN, true); + caps.setCapabilitySupport(Capability.QUERY_AGGREGATES_COUNT, false); capFinder.addCapabilities("pm1", caps); //$NON-NLS-1$ + String sql = "with a as (select x, y, z from (select e1 as x, e2 as y, e3 as z from pm1.g1) v), b as (select e4 from pm1.g3) SELECT count(a.x), max(a.y) from a, a z group by z.x having max(a.y) < (with b as (select e1 from pm1.g1) select a.y from a, b where a.x = z.x)"; //$NON-NLS-1$ + + HardcodedDataManager dataManager = new HardcodedDataManager(RealMetadataFactory.example1Cached()); + List<?>[] expected = new List[] { + Arrays.asList("a", 1, "a"), + }; + + dataManager.addData("WITH a (x, y, z) AS (SELECT g_0.e1, g_0.e2, g_0.e3 FROM g1 AS g_0) SELECT g_1.x, g_0.y, g_0.x FROM a AS g_0, a AS g_1", expected); + dataManager.addData("WITH b (e1) AS (SELECT g_0.e1 FROM g1 AS g_0), a (x, y, z) AS (SELECT g_0.e1, g_0.e2, g_0.e3 FROM g1 AS g_0) SELECT g_0.y FROM a AS g_0, b AS g_1 WHERE g_0.x = 'a'", + new List[] {Arrays.asList(2)}); + + ProcessorPlan plan = TestOptimizer.helpPlan(sql, RealMetadataFactory.example1Cached(), null, capFinder, new String[] {"WITH a (x, y, z) AS (SELECT g_0.e1, g_0.e2, g_0.e3 FROM pm1.g1 AS g_0) SELECT g_1.x, g_0.y, g_0.x FROM a AS g_0, a AS g_1"}, ComparisonMode.EXACT_COMMAND_STRING); + + helpProcess(plan, dataManager, new List[] { + Arrays.asList(1, 1), + }); + } + + /** + * Tests source affinity + */ + @Test public void testWithPushdownNotFullyPushed1() throws TeiidException { + FakeCapabilitiesFinder capFinder = new FakeCapabilitiesFinder(); + BasicSourceCapabilities caps = TestOptimizer.getTypicalCapabilities(); + caps.setCapabilitySupport(Capability.COMMON_TABLE_EXPRESSIONS, true); + caps.setCapabilitySupport(Capability.QUERY_FROM_JOIN_SELFJOIN, true); + caps.setCapabilitySupport(Capability.QUERY_AGGREGATES_COUNT, false); + capFinder.addCapabilities("pm1", caps); //$NON-NLS-1$ + capFinder.addCapabilities("pm2", caps); //$NON-NLS-1$ + + String sql = "with a as (select e1 from pm1.g1), b as (select e1 from pm2.g2), c as (select count(*) as x from pm1.g1) SELECT a.e1, (select max(x) from c), pm1.g1.e2 from pm1.g1, a, b"; //$NON-NLS-1$ + + HardcodedDataManager dataManager = new HardcodedDataManager(RealMetadataFactory.example1Cached()); + + dataManager.addData("WITH a (e1) AS (SELECT g_0.e1 FROM g1 AS g_0) SELECT g_1.e1, g_0.e2 FROM g1 AS g_0, a AS g_1", new List[] { + Arrays.asList("a", 1), + Arrays.asList("a", 2) + }); + dataManager.addData("WITH b (e1) AS (SELECT g_0.e1 FROM g2 AS g_0) SELECT 1 FROM b AS g_0", new List[] { + Arrays.asList("b"), + }); + dataManager.addData("SELECT 1 FROM g1 AS g_0", new List[] { + Arrays.asList(1), Arrays.asList(1), Arrays.asList(1) + }); + + ProcessorPlan plan = TestOptimizer.helpPlan(sql, RealMetadataFactory.example1Cached(), null, capFinder, new String[] { + "WITH a (e1) AS (SELECT g_0.e1 FROM pm1.g1 AS g_0) SELECT g_1.e1, g_0.e2 FROM pm1.g1 AS g_0, a AS g_1", + "WITH b (e1) AS (SELECT g_0.e1 FROM pm2.g2 AS g_0) SELECT 1 FROM b AS g_0"}, ComparisonMode.EXACT_COMMAND_STRING); + + helpProcess(plan, dataManager, new List[] { + Arrays.asList("a", 3, 1), + Arrays.asList("a", 3, 2), + }); + } + + @Test public void testWithPushdownWithConstants() throws TeiidException { + FakeCapabilitiesFinder capFinder = new FakeCapabilitiesFinder(); + BasicSourceCapabilities caps = TestOptimizer.getTypicalCapabilities(); + caps.setCapabilitySupport(Capability.COMMON_TABLE_EXPRESSIONS, true); + caps.setCapabilitySupport(Capability.QUERY_FROM_JOIN_SELFJOIN, true); + capFinder.addCapabilities("pm1", caps); //$NON-NLS-1$ + String sql = "with a (x, y) as (select 1, 2 from pm1.g1) SELECT a.x from a, a z"; //$NON-NLS-1$ - FakeDataManager dataManager = new FakeDataManager(); - sampleData1(dataManager); - TestOptimizer.helpPlan(sql, RealMetadataFactory.example1Cached(), null, capFinder, new String[] {"WITH a (x, y) AS (SELECT 1, 2 FROM pm1.g1 AS g_0) SELECT g_0.x FROM a AS g_0, a AS g_1"}, ComparisonMode.EXACT_COMMAND_STRING); } + + @Test public void testWithOrderBy() throws TeiidException { + FakeCapabilitiesFinder capFinder = new FakeCapabilitiesFinder(); + BasicSourceCapabilities caps = TestOptimizer.getTypicalCapabilities(); + caps.setCapabilitySupport(Capability.COMMON_TABLE_EXPRESSIONS, true); + caps.setCapabilitySupport(Capability.QUERY_FROM_JOIN_SELFJOIN, true); + capFinder.addCapabilities("pm1", caps); //$NON-NLS-1$ + + String sql = "with a (x, y) as (select 1, 2 from pm1.g1) SELECT a.x from a, a z order by x"; //$NON-NLS-1$ + + HardcodedDataManager dataManager = new HardcodedDataManager(RealMetadataFactory.example1Cached()); + dataManager.addData("WITH a (x, y) AS (SELECT 1, 2 FROM g1 AS g_0) SELECT g_0.x AS c_0 FROM a AS g_0, a AS g_1 ORDER BY c_0", new List[0]); + ProcessorPlan plan = TestOptimizer.helpPlan(sql, RealMetadataFactory.example1Cached(), null, capFinder, new String[] {"WITH a (x, y) AS (SELECT 1, 2 FROM pm1.g1 AS g_0) SELECT g_0.x AS c_0 FROM a AS g_0, a AS g_1 ORDER BY c_0"}, ComparisonMode.EXACT_COMMAND_STRING); + //check the full pushdown command + helpProcess(plan, dataManager, new List[0]); + } }