Author: shawkins Date: 2010-12-22 11:13:59 -0500 (Wed, 22 Dec 2010) New Revision: 2791 Modified: trunk/build/kits/jboss-container/teiid-releasenotes.html trunk/documentation/reference/src/main/docbook/en-US/content/federated_planning.xml trunk/engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleAccessPatternValidation.java trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleChooseJoinStrategy.java trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleDecomposeJoin.java trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RulePushAggregates.java trunk/engine/src/main/java/org/teiid/query/resolver/ProcedureContainerResolver.java trunk/engine/src/main/java/org/teiid/query/sql/util/SymbolMap.java trunk/engine/src/main/java/org/teiid/query/validator/UpdateValidator.java trunk/engine/src/test/java/org/teiid/query/optimizer/TestInlineView.java trunk/engine/src/test/java/org/teiid/query/optimizer/TestUnionPlanning.java Log: TEIID-1376 adding support for partition wise aggregation Modified: trunk/build/kits/jboss-container/teiid-releasenotes.html =================================================================== --- trunk/build/kits/jboss-container/teiid-releasenotes.html 2010-12-22 14:39:41 UTC (rev 2790) +++ trunk/build/kits/jboss-container/teiid-releasenotes.html 2010-12-22 16:13:59 UTC (rev 2791) @@ -35,8 +35,8 @@ <LI><B>UDF</B> - new API objects added to teiid-api to support user defined functions that are capable of pushdown to source. <LI><B>Unescape Function</B> - a new system function, unescape, was added to handle common \ escaping in strings including octal, hex/unicode, \b, \r, \n, \f, and \t. <LI><B>Predicate Optimization</B> - expanded optimizations for detecting always false conditions when using IS NULL, IN, or comparison predicates with literal values. - <LI><B>Partition-wise Joins</B> - Views defined by partitioned unions (union alls where each branch has a projected literal or an IN predicate that makes its values mutually exclusive from the other branches) can - be joined against other partitioned unions and the optimizer will reorganize the join of unions into a union of joins. + <LI><B>Partition-wise Optimizations</B> - Views defined by partitioned unions (union alls where each branch has a projected literal or an IN predicate that makes its values mutually exclusive from the other branches) can + be used in aggregation or joins and the optimizer will take advantage of the partitioning information. For example, when a partitioned union is joined against another partitioned union, the optimizer will reorganize the join of unions into a union of joins. </UL> <h2><a name="Compatibility">Compatibility Issues</a></h2> Modified: trunk/documentation/reference/src/main/docbook/en-US/content/federated_planning.xml =================================================================== --- trunk/documentation/reference/src/main/docbook/en-US/content/federated_planning.xml 2010-12-22 14:39:41 UTC (rev 2790) +++ trunk/documentation/reference/src/main/docbook/en-US/content/federated_planning.xml 2010-12-22 16:13:59 UTC (rev 2791) @@ -38,8 +38,7 @@ These rules may trigger in turn trigger the execution of more rules. Within several rules, Teiid also takes advantage of costing information. The logical plan optimization steps can - be seen by using the - <link linkend="option_clause">OPTION DEBUG</link> + be seen by using <link linkend="show_plan">SHOWPLAN DEBUG</link> clause and are described in the <link linkend="query_planner">query planner</link> section. @@ -208,7 +207,7 @@ especially helpful when querying through large intermediate view layers.</para> </section> - <section> + <section id="aggregate_pushdown"> <title>Partial Aggregate Pushdown</title> <para> Partial aggregate pushdown allows for grouping operations above multi-source joins and unions to be decomposed so that some of the @@ -254,7 +253,7 @@ For example the view definition "select 1 as x, y from foo union all select z, a from foo1 where z in (2, 3)" would be considered partitioned on column x, since the first branch can only be the value 1 and the second branch can only be the values 2 or 3. Note that more advanced or explicit partition could be considered in the future. - The concept of a partitioned union is used for performing partition-wise joins and in <xref linkend="updatable_views"/>. + The concept of a partitioned union is used for performing partition-wise joins, in <xref linkend="updatable_views"/>, and <xref linkend="aggregate_pushdown"/>. </para> </section> <section id="standard_relational_techniques"> @@ -316,7 +315,7 @@ engine for executing a command submitted by a user or application. The purpose of the query plan is to execute the user's query in as efficient a way as possible.</para> - <section> + <section id="show_plan"> <title>Getting a Query Plan</title> <para> You can get a query plan any time you execute a command. The SQL options available are as follows:</para> Modified: trunk/engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java 2010-12-22 14:39:41 UTC (rev 2790) +++ trunk/engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java 2010-12-22 16:13:59 UTC (rev 2791) @@ -69,6 +69,7 @@ import org.teiid.query.processor.ProcessorPlan; import org.teiid.query.processor.relational.RelationalPlan; import org.teiid.query.processor.relational.JoinNode.JoinStrategyType; +import org.teiid.query.resolver.ProcedureContainerResolver; import org.teiid.query.resolver.QueryResolver; import org.teiid.query.resolver.util.BindVariableVisitor; import org.teiid.query.resolver.util.ResolverUtil; @@ -120,6 +121,7 @@ import org.teiid.query.sql.visitor.ValueIteratorProviderCollectorVisitor; import org.teiid.query.util.CommandContext; import org.teiid.query.validator.ValidationVisitor; +import org.teiid.query.validator.UpdateValidator.UpdateInfo; /** @@ -777,6 +779,10 @@ } node.addGroup(group); if (nestedCommand != null) { + UpdateInfo info = ProcedureContainerResolver.getUpdateInfo(group, metadata); + if (info != null && info.getPartitionInfo() != null && !info.getPartitionInfo().isEmpty()) { + node.setProperty(NodeConstants.Info.PARTITION_INFO, info.getPartitionInfo()); + } addNestedCommand(node, group, nestedCommand, nestedCommand, true); } parent.addLastChild(node); @@ -816,6 +822,12 @@ } node.addGroup(group); addNestedCommand(node, group, nestedCommand, nestedCommand, true); + if (nestedCommand instanceof SetQuery) { + Map<ElementSymbol, List<Set<Constant>>> partitionInfo = PartitionAnalyzer.extractPartionInfo((SetQuery)nestedCommand, ResolverUtil.resolveElementsInGroup(group, metadata)); + if (!partitionInfo.isEmpty()) { + node.setProperty(NodeConstants.Info.PARTITION_INFO, partitionInfo); + } + } hints.hasVirtualGroups = true; parent.addLastChild(node); } else if (clause instanceof TableFunctionReference) { @@ -886,11 +898,6 @@ List<SingleElementSymbol> projectCols = nestedCommand.getProjectedSymbols(); SymbolMap map = SymbolMap.createSymbolMap(group, projectCols, metadata); node.setProperty(NodeConstants.Info.SYMBOL_MAP, map); - if (nestedCommand instanceof SetQuery) { - //TODO: should cache for views - Map<ElementSymbol, List<Set<Constant>>> partitionInfo = PartitionAnalyzer.extractPartionInfo((SetQuery)nestedCommand, map.getKeys()); - node.setProperty(NodeConstants.Info.PARTITION_INFO, partitionInfo); - } } else { QueryMetadataInterface actualMetadata = metadata; if (actualMetadata instanceof TempMetadataAdapter) { Modified: trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleAccessPatternValidation.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleAccessPatternValidation.java 2010-12-22 14:39:41 UTC (rev 2790) +++ trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleAccessPatternValidation.java 2010-12-22 16:13:59 UTC (rev 2791) @@ -23,7 +23,6 @@ package org.teiid.query.optimizer.relational.rules; import java.util.Collection; -import java.util.Iterator; import java.util.List; import org.teiid.api.exception.query.QueryPlannerException; @@ -39,6 +38,7 @@ import org.teiid.query.sql.lang.Delete; import org.teiid.query.sql.lang.Insert; import org.teiid.query.sql.lang.Update; +import org.teiid.query.sql.symbol.ElementSymbol; import org.teiid.query.sql.visitor.ElementCollectorVisitor; import org.teiid.query.util.CommandContext; @@ -102,11 +102,8 @@ List accessPatterns = (List)node.getProperty(NodeConstants.Info.ACCESS_PATTERNS); if (criteria != null) { - List crits = Criteria.separateCriteriaByAnd(criteria); - - for(Iterator i = crits.iterator(); i.hasNext();) { - Criteria crit = (Criteria)i.next(); - Collection elements = ElementCollectorVisitor.getElements(crit, true); + for(Criteria crit : Criteria.separateCriteriaByAnd(criteria)) { + Collection<ElementSymbol> elements = ElementCollectorVisitor.getElements(crit, true); if (RulePushSelectCriteria.satisfyAccessPatterns(accessPatterns, elements)) { return; Modified: trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleChooseJoinStrategy.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleChooseJoinStrategy.java 2010-12-22 14:39:41 UTC (rev 2790) +++ trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleChooseJoinStrategy.java 2010-12-22 16:13:59 UTC (rev 2791) @@ -168,7 +168,7 @@ private static AtomicInteger EXPRESSION_INDEX = new AtomicInteger(0); - public static List<SingleElementSymbol> createExpressionSymbols(List<Expression> expressions) { + public static List<SingleElementSymbol> createExpressionSymbols(List<? extends Expression> expressions) { HashMap<Expression, ExpressionSymbol> uniqueExpressions = new HashMap<Expression, ExpressionSymbol>(); List<SingleElementSymbol> result = new ArrayList<SingleElementSymbol>(); for (Expression expression : expressions) { Modified: trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleDecomposeJoin.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleDecomposeJoin.java 2010-12-22 14:39:41 UTC (rev 2790) +++ trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleDecomposeJoin.java 2010-12-22 16:13:59 UTC (rev 2791) @@ -51,7 +51,6 @@ import org.teiid.query.sql.lang.Criteria; import org.teiid.query.sql.lang.JoinType; import org.teiid.query.sql.lang.Select; -import org.teiid.query.sql.lang.SetQuery.Operation; import org.teiid.query.sql.symbol.Constant; import org.teiid.query.sql.symbol.ElementSymbol; import org.teiid.query.sql.symbol.Expression; @@ -87,49 +86,45 @@ throws QueryPlannerException, QueryMetadataException, TeiidComponentException { - for (PlanNode unionNode : NodeEditor.findAllNodes(plan, NodeConstants.Types.SET_OP, NodeConstants.Types.SET_OP | NodeConstants.Types.ACCESS)) { - plan = decomposeJoin(unionNode, plan, metadata, context); + for (PlanNode joinNode : NodeEditor.findAllNodes(plan, NodeConstants.Types.JOIN, NodeConstants.Types.ACCESS)) { + plan = decomposeJoin(joinNode, plan, metadata, context); } return plan; } - public PlanNode decomposeJoin(PlanNode unionNode, PlanNode root, QueryMetadataInterface metadata, CommandContext context) throws TeiidComponentException, QueryPlannerException { - Operation op = (Operation)unionNode.getProperty(NodeConstants.Info.SET_OPERATION); - - if (op != Operation.UNION - || unionNode.getParent() == null - || unionNode.getParent().getType() != NodeConstants.Types.SOURCE - || unionNode.getParent().getParent().getType() != NodeConstants.Types.JOIN) { - return root; - } - - PlanNode joinNode = unionNode.getParent().getParent(); - + public PlanNode decomposeJoin(PlanNode joinNode, PlanNode root, QueryMetadataInterface metadata, CommandContext context) throws TeiidComponentException, QueryPlannerException { //TODO: should be done based upon join region to allow more than a 2-way non-ansi join - JoinType joinType = (JoinType)joinNode.getProperty(Info.JOIN_TYPE); if (joinType != JoinType.JOIN_INNER) { return root; } + + PlanNode left = joinNode.getFirstChild(); + if (left.getType() != NodeConstants.Types.SOURCE) { + return root; + } - Map<ElementSymbol, List<Set<Constant>>> partitionInfo = (Map<ElementSymbol, List<Set<Constant>>>)unionNode.getParent().getProperty(Info.PARTITION_INFO); + Map<ElementSymbol, List<Set<Constant>>> partitionInfo = (Map<ElementSymbol, List<Set<Constant>>>)left.getProperty(Info.PARTITION_INFO); if (partitionInfo == null) { return root; } + + PlanNode unionNode = left.getFirstChild(); + if (unionNode.getType() != NodeConstants.Types.SET_OP) { + return root; + } - boolean left = unionNode == unionNode.getParent().getFirstChild(); + PlanNode right = joinNode.getLastChild(); - PlanNode otherSide = left?joinNode.getLastChild():joinNode.getFirstChild(); - - if (otherSide.getType() != NodeConstants.Types.SOURCE) { + if (right.getType() != NodeConstants.Types.SOURCE) { return root; } - Map<ElementSymbol, List<Set<Constant>>> partitionInfoOther = (Map<ElementSymbol, List<Set<Constant>>>)otherSide.getProperty(Info.PARTITION_INFO); + Map<ElementSymbol, List<Set<Constant>>> rightPartionInfo = (Map<ElementSymbol, List<Set<Constant>>>)right.getProperty(Info.PARTITION_INFO); - if (partitionInfoOther == null) { + if (rightPartionInfo == null) { return root; } @@ -137,20 +132,20 @@ List<Expression> expr = new ArrayList<Expression>(); List<Expression> exprOther = new ArrayList<Expression>(); - RuleChooseJoinStrategy.separateCriteria(unionNode.getParent().getGroups(), otherSide.getGroups(), expr, exprOther, criteria, new LinkedList<Criteria>()); + RuleChooseJoinStrategy.separateCriteria(unionNode.getParent().getGroups(), right.getGroups(), expr, exprOther, criteria, new LinkedList<Criteria>()); if (expr.isEmpty()) { return root; //no equi-join } - List<int[]> matches = findMatches(partitionInfo, partitionInfoOther, expr, exprOther); + List<int[]> matches = findMatches(partitionInfo, rightPartionInfo, expr, exprOther); if (matches == null) { return root; //no non-overlapping partitions } int branchSize = partitionInfo.values().iterator().next().size(); - int otherBranchSize = partitionInfoOther.values().iterator().next().size(); + int otherBranchSize = rightPartionInfo.values().iterator().next().size(); if (matches.isEmpty()) { //no matches mean that we can just insert a null node (false criteria) and be done with it @@ -161,6 +156,7 @@ } List<PlanNode> branches = new ArrayList<PlanNode>(); + //TODO: find union children from RulePushAggregates RulePushSelectCriteria.collectUnionChildren(unionNode, branches); if (branches.size() != branchSize) { @@ -168,20 +164,20 @@ } List<PlanNode> otherBranches = new ArrayList<PlanNode>(); - RulePushSelectCriteria.collectUnionChildren(otherSide.getFirstChild(), otherBranches); + RulePushSelectCriteria.collectUnionChildren(right.getFirstChild(), otherBranches); if (otherBranches.size() != otherBranchSize) { return root; //sanity check } - PlanNode newUnion = buildUnion(unionNode, otherSide, criteria, matches, branches, otherBranches); - PlanNode view = rebuild(unionNode, metadata, context, joinNode, otherSide, newUnion); + PlanNode newUnion = buildUnion(unionNode, right, criteria, matches, branches, otherBranches); + PlanNode view = rebuild(left.getGroups().iterator().next(), joinNode, newUnion, metadata, context, left, right); SymbolMap symbolmap = (SymbolMap)view.getProperty(Info.SYMBOL_MAP); HashMap<ElementSymbol, List<Set<Constant>>> newPartitionInfo = new LinkedHashMap<ElementSymbol, List<Set<Constant>>>(); for (int[] match : matches) { updatePartitionInfo(partitionInfo, matches, symbolmap, newPartitionInfo, 0, match[0]); - updatePartitionInfo(partitionInfoOther, matches, symbolmap, newPartitionInfo, partitionInfo.size(), match[1]); + updatePartitionInfo(rightPartionInfo, matches, symbolmap, newPartitionInfo, partitionInfo.size(), match[1]); } view.setProperty(Info.PARTITION_INFO, newPartitionInfo); @@ -207,9 +203,8 @@ /** * Add the new union back in under a view */ - private PlanNode rebuild(PlanNode unionNode, - QueryMetadataInterface metadata, CommandContext context, - PlanNode joinNode, PlanNode otherSide, PlanNode newUnion) + static PlanNode rebuild(GroupSymbol group, PlanNode toReplace, PlanNode newUnion, QueryMetadataInterface metadata, CommandContext context, + PlanNode... toMap) throws TeiidComponentException, QueryPlannerException, QueryMetadataException { Set<String> groups = context.getGroups(); @@ -218,7 +213,6 @@ context.setGroups(groups); } - GroupSymbol group = unionNode.getParent().getGroups().iterator().next(); group = RulePlaceAccess.recontextSymbol(group, groups); PlanNode projectNode = NodeEditor.findNodePreOrder(newUnion, NodeConstants.Types.PROJECT); @@ -228,16 +222,13 @@ SymbolMap newSymbolMap = (SymbolMap)view.getProperty(Info.SYMBOL_MAP); - HashMap<ElementSymbol, ElementSymbol> inverseMap = new HashMap<ElementSymbol, ElementSymbol>(); - List<ElementSymbol> viewSymbols = newSymbolMap.getKeys(); - for (int i = 0; i < projectedSymbols.size(); i++) { - inverseMap.put((ElementSymbol)SymbolMap.getExpression(projectedSymbols.get(i)), viewSymbols.get(i)); + Map<Expression, ElementSymbol> inverseMap = newSymbolMap.inserseMapping(); + toReplace.getParent().replaceChild(toReplace, view); + Set<GroupSymbol> newGroups = Collections.singleton(group); + for (PlanNode node : toMap) { + FrameUtil.convertFrame(view, node.getGroups().iterator().next(), newGroups, inverseMap, metadata); } - joinNode.getParent().replaceChild(joinNode, view); - FrameUtil.convertFrame(view, unionNode.getParent().getGroups().iterator().next(), Collections.singleton(group), inverseMap, metadata); - FrameUtil.convertFrame(view, otherSide.getGroups().iterator().next(), Collections.singleton(group), inverseMap, metadata); - return view; } @@ -302,14 +293,12 @@ for (int i = 0; i < matches.size(); i++) { int[] is = matches.get(i); PlanNode branch = branches.get(is[0]); - PlanNode branchSource = createSource(unionNode, symbolMap); + PlanNode branchSource = createSource(unionNode.getParent().getGroups().iterator().next(), branch, symbolMap); PlanNode otherBranch = otherBranches.get(is[1]); - PlanNode otherBranchSource = createSource(otherSide.getFirstChild(), otherSymbolMap); + PlanNode otherBranchSource = createSource(otherSide.getGroups().iterator().next(), otherBranch, otherSymbolMap); PlanNode newJoinNode = NodeFactory.getNewNode(NodeConstants.Types.JOIN); - branchSource.addFirstChild(branch); - otherBranchSource.addFirstChild(otherBranch); newJoinNode.addLastChild(branchSource); newJoinNode.addLastChild(otherBranchSource); @@ -337,11 +326,12 @@ return newUnion; } - private PlanNode createSource(PlanNode unionNode, SymbolMap symbolMap) { + static PlanNode createSource(GroupSymbol group, PlanNode unionNode, SymbolMap symbolMap) { PlanNode branchSource = NodeFactory.getNewNode(NodeConstants.Types.SOURCE); - branchSource.addGroups(unionNode.getParent().getGroups()); + branchSource.addGroup(group); PlanNode projectNode = NodeEditor.findNodePreOrder(unionNode, NodeConstants.Types.PROJECT); branchSource.setProperty(Info.SYMBOL_MAP, SymbolMap.createSymbolMap(symbolMap.getKeys(), (List<? extends SingleElementSymbol>)projectNode.getProperty(Info.PROJECT_COLS))); + unionNode.addAsParent(branchSource); return branchSource; } Modified: trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RulePushAggregates.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RulePushAggregates.java 2010-12-22 14:39:41 UTC (rev 2790) +++ trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RulePushAggregates.java 2010-12-22 16:13:59 UTC (rev 2791) @@ -58,6 +58,7 @@ import org.teiid.query.resolver.util.ResolverUtil; import org.teiid.query.resolver.util.ResolverVisitor; import org.teiid.query.rewriter.QueryRewriter; +import org.teiid.query.sql.LanguageObject; import org.teiid.query.sql.LanguageObject.Util; import org.teiid.query.sql.lang.CompareCriteria; import org.teiid.query.sql.lang.Criteria; @@ -116,7 +117,7 @@ PlanNode setOp = child.getFirstChild(); try { - pushGroupNodeOverUnion(plan, metadata, capFinder, groupNode, child, groupingExpressions, setOp); + pushGroupNodeOverUnion(plan, metadata, capFinder, groupNode, child, groupingExpressions, setOp, context); } catch (QueryResolverException e) { throw new TeiidComponentException(e); } @@ -154,14 +155,26 @@ * source * child 1 * ... - * + * * Or if the child does not support pushdown we add dummy aggregate projection * count(*) = 1, count(x) = case x is null then 0 else 1 end, avg(x) = x, etc. + * + * if partitioned, then we don't need decomposition or the top level group by + * + * source + * set op + * project + * [select] + * group [agg(x), {a, b}] + * source + * child 1 + * ... + * */ private void pushGroupNodeOverUnion(PlanNode plan, QueryMetadataInterface metadata, CapabilitiesFinder capFinder, PlanNode groupNode, PlanNode child, - List<SingleElementSymbol> groupingExpressions, PlanNode setOp) + List<SingleElementSymbol> groupingExpressions, PlanNode setOp, CommandContext context) throws TeiidComponentException, QueryMetadataException, QueryPlannerException, QueryResolverException { if (setOp == null || setOp.getType() != NodeConstants.Types.SET_OP || setOp.getProperty(NodeConstants.Info.SET_OPERATION) != Operation.UNION) { @@ -169,6 +182,22 @@ } LinkedHashSet<AggregateSymbol> aggregates = collectAggregates(groupNode); + Map<ElementSymbol, List<Set<Constant>>> partitionInfo = (Map<ElementSymbol, List<Set<Constant>>>)child.getProperty(Info.PARTITION_INFO); + + //check to see if any aggregate is dependent upon cardinality + boolean cardinalityDependent = RuleRemoveOptionalJoins.areAggregatesCardinalityDependent(aggregates); + + LinkedList<PlanNode> unionChildren = new LinkedList<PlanNode>(); + findUnionChildren(unionChildren, cardinalityDependent, setOp); + + SymbolMap parentMap = (SymbolMap)child.getProperty(NodeConstants.Info.SYMBOL_MAP); + + //partitioned union + if (partitionInfo != null && !Collections.disjoint(partitionInfo.keySet(), groupingExpressions)) { + decomposeGroupBy(groupNode, child, groupingExpressions, aggregates, unionChildren, parentMap, context, metadata, capFinder); + return; + } + /* * if there are no aggregates, this is just duplicate removal * mark the union as not all, which should be removed later but @@ -181,17 +210,12 @@ return; } - //check to see if any aggregate is dependent upon cardinality - boolean cardinalityDependent = RuleRemoveOptionalJoins.areAggregatesCardinalityDependent(aggregates); + //TODO: merge virtual, plan unions, raise null - change the partition information - LinkedList<PlanNode> unionChildren = new LinkedList<PlanNode>(); - findUnionChildren(unionChildren, cardinalityDependent, setOp); - if (unionChildren.size() < 2) { return; } - SymbolMap parentMap = (SymbolMap)child.getProperty(NodeConstants.Info.SYMBOL_MAP); List<ElementSymbol> virtualElements = parentMap.getKeys(); List<SingleElementSymbol> copy = new ArrayList<SingleElementSymbol>(aggregates); aggregates.clear(); @@ -242,6 +266,53 @@ mapExpressions(groupNode.getParent(), aggMap, metadata); } + private void decomposeGroupBy(PlanNode groupNode, PlanNode sourceNode, + List<SingleElementSymbol> groupingExpressions, + LinkedHashSet<AggregateSymbol> aggregates, + LinkedList<PlanNode> unionChildren, SymbolMap parentMap, CommandContext context, QueryMetadataInterface metadata, CapabilitiesFinder capFinder) throws QueryPlannerException, QueryMetadataException, TeiidComponentException { + // remove the group node + groupNode.getParent().replaceChild(groupNode, groupNode.getFirstChild()); + GroupSymbol group = sourceNode.getGroups().iterator().next(); + + boolean first = true; + List<SingleElementSymbol> symbols = null; + for (PlanNode planNode : unionChildren) { + PlanNode groupClone = NodeFactory.getNewNode(NodeConstants.Types.GROUP); + groupClone.setProperty(Info.GROUP_COLS, LanguageObject.Util.deepClone(groupingExpressions, SingleElementSymbol.class)); + groupClone.addGroups(groupNode.getGroups()); + + PlanNode view = RuleDecomposeJoin.createSource(group, planNode, parentMap); + + view.addAsParent(groupClone); + + PlanNode projectPlanNode = NodeFactory.getNewNode(NodeConstants.Types.PROJECT); + + Select allSymbols = new Select(groupingExpressions); + allSymbols.addSymbols(aggregates); + if (first) { + first = false; + QueryRewriter.makeSelectUnique(allSymbols, false); + symbols = allSymbols.getSymbols(); + } + projectPlanNode.setProperty(NodeConstants.Info.PROJECT_COLS, allSymbols.getSymbols()); + projectPlanNode.addGroups(view.getGroups()); + + groupClone.addAsParent(projectPlanNode); + + if (planNode.getType() == NodeConstants.Types.ACCESS) { + //TODO: temporarily remove the access node so that the inline view could be removed if possible + while (RuleRaiseAccess.raiseAccessNode(planNode, planNode, metadata, capFinder, true, null) != null) { + //continue to raise + } + } + } + + SymbolMap symbolMap = createSymbolMap(group, symbols, sourceNode, metadata); + sourceNode.setProperty(Info.SYMBOL_MAP, symbolMap); + + FrameUtil.convertFrame(sourceNode, group, Collections.singleton(group), symbolMap.inserseMapping(), metadata); + } + private boolean canPushGroupByToUnionChild(QueryMetadataInterface metadata, CapabilitiesFinder capFinder, List<SingleElementSymbol> groupingExpressions, @@ -270,7 +341,7 @@ /** * Recursively searches the union tree for all applicable source nodes */ - private PlanNode findUnionChildren(List<PlanNode> unionChildren, boolean carinalityDependent, PlanNode setOp) { + static PlanNode findUnionChildren(List<PlanNode> unionChildren, boolean carinalityDependent, PlanNode setOp) { if (setOp.getType() != NodeConstants.Types.SET_OP || setOp.getProperty(NodeConstants.Info.SET_OPERATION) != Operation.UNION) { return setOp; } @@ -383,20 +454,28 @@ static PlanNode createView(GroupSymbol group, List<? extends SingleElementSymbol> virtualElements, PlanNode child, QueryMetadataInterface metadata) throws TeiidComponentException { PlanNode intermediateView = NodeFactory.getNewNode(NodeConstants.Types.SOURCE); - TempMetadataStore store = new TempMetadataStore(); + SymbolMap symbolMap = createSymbolMap(group, virtualElements, child, metadata); + intermediateView.setProperty(NodeConstants.Info.SYMBOL_MAP, symbolMap); + child.addAsParent(intermediateView); + intermediateView.addGroup(group); + return intermediateView; + } + + private static SymbolMap createSymbolMap(GroupSymbol group, + List<? extends SingleElementSymbol> virtualElements, + PlanNode child, QueryMetadataInterface metadata) + throws TeiidComponentException, QueryMetadataException { + TempMetadataStore store = new TempMetadataStore(); TempMetadataAdapter tma = new TempMetadataAdapter(metadata, store); try { group.setMetadataID(ResolverUtil.addTempGroup(tma, group, virtualElements, false)); } catch (QueryResolverException e) { throw new TeiidComponentException(e); } - intermediateView.addGroup(group); List<ElementSymbol> projectedSymbols = ResolverUtil.resolveElementsInGroup(group, metadata); SymbolMap symbolMap = SymbolMap.createSymbolMap(projectedSymbols, (List<Expression>)NodeEditor.findNodePreOrder(child, NodeConstants.Types.PROJECT).getProperty(NodeConstants.Info.PROJECT_COLS)); - intermediateView.setProperty(NodeConstants.Info.SYMBOL_MAP, symbolMap); - child.addAsParent(intermediateView); - return intermediateView; + return symbolMap; } private void updateSymbolName(List<SingleElementSymbol> projectCols, int i, Modified: trunk/engine/src/main/java/org/teiid/query/resolver/ProcedureContainerResolver.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/resolver/ProcedureContainerResolver.java 2010-12-22 14:39:41 UTC (rev 2790) +++ trunk/engine/src/main/java/org/teiid/query/resolver/ProcedureContainerResolver.java 2010-12-22 16:13:59 UTC (rev 2791) @@ -245,15 +245,37 @@ } public static UpdateInfo getUpdateInfo(GroupSymbol group, QueryMetadataInterface metadata, int type) throws QueryMetadataException, TeiidComponentException, QueryResolverException { + UpdateInfo info = getUpdateInfo(group, metadata); + + if (info == null) { + return null; + } + + if ((info.isDeleteValidationError() && type == Command.TYPE_DELETE) + || (info.isUpdateValidationError() && type == Command.TYPE_UPDATE) + || (info.isInsertValidationError() && type == Command.TYPE_INSERT)) { + String name = "Delete"; //$NON-NLS-1$ + if (type == Command.TYPE_UPDATE) { + name = "Update"; //$NON-NLS-1$ + } else if (type == Command.TYPE_INSERT) { + name = "Insert"; //$NON-NLS-1$ + } + throw new QueryResolverException("ERR.015.008.0009", QueryPlugin.Util.getString("ERR.015.008.0009", group, name)); //$NON-NLS-1$ //$NON-NLS-2$ + } + return info; + } + + public static UpdateInfo getUpdateInfo(GroupSymbol group, + QueryMetadataInterface metadata) throws TeiidComponentException, + QueryMetadataException, QueryResolverException { //if this is not a view, just return null - if(group.isTempGroupSymbol() || !metadata.isVirtualGroup(group.getMetadataID())) { + if(group.isTempGroupSymbol() || !metadata.isVirtualGroup(group.getMetadataID()) || !metadata.isVirtualModel(metadata.getModelID(group.getMetadataID()))) { return null; } String updatePlan = metadata.getUpdatePlan(group.getMetadataID()); String deletePlan = metadata.getDeletePlan(group.getMetadataID()); String insertPlan = metadata.getInsertPlan(group.getMetadataID()); - UpdateInfo info = (UpdateInfo)metadata.getFromMetadataCache(group.getMetadataID(), "UpdateInfo"); //$NON-NLS-1$ if (info == null) { List<ElementSymbol> elements = ResolverUtil.resolveElementsInGroup(group, metadata); @@ -264,19 +286,7 @@ } metadata.addToMetadataCache(group.getMetadataID(), "UpdateInfo", info); //$NON-NLS-1$ } - - if ((info.isDeleteValidationError() && type == Command.TYPE_DELETE) - || (info.isUpdateValidationError() && type == Command.TYPE_UPDATE) - || (info.isInsertValidationError() && type == Command.TYPE_INSERT)) { - String name = "Delete"; //$NON-NLS-1$ - if (type == Command.TYPE_UPDATE) { - name = "Update"; //$NON-NLS-1$ - } else if (type == Command.TYPE_INSERT) { - name = "Insert"; //$NON-NLS-1$ - } - throw new QueryResolverException("ERR.015.008.0009", QueryPlugin.Util.getString("ERR.015.008.0009", group, name)); //$NON-NLS-1$ //$NON-NLS-2$ - } - return info; + return info; } /** Modified: trunk/engine/src/main/java/org/teiid/query/sql/util/SymbolMap.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/sql/util/SymbolMap.java 2010-12-22 14:39:41 UTC (rev 2790) +++ trunk/engine/src/main/java/org/teiid/query/sql/util/SymbolMap.java 2010-12-22 16:13:59 UTC (rev 2791) @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; @@ -59,6 +60,14 @@ return clonedMap; } + public Map<Expression, ElementSymbol> inserseMapping() { + HashMap<Expression, ElementSymbol> inverseMap = new HashMap<Expression, ElementSymbol>(); + for (Map.Entry<ElementSymbol, Expression> entry : this.map.entrySet()) { + inverseMap.put(entry.getValue(), entry.getKey()); + } + return inverseMap; + } + /** * @return true if the map did not already contained the given symbol */ Modified: trunk/engine/src/main/java/org/teiid/query/validator/UpdateValidator.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/validator/UpdateValidator.java 2010-12-22 14:39:41 UTC (rev 2790) +++ trunk/engine/src/main/java/org/teiid/query/validator/UpdateValidator.java 2010-12-22 16:13:59 UTC (rev 2791) @@ -448,7 +448,7 @@ handleValidationError(QueryPlugin.Util.getString("ERR.015.012.0005"), true, false, true); //$NON-NLS-1$ } if (this.updateInfo.deleteType == UpdateType.INHERENT && this.updateInfo.deleteTarget == null) { - if (this.updateInfo.isSimple) { + if (this.updateInfo.isSimple && updatable) { this.updateInfo.deleteTarget = this.updateInfo.updatableGroups.values().iterator().next(); } else { handleValidationError(QueryPlugin.Util.getString("ERR.015.012.0014"), false, false, true); //$NON-NLS-1$ Modified: trunk/engine/src/test/java/org/teiid/query/optimizer/TestInlineView.java =================================================================== --- trunk/engine/src/test/java/org/teiid/query/optimizer/TestInlineView.java 2010-12-22 14:39:41 UTC (rev 2790) +++ trunk/engine/src/test/java/org/teiid/query/optimizer/TestInlineView.java 2010-12-22 16:13:59 UTC (rev 2791) @@ -41,13 +41,13 @@ public class TestInlineView { - public static QueryMetadataInterface createInlineViewMetadata(FakeCapabilitiesFinder capFinder) { - QueryMetadataInterface metadata = FakeMetadataFactory.exampleBQTCached(); - + public static FakeCapabilitiesFinder getInliveViewCapabilitiesFinder() { + FakeCapabilitiesFinder capFinder = new FakeCapabilitiesFinder(); BasicSourceCapabilities caps = getTypicalCapabilities(); caps.setCapabilitySupport(Capability.QUERY_FROM_INLINE_VIEWS, true); caps.setCapabilitySupport(Capability.QUERY_AGGREGATES, true); caps.setCapabilitySupport(Capability.QUERY_AGGREGATES_COUNT, true); + caps.setCapabilitySupport(Capability.QUERY_AGGREGATES_MAX, true); caps.setCapabilitySupport(Capability.QUERY_SELECT_DISTINCT, true); caps.setCapabilitySupport(Capability.QUERY_FROM_JOIN_OUTER, true); caps.setCapabilitySupport(Capability.QUERY_UNION, true); @@ -59,8 +59,7 @@ caps.setFunctionSupport("+", true); //$NON-NLS-1$ capFinder.addCapabilities("BQT1", caps); //$NON-NLS-1$ capFinder.addCapabilities("BQT2", caps); //$NON-NLS-1$ - - return metadata; + return capFinder; } @Test public void testANSIJoinInlineView() throws Exception { @@ -244,8 +243,8 @@ } protected void runTest(InlineViewCase testCase) throws Exception { - FakeCapabilitiesFinder capFinder = new FakeCapabilitiesFinder(); - QueryMetadataInterface metadata = createInlineViewMetadata(capFinder); + FakeCapabilitiesFinder capFinder = getInliveViewCapabilitiesFinder(); + QueryMetadataInterface metadata = FakeMetadataFactory.exampleBQTCached(); ProcessorPlan plan = TestOptimizer.helpPlan(testCase.userQuery, metadata, null, capFinder, new String[] {testCase.optimizedQuery}, TestOptimizer.ComparisonMode.EXACT_COMMAND_STRING); @@ -253,8 +252,8 @@ } @Test public void testAliasCreationWithInlineView() { - FakeCapabilitiesFinder capFinder = new FakeCapabilitiesFinder(); - QueryMetadataInterface metadata = createInlineViewMetadata(capFinder); + FakeCapabilitiesFinder capFinder = getInliveViewCapabilitiesFinder(); + QueryMetadataInterface metadata = FakeMetadataFactory.exampleBQTCached(); ProcessorPlan plan = helpPlan("select a, b from (select distinct count(intNum) a, count(stringKey), bqt1.smalla.intkey as b from bqt1.smalla group by bqt1.smalla.intkey) q1 order by q1.a", //$NON-NLS-1$ metadata, null, capFinder, new String[] {"SELECT COUNT(g_0.intNum) AS c_0, g_0.intkey AS c_1 FROM bqt1.smalla AS g_0 GROUP BY g_0.intkey ORDER BY c_0"}, true); //$NON-NLS-1$ @@ -263,8 +262,8 @@ } @Test public void testAliasPreservationWithInlineView() { - FakeCapabilitiesFinder capFinder = new FakeCapabilitiesFinder(); - QueryMetadataInterface metadata = createInlineViewMetadata(capFinder); + FakeCapabilitiesFinder capFinder = getInliveViewCapabilitiesFinder(); + QueryMetadataInterface metadata = FakeMetadataFactory.exampleBQTCached(); ProcessorPlan plan = helpPlan("select q1.a + 1, q1.b from (select count(bqt1.smalla.intNum) as a, bqt1.smalla.intkey as b from bqt1.smalla group by bqt1.smalla.intNum, bqt1.smalla.intkey order by b) q1 where q1.a = 1", //$NON-NLS-1$ metadata, null, capFinder, new String[] {"SELECT (q1.a + 1), q1.b FROM (SELECT COUNT(bqt1.smalla.intNum) AS a, bqt1.smalla.intkey AS b FROM bqt1.smalla GROUP BY bqt1.smalla.intNum, bqt1.smalla.intkey HAVING COUNT(bqt1.smalla.intNum) = 1) AS q1"}, true); //$NON-NLS-1$ @@ -276,8 +275,8 @@ * Order by's will be added to the atomic queries */ @Test public void testCrossSourceInlineView() throws Exception { - FakeCapabilitiesFinder capFinder = new FakeCapabilitiesFinder(); - QueryMetadataInterface metadata = createInlineViewMetadata(capFinder); + FakeCapabilitiesFinder capFinder = getInliveViewCapabilitiesFinder(); + QueryMetadataInterface metadata = FakeMetadataFactory.exampleBQTCached(); ProcessorPlan plan = helpPlan("select * from (select count(bqt1.smalla.intkey) as a, bqt1.smalla.intkey from bqt1.smalla group by bqt1.smalla.intkey) q1 inner join (select count(bqt2.smallb.intkey) as a, bqt2.smallb.intkey from bqt2.smallb group by bqt2.smallb.intkey) as q2 on q1.intkey = q2.intkey where q1.a = 1", //$NON-NLS-1$ metadata, null, capFinder, new String[] {"SELECT v_0.c_0, v_0.c_1 FROM (SELECT g_0.intkey AS c_0, COUNT(g_0.intkey) AS c_1 FROM bqt2.smallb AS g_0 GROUP BY g_0.intkey) AS v_0 ORDER BY c_0", //$NON-NLS-1$ Modified: trunk/engine/src/test/java/org/teiid/query/optimizer/TestUnionPlanning.java =================================================================== --- trunk/engine/src/test/java/org/teiid/query/optimizer/TestUnionPlanning.java 2010-12-22 14:39:41 UTC (rev 2790) +++ trunk/engine/src/test/java/org/teiid/query/optimizer/TestUnionPlanning.java 2010-12-22 16:13:59 UTC (rev 2791) @@ -207,4 +207,28 @@ }); } + @Test public void testUnionWithPartitionedAggregate() throws Exception { + ProcessorPlan plan = TestOptimizer.helpPlan("select max(intnum) from (SELECT IntKey, intnum FROM BQT1.SmallA where intkey in (1, 2) UNION ALL SELECT intkey, intnum FROM BQT2.SmallA where intkey in (3, 4)) A group by intkey", FakeMetadataFactory.exampleBQTCached(), null, TestInlineView.getInliveViewCapabilitiesFinder(),//$NON-NLS-1$ + new String[] { "SELECT MAX(v_0.c_1) FROM (SELECT g_0.IntKey AS c_0, g_0.intnum AS c_1 FROM BQT1.SmallA AS g_0 WHERE g_0.intkey IN (1, 2)) AS v_0 GROUP BY v_0.c_0", + "SELECT MAX(v_0.c_1) FROM (SELECT g_0.intkey AS c_0, g_0.intnum AS c_1 FROM BQT2.SmallA AS g_0 WHERE g_0.intkey IN (3, 4)) AS v_0 GROUP BY v_0.c_0" }, ComparisonMode.EXACT_COMMAND_STRING); + + TestOptimizer.checkNodeTypes(plan, new int[] { + 2, // Access + 0, // DependentAccess + 0, // DependentSelect + 0, // DependentProject + 0, // DupRemove + 0, // Grouping + 0, // NestedLoopJoinStrategy + 0, // MergeJoinStrategy + 0, // Null + 0, // PlanExecution + 1, // Project + 0, // Select + 0, // Sort + 1 // UnionAll + }); + } + + }