Author: shawkins Date: 2009-05-11 15:40:52 -0400 (Mon, 11 May 2009) New Revision: 906 Modified: trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/FrameUtil.java trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RuleMergeVirtual.java trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RulePlaceAccess.java trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RuleRemoveOptionalJoins.java trunk/engine/src/main/java/com/metamatrix/query/sql/symbol/Constant.java trunk/engine/src/test/java/com/metamatrix/query/optimizer/relational/rules/TestFrameUtil.java Log: TEIID-568 join groups were not appropriately reestablished after a merge. also converted optional joins to use the common frame logic. Modified: trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/FrameUtil.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/FrameUtil.java 2009-05-11 14:50:00 UTC (rev 905) +++ trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/FrameUtil.java 2009-05-11 19:40:52 UTC (rev 906) @@ -69,17 +69,17 @@ public class FrameUtil { - static void convertFrame(PlanNode startNode, GroupSymbol oldGroup, GroupSymbol newGroup, Map symbolMap, QueryMetadataInterface metadata) + static void convertFrame(PlanNode startNode, GroupSymbol oldGroup, Set<GroupSymbol> newGroups, Map symbolMap, QueryMetadataInterface metadata) throws QueryPlannerException { PlanNode current = startNode; - PlanNode endNode = NodeEditor.findParent(startNode.getParent(), NodeConstants.Types.SOURCE); + PlanNode endNode = NodeEditor.findParent(startNode.getType()==NodeConstants.Types.SOURCE?startNode.getParent():startNode, NodeConstants.Types.SOURCE); while(current != endNode) { - // Make translations as defined in vnode in each current node - convertNode(current, oldGroup, newGroup, symbolMap); + // Make translations as defined in node in each current node + convertNode(current, oldGroup, newGroups, symbolMap); PlanNode parent = current.getParent(); @@ -160,7 +160,7 @@ // symbols. In that case, some additional work can be done because we can assume // that an oldElement isn't being replaced by an expression using elements from // multiple new groups. - static void convertNode(PlanNode node, GroupSymbol oldGroup, GroupSymbol newGroup, Map symbolMap) + static void convertNode(PlanNode node, GroupSymbol oldGroup, Set<GroupSymbol> newGroups, Map symbolMap) throws QueryPlannerException { // Update groups for current node @@ -182,12 +182,18 @@ } } - if(newGroup != null) { + boolean singleMapping = newGroups != null && newGroups.size() == 1; + + if(singleMapping) { if (!hasOld) { return; } - groups.add(newGroup); - } else if (type != NodeConstants.Types.SOURCE && type != NodeConstants.Types.ACCESS) { + groups.addAll(newGroups); + } else if ((type & (NodeConstants.Types.ACCESS | NodeConstants.Types.JOIN | NodeConstants.Types.SOURCE)) == type) { + if (newGroups != null) { + groups.addAll(newGroups); + } + } else { groups.clear(); } @@ -200,7 +206,7 @@ crit = convertCriteria(crit, symbolMap); node.setProperty(NodeConstants.Info.SELECT_CRITERIA, crit); - if (newGroup == null) { + if (!singleMapping) { GroupsUsedByElementsVisitor.getGroups(crit, groups); } @@ -209,7 +215,7 @@ Select select = new Select(projectedSymbols); ExpressionMappingVisitor.mapExpressions(select, symbolMap); node.setProperty(NodeConstants.Info.PROJECT_COLS, select.getSymbols()); - if (newGroup == null) { + if (!singleMapping) { GroupsUsedByElementsVisitor.getGroups(select, groups); } } else if(type == NodeConstants.Types.JOIN) { @@ -225,9 +231,6 @@ joinCrits.add(crit); node.setProperty(NodeConstants.Info.JOIN_CRITERIA, joinCrits); } - if (newGroup == null) { - GroupsUsedByElementsVisitor.getGroups(crit, groups); - } } convertAccessPatterns(symbolMap, node); @@ -237,7 +240,7 @@ OrderBy orderBy = new OrderBy(sortCols); ExpressionMappingVisitor.mapExpressions(orderBy, symbolMap); node.setProperty(NodeConstants.Info.SORT_ORDER, orderBy.getVariables()); - if (newGroup == null) { + if (!singleMapping) { GroupsUsedByElementsVisitor.getGroups(orderBy, groups); } } else if(type == NodeConstants.Types.GROUP) { @@ -246,7 +249,7 @@ GroupBy groupBy= new GroupBy(groupCols); ExpressionMappingVisitor.mapExpressions(groupBy, symbolMap); node.setProperty(NodeConstants.Info.GROUP_COLS, groupBy.getSymbols()); - if (newGroup == null) { + if (!singleMapping) { GroupsUsedByElementsVisitor.getGroups(groupCols, groups); } } Modified: trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RuleMergeVirtual.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RuleMergeVirtual.java 2009-05-11 14:50:00 UTC (rev 905) +++ trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RuleMergeVirtual.java 2009-05-11 19:40:52 UTC (rev 906) @@ -130,10 +130,10 @@ // Convert parent frame before merge SymbolMap symbolMap = (SymbolMap)frame.getProperty(NodeConstants.Info.SYMBOL_MAP); - FrameUtil.convertFrame(frame, virtualGroup, null, symbolMap.asMap(), metadata); + FrameUtil.convertFrame(frame, virtualGroup, FrameUtil.findJoinSourceNode(projectNode).getGroups(), symbolMap.asMap(), metadata); PlanNode parentBottom = frame.getParent(); - prepareFrame(frame, parentJoin); + prepareFrame(frame); // Remove top 2 nodes (SOURCE, PROJECT) of virtual group - they're no longer needed NodeEditor.removeChildNode(parentBottom, frame); @@ -142,8 +142,7 @@ return root; } - private static void prepareFrame(PlanNode frame, - PlanNode parentJoin) { + private static void prepareFrame(PlanNode frame) { // find the new root of the frame so that access patterns can be propagated PlanNode newRoot = FrameUtil.findJoinSourceNode(frame.getFirstChild()); if (newRoot != null) { @@ -157,18 +156,6 @@ } } RulePlaceAccess.copyDependentHints(frame, newRoot); - - // correct the upper join with the groups introduced in the lower join - if (parentJoin != null) { - PlanNode upperJoin = newRoot.getParent(); - while (upperJoin != parentJoin) { - if (upperJoin.getType() == NodeConstants.Types.JOIN) { - upperJoin.addGroups(newRoot.getGroups()); - } - upperJoin = upperJoin.getParent(); - } - upperJoin.addGroups(newRoot.getGroups()); - } } } @@ -257,7 +244,7 @@ sort.addGroups(GroupsUsedByElementsVisitor.getGroups(newElements)); } - prepareFrame(frame, null); + prepareFrame(frame); //remove the parent project and the source node NodeEditor.removeChildNode(parentProject, frame); @@ -300,7 +287,7 @@ checkForNullDependent = true; break; } - joinToTest = NodeEditor.findParent(joinToTest.getParent(), NodeConstants.Types.JOIN); + joinToTest = NodeEditor.findParent(joinToTest.getParent(), NodeConstants.Types.JOIN, NodeConstants.Types.SOURCE); } } Modified: trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RulePlaceAccess.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RulePlaceAccess.java 2009-05-11 14:50:00 UTC (rev 905) +++ trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RulePlaceAccess.java 2009-05-11 19:40:52 UTC (rev 906) @@ -22,7 +22,9 @@ package com.metamatrix.query.optimizer.relational.rules; +import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.Iterator; import java.util.LinkedList; import java.util.List; @@ -199,7 +201,7 @@ //the expressions in the map will all be element symbols Map<ElementSymbol, Expression> replacementSymbols = FrameUtil.buildSymbolMap(group, newGroup, metadata); - FrameUtil.convertFrame(sourceNode, group, newGroup, replacementSymbols, metadata); + FrameUtil.convertFrame(sourceNode, group, new HashSet<GroupSymbol>(Arrays.asList(newGroup)), replacementSymbols, metadata); // correct the lower symbol map if (childProjects != null) { Modified: trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RuleRemoveOptionalJoins.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RuleRemoveOptionalJoins.java 2009-05-11 14:50:00 UTC (rev 905) +++ trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RuleRemoveOptionalJoins.java 2009-05-11 19:40:52 UTC (rev 906) @@ -22,11 +22,11 @@ package com.metamatrix.query.optimizer.relational.rules; +import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Set; import com.metamatrix.api.exception.MetaMatrixComponentException; @@ -48,12 +48,12 @@ import com.metamatrix.query.sql.lang.Criteria; import com.metamatrix.query.sql.lang.JoinType; import com.metamatrix.query.sql.symbol.AggregateSymbol; +import com.metamatrix.query.sql.symbol.Constant; import com.metamatrix.query.sql.symbol.ElementSymbol; import com.metamatrix.query.sql.symbol.Expression; import com.metamatrix.query.sql.symbol.GroupSymbol; import com.metamatrix.query.sql.util.SymbolMap; import com.metamatrix.query.sql.visitor.ElementCollectorVisitor; -import com.metamatrix.query.sql.visitor.GroupsUsedByElementsVisitor; import com.metamatrix.query.util.CommandContext; /** @@ -201,9 +201,10 @@ /** * remove the optional node if possible + * @throws QueryPlannerException */ private boolean removedJoin(PlanNode joinNode, PlanNode optionalNode, - Set elements, QueryMetadataInterface metadata) throws QueryMetadataException, MetaMatrixComponentException { + Set elements, QueryMetadataInterface metadata) throws QueryMetadataException, MetaMatrixComponentException, QueryPlannerException { Set groups = optionalNode.getGroups(); Assertion.isNotNull(elements); @@ -227,13 +228,15 @@ NodeEditor.removeChildNode(parentNode, joinNode); // correct the parent nodes that may be using optional elements - HashSet optionElements = new HashSet(); - for (Iterator i = optionalNode.getGroups().iterator(); i.hasNext();) { - optionElements.addAll(ResolverUtil.resolveElementsInGroup((GroupSymbol) i.next(), metadata)); + for (GroupSymbol optionalGroup : optionalNode.getGroups()) { + List<ElementSymbol> optionalElements = ResolverUtil.resolveElementsInGroup(optionalGroup, metadata); + List<Constant> replacements = new ArrayList<Constant>(optionalElements.size()); + for (ElementSymbol elementSymbol : optionalElements) { + replacements.add(new Constant(null, elementSymbol.getType())); + } + FrameUtil.convertFrame(parentNode, optionalGroup, null, SymbolMap.createSymbolMap(optionalElements, replacements).asMap(), metadata); } - correctParents(optionalNode.getGroups(), parentNode, optionElements); - return true; } @@ -269,88 +272,6 @@ return false; } - private void correctParents(Set groups, - PlanNode parentNode, - HashSet optionElements) { - boolean done = false; - boolean correctJoinGroups = true; //true until the first source node is reached - - while (!done && parentNode != null) { - - switch (parentNode.getType()) { - - case NodeConstants.Types.SET_OP: - { - done = true; - break; - } - case NodeConstants.Types.SOURCE: - { - HashSet parentOptionalElements = new HashSet(); - SymbolMap symbolMap = (SymbolMap)parentNode.getProperty(NodeConstants.Info.SYMBOL_MAP); - for (Map.Entry<ElementSymbol, Expression> entry : symbolMap.asMap().entrySet()) { - Expression parentExpression = entry.getValue(); - Collection parentElements = ElementCollectorVisitor.getElements(parentExpression, true); - parentElements.retainAll(optionElements); - if (!parentElements.isEmpty()) { - parentOptionalElements.add(entry.getKey()); - } - } - correctJoinGroups = false; - optionElements = parentOptionalElements; - if (optionElements.isEmpty()) { - done = true; - } - break; - } - case NodeConstants.Types.JOIN: - { - List joinCriteria = (List)parentNode.getProperty(NodeConstants.Info.JOIN_CRITERIA); - removeOptionalEntries(optionElements, joinCriteria, null); - if (correctJoinGroups) { - parentNode.getGroups().removeAll(groups); - } - break; - } - case NodeConstants.Types.PROJECT: - { - if (parentNode.getParent() == null || parentNode.getProperty(NodeConstants.Info.INTO_GROUP) != null) { - done = true; - } - break; - } - } - - parentNode = parentNode.getParent(); - } - } - - /** - * Will remove the optional entries from the list of languageObjects. This will - * also correct the groups on the parentNode if it is non-null. - * - * @param optionElements - * @param languageObjects - */ - private void removeOptionalEntries(HashSet optionElements, - List languageObjects, PlanNode parentNode) { - if (languageObjects != null && !languageObjects.isEmpty()) { - if (parentNode != null) { - parentNode.getGroups().clear(); - } - for (Iterator i = languageObjects.iterator(); i.hasNext();) { - LanguageObject object = (LanguageObject)i.next(); - if (isOptional(optionElements, object)) { - i.remove(); - continue; - } - if (parentNode != null) { - parentNode.addGroups(GroupsUsedByElementsVisitor.getGroups(object)); - } - } - } - } - private boolean isOptional(HashSet optionElements, LanguageObject languageObject) { Collection elementsUsed = ElementCollectorVisitor.getElements(languageObject, true); Modified: trunk/engine/src/main/java/com/metamatrix/query/sql/symbol/Constant.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/sql/symbol/Constant.java 2009-05-11 14:50:00 UTC (rev 905) +++ trunk/engine/src/main/java/com/metamatrix/query/sql/symbol/Constant.java 2009-05-11 19:40:52 UTC (rev 906) @@ -148,7 +148,8 @@ if(other.isNull() || this.isNull()) { return false; } - return other.getValue().equals(this.getValue()); + + return multiValued == other.multiValued && other.getValue().equals(this.getValue()); } /** Modified: trunk/engine/src/test/java/com/metamatrix/query/optimizer/relational/rules/TestFrameUtil.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/query/optimizer/relational/rules/TestFrameUtil.java 2009-05-11 14:50:00 UTC (rev 905) +++ trunk/engine/src/test/java/com/metamatrix/query/optimizer/relational/rules/TestFrameUtil.java 2009-05-11 19:40:52 UTC (rev 906) @@ -22,24 +22,33 @@ package com.metamatrix.query.optimizer.relational.rules; +import static org.junit.Assert.*; + +import java.util.Arrays; import java.util.HashSet; import java.util.Set; +import org.junit.Test; + import com.metamatrix.query.optimizer.relational.plantree.NodeConstants; +import com.metamatrix.query.optimizer.relational.plantree.NodeEditor; import com.metamatrix.query.optimizer.relational.plantree.NodeFactory; import com.metamatrix.query.optimizer.relational.plantree.PlanNode; +import com.metamatrix.query.optimizer.relational.plantree.NodeConstants.Info; +import com.metamatrix.query.sql.lang.IsNullCriteria; import com.metamatrix.query.sql.lang.JoinType; +import com.metamatrix.query.sql.symbol.Constant; +import com.metamatrix.query.sql.symbol.ElementSymbol; import com.metamatrix.query.sql.symbol.GroupSymbol; +import com.metamatrix.query.sql.util.SymbolMap; -import junit.framework.TestCase; - -public class TestFrameUtil extends TestCase { +public class TestFrameUtil { static GroupSymbol getGroup(int id) { return new GroupSymbol(String.valueOf(id)); } - public void testFindJoinSourceNode() { + @Test public void testFindJoinSourceNode() { PlanNode root = getExamplePlan(); PlanNode joinSource = FrameUtil.findJoinSourceNode(root); @@ -47,7 +56,7 @@ assertSame(root, joinSource); } - public void testFindJoinSourceNode1() { + @Test public void testFindJoinSourceNode1() { PlanNode root = getExamplePlan(); PlanNode joinSource = FrameUtil.findJoinSourceNode(root.getLastChild()); @@ -55,10 +64,10 @@ assertEquals(NodeConstants.Types.JOIN, joinSource.getType()); } - public void testFindSourceNode() { + @Test public void testFindSourceNode() { PlanNode root = getExamplePlan(); - Set groups = new HashSet(); + Set<GroupSymbol> groups = new HashSet<GroupSymbol>(); groups.add(getGroup(1)); @@ -70,10 +79,10 @@ /** * Access nodes are not eligible originating nodes */ - public void testFindSourceNodeWithAccessSource() { + @Test public void testFindSourceNodeWithAccessSource() { PlanNode root = getExamplePlan(); - Set groups = new HashSet(); + Set<GroupSymbol> groups = new HashSet<GroupSymbol>(); groups.add(getGroup(2)); @@ -82,10 +91,10 @@ assertEquals(NodeConstants.Types.JOIN, originatingNode.getType()); } - public void testFindSourceNode2() { + @Test public void testFindSourceNode2() { PlanNode root = getExamplePlan(); - Set groups = new HashSet(); + Set<GroupSymbol> groups = new HashSet<GroupSymbol>(); groups.add(getGroup(3)); @@ -94,10 +103,10 @@ assertEquals(NodeConstants.Types.SOURCE, originatingNode.getType()); } - public void testNonExistentSource() { + @Test public void testNonExistentSource() { PlanNode root = getExamplePlan(); - Set groups = new HashSet(); + Set<GroupSymbol> groups = new HashSet<GroupSymbol>(); groups.add(getGroup(4)); @@ -105,6 +114,66 @@ assertNull(originatingNode); } + + @Test public void testJoinGroups() throws Exception { + PlanNode joinNode = getExamplePlan(); + PlanNode projectNode = NodeFactory.getNewNode(NodeConstants.Types.PROJECT); + ElementSymbol e1 = new ElementSymbol("e1"); //$NON-NLS-1$ + e1.setGroupSymbol(getGroup(3)); + projectNode.setProperty(Info.PROJECT_COLS, Arrays.asList(e1)); + projectNode.addFirstChild(joinNode); + projectNode.addGroup(getGroup(3)); + PlanNode sourceNode = NodeFactory.getNewNode(NodeConstants.Types.SOURCE); + sourceNode.addFirstChild(projectNode); + GroupSymbol four = getGroup(4); + sourceNode.addGroup(four); + ElementSymbol e2 = new ElementSymbol("e2"); //$NON-NLS-1$ + e2.setGroupSymbol(four); + SymbolMap sm = SymbolMap.createSymbolMap(Arrays.asList(e2), Arrays.asList(e1)); + sourceNode.setProperty(Info.SYMBOL_MAP, sm); + PlanNode projectNode1 = NodeFactory.getNewNode(NodeConstants.Types.PROJECT); + projectNode1.addFirstChild(sourceNode); + projectNode1.addGroup(four); + projectNode1.setProperty(Info.PROJECT_COLS, Arrays.asList(e2)); + + //removing source node 3 completely + SymbolMap replacement = SymbolMap.createSymbolMap(Arrays.asList(e1), Arrays.asList(new Constant(null))); + FrameUtil.convertFrame(NodeEditor.findNodePreOrder(joinNode, NodeConstants.Types.SOURCE), getGroup(3), null, replacement.asMap(), null); + assertEquals(2, joinNode.getGroups().size()); //even though this is a cross join it should still retain its groups + assertEquals(0, NodeEditor.findNodePreOrder(joinNode, NodeConstants.Types.SELECT).getGroups().size()); + assertEquals(1, projectNode1.getGroups().size()); + assertEquals(0, projectNode.getGroups().size()); + } + + @Test public void testJoinGroups1() throws Exception { + PlanNode joinNode = getExamplePlan(); + PlanNode projectNode = NodeFactory.getNewNode(NodeConstants.Types.PROJECT); + ElementSymbol e1 = new ElementSymbol("e1"); //$NON-NLS-1$ + e1.setGroupSymbol(getGroup(3)); + projectNode.setProperty(Info.PROJECT_COLS, Arrays.asList(e1)); + projectNode.addFirstChild(joinNode); + projectNode.addGroup(getGroup(3)); + PlanNode sourceNode = NodeFactory.getNewNode(NodeConstants.Types.SOURCE); + sourceNode.addFirstChild(projectNode); + GroupSymbol four = getGroup(4); + sourceNode.addGroup(four); + ElementSymbol e2 = new ElementSymbol("e2"); //$NON-NLS-1$ + e2.setGroupSymbol(four); + SymbolMap sm = SymbolMap.createSymbolMap(Arrays.asList(e2), Arrays.asList(e1)); + sourceNode.setProperty(Info.SYMBOL_MAP, sm); + PlanNode projectNode1 = NodeFactory.getNewNode(NodeConstants.Types.PROJECT); + projectNode1.addFirstChild(sourceNode); + projectNode1.addGroup(four); + projectNode1.setProperty(Info.PROJECT_COLS, Arrays.asList(e2)); + + //replace source 3 with groups 5, 6 + SymbolMap replacement = SymbolMap.createSymbolMap(Arrays.asList(e1), Arrays.asList(new Constant(null))); + FrameUtil.convertFrame(NodeEditor.findNodePreOrder(joinNode, NodeConstants.Types.SOURCE), getGroup(3), new HashSet<GroupSymbol>(Arrays.asList(getGroup(5), getGroup(6))), replacement.asMap(), null); + assertEquals(4, joinNode.getGroups().size()); //even though this is a cross join it should still retain its groups + assertEquals(0, NodeEditor.findNodePreOrder(joinNode, NodeConstants.Types.SELECT).getGroups().size()); + assertEquals(1, projectNode1.getGroups().size()); + assertEquals(0, projectNode.getGroups().size()); + } /** * <pre> @@ -129,7 +198,7 @@ joinNode.addFirstChild(nullNode); PlanNode childCriteria = NodeFactory.getNewNode(NodeConstants.Types.SELECT); - + childCriteria.setProperty(Info.SELECT_CRITERIA, new IsNullCriteria(new Constant(1))); childCriteria.addGroup(getGroup(2)); joinNode.addLastChild(childCriteria);