Author: shawkins Date: 2009-05-13 16:18:25 -0400 (Wed, 13 May 2009) New Revision: 934 Modified: branches/6.0.x/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/FrameUtil.java branches/6.0.x/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RuleMergeVirtual.java branches/6.0.x/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RulePlaceAccess.java branches/6.0.x/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RuleRemoveOptionalJoins.java branches/6.0.x/engine/src/test/java/com/metamatrix/query/optimizer/relational/rules/TestFrameUtil.java Log: TEIID-568 porting join group fix from 6.1.0 Modified: branches/6.0.x/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/FrameUtil.java =================================================================== --- branches/6.0.x/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/FrameUtil.java 2009-05-13 20:07:21 UTC (rev 933) +++ branches/6.0.x/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/FrameUtil.java 2009-05-13 20:18:25 UTC (rev 934) @@ -72,17 +72,17 @@ public class FrameUtil { - static void convertFrame(PlanNode startNode, GroupSymbol oldGroup, GroupSymbol newGroup, Map symbolMap, QueryMetadataInterface metadata) + static void convertFrame(PlanNode startNode, GroupSymbol oldGroup, Set<GroupSymbol> newGroups, Map symbolMap, QueryMetadataInterface metadata) throws QueryPlannerException { PlanNode current = startNode; - PlanNode endNode = NodeEditor.findParent(startNode.getParent(), NodeConstants.Types.SOURCE); + PlanNode endNode = NodeEditor.findParent(startNode.getType()==NodeConstants.Types.SOURCE?startNode.getParent():startNode, NodeConstants.Types.SOURCE); while(current != endNode) { - // Make translations as defined in vnode in each current node - convertNode(current, oldGroup, newGroup, symbolMap); + // Make translations as defined in node in each current node + convertNode(current, oldGroup, newGroups, symbolMap); PlanNode parent = current.getParent(); @@ -164,12 +164,16 @@ // symbols. In that case, some additional work can be done because we can assume // that an oldElement isn't being replaced by an expression using elements from // multiple new groups. - static void convertNode(PlanNode node, GroupSymbol oldGroup, GroupSymbol newGroup, Map symbolMap) + static void convertNode(PlanNode node, GroupSymbol oldGroup, Set<GroupSymbol> newGroups, Map symbolMap) throws QueryPlannerException { // Update groups for current node Set groups = node.getGroups(); + boolean hasOld = groups.remove(oldGroup); + + int type = node.getType(); + // Convert expressions from correlated subquery references; // currently only for SELECT or PROJECT nodes List refs = (List)node.getProperty(NodeConstants.Info.CORRELATED_REFERENCES); @@ -183,28 +187,27 @@ } } - boolean hasOld = groups.remove(oldGroup); + boolean singleMapping = newGroups != null && newGroups.size() == 1; - if(newGroup != null) { + if(singleMapping) { if (!hasOld) { return; } - groups.add(newGroup); + groups.addAll(newGroups); + } else if ((type & (NodeConstants.Types.ACCESS | NodeConstants.Types.JOIN | NodeConstants.Types.SOURCE)) == type) { + if (newGroups != null) { + groups.addAll(newGroups); + } + } else { + groups.clear(); } - // Updated elements - List newElementSymbols = null; - if (newGroup == null) { - newElementSymbols = new ArrayList(); - } - - int type = node.getType(); if(type == NodeConstants.Types.SELECT) { Criteria crit = (Criteria) node.getProperty(NodeConstants.Info.SELECT_CRITERIA); convertCriteria(crit, symbolMap); - if (newGroup == null) { - ElementCollectorVisitor.getElements(crit, newElementSymbols); + if (!singleMapping) { + GroupsUsedByElementsVisitor.getGroups(crit, groups); } } else if(type == NodeConstants.Types.PROJECT) { @@ -217,8 +220,8 @@ SingleElementSymbol mappedSymbol = convertSingleElementSymbol(symbol, symbolMap, true); newElements.add(mappedSymbol); - if (newGroup == null) { - ElementCollectorVisitor.getElements(mappedSymbol, newElementSymbols); + if (!singleMapping) { + GroupsUsedByElementsVisitor.getGroups(mappedSymbol, groups); } } @@ -232,10 +235,6 @@ while(critIter.hasNext()) { Criteria crit = (Criteria) critIter.next(); convertCriteria(crit, symbolMap); - - if (newGroup == null) { - ElementCollectorVisitor.getElements(crit, newElementSymbols); - } } } @@ -251,8 +250,8 @@ SingleElementSymbol mappedSymbol = convertSingleElementSymbol(symbol, symbolMap, true); newElements.add( mappedSymbol ); - if (newGroup == null) { - ElementCollectorVisitor.getElements(mappedSymbol, newElementSymbols); + if (!singleMapping) { + GroupsUsedByElementsVisitor.getGroups(mappedSymbol, groups); } } @@ -269,8 +268,8 @@ Expression mappedCol = convertSingleElementSymbol(groupCol, symbolMap, false); newGroupCols.add( mappedCol ); - if (newGroup == null) { - ElementCollectorVisitor.getElements(mappedCol, newElementSymbols); + if (!singleMapping) { + GroupsUsedByElementsVisitor.getGroups(mappedCol, groups); } } node.setProperty(NodeConstants.Info.GROUP_COLS, newGroupCols); @@ -278,10 +277,6 @@ } else if (type == NodeConstants.Types.SOURCE || type == NodeConstants.Types.ACCESS) { convertAccessPatterns(symbolMap, node); } - - if (newGroup == null) { - GroupsUsedByElementsVisitor.getGroups(newElementSymbols, groups); - } } static SingleElementSymbol convertSingleElementSymbol(SingleElementSymbol symbol, Map symbolMap, boolean shouldAlias) { Modified: branches/6.0.x/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RuleMergeVirtual.java =================================================================== --- branches/6.0.x/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RuleMergeVirtual.java 2009-05-13 20:07:21 UTC (rev 933) +++ branches/6.0.x/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RuleMergeVirtual.java 2009-05-13 20:18:25 UTC (rev 934) @@ -130,10 +130,10 @@ // Convert parent frame before merge SymbolMap symbolMap = (SymbolMap)frame.getProperty(NodeConstants.Info.SYMBOL_MAP); - FrameUtil.convertFrame(frame, virtualGroup, null, symbolMap.asMap(), metadata); + FrameUtil.convertFrame(frame, virtualGroup, FrameUtil.findJoinSourceNode(projectNode).getGroups(), symbolMap.asMap(), metadata); PlanNode parentBottom = frame.getParent(); - prepareFrame(frame, parentJoin); + prepareFrame(frame); // Remove top 2 nodes (SOURCE, PROJECT) of virtual group - they're no longer needed NodeEditor.removeChildNode(parentBottom, frame); @@ -142,8 +142,7 @@ return root; } - private static void prepareFrame(PlanNode frame, - PlanNode parentJoin) { + private static void prepareFrame(PlanNode frame) { // find the new root of the frame so that access patterns can be propagated PlanNode newRoot = FrameUtil.findJoinSourceNode(frame.getFirstChild()); if (newRoot != null) { @@ -157,18 +156,6 @@ } } RulePlaceAccess.copyDependentHints(frame, newRoot); - - // correct the upper join with the groups introduced in the lower join - if (parentJoin != null) { - PlanNode upperJoin = newRoot.getParent(); - while (upperJoin != parentJoin) { - if (upperJoin.getType() == NodeConstants.Types.JOIN) { - upperJoin.addGroups(newRoot.getGroups()); - } - upperJoin = upperJoin.getParent(); - } - upperJoin.addGroups(newRoot.getGroups()); - } } } @@ -257,7 +244,7 @@ sort.addGroups(GroupsUsedByElementsVisitor.getGroups(newElements)); } - prepareFrame(frame, null); + prepareFrame(frame); //remove the parent project and the source node NodeEditor.removeChildNode(parentProject, frame); @@ -300,7 +287,7 @@ checkForNullDependent = true; break; } - joinToTest = NodeEditor.findParent(joinToTest.getParent(), NodeConstants.Types.JOIN); + joinToTest = NodeEditor.findParent(joinToTest.getParent(), NodeConstants.Types.JOIN, NodeConstants.Types.SOURCE); } } Modified: branches/6.0.x/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RulePlaceAccess.java =================================================================== --- branches/6.0.x/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RulePlaceAccess.java 2009-05-13 20:07:21 UTC (rev 933) +++ branches/6.0.x/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RulePlaceAccess.java 2009-05-13 20:18:25 UTC (rev 934) @@ -22,7 +22,9 @@ package com.metamatrix.query.optimizer.relational.rules; +import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.Iterator; import java.util.LinkedList; import java.util.List; @@ -199,7 +201,7 @@ //the expressions in the map will all be element symbols Map<ElementSymbol, Expression> replacementSymbols = FrameUtil.buildSymbolMap(group, newGroup, metadata); - FrameUtil.convertFrame(sourceNode, group, newGroup, replacementSymbols, metadata); + FrameUtil.convertFrame(sourceNode, group, new HashSet<GroupSymbol>(Arrays.asList(newGroup)), replacementSymbols, metadata); // correct the lower symbol map if (childProjects != null) { Modified: branches/6.0.x/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RuleRemoveOptionalJoins.java =================================================================== --- branches/6.0.x/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RuleRemoveOptionalJoins.java 2009-05-13 20:07:21 UTC (rev 933) +++ branches/6.0.x/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RuleRemoveOptionalJoins.java 2009-05-13 20:18:25 UTC (rev 934) @@ -22,11 +22,11 @@ package com.metamatrix.query.optimizer.relational.rules; +import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Set; import com.metamatrix.api.exception.MetaMatrixComponentException; @@ -48,13 +48,13 @@ import com.metamatrix.query.sql.lang.Criteria; import com.metamatrix.query.sql.lang.JoinType; import com.metamatrix.query.sql.symbol.AggregateSymbol; +import com.metamatrix.query.sql.symbol.Constant; import com.metamatrix.query.sql.symbol.ElementSymbol; import com.metamatrix.query.sql.symbol.Expression; import com.metamatrix.query.sql.symbol.GroupSymbol; import com.metamatrix.query.sql.symbol.Reference; import com.metamatrix.query.sql.util.SymbolMap; import com.metamatrix.query.sql.visitor.ElementCollectorVisitor; -import com.metamatrix.query.sql.visitor.GroupsUsedByElementsVisitor; import com.metamatrix.query.util.CommandContext; /** @@ -214,9 +214,10 @@ /** * remove the optional node if possible + * @throws QueryPlannerException */ private boolean removedJoin(PlanNode joinNode, PlanNode optionalNode, - Set elements, QueryMetadataInterface metadata) throws QueryMetadataException, MetaMatrixComponentException { + Set elements, QueryMetadataInterface metadata) throws QueryMetadataException, MetaMatrixComponentException, QueryPlannerException { Set groups = optionalNode.getGroups(); Assertion.isNotNull(elements); @@ -240,13 +241,15 @@ NodeEditor.removeChildNode(parentNode, joinNode); // correct the parent nodes that may be using optional elements - HashSet optionElements = new HashSet(); - for (Iterator i = optionalNode.getGroups().iterator(); i.hasNext();) { - optionElements.addAll(ResolverUtil.resolveElementsInGroup((GroupSymbol) i.next(), metadata)); + for (GroupSymbol optionalGroup : optionalNode.getGroups()) { + List<ElementSymbol> optionalElements = ResolverUtil.resolveElementsInGroup(optionalGroup, metadata); + List<Constant> replacements = new ArrayList<Constant>(optionalElements.size()); + for (ElementSymbol elementSymbol : optionalElements) { + replacements.add(new Constant(null, elementSymbol.getType())); + } + FrameUtil.convertFrame(parentNode, optionalGroup, null, SymbolMap.createSymbolMap(optionalElements, replacements).asMap(), metadata); } - correctParents(optionalNode.getGroups(), parentNode, optionElements); - return true; } @@ -282,88 +285,6 @@ return false; } - private void correctParents(Set groups, - PlanNode parentNode, - HashSet optionElements) { - boolean done = false; - boolean correctJoinGroups = true; //true until the first source node is reached - - while (!done && parentNode != null) { - - switch (parentNode.getType()) { - - case NodeConstants.Types.SET_OP: - { - done = true; - break; - } - case NodeConstants.Types.SOURCE: - { - HashSet parentOptionalElements = new HashSet(); - SymbolMap symbolMap = (SymbolMap)parentNode.getProperty(NodeConstants.Info.SYMBOL_MAP); - for (Map.Entry<ElementSymbol, Expression> entry : symbolMap.asMap().entrySet()) { - Expression parentExpression = entry.getValue(); - Collection parentElements = ElementCollectorVisitor.getElements(parentExpression, true); - parentElements.retainAll(optionElements); - if (!parentElements.isEmpty()) { - parentOptionalElements.add(entry.getKey()); - } - } - correctJoinGroups = false; - optionElements = parentOptionalElements; - if (optionElements.isEmpty()) { - done = true; - } - break; - } - case NodeConstants.Types.JOIN: - { - List joinCriteria = (List)parentNode.getProperty(NodeConstants.Info.JOIN_CRITERIA); - removeOptionalEntries(optionElements, joinCriteria, null); - if (correctJoinGroups) { - parentNode.getGroups().removeAll(groups); - } - break; - } - case NodeConstants.Types.PROJECT: - { - if (parentNode.getParent() == null || parentNode.getProperty(NodeConstants.Info.INTO_GROUP) != null) { - done = true; - } - break; - } - } - - parentNode = parentNode.getParent(); - } - } - - /** - * Will remove the optional entries from the list of languageObjects. This will - * also correct the groups on the parentNode if it is non-null. - * - * @param optionElements - * @param languageObjects - */ - private void removeOptionalEntries(HashSet optionElements, - List languageObjects, PlanNode parentNode) { - if (languageObjects != null && !languageObjects.isEmpty()) { - if (parentNode != null) { - parentNode.getGroups().clear(); - } - for (Iterator i = languageObjects.iterator(); i.hasNext();) { - LanguageObject object = (LanguageObject)i.next(); - if (isOptional(optionElements, object)) { - i.remove(); - continue; - } - if (parentNode != null) { - parentNode.addGroups(GroupsUsedByElementsVisitor.getGroups(object)); - } - } - } - } - private boolean isOptional(HashSet optionElements, LanguageObject languageObject) { Collection elementsUsed = ElementCollectorVisitor.getElements(languageObject, true); Modified: branches/6.0.x/engine/src/test/java/com/metamatrix/query/optimizer/relational/rules/TestFrameUtil.java =================================================================== --- branches/6.0.x/engine/src/test/java/com/metamatrix/query/optimizer/relational/rules/TestFrameUtil.java 2009-05-13 20:07:21 UTC (rev 933) +++ branches/6.0.x/engine/src/test/java/com/metamatrix/query/optimizer/relational/rules/TestFrameUtil.java 2009-05-13 20:18:25 UTC (rev 934) @@ -22,24 +22,33 @@ package com.metamatrix.query.optimizer.relational.rules; +import static org.junit.Assert.*; + +import java.util.Arrays; import java.util.HashSet; import java.util.Set; +import org.junit.Test; + import com.metamatrix.query.optimizer.relational.plantree.NodeConstants; +import com.metamatrix.query.optimizer.relational.plantree.NodeEditor; import com.metamatrix.query.optimizer.relational.plantree.NodeFactory; import com.metamatrix.query.optimizer.relational.plantree.PlanNode; +import com.metamatrix.query.optimizer.relational.plantree.NodeConstants.Info; +import com.metamatrix.query.sql.lang.IsNullCriteria; import com.metamatrix.query.sql.lang.JoinType; +import com.metamatrix.query.sql.symbol.Constant; +import com.metamatrix.query.sql.symbol.ElementSymbol; import com.metamatrix.query.sql.symbol.GroupSymbol; +import com.metamatrix.query.sql.util.SymbolMap; -import junit.framework.TestCase; - -public class TestFrameUtil extends TestCase { +public class TestFrameUtil { static GroupSymbol getGroup(int id) { return new GroupSymbol(String.valueOf(id)); } - public void testFindJoinSourceNode() { + @Test public void testFindJoinSourceNode() { PlanNode root = getExamplePlan(); PlanNode joinSource = FrameUtil.findJoinSourceNode(root); @@ -47,7 +56,7 @@ assertSame(root, joinSource); } - public void testFindJoinSourceNode1() { + @Test public void testFindJoinSourceNode1() { PlanNode root = getExamplePlan(); PlanNode joinSource = FrameUtil.findJoinSourceNode(root.getLastChild()); @@ -55,10 +64,10 @@ assertEquals(NodeConstants.Types.JOIN, joinSource.getType()); } - public void testFindSourceNode() { + @Test public void testFindSourceNode() { PlanNode root = getExamplePlan(); - Set groups = new HashSet(); + Set<GroupSymbol> groups = new HashSet<GroupSymbol>(); groups.add(getGroup(1)); @@ -70,10 +79,10 @@ /** * Access nodes are not eligible originating nodes */ - public void testFindSourceNodeWithAccessSource() { + @Test public void testFindSourceNodeWithAccessSource() { PlanNode root = getExamplePlan(); - Set groups = new HashSet(); + Set<GroupSymbol> groups = new HashSet<GroupSymbol>(); groups.add(getGroup(2)); @@ -82,10 +91,10 @@ assertEquals(NodeConstants.Types.JOIN, originatingNode.getType()); } - public void testFindSourceNode2() { + @Test public void testFindSourceNode2() { PlanNode root = getExamplePlan(); - Set groups = new HashSet(); + Set<GroupSymbol> groups = new HashSet<GroupSymbol>(); groups.add(getGroup(3)); @@ -94,10 +103,10 @@ assertEquals(NodeConstants.Types.SOURCE, originatingNode.getType()); } - public void testNonExistentSource() { + @Test public void testNonExistentSource() { PlanNode root = getExamplePlan(); - Set groups = new HashSet(); + Set<GroupSymbol> groups = new HashSet<GroupSymbol>(); groups.add(getGroup(4)); @@ -105,6 +114,66 @@ assertNull(originatingNode); } + + @Test public void testJoinGroups() throws Exception { + PlanNode joinNode = getExamplePlan(); + PlanNode projectNode = NodeFactory.getNewNode(NodeConstants.Types.PROJECT); + ElementSymbol e1 = new ElementSymbol("e1"); //$NON-NLS-1$ + e1.setGroupSymbol(getGroup(3)); + projectNode.setProperty(Info.PROJECT_COLS, Arrays.asList(e1)); + projectNode.addFirstChild(joinNode); + projectNode.addGroup(getGroup(3)); + PlanNode sourceNode = NodeFactory.getNewNode(NodeConstants.Types.SOURCE); + sourceNode.addFirstChild(projectNode); + GroupSymbol four = getGroup(4); + sourceNode.addGroup(four); + ElementSymbol e2 = new ElementSymbol("e2"); //$NON-NLS-1$ + e2.setGroupSymbol(four); + SymbolMap sm = SymbolMap.createSymbolMap(Arrays.asList(e2), Arrays.asList(e1)); + sourceNode.setProperty(Info.SYMBOL_MAP, sm); + PlanNode projectNode1 = NodeFactory.getNewNode(NodeConstants.Types.PROJECT); + projectNode1.addFirstChild(sourceNode); + projectNode1.addGroup(four); + projectNode1.setProperty(Info.PROJECT_COLS, Arrays.asList(e2)); + + //removing source node 3 completely + SymbolMap replacement = SymbolMap.createSymbolMap(Arrays.asList(e1), Arrays.asList(new Constant(null))); + FrameUtil.convertFrame(NodeEditor.findNodePreOrder(joinNode, NodeConstants.Types.SOURCE), getGroup(3), null, replacement.asMap(), null); + assertEquals(2, joinNode.getGroups().size()); //even though this is a cross join it should still retain its groups + assertEquals(0, NodeEditor.findNodePreOrder(joinNode, NodeConstants.Types.SELECT).getGroups().size()); + assertEquals(1, projectNode1.getGroups().size()); + assertEquals(0, projectNode.getGroups().size()); + } + + @Test public void testJoinGroups1() throws Exception { + PlanNode joinNode = getExamplePlan(); + PlanNode projectNode = NodeFactory.getNewNode(NodeConstants.Types.PROJECT); + ElementSymbol e1 = new ElementSymbol("e1"); //$NON-NLS-1$ + e1.setGroupSymbol(getGroup(3)); + projectNode.setProperty(Info.PROJECT_COLS, Arrays.asList(e1)); + projectNode.addFirstChild(joinNode); + projectNode.addGroup(getGroup(3)); + PlanNode sourceNode = NodeFactory.getNewNode(NodeConstants.Types.SOURCE); + sourceNode.addFirstChild(projectNode); + GroupSymbol four = getGroup(4); + sourceNode.addGroup(four); + ElementSymbol e2 = new ElementSymbol("e2"); //$NON-NLS-1$ + e2.setGroupSymbol(four); + SymbolMap sm = SymbolMap.createSymbolMap(Arrays.asList(e2), Arrays.asList(e1)); + sourceNode.setProperty(Info.SYMBOL_MAP, sm); + PlanNode projectNode1 = NodeFactory.getNewNode(NodeConstants.Types.PROJECT); + projectNode1.addFirstChild(sourceNode); + projectNode1.addGroup(four); + projectNode1.setProperty(Info.PROJECT_COLS, Arrays.asList(e2)); + + //replace source 3 with groups 5, 6 + SymbolMap replacement = SymbolMap.createSymbolMap(Arrays.asList(e1), Arrays.asList(new Constant(null))); + FrameUtil.convertFrame(NodeEditor.findNodePreOrder(joinNode, NodeConstants.Types.SOURCE), getGroup(3), new HashSet<GroupSymbol>(Arrays.asList(getGroup(5), getGroup(6))), replacement.asMap(), null); + assertEquals(4, joinNode.getGroups().size()); //even though this is a cross join it should still retain its groups + assertEquals(0, NodeEditor.findNodePreOrder(joinNode, NodeConstants.Types.SELECT).getGroups().size()); + assertEquals(1, projectNode1.getGroups().size()); + assertEquals(0, projectNode.getGroups().size()); + } /** * <pre> @@ -129,7 +198,7 @@ joinNode.addFirstChild(nullNode); PlanNode childCriteria = NodeFactory.getNewNode(NodeConstants.Types.SELECT); - + childCriteria.setProperty(Info.SELECT_CRITERIA, new IsNullCriteria(new Constant(1))); childCriteria.addGroup(getGroup(2)); joinNode.addLastChild(childCriteria);