Author: shawkins Date: 2009-05-04 09:21:45 -0400 (Mon, 04 May 2009) New Revision: 857 Added: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/PartitionedSortJoin.java trunk/engine/src/test/java/com/metamatrix/query/optimizer/TestPartitionedJoinPlanning.java Removed: trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/plantree/JoinStrategyType.java trunk/engine/src/main/java/com/metamatrix/query/sql/visitor/DependentSetCriteriaCollectorVisitor.java trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestJoinNodeWithProcessorBatchSize1.java trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestJoinNodeWithProcessorBatchSize10.java trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestJoinNodeWithProcessorBatchSizeSameAsTupleCount.java Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/IndexedTupleSource.java trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleSourceImpl.java trunk/engine/src/main/java/com/metamatrix/query/eval/Evaluator.java trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/GenerateCanonical.java trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/PlanToProcessConverter.java trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/plantree/NodeConstants.java trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/NewCalculateCostUtil.java trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RuleChooseDependent.java trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RuleChooseJoinStrategy.java trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RuleImplementJoinStrategy.java trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RulePlanJoins.java trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RulePlanSorts.java trunk/engine/src/main/java/com/metamatrix/query/processor/relational/BatchIterator.java trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentValueSource.java trunk/engine/src/main/java/com/metamatrix/query/processor/relational/GroupingNode.java trunk/engine/src/main/java/com/metamatrix/query/processor/relational/JoinNode.java trunk/engine/src/main/java/com/metamatrix/query/processor/relational/JoinStrategy.java trunk/engine/src/main/java/com/metamatrix/query/processor/relational/ListNestedSortComparator.java trunk/engine/src/main/java/com/metamatrix/query/processor/relational/MergeJoinStrategy.java trunk/engine/src/main/java/com/metamatrix/query/processor/relational/NestedLoopJoinStrategy.java trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SortUtility.java trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SourceState.java trunk/engine/src/main/java/com/metamatrix/query/processor/relational/TupleCollector.java trunk/engine/src/main/java/com/metamatrix/query/sql/util/ValueIteratorSource.java trunk/engine/src/main/java/com/metamatrix/query/util/CommandContext.java trunk/engine/src/test/java/com/metamatrix/query/optimizer/TestLimit.java trunk/engine/src/test/java/com/metamatrix/query/optimizer/TestOptimizer.java trunk/engine/src/test/java/com/metamatrix/query/optimizer/relational/rules/TestRuleChooseDependent.java trunk/engine/src/test/java/com/metamatrix/query/processor/TestProcessor.java trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestJoinNode.java trunk/test-integration/src/test/java/com/metamatrix/systemmodel/TestSystemVirtualModel.java trunk/test-integration/src/test/resources/partssupplier/expected/CrossReference.txt trunk/test-integration/src/test/resources/partssupplier/expected/ExportedKeys.txt Log: TEIID-545, TEIID-541, TEIID-238 peformance enhancements for unbalanced joins, non-pushed dependentset criteria, and detection of distinct join expressions. Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/IndexedTupleSource.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/IndexedTupleSource.java 2009-05-02 00:13:45 UTC (rev 856) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/IndexedTupleSource.java 2009-05-04 13:21:45 UTC (rev 857) @@ -56,4 +56,7 @@ * @return */ int getCurrentIndex(); + + TupleBatch getBatch() + throws MetaMatrixComponentException; } Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleSourceImpl.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleSourceImpl.java 2009-05-02 00:13:45 UTC (rev 856) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleSourceImpl.java 2009-05-04 13:21:45 UTC (rev 857) @@ -132,11 +132,11 @@ } // Retrieves the necessary batch based on the currentRow - private TupleBatch getBatch() + public TupleBatch getBatch() throws MetaMatrixComponentException{ TupleBatch batch = getCurrentBatch(); if (batch != null) { - if (currentRow < batch.getEndRow() && currentRow > batch.getBeginRow()) { + if (currentRow <= batch.getEndRow() && currentRow >= batch.getBeginRow()) { return batch; } unpinCurrentBatch(); Modified: trunk/engine/src/main/java/com/metamatrix/query/eval/Evaluator.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/eval/Evaluator.java 2009-05-02 00:13:45 UTC (rev 856) +++ trunk/engine/src/main/java/com/metamatrix/query/eval/Evaluator.java 2009-05-04 13:21:45 UTC (rev 857) @@ -24,6 +24,7 @@ import java.sql.SQLException; import java.util.Collections; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -74,6 +75,7 @@ import com.metamatrix.query.sql.symbol.SearchedCaseExpression; import com.metamatrix.query.sql.symbol.SingleElementSymbol; import com.metamatrix.query.sql.util.ValueIterator; +import com.metamatrix.query.sql.util.ValueIteratorSource; import com.metamatrix.query.util.CommandContext; import com.metamatrix.query.util.ErrorMessageKeys; @@ -314,7 +316,19 @@ valueIter = new CollectionValueIterator(((SetCriteria)criteria).getValues()); } else if (criteria instanceof DependentSetCriteria){ ContextReference ref = (ContextReference)criteria; - valueIter = getContext(criteria).getValueIterator(ref); + ValueIteratorSource vis = (ValueIteratorSource)getContext(criteria).getVariableContext().getGlobalValue(ref.getContextSymbol()); + HashSet<Object> values; + try { + values = vis.getCachedSet(ref.getValueExpression()); + } catch (MetaMatrixProcessingException e) { + throw new CriteriaEvaluationException(e, e.getMessage()); + } + if (values != null) { + return values.contains(leftValue); + } + //there are too many values to justify a linear search or holding + //them in memory + return true; } else if (criteria instanceof SubquerySetCriteria) { try { valueIter = evaluateSubquery((SubquerySetCriteria)criteria, tuple); Modified: trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/GenerateCanonical.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/GenerateCanonical.java 2009-05-02 00:13:45 UTC (rev 856) +++ trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/GenerateCanonical.java 2009-05-04 13:21:45 UTC (rev 857) @@ -30,10 +30,10 @@ import com.metamatrix.api.exception.query.QueryPlannerException; import com.metamatrix.query.execution.QueryExecPlugin; import com.metamatrix.query.metadata.QueryMetadataInterface; -import com.metamatrix.query.optimizer.relational.plantree.JoinStrategyType; import com.metamatrix.query.optimizer.relational.plantree.NodeConstants; import com.metamatrix.query.optimizer.relational.plantree.NodeFactory; import com.metamatrix.query.optimizer.relational.plantree.PlanNode; +import com.metamatrix.query.processor.relational.JoinNode.JoinStrategyType; import com.metamatrix.query.sql.lang.Command; import com.metamatrix.query.sql.lang.Criteria; import com.metamatrix.query.sql.lang.From; Modified: trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/PlanToProcessConverter.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/PlanToProcessConverter.java 2009-05-02 00:13:45 UTC (rev 856) +++ trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/PlanToProcessConverter.java 2009-05-04 13:21:45 UTC (rev 857) @@ -40,7 +40,6 @@ import com.metamatrix.query.optimizer.capabilities.CapabilitiesFinder; import com.metamatrix.query.optimizer.capabilities.SourceCapabilities; import com.metamatrix.query.optimizer.capabilities.SourceCapabilities.Capability; -import com.metamatrix.query.optimizer.relational.plantree.JoinStrategyType; import com.metamatrix.query.optimizer.relational.plantree.NodeConstants; import com.metamatrix.query.optimizer.relational.plantree.PlanNode; import com.metamatrix.query.optimizer.relational.rules.CapabilitiesUtil; @@ -55,6 +54,7 @@ import com.metamatrix.query.processor.relational.MergeJoinStrategy; import com.metamatrix.query.processor.relational.NestedLoopJoinStrategy; import com.metamatrix.query.processor.relational.NullNode; +import com.metamatrix.query.processor.relational.PartitionedSortJoin; import com.metamatrix.query.processor.relational.PlanExecutionNode; import com.metamatrix.query.processor.relational.ProjectIntoNode; import com.metamatrix.query.processor.relational.ProjectNode; @@ -63,6 +63,7 @@ import com.metamatrix.query.processor.relational.SelectNode; import com.metamatrix.query.processor.relational.SortNode; import com.metamatrix.query.processor.relational.UnionAllNode; +import com.metamatrix.query.processor.relational.JoinNode.JoinStrategyType; import com.metamatrix.query.processor.relational.MergeJoinStrategy.SortOption; import com.metamatrix.query.processor.relational.SortUtility.Mode; import com.metamatrix.query.resolver.util.ResolverUtil; @@ -195,11 +196,18 @@ JoinNode jnode = new JoinNode(getID()); jnode.setJoinType(jtype); - + jnode.setLeftDistinct(node.hasBooleanProperty(NodeConstants.Info.IS_LEFT_DISTINCT)); + jnode.setRightDistinct(node.hasBooleanProperty(NodeConstants.Info.IS_RIGHT_DISTINCT)); List joinCrits = (List) node.getProperty(NodeConstants.Info.JOIN_CRITERIA); - - if(stype.equals(JoinStrategyType.MERGE)) { - MergeJoinStrategy mjStrategy = new MergeJoinStrategy((SortOption)node.getProperty(NodeConstants.Info.SORT_LEFT), (SortOption)node.getProperty(NodeConstants.Info.SORT_RIGHT), false); + String depValueSource = (String) node.getProperty(NodeConstants.Info.DEPENDENT_VALUE_SOURCE); + SortOption leftSort = (SortOption)node.getProperty(NodeConstants.Info.SORT_LEFT); + if(stype.equals(JoinStrategyType.MERGE) || stype.equals(JoinStrategyType.PARTITIONED_SORT)) { + MergeJoinStrategy mjStrategy = null; + if (stype.equals(JoinStrategyType.PARTITIONED_SORT)) { + mjStrategy = new PartitionedSortJoin(leftSort, (SortOption)node.getProperty(NodeConstants.Info.SORT_RIGHT)); + } else { + mjStrategy = new MergeJoinStrategy(leftSort, (SortOption)node.getProperty(NodeConstants.Info.SORT_RIGHT), false); + } jnode.setJoinStrategy(mjStrategy); List leftExpressions = (List) node.getProperty(NodeConstants.Info.LEFT_EXPRESSIONS); List rightExpressions = (List) node.getProperty(NodeConstants.Info.RIGHT_EXPRESSIONS); @@ -214,7 +222,6 @@ processNode = jnode; - String depValueSource = (String) node.getProperty(NodeConstants.Info.DEPENDENT_VALUE_SOURCE); jnode.setDependentValueSource(depValueSource); break; Deleted: trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/plantree/JoinStrategyType.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/plantree/JoinStrategyType.java 2009-05-02 00:13:45 UTC (rev 856) +++ trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/plantree/JoinStrategyType.java 2009-05-04 13:21:45 UTC (rev 857) @@ -1,29 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.query.optimizer.relational.plantree; - -public enum JoinStrategyType { - MERGE, - HASH, - NESTED_LOOP -} Modified: trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/plantree/NodeConstants.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/plantree/NodeConstants.java 2009-05-02 00:13:45 UTC (rev 856) +++ trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/plantree/NodeConstants.java 2009-05-04 13:21:45 UTC (rev 857) @@ -90,8 +90,9 @@ SORT_LEFT, // SortOption SORT_RIGHT, // SortOption REMOVED_JOIN_GROUPS, //Set<GroupSymbol> - IS_OPTIONAL, // Boolean + IS_LEFT_DISTINCT, // Boolean + IS_RIGHT_DISTINCT, // Boolean // Project node properties PROJECT_COLS, // List <SingleElementSymbol> Modified: trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/NewCalculateCostUtil.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/NewCalculateCostUtil.java 2009-05-02 00:13:45 UTC (rev 856) +++ trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/NewCalculateCostUtil.java 2009-05-04 13:21:45 UTC (rev 857) @@ -41,10 +41,10 @@ import com.metamatrix.query.metadata.QueryMetadataInterface; import com.metamatrix.query.metadata.SupportConstants; import com.metamatrix.query.optimizer.capabilities.CapabilitiesFinder; -import com.metamatrix.query.optimizer.relational.plantree.JoinStrategyType; import com.metamatrix.query.optimizer.relational.plantree.NodeConstants; import com.metamatrix.query.optimizer.relational.plantree.NodeEditor; import com.metamatrix.query.optimizer.relational.plantree.PlanNode; +import com.metamatrix.query.processor.relational.JoinNode.JoinStrategyType; import com.metamatrix.query.sql.lang.AbstractSetCriteria; import com.metamatrix.query.sql.lang.CompareCriteria; import com.metamatrix.query.sql.lang.CompoundCriteria; @@ -61,6 +61,7 @@ import com.metamatrix.query.sql.symbol.ElementSymbol; import com.metamatrix.query.sql.symbol.Expression; import com.metamatrix.query.sql.symbol.GroupSymbol; +import com.metamatrix.query.sql.symbol.SingleElementSymbol; import com.metamatrix.query.sql.util.SymbolMap; import com.metamatrix.query.sql.visitor.ElementCollectorVisitor; import com.metamatrix.query.sql.visitor.EvaluateExpressionVisitor; @@ -786,7 +787,7 @@ return cost; } - static boolean usesKey(Collection allElements, QueryMetadataInterface metadata) + static boolean usesKey(Collection<SingleElementSymbol> allElements, QueryMetadataInterface metadata) throws QueryMetadataException, MetaMatrixComponentException { if(allElements == null || allElements.size() == 0) { @@ -797,7 +798,11 @@ Map groupMap = new HashMap(); Iterator elementIter = allElements.iterator(); while(elementIter.hasNext()) { - ElementSymbol element = (ElementSymbol) elementIter.next(); + SingleElementSymbol ses = (SingleElementSymbol) elementIter.next(); + if (!(ses instanceof ElementSymbol)) { + continue; + } + ElementSymbol element = (ElementSymbol)ses; GroupSymbol group = element.getGroupSymbol(); List elements = (List) groupMap.get(group); if(elements == null) { Modified: trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RuleChooseDependent.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RuleChooseDependent.java 2009-05-02 00:13:45 UTC (rev 856) +++ trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RuleChooseDependent.java 2009-05-04 13:21:45 UTC (rev 857) @@ -39,10 +39,10 @@ import com.metamatrix.query.optimizer.relational.GenerateCanonical; import com.metamatrix.query.optimizer.relational.OptimizerRule; import com.metamatrix.query.optimizer.relational.RuleStack; -import com.metamatrix.query.optimizer.relational.plantree.JoinStrategyType; import com.metamatrix.query.optimizer.relational.plantree.NodeConstants; import com.metamatrix.query.optimizer.relational.plantree.NodeEditor; import com.metamatrix.query.optimizer.relational.plantree.PlanNode; +import com.metamatrix.query.processor.relational.JoinNode.JoinStrategyType; import com.metamatrix.query.sql.lang.CompareCriteria; import com.metamatrix.query.sql.lang.Criteria; import com.metamatrix.query.sql.lang.DependentSetCriteria; Modified: trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RuleChooseJoinStrategy.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RuleChooseJoinStrategy.java 2009-05-02 00:13:45 UTC (rev 856) +++ trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RuleChooseJoinStrategy.java 2009-05-04 13:21:45 UTC (rev 857) @@ -36,10 +36,10 @@ import com.metamatrix.query.optimizer.capabilities.CapabilitiesFinder; import com.metamatrix.query.optimizer.relational.OptimizerRule; import com.metamatrix.query.optimizer.relational.RuleStack; -import com.metamatrix.query.optimizer.relational.plantree.JoinStrategyType; import com.metamatrix.query.optimizer.relational.plantree.NodeConstants; import com.metamatrix.query.optimizer.relational.plantree.NodeEditor; import com.metamatrix.query.optimizer.relational.plantree.PlanNode; +import com.metamatrix.query.processor.relational.JoinNode.JoinStrategyType; import com.metamatrix.query.sql.lang.CompareCriteria; import com.metamatrix.query.sql.lang.Criteria; import com.metamatrix.query.sql.lang.JoinType; Modified: trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RuleImplementJoinStrategy.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RuleImplementJoinStrategy.java 2009-05-02 00:13:45 UTC (rev 856) +++ trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RuleImplementJoinStrategy.java 2009-05-04 13:21:45 UTC (rev 857) @@ -37,11 +37,11 @@ import com.metamatrix.query.optimizer.capabilities.CapabilitiesFinder; import com.metamatrix.query.optimizer.relational.OptimizerRule; import com.metamatrix.query.optimizer.relational.RuleStack; -import com.metamatrix.query.optimizer.relational.plantree.JoinStrategyType; import com.metamatrix.query.optimizer.relational.plantree.NodeConstants; import com.metamatrix.query.optimizer.relational.plantree.NodeEditor; import com.metamatrix.query.optimizer.relational.plantree.NodeFactory; import com.metamatrix.query.optimizer.relational.plantree.PlanNode; +import com.metamatrix.query.processor.relational.JoinNode.JoinStrategyType; import com.metamatrix.query.processor.relational.MergeJoinStrategy.SortOption; import com.metamatrix.query.sql.lang.JoinType; import com.metamatrix.query.sql.lang.OrderBy; @@ -70,20 +70,31 @@ if (!JoinStrategyType.MERGE.equals(stype)) { continue; } -/* if (joinNode.getProperty(NodeConstants.Info.JOIN_TYPE) == JoinType.JOIN_INNER) { - //there is a possible optimization at runtime here based upon the cardinality + + /** + * Don't push sorts for unbalanced inner joins, we prefer to use partitioning + */ + boolean pushLeft = true; + boolean pushRight = true; + if (joinNode.getProperty(NodeConstants.Info.JOIN_TYPE) == JoinType.JOIN_INNER && context != null) { float leftCost = NewCalculateCostUtil.computeCostForTree(joinNode.getFirstChild(), metadata); float rightCost = NewCalculateCostUtil.computeCostForTree(joinNode.getLastChild(), metadata); - if (leftCost != NewCalculateCostUtil.UNKNOWN_VALUE && leftCost < context.getProcessorBatchSize() * context.getProcessorBatchSize() - && rightCost != NewCalculateCostUtil.UNKNOWN_VALUE && rightCost > context.getProcessorBatchSize()) { - joinNode.setProperty(NodeConstants.Info.SORT_LEFT, SortOption.SORT); - joinNode.setProperty(NodeConstants.Info.SORT_RIGHT, SortOption.SORT); - continue; + boolean leftSmall = leftCost < context.getProcessorBatchSize() / 4; + boolean rightSmall = rightCost < context.getProcessorBatchSize() / 4; + boolean leftLarge = leftCost > context.getProcessorBatchSize(); + boolean rightLarge = rightCost > context.getProcessorBatchSize(); + if (leftLarge || rightLarge) { + pushLeft = leftCost == NewCalculateCostUtil.UNKNOWN_VALUE || leftSmall || rightLarge; + pushRight = rightCost == NewCalculateCostUtil.UNKNOWN_VALUE || rightSmall || leftLarge || joinNode.getProperty(NodeConstants.Info.DEPENDENT_VALUE_SOURCE) != null; } - } -*/ - insertSort(joinNode.getFirstChild(), (List<SingleElementSymbol>) joinNode.getProperty(NodeConstants.Info.LEFT_EXPRESSIONS), joinNode, metadata, capabilitiesFinder); - insertSort(joinNode.getLastChild(), (List<SingleElementSymbol>) joinNode.getProperty(NodeConstants.Info.RIGHT_EXPRESSIONS), joinNode, metadata, capabilitiesFinder); + } + + boolean pushedLeft = insertSort(joinNode.getFirstChild(), (List<SingleElementSymbol>) joinNode.getProperty(NodeConstants.Info.LEFT_EXPRESSIONS), joinNode, metadata, capabilitiesFinder, pushLeft); + insertSort(joinNode.getLastChild(), (List<SingleElementSymbol>) joinNode.getProperty(NodeConstants.Info.RIGHT_EXPRESSIONS), joinNode, metadata, capabilitiesFinder, pushRight); + + if (joinNode.getProperty(NodeConstants.Info.JOIN_TYPE) == JoinType.JOIN_INNER && (!pushRight || !pushedLeft)) { + joinNode.setProperty(NodeConstants.Info.JOIN_STRATEGY, JoinStrategyType.PARTITIONED_SORT); + } } return plan; @@ -98,7 +109,8 @@ * @throws MetaMatrixComponentException * @throws QueryMetadataException */ - private static void insertSort(PlanNode childNode, List<SingleElementSymbol> expressions, PlanNode jnode, QueryMetadataInterface metadata, CapabilitiesFinder capFinder) throws QueryMetadataException, MetaMatrixComponentException { + private static boolean insertSort(PlanNode childNode, List<SingleElementSymbol> expressions, PlanNode jnode, QueryMetadataInterface metadata, CapabilitiesFinder capFinder, + boolean attemptPush) throws QueryMetadataException, MetaMatrixComponentException { Set<SingleElementSymbol> orderSymbols = new LinkedHashSet<SingleElementSymbol>(expressions); PlanNode sourceNode = FrameUtil.findJoinSourceNode(childNode); @@ -116,14 +128,19 @@ PlanNode sortNode = createSortNode(orderSymbols, outputSymbols, directions); - if (sourceNode.getType() == NodeConstants.Types.ACCESS - && RuleRaiseAccess.canRaiseOverSort(sourceNode, metadata, capFinder, sortNode)) { - sourceNode.getFirstChild().addAsParent(sortNode); - - if (needsCorrection) { - correctOutputElements(joinNode, outputSymbols, sortNode); - } - return; + if (sourceNode.getType() == NodeConstants.Types.ACCESS) { + if (NodeEditor.findAllNodes(sourceNode, NodeConstants.Types.SOURCE).size() == 1 + && NewCalculateCostUtil.usesKey(expressions, metadata)) { + joinNode.setProperty(joinNode.getFirstChild() == childNode ? NodeConstants.Info.IS_LEFT_DISTINCT : NodeConstants.Info.IS_RIGHT_DISTINCT, true); + } + if (attemptPush && RuleRaiseAccess.canRaiseOverSort(sourceNode, metadata, capFinder, sortNode)) { + sourceNode.getFirstChild().addAsParent(sortNode); + + if (needsCorrection) { + correctOutputElements(joinNode, outputSymbols, sortNode); + } + return true; + } } joinNode.setProperty(joinNode.getFirstChild() == childNode ? NodeConstants.Info.SORT_LEFT : NodeConstants.Info.SORT_RIGHT, SortOption.SORT); @@ -134,6 +151,7 @@ childNode.addAsParent(projectNode); correctOutputElements(joinNode, outputSymbols, projectNode); } + return false; } private static PlanNode createSortNode(Collection orderSymbols, Modified: trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RulePlanJoins.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RulePlanJoins.java 2009-05-02 00:13:45 UTC (rev 856) +++ trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RulePlanJoins.java 2009-05-04 13:21:45 UTC (rev 857) @@ -44,10 +44,10 @@ import com.metamatrix.query.optimizer.capabilities.CapabilitiesFinder; import com.metamatrix.query.optimizer.relational.OptimizerRule; import com.metamatrix.query.optimizer.relational.RuleStack; -import com.metamatrix.query.optimizer.relational.plantree.JoinStrategyType; import com.metamatrix.query.optimizer.relational.plantree.NodeConstants; import com.metamatrix.query.optimizer.relational.plantree.NodeFactory; import com.metamatrix.query.optimizer.relational.plantree.PlanNode; +import com.metamatrix.query.processor.relational.JoinNode.JoinStrategyType; import com.metamatrix.query.resolver.util.AccessPattern; import com.metamatrix.query.sql.lang.JoinType; import com.metamatrix.query.sql.symbol.ElementSymbol; Modified: trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RulePlanSorts.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RulePlanSorts.java 2009-05-02 00:13:45 UTC (rev 856) +++ trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/rules/RulePlanSorts.java 2009-05-04 13:21:45 UTC (rev 857) @@ -32,13 +32,12 @@ import com.metamatrix.query.optimizer.capabilities.CapabilitiesFinder; import com.metamatrix.query.optimizer.relational.OptimizerRule; import com.metamatrix.query.optimizer.relational.RuleStack; -import com.metamatrix.query.optimizer.relational.plantree.JoinStrategyType; import com.metamatrix.query.optimizer.relational.plantree.NodeConstants; import com.metamatrix.query.optimizer.relational.plantree.NodeEditor; import com.metamatrix.query.optimizer.relational.plantree.PlanNode; import com.metamatrix.query.optimizer.relational.plantree.NodeConstants.Info; +import com.metamatrix.query.processor.relational.JoinNode.JoinStrategyType; import com.metamatrix.query.processor.relational.MergeJoinStrategy.SortOption; -import com.metamatrix.query.sql.lang.JoinType; import com.metamatrix.query.sql.lang.SetQuery; import com.metamatrix.query.util.CommandContext; @@ -76,21 +75,13 @@ node.setProperty(NodeConstants.Info.IS_DUP_REMOVAL, true); } List orderColumns = (List)node.getProperty(NodeConstants.Info.SORT_ORDER); - PlanNode possibleSort = NodeEditor.findNodePreOrder(node, NodeConstants.Types.GROUP | NodeConstants.Types.JOIN, NodeConstants.Types.SOURCE | NodeConstants.Types.ACCESS); + PlanNode possibleSort = NodeEditor.findNodePreOrder(node, NodeConstants.Types.GROUP, NodeConstants.Types.SOURCE | NodeConstants.Types.ACCESS); if (possibleSort != null) { - NodeConstants.Info expr = Info.GROUP_COLS; - if (possibleSort.getType() == NodeConstants.Types.JOIN) { - if (possibleSort.getProperty(NodeConstants.Info.JOIN_STRATEGY) != JoinStrategyType.MERGE - || possibleSort.getProperty(NodeConstants.Info.JOIN_TYPE) != JoinType.JOIN_INNER) { - break; - } - expr = Info.LEFT_EXPRESSIONS; - } - List exprs = (List)possibleSort.getProperty(expr); + List exprs = (List)possibleSort.getProperty(Info.GROUP_COLS); if (exprs != null && exprs.containsAll(orderColumns)) { exprs.removeAll(orderColumns); orderColumns.addAll(exprs); - possibleSort.setProperty(expr, orderColumns); + possibleSort.setProperty(Info.GROUP_COLS, orderColumns); if (node.getParent() == null) { root = node.getFirstChild(); root.removeFromParent(); @@ -101,7 +92,56 @@ node = nextNode; } } + break; + } +/* possibleSort = NodeEditor.findNodePreOrder(node, NodeConstants.Types.JOIN, NodeConstants.Types.SOURCE | NodeConstants.Types.ACCESS); + if (possibleSort == null) { + break; } + boolean left = false; + if (possibleSort.getType() == NodeConstants.Types.JOIN) { + if (possibleSort.getProperty(NodeConstants.Info.JOIN_STRATEGY) != JoinStrategyType.MERGE + || possibleSort.getProperty(NodeConstants.Info.JOIN_TYPE) != JoinType.JOIN_INNER) { + break; + } + if (FrameUtil.findJoinSourceNode(possibleSort.getFirstChild()).getGroups().containsAll(node.getGroups()) + && possibleSort.getProperty(NodeConstants.Info.SORT_LEFT) == SortOption.SORT) { + left = true; + } else if (!FrameUtil.findJoinSourceNode(possibleSort.getLastChild()).getGroups().containsAll(node.getGroups()) + || possibleSort.getProperty(NodeConstants.Info.SORT_RIGHT) != SortOption.SORT) { + break; + } + } + List exprs = (List)possibleSort.getProperty(left?Info.LEFT_EXPRESSIONS:Info.RIGHT_EXPRESSIONS); + if (exprs != null && exprs.containsAll(orderColumns)) { + List<Integer> indexes = new ArrayList<Integer>(orderColumns.size()); + for (Expression expr : (List<Expression>)orderColumns) { + indexes.add(0, exprs.indexOf(expr)); + } + exprs.removeAll(orderColumns); + List newExprs = new ArrayList(orderColumns); + newExprs.addAll(exprs); + possibleSort.setProperty(left?Info.LEFT_EXPRESSIONS:Info.RIGHT_EXPRESSIONS, newExprs); + if (node.getParent() == null) { + root = node.getFirstChild(); + root.removeFromParent(); + node = root; + } else { + PlanNode nextNode = node.getFirstChild(); + NodeEditor.removeChildNode(node.getParent(), node); + node = nextNode; + } + exprs = (List)possibleSort.getProperty(left?Info.RIGHT_EXPRESSIONS:Info.LEFT_EXPRESSIONS); + List toRemove = new ArrayList(); + for (Integer index : indexes) { + Object o = exprs.get(index); + exprs.add(0, o); + toRemove.add(o); + } + exprs.subList(indexes.size(), exprs.size()).removeAll(toRemove); + possibleSort.setProperty(left?NodeConstants.Info.SORT_LEFT:NodeConstants.Info.SORT_RIGHT, SortOption.SORT_REQUIRED); + } +*/ break; case NodeConstants.Types.DUP_REMOVE: if (parentBlocking) { @@ -120,7 +160,7 @@ parentBlocking = true; break; case NodeConstants.Types.JOIN: - if (node.getProperty(NodeConstants.Info.JOIN_STRATEGY) != JoinStrategyType.MERGE) { + if (node.getProperty(NodeConstants.Info.JOIN_STRATEGY) == JoinStrategyType.NESTED_LOOP) { break; } /* @@ -134,10 +174,16 @@ PlanNode toTest = node.getFirstChild(); if (mergeSortWithDupRemovalAcrossSource(toTest)) { node.setProperty(NodeConstants.Info.SORT_LEFT, SortOption.SORT_DISTINCT); + if (node.getProperty(NodeConstants.Info.SORT_RIGHT) != SortOption.SORT) { + node.setProperty(NodeConstants.Info.JOIN_STRATEGY, JoinStrategyType.MERGE); + } } toTest = node.getLastChild(); if (mergeSortWithDupRemovalAcrossSource(toTest)) { node.setProperty(NodeConstants.Info.SORT_RIGHT, SortOption.SORT_DISTINCT); + if (node.getProperty(NodeConstants.Info.SORT_LEFT) != SortOption.SORT) { + node.setProperty(NodeConstants.Info.JOIN_STRATEGY, JoinStrategyType.MERGE); + } } break; case NodeConstants.Types.SET_OP: Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/BatchIterator.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/BatchIterator.java 2009-05-02 00:13:45 UTC (rev 856) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/BatchIterator.java 2009-05-04 13:21:45 UTC (rev 857) @@ -114,5 +114,10 @@ this.currentBatch = null; } } + + @Override + public TupleBatch getBatch() throws MetaMatrixComponentException { + return currentBatch; + } } Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentValueSource.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentValueSource.java 2009-05-02 00:13:45 UTC (rev 856) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentValueSource.java 2009-05-04 13:21:45 UTC (rev 857) @@ -22,11 +22,17 @@ package com.metamatrix.query.processor.relational; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; + import com.metamatrix.api.exception.MetaMatrixComponentException; +import com.metamatrix.api.exception.MetaMatrixProcessingException; import com.metamatrix.common.buffer.BufferManager; import com.metamatrix.common.buffer.IndexedTupleSource; import com.metamatrix.common.buffer.TupleSourceID; import com.metamatrix.common.buffer.TupleSourceNotFoundException; +import com.metamatrix.core.util.Assertion; import com.metamatrix.query.sql.symbol.Expression; import com.metamatrix.query.sql.util.ValueIterator; import com.metamatrix.query.sql.util.ValueIteratorSource; @@ -39,6 +45,7 @@ private TupleSourceID tupleSourceID; private BufferManager bm; + private Map<Expression, HashSet<Object>> cachedSets; public DependentValueSource(TupleSourceID tupleSourceID, BufferManager bm) { this.tupleSourceID = tupleSourceID; @@ -64,8 +71,44 @@ int index = 0; if (valueExpression != null) { index = its.getSchema().indexOf(valueExpression); + Assertion.assertTrue(index != -1); } return new TupleSourceValueIterator(its, index); } + + public HashSet<Object> getCachedSet(Expression valueExpression) throws MetaMatrixComponentException, MetaMatrixProcessingException { + HashSet<Object> result = null; + if (cachedSets != null) { + result = cachedSets.get(valueExpression); + } + if (result == null) { + IndexedTupleSource its; + try { + if (bm.getRowCount(tupleSourceID) > bm.getProcessorBatchSize() / 2) { + return null; + } + its = bm.getTupleSource(tupleSourceID); + } catch (TupleSourceNotFoundException e) { + throw new MetaMatrixComponentException(e); + } + int index = 0; + if (valueExpression != null) { + index = its.getSchema().indexOf(valueExpression); + } + Assertion.assertTrue(index != -1); + result = new HashSet<Object>(); + while (its.hasNext()) { + Object value = its.nextTuple().get(index); + if (value != null) { + result.add(value); + } + } + if (cachedSets == null) { + cachedSets = new HashMap<Expression, HashSet<Object>>(); + } + cachedSets.put(valueExpression, result); + } + return result; + } } Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/GroupingNode.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/GroupingNode.java 2009-05-02 00:13:45 UTC (rev 856) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/GroupingNode.java 2009-05-04 13:21:45 UTC (rev 857) @@ -47,6 +47,7 @@ import com.metamatrix.query.function.aggregate.Min; import com.metamatrix.query.function.aggregate.NullFilter; import com.metamatrix.query.function.aggregate.Sum; +import com.metamatrix.query.processor.relational.SortUtility.Mode; import com.metamatrix.query.sql.ReservedWords; import com.metamatrix.query.sql.lang.OrderBy; import com.metamatrix.query.sql.symbol.AggregateSymbol; @@ -311,8 +312,8 @@ this.phase = GROUP; } else { this.sortUtility = new SortUtility(collectionID, sortElements, - sortTypes, removeDuplicates, getBufferManager(), - getConnectionID()); + sortTypes, removeDuplicates?Mode.DUP_REMOVE_SORT:Mode.SORT, getBufferManager(), + getConnectionID(), removeDuplicates); this.phase = SORT; } } Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/JoinNode.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/JoinNode.java 2009-05-02 00:13:45 UTC (rev 856) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/JoinNode.java 2009-05-04 13:21:45 UTC (rev 857) @@ -47,8 +47,14 @@ * @since 4.2 */ public class JoinNode extends SubqueryAwareRelationalNode { + + public enum JoinStrategyType { + MERGE, + PARTITIONED_SORT, + NESTED_LOOP + } - private enum State { LOAD_LEFT, LOAD_RIGHT, EXECUTE } + private enum State { LOAD_LEFT, LOAD_RIGHT, POST_LOAD_LEFT, POST_LOAD_RIGHT, EXECUTE } private State state = State.LOAD_LEFT; private boolean leftOpened; @@ -60,6 +66,8 @@ private List leftExpressions; private List rightExpressions; + private boolean leftDistinct; + private boolean rightDistinct; private Criteria joinCriteria; private Map combinedElementMap; @@ -85,6 +93,22 @@ this.rightExpressions = rightExpressions; } + public boolean isLeftDistinct() { + return leftDistinct; + } + + public void setLeftDistinct(boolean leftDistinct) { + this.leftDistinct = leftDistinct; + } + + public boolean isRightDistinct() { + return rightDistinct; + } + + public void setRightDistinct(boolean rightDistinct) { + this.rightDistinct = rightDistinct; + } + public void setJoinCriteria(Criteria joinCriteria) { this.joinCriteria = joinCriteria; } @@ -147,6 +171,8 @@ clonedNode.rightExpressions = rightExpressions; clonedNode.dependentValueSource = this.dependentValueSource; + clonedNode.rightDistinct = rightDistinct; + clonedNode.leftDistinct = leftDistinct; return clonedNode; } @@ -172,8 +198,17 @@ this.rightOpened = true; } this.joinStrategy.loadRight(); - state = State.EXECUTE; + state = State.POST_LOAD_LEFT; } + if (state == State.POST_LOAD_LEFT) { + this.joinStrategy.postLoadLeft(); + state = State.POST_LOAD_RIGHT; + } + if (state == State.POST_LOAD_RIGHT) { + this.joinStrategy.postLoadRight(); + state = State.EXECUTE; + } + while(true) { if(super.isBatchFull()) { return super.pullBatch(); @@ -270,6 +305,9 @@ } catch (TupleSourceNotFoundException err) { //ignore } + if (this.isDependent()) { + this.getContext().getVariableContext().setGlobalValue(this.dependentValueSource, null); + } } public JoinType getJoinType() { Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/JoinStrategy.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/JoinStrategy.java 2009-05-02 00:13:45 UTC (rev 856) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/JoinStrategy.java 2009-05-04 13:21:45 UTC (rev 857) @@ -57,16 +57,24 @@ throws MetaMatrixComponentException { this.joinNode = joinNode; this.leftSource = new SourceState(joinNode.getChildren()[0], joinNode.getLeftExpressions()); + this.leftSource.markDistinct(this.joinNode.isLeftDistinct()); this.rightSource = new SourceState(joinNode.getChildren()[1], joinNode.getRightExpressions()); + this.rightSource.markDistinct(this.joinNode.isRightDistinct()); } protected void loadLeft() throws MetaMatrixComponentException, MetaMatrixProcessingException { this.leftSource.collectTuples(); } + protected void postLoadLeft() throws MetaMatrixComponentException, MetaMatrixProcessingException { + } + protected void loadRight() throws MetaMatrixComponentException, MetaMatrixProcessingException { this.rightSource.collectTuples(); } + + protected void postLoadRight() throws MetaMatrixComponentException, MetaMatrixProcessingException { + } /** * Output a combined, projected tuple based on tuple parts from the left and right. Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/ListNestedSortComparator.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/ListNestedSortComparator.java 2009-05-02 00:13:45 UTC (rev 856) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/ListNestedSortComparator.java 2009-05-04 13:21:45 UTC (rev 857) @@ -22,7 +22,7 @@ package com.metamatrix.query.processor.relational; -import java.util.*; +import java.util.List; import com.metamatrix.core.util.Assertion; @@ -60,18 +60,21 @@ /** * Specifies which fields to sort on. */ - int[] sortParameters; + private int[] sortParameters; /** * Indicates whether comparison should be based on ascending or descending * order. */ - boolean ascendingOrder = false; + private boolean ascendingOrder = false; /** * List of booleans indicating the order in which each column should be sorted */ - List orderTypes = null; + private List orderTypes = null; + + private boolean isDistinct = true; + private int distinctIndex; /** * Constructs an instance of this class given the indicies of the parameters @@ -101,8 +104,15 @@ this.sortParameters = sortParameters; this.orderTypes = orderTypes; } + + public boolean isDistinct() { + return isDistinct; + } + + public void setDistinctIndex(int distinctIndex) { + this.distinctIndex = distinctIndex; + } - /** * Compares its two arguments for order. Returns a negative integer, * zero, or a positive integer as the first argument is less than, @@ -119,20 +129,13 @@ * @param o2 The second object being compared */ public int compare( Object o1, Object o2 ) { - // Cast input objects to Lists... List list1 = (List)o1; List list2 = (List)o2; int compare = 0; - int k = 0; - while ( k < sortParameters.length && compare == 0 ) { + for (int k = 0; k < sortParameters.length; k++) { Object param1 = list1.get(sortParameters[k]); Object param2 = list2.get(sortParameters[k]); - // if orderTypes is not set - if(orderTypes != null) { - // getting ordertype for each column - ascendingOrder = ((Boolean)orderTypes.get(k)).booleanValue(); - } if( param1 == null ) { if(param2 == null ) { @@ -150,11 +153,15 @@ } else { Assertion.failed("Expected comparable types"); //$NON-NLS-1$ } - k++; + if (compare != 0) { + boolean asc = orderTypes != null?((Boolean)orderTypes.get(k)).booleanValue():this.ascendingOrder; + return asc ? compare : -compare; + } else if (k == distinctIndex) { + isDistinct = false; + } } - return ascendingOrder ? compare : -compare; + return 0; } } // END CLASS - Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/MergeJoinStrategy.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/MergeJoinStrategy.java 2009-05-02 00:13:45 UTC (rev 856) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/MergeJoinStrategy.java 2009-05-04 13:21:45 UTC (rev 857) @@ -28,6 +28,7 @@ import com.metamatrix.api.exception.MetaMatrixComponentException; import com.metamatrix.api.exception.MetaMatrixProcessingException; import com.metamatrix.api.exception.query.CriteriaEvaluationException; +import com.metamatrix.common.buffer.BlockedOnMemoryException; import com.metamatrix.common.buffer.TupleSourceNotFoundException; import com.metamatrix.query.processor.relational.SortUtility.Mode; import com.metamatrix.query.sql.lang.JoinType; @@ -46,7 +47,7 @@ * */ public class MergeJoinStrategy extends JoinStrategy { - + private enum MergeState { SCAN, MATCH, DONE } @@ -76,32 +77,33 @@ //planning time information public enum SortOption { - SKIP_SORT, SORT, SORT_DISTINCT + ALREADY_SORTED, SORT, SORT_DISTINCT, PARTITION } - private SortOption sortLeft; - private SortOption sortRight; + protected SortOption sortLeft; + protected SortOption sortRight; /** false if three-level comparison, true if grouping comparison (null == null) */ private boolean grouping; - + //load time state private SortUtility leftSort; private SortUtility rightSort; - private SortOption processingSortRight; + protected SortOption processingSortLeft; + protected SortOption processingSortRight; public MergeJoinStrategy(SortOption sortLeft, SortOption sortRight, boolean grouping) { if (sortLeft == null) { - sortLeft = SortOption.SKIP_SORT; + sortLeft = SortOption.ALREADY_SORTED; } if (sortRight == null) { - sortRight = SortOption.SKIP_SORT; + sortRight = SortOption.ALREADY_SORTED; } this.sortLeft = sortLeft; this.sortRight = sortRight; this.grouping = grouping; } - + /** * @see com.metamatrix.query.processor.relational.JoinStrategy#clone() */ @@ -125,6 +127,7 @@ this.rightScanState = ScanState.READ; this.outerMatched = false; this.processingSortRight = this.sortRight; + this.processingSortLeft = this.sortLeft; } /** @@ -139,10 +142,10 @@ this.leftSort = null; this.rightSort = null; } - + + @Override protected List nextTuple() throws MetaMatrixComponentException, - CriteriaEvaluationException, - MetaMatrixProcessingException { + CriteriaEvaluationException, MetaMatrixProcessingException { while (this.mergeState != MergeState.DONE) { while (this.mergeState == MergeState.SCAN) { @@ -334,51 +337,68 @@ @Override protected void loadLeft() throws MetaMatrixComponentException, MetaMatrixProcessingException { - if (sortLeft != SortOption.SKIP_SORT) { + if (sortLeft == SortOption.ALREADY_SORTED && !this.joinNode.isDependent() && !JoinType.JOIN_FULL_OUTER.equals(joinNode.getJoinType())) { + return; // don't buffer + } + super.loadLeft(); + } + + @Override + protected void postLoadLeft() throws MetaMatrixComponentException, + MetaMatrixProcessingException { + if (this.processingSortLeft == SortOption.SORT || this.processingSortLeft == SortOption.SORT_DISTINCT) { if (this.leftSort == null) { - List expressions = this.joinNode.getLeftExpressions(); - this.leftSort = new SortUtility(this.leftSource.collectTuples(), - expressions, Collections.nCopies(expressions.size(), OrderBy.ASC), sortLeft == SortOption.SORT_DISTINCT?Mode.DUP_REMOVE_SORT:Mode.SORT, + List expressions = this.joinNode.getLeftExpressions(); + this.leftSort = new SortUtility(this.leftSource.getTupleSourceID(), + expressions, Collections.nCopies(expressions.size(), OrderBy.ASC), processingSortLeft == SortOption.SORT_DISTINCT?Mode.DUP_REMOVE_SORT:Mode.SORT, this.joinNode.getBufferManager(), this.joinNode.getConnectionID(), true); - this.leftSource.setDistinct(sortLeft == SortOption.SORT_DISTINCT && expressions.size() == this.leftSource.getOuterVals().size()); + this.leftSource.markDistinct(processingSortLeft == SortOption.SORT_DISTINCT && expressions.size() == this.leftSource.getOuterVals().size()); } this.leftSource.setTupleSource(leftSort.sort()); - } else if (this.joinNode.isDependent() || JoinType.JOIN_FULL_OUTER.equals(joinNode.getJoinType())) { - super.loadLeft(); //buffer only for dependent and full outer joins - } + this.leftSource.markDistinct(leftSort.isDistinct()); + } } - - /** - * @see com.metamatrix.query.processor.relational.JoinStrategy#loadRight() - */ + @Override - protected void loadRight() throws MetaMatrixComponentException, - MetaMatrixProcessingException { - super.loadRight(); - if (processingSortRight != SortOption.SKIP_SORT) { - if (this.rightSort == null) { - List expressions = this.joinNode.getRightExpressions(); - this.rightSort = new SortUtility(this.rightSource.getTupleSourceID(), - expressions, Collections.nCopies(expressions.size(), OrderBy.ASC), processingSortRight == SortOption.SORT_DISTINCT?Mode.DUP_REMOVE_SORT:Mode.SORT, - this.joinNode.getBufferManager(), this.joinNode.getConnectionID(), true); - this.rightSource.setDistinct(processingSortRight == SortOption.SORT_DISTINCT && expressions.size() == this.rightSource.getOuterVals().size()); - } - this.rightSource.setTupleSource(rightSort.sort()); - } + protected void postLoadRight() throws MetaMatrixComponentException, + MetaMatrixProcessingException { + sortRight(); } - + + protected void sortRight() throws MetaMatrixComponentException, + TupleSourceNotFoundException, BlockedOnMemoryException { + if (this.processingSortRight == SortOption.SORT || this.processingSortRight == SortOption.SORT_DISTINCT) { + if (this.rightSort == null) { + List expressions = this.joinNode.getRightExpressions(); + this.rightSort = new SortUtility(this.rightSource.getTupleSourceID(), + expressions, Collections.nCopies(expressions.size(), OrderBy.ASC), processingSortRight == SortOption.SORT_DISTINCT?Mode.DUP_REMOVE_SORT:Mode.SORT, + this.joinNode.getBufferManager(), this.joinNode.getConnectionID(), true); + this.rightSource.markDistinct(processingSortRight == SortOption.SORT_DISTINCT && expressions.size() == this.rightSource.getOuterVals().size()); + } + this.rightSource.setTupleSource(rightSort.sort()); + this.rightSource.markDistinct(rightSort.isDistinct()); + } + } + public void setProcessingSortRight(boolean processingSortRight) { - if (processingSortRight && this.processingSortRight == SortOption.SKIP_SORT) { + if (processingSortRight && this.processingSortRight == SortOption.ALREADY_SORTED) { this.processingSortRight = SortOption.SORT; } } + + public String getName() { + return "MERGE JOIN"; //$NON-NLS-1$ + } /** * @see java.lang.Object#toString() */ @Override public String toString() { - return "MERGE JOIN"; //$NON-NLS-1$ - } + StringBuffer sb = new StringBuffer(); + return sb + .append(getName()) + .append(" (").append(sortLeft).append("/").append(sortRight).append(")").toString(); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + } } \ No newline at end of file Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/NestedLoopJoinStrategy.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/NestedLoopJoinStrategy.java 2009-05-02 00:13:45 UTC (rev 856) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/NestedLoopJoinStrategy.java 2009-05-04 13:21:45 UTC (rev 857) @@ -33,7 +33,7 @@ public class NestedLoopJoinStrategy extends MergeJoinStrategy { public NestedLoopJoinStrategy() { - super(SortOption.SKIP_SORT, SortOption.SKIP_SORT, false); + super(SortOption.ALREADY_SORTED, SortOption.ALREADY_SORTED, false); } /** Added: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/PartitionedSortJoin.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/PartitionedSortJoin.java (rev 0) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/PartitionedSortJoin.java 2009-05-04 13:21:45 UTC (rev 857) @@ -0,0 +1,336 @@ +/* + * JBoss, Home of Professional Open Source. + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA. + */ + +package com.metamatrix.query.processor.relational; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + +import com.metamatrix.api.exception.MetaMatrixComponentException; +import com.metamatrix.api.exception.MetaMatrixProcessingException; +import com.metamatrix.api.exception.query.CriteriaEvaluationException; +import com.metamatrix.common.buffer.BlockedOnMemoryException; +import com.metamatrix.common.buffer.IndexedTupleSource; +import com.metamatrix.common.buffer.MemoryNotAvailableException; +import com.metamatrix.common.buffer.TupleBatch; +import com.metamatrix.common.buffer.TupleSourceID; +import com.metamatrix.common.buffer.TupleSourceNotFoundException; + +/** + * Extends the basic fully sorted merge join to check for conditions necessary + * to not fully sort one of the sides + * + * Will be used for inner joins and only if both sorts are not required. + * Degrades to a normal merge join if the tuples are balanced. + */ +public class PartitionedSortJoin extends MergeJoinStrategy { + + /** + * This is a compromise between the max size of the smaller side + * and effective partitioning assuming that we only want to hold + * two batches in memory during partitioning. + * + * TODO: apply partitioning recursively and/or have a better mechanism + * for buffermanager reserve/release of memory + * (would also help the sort utility) + */ + public static final int MAX_PARTITIONS = 8; + + private List[] endTuples; + private List<Boolean> overlap = new ArrayList<Boolean>(); + private List<Integer> endRows = new ArrayList<Integer>(); + private List<TupleSourceID> partitionIds = new ArrayList<TupleSourceID>(); + private List<TupleCollector> tupleCollectors = new ArrayList<TupleCollector>(); + private int currentPartition; + private IndexedTupleSource currentSource; + private SourceState sortedSource; + private SourceState partitionedSource; + private boolean partitioned; + private List<?> partitionedTuple; + private int matchBegin = -1; + private int matchEnd = -1; + + public PartitionedSortJoin(SortOption sortLeft, SortOption sortRight) { + super(sortLeft, sortRight, false); + } + + @Override + public void close() throws TupleSourceNotFoundException, + MetaMatrixComponentException { + super.close(); + for (TupleSourceID tupleSourceID : this.partitionIds) { + try { + this.joinNode.getBufferManager().removeTupleSource(tupleSourceID); + } catch (TupleSourceNotFoundException e) { + + } + } + this.endTuples = null; + this.overlap.clear(); + this.endRows.clear(); + this.partitionIds.clear(); + this.tupleCollectors.clear(); + this.currentSource = null; + this.sortedSource = null; + this.partitionedSource = null; + this.partitionedTuple = null; + } + + @Override + public void initialize(JoinNode joinNode) + throws MetaMatrixComponentException { + super.initialize(joinNode); + this.currentPartition = 0; + this.partitioned = false; + this.matchBegin = -1; + this.matchEnd = -1; + } + + //TODO: save partial work + public void computeBatchBounds(SourceState state) throws TupleSourceNotFoundException, MetaMatrixComponentException { + if (endTuples != null) { + return; + } + Comparator comp = new ListNestedSortComparator(state.getExpressionIndexes(), true); + ArrayList<List<?>> bounds = new ArrayList<List<?>>(); + int beginRow = 1; + while (beginRow <= state.getRowCount()) { + TupleBatch batch = null; + try { + batch = this.joinNode.getBufferManager().pinTupleBatch(state.getTupleSourceID(), beginRow, beginRow + this.joinNode.getBatchSize() - 1); + if (batch.getRowCount() == 0) { + break; + } + beginRow = batch.getEndRow() + 1; + this.joinNode.getBufferManager().unpinTupleBatch(state.getTupleSourceID(), batch.getBeginRow(), batch.getEndRow()); + if (!bounds.isEmpty()) { + overlap.add(comp.compare(bounds.get(bounds.size() - 1), batch.getTuple(batch.getBeginRow())) == 0); + } + bounds.add(batch.getTuple(batch.getEndRow())); + endRows.add(batch.getEndRow()); + } catch (MemoryNotAvailableException e) { + throw BlockedOnMemoryException.INSTANCE; + } + } + this.endTuples = bounds.toArray(new List[bounds.size()]); + } + + @Override + protected void loadLeft() throws MetaMatrixComponentException, + MetaMatrixProcessingException { + //always buffer to determine row counts + this.leftSource.collectTuples(); + } + + @Override + protected void loadRight() throws MetaMatrixComponentException, + MetaMatrixProcessingException { + super.loadRight(); + int maxRows = this.joinNode.getBatchSize() * MAX_PARTITIONS; + if (processingSortRight == SortOption.SORT + && this.leftSource.getRowCount() < maxRows + && this.leftSource.getRowCount() * 4 < this.rightSource.getRowCount()) { + this.processingSortRight = SortOption.PARTITION; + } else if (processingSortLeft == SortOption.SORT + && this.rightSource.getRowCount() < maxRows + && this.rightSource.getRowCount() * 4 < this.leftSource.getRowCount()) { + this.processingSortLeft = SortOption.PARTITION; + } + sortRight(); + if (this.processingSortLeft == SortOption.PARTITION) { + computeBatchBounds(this.rightSource); + this.sortedSource = this.rightSource; + this.partitionedSource = this.leftSource; + } + } + + @Override + protected void postLoadLeft() throws MetaMatrixComponentException, + MetaMatrixProcessingException { + super.postLoadLeft(); + + if (this.processingSortRight == SortOption.PARTITION) { + computeBatchBounds(this.leftSource); + this.sortedSource = this.leftSource; + this.partitionedSource = this.rightSource; + } + + if (this.processingSortLeft == SortOption.PARTITION) { + partitionSource(true); + } + } + + @Override + protected void postLoadRight() throws MetaMatrixComponentException, + MetaMatrixProcessingException { + if (this.processingSortRight == SortOption.PARTITION) { + partitionSource(false); + if (this.processingSortRight == SortOption.SORT) { + //degrade to a merge join + sortRight(); + } + } + } + + private void partitionSource(boolean left) throws MetaMatrixComponentException, + MetaMatrixProcessingException { + if (partitioned) { + return; + } + if (endTuples.length > MAX_PARTITIONS + 1) { + if (left) { + this.processingSortLeft = SortOption.SORT; + } else { + this.processingSortRight = SortOption.SORT; + } + return; + } + if (endTuples.length < 2) { + partitionIds.add(this.partitionedSource.getTupleSourceID()); + } else { + if (partitionIds.isEmpty()) { + for (int i = 0; i < endTuples.length; i++) { + TupleCollector tc = new TupleCollector(this.partitionedSource.createSourceTupleSource(), this.joinNode.getBufferManager()); + tc.setBatchSize(Math.max(1, this.joinNode.getBatchSize()/4)); + this.tupleCollectors.add(tc); + this.partitionIds.add(tc.getTupleSourceID()); + } + } + while (this.partitionedSource.getIterator().hasNext()) { + List<?> tuple = this.partitionedSource.getIterator().nextTuple(); + int index = binarySearch(tuple, this.endTuples, this.partitionedSource.getExpressionIndexes(), this.sortedSource.getExpressionIndexes()); + if (index < 0) { + index = -index - 1; + } + if (index > this.tupleCollectors.size() -1) { + continue; + } + while (index > 0 && this.overlap.get(index - 1) + && compare(tuple, this.endTuples[index - 1], this.partitionedSource.getExpressionIndexes(), this.sortedSource.getExpressionIndexes()) == 0) { + index--; + } + this.tupleCollectors.get(index).addTuple(tuple); + } + this.partitionedSource.getIterator().setPosition(1); + } + partitioned = true; + } + + @Override + protected List nextTuple() throws MetaMatrixComponentException, + CriteriaEvaluationException, MetaMatrixProcessingException { + if (this.processingSortLeft != SortOption.PARTITION && this.processingSortRight != SortOption.PARTITION) { + return super.nextTuple(); + } + if (endRows.isEmpty()) { + return null; //no rows on the sorted side + } + while (currentPartition < partitionIds.size()) { + if (currentSource == null) { + if (!this.tupleCollectors.isEmpty()) { + this.tupleCollectors.get(currentPartition).close(); + } + currentSource = this.joinNode.getBufferManager().getTupleSource(partitionIds.get(currentPartition)); + } + + int beginIndex = currentPartition>0?endRows.get(currentPartition - 1)+1:1; + + this.sortedSource.getIterator().setPosition(beginIndex); + List[] batch = this.sortedSource.getIterator().getBatch().getAllTuples(); + + while (partitionedTuple != null || currentSource.hasNext()) { + if (partitionedTuple == null) { + partitionedTuple = currentSource.nextTuple(); + int index = binarySearch(partitionedTuple, batch, this.partitionedSource.getExpressionIndexes(), this.sortedSource.getExpressionIndexes()); + if (index < 0) { + partitionedTuple = null; + continue; + } + matchBegin = index; + matchEnd = index; + if (!this.sortedSource.isDistinct()) { + while (matchBegin > 0) { + if (compare(partitionedTuple, batch[matchBegin - 1], this.partitionedSource.getExpressionIndexes(), this.sortedSource.getExpressionIndexes()) != 0) { + break; + } + matchBegin--; + } + while (matchEnd < batch.length - 1) { + if (compare(partitionedTuple, batch[matchEnd + 1], this.partitionedSource.getExpressionIndexes(), this.sortedSource.getExpressionIndexes()) != 0) { + break; + } + matchEnd++; + } + } + if (matchEnd == batch.length - 1 && currentPartition < overlap.size() && overlap.get(currentPartition)) { + this.tupleCollectors.get(currentPartition + 1).addTuple(partitionedTuple); + } + } + while (matchBegin <= matchEnd) { + List outputTuple = outputTuple(this.processingSortLeft==SortOption.PARTITION?partitionedTuple:batch[matchBegin], + this.processingSortLeft==SortOption.PARTITION?batch[matchBegin]:partitionedTuple); + boolean matches = this.joinNode.matchesCriteria(outputTuple); + matchBegin++; + if (matches) { + return outputTuple; + } + } + matchBegin = -1; + matchEnd = -1; + partitionedTuple = null; + } + currentSource = null; + currentPartition++; + } + return null; + } + + public int binarySearch(List<?> tuple, List[] tuples, int[] leftIndexes, int[] rightIndexes) { + int begin = 0; + int end = tuples.length - 1; + while (begin <= end) { + int mid = (begin + end)/2; + int compare = compare(tuples[mid], tuple, rightIndexes, leftIndexes); + if (compare == 0) { + return mid; + } + if (compare < 0) { + end = mid - 1; + } else { + begin = mid + 1; + } + } + return -begin -1; + } + + @Override + public Object clone() { + return new PartitionedSortJoin(this.sortLeft, this.sortRight); + } + + @Override + public String getName() { + return "PARTITIONED SORT JOIN"; //$NON-NLS-1$ + } + +} Property changes on: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/PartitionedSortJoin.java ___________________________________________________________________ Name: svn:mime-type + text/plain Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SortUtility.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SortUtility.java 2009-05-02 00:13:45 UTC (rev 856) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SortUtility.java 2009-05-04 13:21:45 UTC (rev 857) @@ -25,7 +25,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.Comparator; import java.util.Iterator; import java.util.List; @@ -69,7 +68,7 @@ protected List schema; private String[] schemaTypes; protected int[] sortCols; - private Comparator comparator; + private ListNestedSortComparator comparator; private TupleSourceID outputID; private IndexedTupleSource outTs; @@ -78,7 +77,6 @@ private int phase = INITIAL_SORT; protected List<TupleSourceID> activeTupleIDs = new ArrayList<TupleSourceID>(); private List<TupleBatch> workingBatches; - private int[] workingPointers; private int masterSortIndex; private TupleSourceID mergedID; private TupleSourceID tempOutId; @@ -260,8 +258,8 @@ } // Initialize pointers into working batches - this.workingPointers = new int[workingBatches.size()]; - Arrays.fill(this.workingPointers, 1); + int[] workingPointers = new int[workingBatches.size()]; + Arrays.fill(workingPointers, 1); mergedID = createTupleSource(); @@ -288,7 +286,7 @@ chosenBatchIndex = i; chosenBatch = batch; } else if (compare == 0 && this.mode != Mode.SORT) { - incrementWorkingBatch(i, batch); + incrementWorkingBatch(i, workingPointers, batch); } } @@ -305,11 +303,11 @@ } tempCollector.addTuple(currentRow); } - incrementWorkingBatch(chosenBatchIndex, chosenBatch); + incrementWorkingBatch(chosenBatchIndex, workingPointers, chosenBatch); } // Save without closing - collector.saveBatch(false); + collector.saveBatch(); // Remove merged sublists for(int i=0; i<sortedIndex; i++) { @@ -363,7 +361,7 @@ throw new MetaMatrixComponentException(e); } } catch (BlockedOnMemoryException e) { - tc.saveBatch(false); + tc.saveBatch(); throw e; } outTs = null; @@ -375,7 +373,7 @@ * for that batchIndex, which we already happen to have. Return whether the batch * was changed or not. True = changed. */ - private void incrementWorkingBatch(int batchIndex, TupleBatch currentBatch) throws BlockedOnMemoryException, TupleSourceNotFoundException, MetaMatrixComponentException { + private void incrementWorkingBatch(int batchIndex, int[] workingPointers, TupleBatch currentBatch) throws BlockedOnMemoryException, TupleSourceNotFoundException, MetaMatrixComponentException { workingPointers[batchIndex] += 1; if(workingPointers[batchIndex] > currentBatch.getEndRow()) { TupleSourceID tsID = unpinWorkingBatch(batchIndex, currentBatch); @@ -411,7 +409,7 @@ this.schema = this.bufferManager.getTupleSchema(this.sourceID); this.schemaTypes = TypeRetrievalUtil.getTypeNames(schema); this.batchSize = bufferManager.getProcessorBatchSize(); - + int distinctIndex = sortElements != null? sortElements.size() - 1:0; if (useAllColumns && mode != Mode.SORT) { if (this.sortElements != null) { this.sortElements = new ArrayList(this.sortElements); @@ -438,6 +436,11 @@ } this.sortCols = cols; this.comparator = new ListNestedSortComparator(sortCols, sortTypes); + this.comparator.setDistinctIndex(distinctIndex); } + public boolean isDistinct() { + return this.comparator.isDistinct(); + } + } Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SourceState.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SourceState.java 2009-05-02 00:13:45 UTC (rev 856) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SourceState.java 2009-05-04 13:21:45 UTC (rev 857) @@ -30,6 +30,8 @@ import com.metamatrix.common.buffer.IndexedTupleSource; import com.metamatrix.common.buffer.TupleSourceID; import com.metamatrix.common.buffer.TupleSourceNotFoundException; +import com.metamatrix.common.buffer.BufferManager.TupleSourceType; +import com.metamatrix.query.util.TypeRetrievalUtil; class SourceState { @@ -63,6 +65,10 @@ return indecies; } + TupleSourceID createSourceTupleSource() throws MetaMatrixComponentException { + return this.source.getBufferManager().createTupleSource(source.getElements(), TypeRetrievalUtil.getTypeNames(source.getElements()), source.getConnectionID(), TupleSourceType.PROCESSOR); + } + public List saveNext() throws MetaMatrixComponentException, MetaMatrixProcessingException { this.currentTuple = this.getIterator().nextTuple(); return currentTuple; @@ -85,6 +91,10 @@ this.tsID = null; } } + + public int getRowCount() { + return this.collector.getRowCount(); + } /** * Collect the underlying batches into a tuple source. Subsequent calls will return that tuple source @@ -151,8 +161,8 @@ return this.distinct; } - public void setDistinct(boolean distinct) { - this.distinct = distinct; + public void markDistinct(boolean distinct) { + this.distinct |= distinct; } } Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/TupleCollector.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/TupleCollector.java 2009-05-02 00:13:45 UTC (rev 856) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/TupleCollector.java 2009-05-04 13:21:45 UTC (rev 857) @@ -36,12 +36,10 @@ private TupleSourceID tsid; private BufferManager bm; - private boolean captureBounds; private int batchSize; private ArrayList<List<?>> batch; private int index; - private ArrayList<List<?>> bounds; public TupleCollector(TupleSourceID tsid, BufferManager bm) throws TupleSourceNotFoundException, MetaMatrixComponentException { this.tsid = tsid; @@ -50,48 +48,64 @@ this.index = bm.getRowCount(tsid) + 1; } - public void setCaptureBounds(boolean captureBounds) { - this.captureBounds = captureBounds; + public TupleCollector(BufferManager bm) { + this.batchSize = bm.getProcessorBatchSize(); + this.bm = bm; + this.index = 1; } + public void setBatchSize(int batchSize) { + this.batchSize = batchSize; + } + + public void setTupleSourceID(TupleSourceID tsid) { + this.tsid = tsid; + } + + public boolean isEmpty() { + return this.batch == null || this.batch.isEmpty(); + } + public TupleSourceID getTupleSourceID() { return tsid; } + public ArrayList<List<?>> getBatch() { + if (batch == null) { + batch = new ArrayList<List<?>>(batchSize/4); + } + return batch; + } + public void addTuple(List<?> tuple) throws TupleSourceNotFoundException, MetaMatrixComponentException { if (batch == null) { batch = new ArrayList<List<?>>(batchSize/4); } batch.add(tuple); if (batch.size() == batchSize) { - saveBatch(false); + saveBatch(); } } - public void saveBatch(boolean isLast) throws TupleSourceNotFoundException, + public void saveBatch() throws TupleSourceNotFoundException, MetaMatrixComponentException { - ArrayList<List<?>> toSave = batch; - if (toSave == null || toSave.isEmpty()) { - if (!isLast) { - return; - } - toSave = new ArrayList<List<?>>(0); - } /*else if (captureBounds) { - if (bounds.isEmpty()) { - bounds.add(toSave.get(0)); - } - if (bounds ) - bounds.add(toSave.get(toSave.size() -1)); - }*/ - TupleBatch tb = new TupleBatch(index, toSave); - tb.setTerminationFlag(isLast); - this.bm.addTupleBatch(tsid, tb); - this.index += toSave.size(); - batch = null; + if (batch == null || batch.isEmpty()) { + return; + } + int batchIndex = 0; + while(batchIndex < batch.size()) { + int writeEnd = Math.min(batch.size(), batchIndex + batchSize); + + TupleBatch writeBatch = new TupleBatch(index, batch.subList(batchIndex, writeEnd)); + bm.addTupleBatch(tsid, writeBatch); + index += writeBatch.getRowCount(); + batchIndex += writeBatch.getRowCount(); + } + batch = null; } public void close() throws TupleSourceNotFoundException, MetaMatrixComponentException { - saveBatch(true); + saveBatch(); this.bm.setStatus(this.tsid, TupleSourceStatus.FULL); } Modified: trunk/engine/src/main/java/com/metamatrix/query/sql/util/ValueIteratorSource.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/sql/util/ValueIteratorSource.java 2009-05-02 00:13:45 UTC (rev 856) +++ trunk/engine/src/main/java/com/metamatrix/query/sql/util/ValueIteratorSource.java 2009-05-04 13:21:45 UTC (rev 857) @@ -22,7 +22,10 @@ package com.metamatrix.query.sql.util; +import java.util.HashSet; + import com.metamatrix.api.exception.MetaMatrixComponentException; +import com.metamatrix.api.exception.MetaMatrixProcessingException; import com.metamatrix.common.buffer.TupleSourceNotFoundException; import com.metamatrix.query.sql.symbol.Expression; @@ -47,4 +50,6 @@ */ ValueIterator getValueIterator(Expression valueExpression) throws MetaMatrixComponentException; + HashSet<Object> getCachedSet(Expression valueExpression) throws MetaMatrixComponentException, MetaMatrixProcessingException; + } Deleted: trunk/engine/src/main/java/com/metamatrix/query/sql/visitor/DependentSetCriteriaCollectorVisitor.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/sql/visitor/DependentSetCriteriaCollectorVisitor.java 2009-05-02 00:13:45 UTC (rev 856) +++ trunk/engine/src/main/java/com/metamatrix/query/sql/visitor/DependentSetCriteriaCollectorVisitor.java 2009-05-04 13:21:45 UTC (rev 857) @@ -1,84 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.query.sql.visitor; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import com.metamatrix.query.sql.LanguageObject; -import com.metamatrix.query.sql.LanguageVisitor; -import com.metamatrix.query.sql.lang.DependentSetCriteria; -import com.metamatrix.query.sql.navigator.DeepPreOrderNavigator; - - -/** - * Visitor to collect DependentSetCriteria. - */ -public class DependentSetCriteriaCollectorVisitor extends LanguageVisitor { - - // List<DependentSetCriteria> - private List crits = new ArrayList(); - - /** - * - */ - public DependentSetCriteriaCollectorVisitor() { - super(); - } - - /** - * Get the criteria collected by the visitor. This should be called - * after the visitor has been run on the language object tree. - * @return List of {@link com.metamatrix.query.sql.lang.DependentSetCriteria} - */ - public List getCriteria() { - return this.crits; - } - - /** - * Visit a language object and collect symbols. This method should <b>NOT</b> be - * called directly. - * @param obj Language object - */ - public void visit(DependentSetCriteria obj) { - this.crits.add(obj); - } - - /** - * Get all DependentSetCriteria found in a deep search from the input LanguageObject. - * @param obj Starting language object (criteria, command, etc) - * @return List of DependentSetCriteria - */ - public static List getDependentSetCriteria(LanguageObject obj) { - if(obj == null) { - return Collections.EMPTY_LIST; - } - - DependentSetCriteriaCollectorVisitor visitor = new DependentSetCriteriaCollectorVisitor(); - DeepPreOrderNavigator.doVisit(obj, visitor); - return visitor.getCriteria(); - - } - -} Modified: trunk/engine/src/main/java/com/metamatrix/query/util/CommandContext.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/util/CommandContext.java 2009-05-02 00:13:45 UTC (rev 856) +++ trunk/engine/src/main/java/com/metamatrix/query/util/CommandContext.java 2009-05-04 13:21:45 UTC (rev 857) @@ -448,13 +448,4 @@ return value; } - public ValueIterator getValueIterator(ContextReference ref) - throws MetaMatrixComponentException { - if (variableContext == null) { - throw new MetaMatrixComponentException(ErrorMessageKeys.PROCESSOR_0033, QueryPlugin.Util.getString(ErrorMessageKeys.PROCESSOR_0033, ref.getContextSymbol(), "No value was available")); //$NON-NLS-1$ - } - ValueIteratorSource dvs = (ValueIteratorSource) this.variableContext.getGlobalValue(ref.getContextSymbol()); - return dvs.getValueIterator(ref.getValueExpression()); - } - } Modified: trunk/engine/src/test/java/com/metamatrix/query/optimizer/TestLimit.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/query/optimizer/TestLimit.java 2009-05-02 00:13:45 UTC (rev 856) +++ trunk/engine/src/test/java/com/metamatrix/query/optimizer/TestLimit.java 2009-05-04 13:21:45 UTC (rev 857) @@ -742,7 +742,7 @@ 0, // PlanExecution 1, // Project 0, // Select - 0, // Sort + 1, // Sort 0 // UnionAll }, NODE_TYPES); Modified: trunk/engine/src/test/java/com/metamatrix/query/optimizer/TestOptimizer.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/query/optimizer/TestOptimizer.java 2009-05-02 00:13:45 UTC (rev 856) +++ trunk/engine/src/test/java/com/metamatrix/query/optimizer/TestOptimizer.java 2009-05-04 13:21:45 UTC (rev 857) @@ -60,9 +60,11 @@ import com.metamatrix.query.processor.relational.DependentAccessNode; import com.metamatrix.query.processor.relational.GroupingNode; import com.metamatrix.query.processor.relational.JoinNode; +import com.metamatrix.query.processor.relational.JoinStrategy; import com.metamatrix.query.processor.relational.MergeJoinStrategy; import com.metamatrix.query.processor.relational.NestedLoopJoinStrategy; import com.metamatrix.query.processor.relational.NullNode; +import com.metamatrix.query.processor.relational.PartitionedSortJoin; import com.metamatrix.query.processor.relational.PlanExecutionNode; import com.metamatrix.query.processor.relational.ProjectIntoNode; import com.metamatrix.query.processor.relational.ProjectNode; @@ -423,11 +425,15 @@ public static void collectCounts(RelationalNode relationalNode, int[] counts, Class<?>[] types) { Class<?> nodeType = relationalNode.getClass(); if(nodeType.equals(JoinNode.class)) { - if (((JoinNode)relationalNode).getJoinStrategy() instanceof NestedLoopJoinStrategy) { + JoinStrategy strategy = ((JoinNode)relationalNode).getJoinStrategy(); + if (strategy instanceof NestedLoopJoinStrategy) { updateCounts(NestedLoopJoinStrategy.class, counts, types); - }else { + } else if (strategy instanceof MergeJoinStrategy) { updateCounts(MergeJoinStrategy.class, counts, types); - } + if (strategy instanceof PartitionedSortJoin) { + updateCounts(PartitionedSortJoin.class, counts, types); + } + } if (((JoinNode)relationalNode).isDependent()) { updateCounts(DependentJoin.class, counts, types); } Added: trunk/engine/src/test/java/com/metamatrix/query/optimizer/TestPartitionedJoinPlanning.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/query/optimizer/TestPartitionedJoinPlanning.java (rev 0) +++ trunk/engine/src/test/java/com/metamatrix/query/optimizer/TestPartitionedJoinPlanning.java 2009-05-04 13:21:45 UTC (rev 857) @@ -0,0 +1,84 @@ +/* + * JBoss, Home of Professional Open Source. + * Copyright (C) 2009 Red Hat, Inc. + * Licensed to Red Hat, Inc. under one or more contributor + * license agreements. See the copyright.txt file in the + * distribution for a full listing of individual contributors. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA. + */ + +package com.metamatrix.query.optimizer; + +import static com.metamatrix.query.optimizer.TestOptimizer.*; + +import org.junit.Test; + +import com.metamatrix.query.optimizer.capabilities.BasicSourceCapabilities; +import com.metamatrix.query.optimizer.capabilities.FakeCapabilitiesFinder; +import com.metamatrix.query.optimizer.capabilities.SourceCapabilities.Capability; +import com.metamatrix.query.processor.ProcessorPlan; +import com.metamatrix.query.processor.relational.PartitionedSortJoin; +import com.metamatrix.query.unittest.FakeMetadataFacade; +import com.metamatrix.query.unittest.FakeMetadataFactory; +import com.metamatrix.query.unittest.FakeMetadataObject; + +public class TestPartitionedJoinPlanning { + + @Test public void testUsePartitionedMergeJoin(){ + // Create query + String sql = "SELECT pm1.g1.e1 FROM pm1.g1, pm1.g2 WHERE pm1.g1.e1 = pm1.g2.e1";//$NON-NLS-1$ + + FakeCapabilitiesFinder capFinder = new FakeCapabilitiesFinder(); + BasicSourceCapabilities caps = new BasicSourceCapabilities(); + caps.setCapabilitySupport(Capability.QUERY_WHERE, true); + caps.setCapabilitySupport(Capability.QUERY_WHERE_COMPARE, true); + caps.setCapabilitySupport(Capability.QUERY_WHERE_COMPARE_EQ, true); + caps.setCapabilitySupport(Capability.QUERY_WHERE_AND, true); + caps.setCapabilitySupport(Capability.QUERY_FROM_GROUP_ALIAS, true); + caps.setCapabilitySupport(Capability.QUERY_ORDERBY, true); + capFinder.addCapabilities("pm1", caps); //$NON-NLS-1$ + + FakeMetadataFacade metadata = FakeMetadataFactory.example1(); + FakeMetadataObject g1 = metadata.getStore().findObject("pm1.g1", FakeMetadataObject.GROUP); //$NON-NLS-1$ + g1.putProperty(FakeMetadataObject.Props.CARDINALITY, 600); + FakeMetadataObject g2 = metadata.getStore().findObject("pm1.g2", FakeMetadataObject.GROUP); //$NON-NLS-1$ + g2.putProperty(FakeMetadataObject.Props.CARDINALITY, 3000); + + ProcessorPlan plan = helpPlan(sql, metadata, + null, capFinder, + new String[] { "SELECT pm1.g1.e1 FROM pm1.g1 ORDER BY pm1.g1.e1", "SELECT pm1.g2.e1 FROM pm1.g2" }, SHOULD_SUCCEED); //$NON-NLS-1$ //$NON-NLS-2$ + checkNodeTypes(plan, new int[] { + 2, // Access + 0, // DependentAccess + 0, // DependentSelect + 0, // DependentProject + 0, // DupRemove + 0, // Grouping + 0, // NestedLoopJoinStrategy + 1, // MergeJoinStrategy + 0, // Null + 0, // PlanExecution + 1, // Project + 0, // Select + 0, // Sort + 0 // UnionAll + }); + checkNodeTypes(plan, new int[] {1}, new Class[] {PartitionedSortJoin.class}); + } + + +} Property changes on: trunk/engine/src/test/java/com/metamatrix/query/optimizer/TestPartitionedJoinPlanning.java ___________________________________________________________________ Name: svn:mergeinfo + Modified: trunk/engine/src/test/java/com/metamatrix/query/optimizer/relational/rules/TestRuleChooseDependent.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/query/optimizer/relational/rules/TestRuleChooseDependent.java 2009-05-02 00:13:45 UTC (rev 856) +++ trunk/engine/src/test/java/com/metamatrix/query/optimizer/relational/rules/TestRuleChooseDependent.java 2009-05-04 13:21:45 UTC (rev 857) @@ -38,10 +38,10 @@ import com.metamatrix.query.optimizer.TestOptimizer; import com.metamatrix.query.optimizer.capabilities.FakeCapabilitiesFinder; import com.metamatrix.query.optimizer.relational.RuleStack; -import com.metamatrix.query.optimizer.relational.plantree.JoinStrategyType; import com.metamatrix.query.optimizer.relational.plantree.NodeConstants; import com.metamatrix.query.optimizer.relational.plantree.NodeFactory; import com.metamatrix.query.optimizer.relational.plantree.PlanNode; +import com.metamatrix.query.processor.relational.JoinNode.JoinStrategyType; import com.metamatrix.query.rewriter.QueryRewriter; import com.metamatrix.query.sql.lang.CompareCriteria; import com.metamatrix.query.sql.lang.CompoundCriteria; Modified: trunk/engine/src/test/java/com/metamatrix/query/processor/TestProcessor.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/query/processor/TestProcessor.java 2009-05-02 00:13:45 UTC (rev 856) +++ trunk/engine/src/test/java/com/metamatrix/query/processor/TestProcessor.java 2009-05-04 13:21:45 UTC (rev 857) @@ -95,10 +95,6 @@ public class TestProcessor extends TestCase { - private TimestampUtil tsUtil = new TimestampUtil(); - - // ################################## FRAMEWORK ################################ - public TestProcessor(String name) { super(name); } @@ -4615,7 +4611,7 @@ // Create expected results List[] expected = new List[] { - Arrays.asList(new Object[] { new Integer(1), Boolean.TRUE, new Double(2.0), tsUtil.createDate(103, 10, 4) }) + Arrays.asList(new Object[] { new Integer(1), Boolean.TRUE, new Double(2.0), TimestampUtil.createDate(103, 10, 4) }) }; // Construct data manager with data Modified: trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestJoinNode.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestJoinNode.java 2009-05-02 00:13:45 UTC (rev 856) +++ trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestJoinNode.java 2009-05-04 13:21:45 UTC (rev 857) @@ -22,13 +22,16 @@ package com.metamatrix.query.processor.relational; +import static org.junit.Assert.*; + import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; -import junit.framework.TestCase; +import org.junit.Before; +import org.junit.Test; import com.metamatrix.api.exception.MetaMatrixComponentException; import com.metamatrix.api.exception.MetaMatrixProcessingException; @@ -39,6 +42,7 @@ import com.metamatrix.query.function.FunctionDescriptor; import com.metamatrix.query.function.FunctionLibraryManager; import com.metamatrix.query.processor.FakeDataManager; +import com.metamatrix.query.processor.relational.MergeJoinStrategy.SortOption; import com.metamatrix.query.sql.lang.CompareCriteria; import com.metamatrix.query.sql.lang.JoinType; import com.metamatrix.query.sql.symbol.Constant; @@ -47,39 +51,33 @@ import com.metamatrix.query.sql.symbol.Function; import com.metamatrix.query.util.CommandContext; -public class TestJoinNode extends TestCase { +public class TestJoinNode { private static final int NO_CRITERIA = 0; private static final int EQUAL_CRITERIA = 1; private static final int FUNCTION_CRITERIA = 2; private int criteriaType = EQUAL_CRITERIA; - private JoinType joinType; + protected JoinType joinType; - private List[] leftTuples = createTuples1(); - private List[] rightTuples = createTuples2(); + protected List[] leftTuples; + protected List[] rightTuples; - private List[] expected; + protected List[] expected; private List[] expectedReversed; - private boolean expectSwap = false; - private boolean expectSwapReversed = false; - - private JoinNode join; - private JoinStrategy joinStrategy; + protected JoinNode join; + protected JoinStrategy joinStrategy; private RelationalNode leftNode; private RelationalNode rightNode; - + private FakeDataManager dataMgr; - public TestJoinNode(String testName) { - super(testName); + @Before public void setup() { + leftTuples = createTuples1(); + rightTuples = createTuples2(); } - - protected int getProcessorBatchSize() { - return 100; - } - + protected List[] createTuples1() { return new List[] { Arrays.asList(new Object[] { new Integer(5) }), @@ -166,7 +164,7 @@ return createTuples(startingValue, count, true); } - private void helpCreateJoin() { + protected void helpCreateJoin() { ElementSymbol es1 = new ElementSymbol("e1"); //$NON-NLS-1$ es1.setType(DataTypeManager.DefaultDataClasses.INTEGER); @@ -189,7 +187,7 @@ join = new JoinNode(3); joinStrategy = new NestedLoopJoinStrategy(); - join.setJoinStrategy(joinStrategy); + join.setJoinStrategy(joinStrategy); join.setElements(joinElements); join.setJoinType(joinType); @@ -198,7 +196,9 @@ break; case EQUAL_CRITERIA : - join.setJoinCriteria(new CompareCriteria(es1, CompareCriteria.EQ, es2)); + join.setJoinExpressions(Arrays.asList(es1), Arrays.asList(es2)); + joinStrategy = new MergeJoinStrategy(SortOption.SORT, SortOption.SORT, false); + join.setJoinStrategy(joinStrategy); break; case FUNCTION_CRITERIA : @@ -213,20 +213,22 @@ } public void helpTestJoin() throws MetaMatrixComponentException, MetaMatrixProcessingException { - helpTestJoinDirect(expected, expectSwap); - helpTestJoinReversed(); + for (int batchSize : new int[] {1, 10, leftTuples.length, 100}) { + helpCreateJoin(); + helpTestJoinDirect(expected, batchSize); + List[] temp = leftTuples; + leftTuples = rightTuples; + rightTuples = temp; + helpCreateJoin(); + helpTestJoinDirect(expectedReversed, batchSize); + temp = leftTuples; + leftTuples = rightTuples; + rightTuples = temp; + } } - - private void helpTestJoinReversed() throws MetaMatrixComponentException, MetaMatrixProcessingException { - List[] temp = leftTuples; - leftTuples = rightTuples; - rightTuples = temp; - helpTestJoinDirect(expectedReversed, expectSwapReversed); - } - public void helpTestJoinDirect(List[] expectedResults, boolean swapExpected) throws MetaMatrixComponentException, MetaMatrixProcessingException { - helpCreateJoin(); - BufferManager mgr = NodeTestUtil.getTestBufferManager(1, getProcessorBatchSize()); + public void helpTestJoinDirect(List[] expectedResults, int batchSize) throws MetaMatrixComponentException, MetaMatrixProcessingException { + BufferManager mgr = NodeTestUtil.getTestBufferManager(1, batchSize); CommandContext context = new CommandContext("pid", "test", null, null, null); //$NON-NLS-1$ //$NON-NLS-2$ join.addChild(leftNode); @@ -244,23 +246,21 @@ TupleBatch batch = join.nextBatch(); for(int row = currentRow; row <= batch.getEndRow(); row++) { List tuple = batch.getTuple(row); - //System.out.println(tuple); assertEquals("Rows don't match at " + row, expectedResults[row-1], tuple); //$NON-NLS-1$ } - + currentRow += batch.getRowCount(); if(batch.getTerminationFlag()) { break; } - currentRow += batch.getRowCount(); } catch(BlockedException e) { // ignore and retry } } - + assertEquals(expectedResults.length, currentRow - 1); join.close(); } - public void testNoRows() throws Exception { + @Test public void testNoRows() throws Exception { leftTuples = new List[0]; rightTuples = new List[0]; joinType = JoinType.JOIN_INNER; @@ -269,105 +269,64 @@ helpTestJoin(); } - public void testInnerJoin() throws Exception { + @Test public void testInnerJoin() throws Exception { joinType = JoinType.JOIN_INNER; expected = new List[] { + Arrays.asList(new Object[] { new Integer(1), new Integer(1) }), Arrays.asList(new Object[] { new Integer(2), new Integer(2) }), Arrays.asList(new Object[] { new Integer(2), new Integer(2) }), Arrays.asList(new Object[] { new Integer(4), new Integer(4) }), Arrays.asList(new Object[] { new Integer(4), new Integer(4) }), - Arrays.asList(new Object[] { new Integer(1), new Integer(1) }), Arrays.asList(new Object[] { new Integer(4), new Integer(4) }), Arrays.asList(new Object[] { new Integer(4), new Integer(4) }) }; - expectedReversed = new List[] { - Arrays.asList(new Object[] { new Integer(1), new Integer(1) }), - Arrays.asList(new Object[] { new Integer(4), new Integer(4) }), - Arrays.asList(new Object[] { new Integer(4), new Integer(4) }), - Arrays.asList(new Object[] { new Integer(2), new Integer(2) }), - Arrays.asList(new Object[] { new Integer(2), new Integer(2) }), - Arrays.asList(new Object[] { new Integer(4), new Integer(4) }), - Arrays.asList(new Object[] { new Integer(4), new Integer(4) }) - }; + expectedReversed = expected; helpTestJoin(); } -// public void testRightOuterJoin() { -// joinType = JoinType.JOIN_RIGHT_OUTER; -// expected = new List[] { -// Arrays.asList(new Object[] { new Integer(2), new Integer(2) }), -// Arrays.asList(new Object[] { new Integer(2), new Integer(2) }), -// Arrays.asList(new Object[] { new Integer(4), new Integer(4) }), -// Arrays.asList(new Object[] { new Integer(4), new Integer(4) }), -// Arrays.asList(new Object[] { new Integer(1), new Integer(1) }), -// Arrays.asList(new Object[] { new Integer(4), new Integer(4) }), -// Arrays.asList(new Object[] { new Integer(4), new Integer(4) }), -// Arrays.asList(new Object[] { null, null }), -// Arrays.asList(new Object[] { null, new Integer(7) }), -// Arrays.asList(new Object[] { null, new Integer(7) }), -// Arrays.asList(new Object[] { null, new Integer(6) }) -// }; -// expectedReversed = new List[] { -// Arrays.asList(new Object[] { new Integer(1), new Integer(1) }), -// Arrays.asList(new Object[] { new Integer(4), new Integer(4) }), -// Arrays.asList(new Object[] { new Integer(4), new Integer(4) }), -// Arrays.asList(new Object[] { new Integer(2), new Integer(2) }), -// Arrays.asList(new Object[] { new Integer(2), new Integer(2) }), -// Arrays.asList(new Object[] { new Integer(4), new Integer(4) }), -// Arrays.asList(new Object[] { new Integer(4), new Integer(4) }), -// Arrays.asList(new Object[] { null, new Integer(5) }), -// Arrays.asList(new Object[] { null, new Integer(3) }), -// Arrays.asList(new Object[] { null, new Integer(10) }), -// Arrays.asList(new Object[] { null, new Integer(11) }), -// Arrays.asList(new Object[] { null, new Integer(11) }) -// }; -// helpTestJoin(); -// } - - public void testLeftOuterJoin() throws Exception { + @Test public void testLeftOuterJoin() throws Exception { joinType = JoinType.JOIN_LEFT_OUTER; expected = new List[] { - Arrays.asList(new Object[] { new Integer(5), null }), - Arrays.asList(new Object[] { new Integer(3), null }), + Arrays.asList(new Object[] { new Integer(1), new Integer(1) }), Arrays.asList(new Object[] { new Integer(2), new Integer(2) }), Arrays.asList(new Object[] { new Integer(2), new Integer(2) }), + Arrays.asList(new Object[] { new Integer(3), null }), Arrays.asList(new Object[] { new Integer(4), new Integer(4) }), Arrays.asList(new Object[] { new Integer(4), new Integer(4) }), - Arrays.asList(new Object[] { new Integer(1), new Integer(1) }), Arrays.asList(new Object[] { new Integer(4), new Integer(4) }), Arrays.asList(new Object[] { new Integer(4), new Integer(4) }), + Arrays.asList(new Object[] { new Integer(5), null }), Arrays.asList(new Object[] { new Integer(10), null }), Arrays.asList(new Object[] { new Integer(11), null }), Arrays.asList(new Object[] { new Integer(11), null }) }; expectedReversed = new List[] { + Arrays.asList(new Object[] { null, null }), Arrays.asList(new Object[] { new Integer(1), new Integer(1) }), - Arrays.asList(new Object[] { new Integer(4), new Integer(4) }), - Arrays.asList(new Object[] { new Integer(4), new Integer(4) }), Arrays.asList(new Object[] { new Integer(2), new Integer(2) }), Arrays.asList(new Object[] { new Integer(2), new Integer(2) }), Arrays.asList(new Object[] { new Integer(4), new Integer(4) }), + Arrays.asList(new Object[] { new Integer(4), new Integer(4) }), Arrays.asList(new Object[] { new Integer(4), new Integer(4) }), - Arrays.asList(new Object[] { null, null }), + Arrays.asList(new Object[] { new Integer(4), new Integer(4) }), + Arrays.asList(new Object[] { new Integer(6), null }), Arrays.asList(new Object[] { new Integer(7), null }), - Arrays.asList(new Object[] { new Integer(7), null }), - Arrays.asList(new Object[] { new Integer(6), null }) + Arrays.asList(new Object[] { new Integer(7), null }) }; helpTestJoin(); } - public void testLeftOuterJoinWithSwap() throws Exception { - int outerSize = getProcessorBatchSize() + 1; + @Test public void testLeftOuterJoinWithSwap() throws Exception { + int outerSize = 11; leftTuples = createTuples(1,outerSize); rightTuples = createTuples(201,outerSize+1); joinType = JoinType.JOIN_LEFT_OUTER; - expectSwap = true; expected = createResults(1, outerSize); expectedReversed = createResults(201, outerSize+1); helpTestJoin(); } - public void testCrossJoin() throws Exception { + @Test public void testCrossJoin() throws Exception { joinType = JoinType.JOIN_CROSS; criteriaType = NO_CRITERIA; expected = new List[] { @@ -548,7 +507,7 @@ helpTestJoin(); } - public void testInnerJoinWithLookupFunction() throws Exception { + @Test public void testInnerJoinWithLookupFunction() throws Exception { criteriaType = FUNCTION_CRITERIA; joinType = JoinType.JOIN_INNER; expected = new List[] { @@ -565,9 +524,7 @@ Arrays.asList(new Object[] { new Integer(4), new Integer(5) }), Arrays.asList(new Object[] { new Integer(2), new Integer(3) }), Arrays.asList(new Object[] { new Integer(2), new Integer(3) }), - Arrays.asList(new Object[] { new Integer(4), new Integer(5) }), - Arrays.asList(new Object[] { new Integer(10), new Integer(7) }), - Arrays.asList(new Object[] { new Integer(10), new Integer(7) }) + Arrays.asList(new Object[] { new Integer(4), new Integer(5) }) }; dataMgr = new FakeDataManager(); @@ -585,34 +542,179 @@ helpTestJoin(); } - public void testFullOuterJoin() throws Exception { + @Test public void testFullOuterJoin() throws Exception { this.joinType = JoinType.JOIN_FULL_OUTER; this.leftTuples = createTuples3(); this.rightTuples = createTuples4(); expected = new List[] { Arrays.asList(new Object[] { null, null }), Arrays.asList(new Object[] { null, null }), - Arrays.asList(new Object[] { new Integer(10), new Integer(10) }), - Arrays.asList(new Object[] { new Integer(10), new Integer(10) }), - Arrays.asList(new Object[] { new Integer(9), new Integer(9) }), - Arrays.asList(new Object[] { new Integer(9), new Integer(9) }), - Arrays.asList(new Object[] { new Integer(9), new Integer(9) }), Arrays.asList(new Object[] { null, null }), + Arrays.asList(new Object[] { null, null }), + Arrays.asList(new Object[] { null, null }), + Arrays.asList(new Object[] { null, null }), Arrays.asList(new Object[] { new Integer(1), new Integer(1) }), Arrays.asList(new Object[] { new Integer(2), new Integer(2) }), Arrays.asList(new Object[] { new Integer(2), new Integer(2) }), Arrays.asList(new Object[] { new Integer(3), null }), - Arrays.asList(new Object[] { new Integer(5), new Integer(5) }), - Arrays.asList(new Object[] { new Integer(15), null }), Arrays.asList(new Object[] { null, new Integer(4) }), Arrays.asList(new Object[] { null, new Integer(4) }), - Arrays.asList(new Object[] { null, null }), + Arrays.asList(new Object[] { new Integer(5), new Integer(5) }), + Arrays.asList(new Object[] { null, new Integer(6) }), Arrays.asList(new Object[] { null, new Integer(7) }), - Arrays.asList(new Object[] { null, new Integer(6) }), - Arrays.asList(new Object[] { null, null }), - Arrays.asList(new Object[] { null, null }) + Arrays.asList(new Object[] { new Integer(9), new Integer(9) }), + Arrays.asList(new Object[] { new Integer(9), new Integer(9) }), + Arrays.asList(new Object[] { new Integer(9), new Integer(9) }), + Arrays.asList(new Object[] { new Integer(10), new Integer(10) }), + Arrays.asList(new Object[] { new Integer(10), new Integer(10) }), + Arrays.asList(new Object[] { new Integer(15), null }) }; - - helpTestJoinDirect(expected, false); + expectedReversed = new List[] { + Arrays.asList(new Object[] { null, null }), + Arrays.asList(new Object[] { null, null }), + Arrays.asList(new Object[] { null, null }), + Arrays.asList(new Object[] { null, null }), + Arrays.asList(new Object[] { null, null }), + Arrays.asList(new Object[] { null, null }), + Arrays.asList(new Object[] { new Integer(1), new Integer(1) }), + Arrays.asList(new Object[] { new Integer(2), new Integer(2) }), + Arrays.asList(new Object[] { new Integer(2), new Integer(2) }), + Arrays.asList(new Object[] { null, 3 }), + Arrays.asList(new Object[] { 4, null }), + Arrays.asList(new Object[] { 4, null }), + Arrays.asList(new Object[] { new Integer(5), new Integer(5) }), + Arrays.asList(new Object[] { 6, null }), + Arrays.asList(new Object[] { 7, null }), + Arrays.asList(new Object[] { new Integer(9), new Integer(9) }), + Arrays.asList(new Object[] { new Integer(9), new Integer(9) }), + Arrays.asList(new Object[] { new Integer(9), new Integer(9) }), + Arrays.asList(new Object[] { new Integer(10), new Integer(10) }), + Arrays.asList(new Object[] { new Integer(10), new Integer(10) }), + Arrays.asList(new Object[] { null, 15 }) + }; + helpTestJoin(); } + + @Test public void testMergeJoinOptimization() throws Exception { + this.joinType = JoinType.JOIN_INNER; + int rows = 100; + List[] data = new List[rows]; + for(int i=0; i<rows; i++) { + data[i] = new ArrayList(); + Integer value = new Integer((i*17) % 47); + data[i].add(value); + } + this.leftTuples = data; + this.rightTuples = createTuples2(); + expected = new List[] { + Arrays.asList(new Object[] { 4, 4 }), + Arrays.asList(new Object[] { 4, 4 }), + Arrays.asList(new Object[] { 7, 7 }), + Arrays.asList(new Object[] { 7, 7 }), + Arrays.asList(new Object[] { 2, 2 }), + Arrays.asList(new Object[] { 2, 2 }), + Arrays.asList(new Object[] { 6, 6 }), + Arrays.asList(new Object[] { 1, 1 }), + Arrays.asList(new Object[] { 4, 4 }), + Arrays.asList(new Object[] { 4, 4 }), + Arrays.asList(new Object[] { 7, 7 }), + Arrays.asList(new Object[] { 7, 7 }), + Arrays.asList(new Object[] { 2, 2 }), + Arrays.asList(new Object[] { 2, 2 }), + Arrays.asList(new Object[] { 6, 6 }), + Arrays.asList(new Object[] { 1, 1 }), + Arrays.asList(new Object[] { 4, 4 }), + Arrays.asList(new Object[] { 4, 4 }), + }; + helpCreateJoin(); + this.joinStrategy = new PartitionedSortJoin(SortOption.SORT, SortOption.SORT); + this.join.setJoinStrategy(joinStrategy); + helpTestJoinDirect(expected, 100); + } + + @Test public void testMergeJoinOptimizationNoRows() throws Exception { + this.joinType = JoinType.JOIN_INNER; + this.leftTuples = createTuples1(); + this.rightTuples = new List[] {}; + expected = new List[] {}; + helpCreateJoin(); + this.joinStrategy = new PartitionedSortJoin(SortOption.SORT, SortOption.SORT); + this.join.setJoinStrategy(joinStrategy); + helpTestJoinDirect(expected, 100); + } + + @Test public void testMergeJoinOptimizationWithDistinct() throws Exception { + this.joinType = JoinType.JOIN_INNER; + int rows = 50; + List[] data = new List[rows]; + for(int i=0; i<rows; i++) { + data[i] = new ArrayList(); + Integer value = new Integer((i*17) % 47); + data[i].add(value); + } + this.leftTuples = data; + this.rightTuples = new List[] { + Arrays.asList(4), + Arrays.asList(7), + Arrays.asList(2), + Arrays.asList(6), + Arrays.asList(1), + Arrays.asList(8), + }; + expected = new List[] { + Arrays.asList(new Object[] { 4, 4 }), + Arrays.asList(new Object[] { 8, 8 }), + Arrays.asList(new Object[] { 7, 7 }), + Arrays.asList(new Object[] { 2, 2 }), + Arrays.asList(new Object[] { 6, 6 }), + Arrays.asList(new Object[] { 1, 1 }) + }; + helpCreateJoin(); + this.joinStrategy = new PartitionedSortJoin(SortOption.SORT, SortOption.SORT); + this.join.setJoinStrategy(joinStrategy); + this.join.setRightDistinct(true); + helpTestJoinDirect(expected, 100); + } + + @Test public void testMergeJoinOptimizationWithMultiplePartitions() throws Exception { + this.joinType = JoinType.JOIN_INNER; + int rows = 30; + List[] data = new List[rows]; + for(int i=0; i<rows; i++) { + data[i] = new ArrayList(); + Integer value = new Integer(i % 17); + data[i].add(value); + } + this.rightTuples = data; + this.leftTuples = new List[] { + Arrays.asList(4), + Arrays.asList(7), + Arrays.asList(2), + Arrays.asList(6), + Arrays.asList(6), + Arrays.asList(1), + Arrays.asList(8), + }; + expected = new List[] { + Arrays.asList(new Object[] { 1, 1 }), + Arrays.asList(new Object[] { 2, 2 }), + Arrays.asList(new Object[] { 4, 4 }), + Arrays.asList(new Object[] { 6, 6 }), + Arrays.asList(new Object[] { 1, 1 }), + Arrays.asList(new Object[] { 2, 2 }), + Arrays.asList(new Object[] { 4, 4 }), + Arrays.asList(new Object[] { 6, 6 }), + Arrays.asList(new Object[] { 7, 7 }), + Arrays.asList(new Object[] { 8, 8 }), + Arrays.asList(new Object[] { 7, 7 }), + Arrays.asList(new Object[] { 8, 8 }), + Arrays.asList(new Object[] { 6, 6 }), + Arrays.asList(new Object[] { 6, 6 }), + }; + helpCreateJoin(); + this.joinStrategy = new PartitionedSortJoin(SortOption.SORT, SortOption.SORT); + this.join.setJoinStrategy(joinStrategy); + helpTestJoinDirect(expected, 4); + } + } Deleted: trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestJoinNodeWithProcessorBatchSize1.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestJoinNodeWithProcessorBatchSize1.java 2009-05-02 00:13:45 UTC (rev 856) +++ trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestJoinNodeWithProcessorBatchSize1.java 2009-05-04 13:21:45 UTC (rev 857) @@ -1,34 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.query.processor.relational; - -public class TestJoinNodeWithProcessorBatchSize1 extends TestJoinNode { - - public TestJoinNodeWithProcessorBatchSize1(String testName) { - super(testName); - } - - protected int getProcessorBatchSize() { - return 1; - } -} Deleted: trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestJoinNodeWithProcessorBatchSize10.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestJoinNodeWithProcessorBatchSize10.java 2009-05-02 00:13:45 UTC (rev 856) +++ trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestJoinNodeWithProcessorBatchSize10.java 2009-05-04 13:21:45 UTC (rev 857) @@ -1,34 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.query.processor.relational; - -public class TestJoinNodeWithProcessorBatchSize10 extends TestJoinNode { - - public TestJoinNodeWithProcessorBatchSize10(String testName) { - super(testName); - } - - protected int getProcessorBatchSize() { - return 10; - } -} Deleted: trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestJoinNodeWithProcessorBatchSizeSameAsTupleCount.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestJoinNodeWithProcessorBatchSizeSameAsTupleCount.java 2009-05-02 00:13:45 UTC (rev 856) +++ trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestJoinNodeWithProcessorBatchSizeSameAsTupleCount.java 2009-05-04 13:21:45 UTC (rev 857) @@ -1,34 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.query.processor.relational; - -public class TestJoinNodeWithProcessorBatchSizeSameAsTupleCount extends TestJoinNode { - - public TestJoinNodeWithProcessorBatchSizeSameAsTupleCount(String testName) { - super(testName); - } - - protected int getProcessorBatchSize() { - return createTuples1().length; - } -} Modified: trunk/test-integration/src/test/java/com/metamatrix/systemmodel/TestSystemVirtualModel.java =================================================================== --- trunk/test-integration/src/test/java/com/metamatrix/systemmodel/TestSystemVirtualModel.java 2009-05-02 00:13:45 UTC (rev 856) +++ trunk/test-integration/src/test/java/com/metamatrix/systemmodel/TestSystemVirtualModel.java 2009-05-04 13:21:45 UTC (rev 857) @@ -272,14 +272,14 @@ "PartsSupplier PartsSupplier.PARTSSUPPLIER.STATUS STATUS_ID PK_STATUS Primary STATUS STATUS null mmuuid:25a8a740-73ff-1edc-a81c-ecf397b10590 1", //$NON-NLS-1$ "PartsSupplier PartsSupplier.PARTSSUPPLIER.SUPPLIER SUPPLIER_ID PK_SUPPLIER Primary SUPPLIER SUPPLIER null mmuuid:375c8380-73ff-1edc-a81c-ecf397b10590 1", //$NON-NLS-1$ "PartsSupplier PartsSupplier.PARTSSUPPLIER.SUPPLIER SUPPLIER_STATUS FK_SPLIER_STATS Foreign SUPPLIER SUPPLIER mmuuid:25a8a740-73ff-1edc-a81c-ecf397b10590 mmuuid:5ac43c00-73ff-1edc-a81c-ecf397b10590 1", //$NON-NLS-1$ + "PartsSupplier PartsSupplier.PARTSSUPPLIER.SUPPLIER_PARTS PART_ID FK_SPLIER_PRTS_PRTS Foreign SUPPLIER_PARTS SUPPLIER_PARTS mmuuid:07db4240-73ff-1edc-a81c-ecf397b10590 mmuuid:66ddc4c0-73ff-1edc-a81c-ecf397b10590 1", //$NON-NLS-1$ "PartsSupplier PartsSupplier.PARTSSUPPLIER.SUPPLIER_PARTS PART_ID PK_SUPPLIER_PARTS Primary SUPPLIER_PARTS SUPPLIER_PARTS null mmuuid:455e5440-73ff-1edc-a81c-ecf397b10590 2", //$NON-NLS-1$ - "PartsSupplier PartsSupplier.PARTSSUPPLIER.SUPPLIER_PARTS PART_ID FK_SPLIER_PRTS_PRTS Foreign SUPPLIER_PARTS SUPPLIER_PARTS mmuuid:07db4240-73ff-1edc-a81c-ecf397b10590 mmuuid:66ddc4c0-73ff-1edc-a81c-ecf397b10590 1", //$NON-NLS-1$ + "PartsSupplier PartsSupplier.PARTSSUPPLIER.SUPPLIER_PARTS SUPPLIER_ID FK_SPLY_PRTS_SPLY Foreign SUPPLIER_PARTS SUPPLIER_PARTS mmuuid:375c8380-73ff-1edc-a81c-ecf397b10590 mmuuid:66ddc4c1-73ff-1edc-a81c-ecf397b10590 1", //$NON-NLS-1$ "PartsSupplier PartsSupplier.PARTSSUPPLIER.SUPPLIER_PARTS SUPPLIER_ID PK_SUPPLIER_PARTS Primary SUPPLIER_PARTS SUPPLIER_PARTS null mmuuid:455e5440-73ff-1edc-a81c-ecf397b10590 1", //$NON-NLS-1$ - "PartsSupplier PartsSupplier.PARTSSUPPLIER.SUPPLIER_PARTS SUPPLIER_ID FK_SPLY_PRTS_SPLY Foreign SUPPLIER_PARTS SUPPLIER_PARTS mmuuid:375c8380-73ff-1edc-a81c-ecf397b10590 mmuuid:66ddc4c1-73ff-1edc-a81c-ecf397b10590 1", //$NON-NLS-1$ }; executeAndAssertResults( - "select* from System.KeyElements order by GroupFullName, Name", //$NON-NLS-1$ + "select* from System.KeyElements order by GroupFullName, Name, KeyName", //$NON-NLS-1$ expected); } @@ -1001,18 +1001,18 @@ @Test public void testOASTATISTICS() { String[] expected = { "TABLE_QUALIFIER[string] TABLE_OWNER[string] TABLE_NAME[string] NON_UNIQUE[short] INDEX_QUALIFIER[string] INDEX_NAME[string] OA_TYPE[short] SEQ_IN_INDEX[short] COLUMN_NAME[string] OA_COLLATION[string] OA_CARDINALITY[string] OA_PAGES[string] FILTER_CONDITIONS[string]", //$NON-NLS-1$ + "PartsSupplier PartsSupplier/PARTSSUPPLIER PARTS 0 PK_PARTS 3 1 PART_ID null null null", //$NON-NLS-1$ "PartsSupplier PartsSupplier/PARTSSUPPLIER SHIP_VIA 0 PK_SHIP_VIA 3 1 SHIPPER_ID null null null ", //$NON-NLS-1$ "PartsSupplier PartsSupplier/PARTSSUPPLIER STATUS 0 PK_STATUS 3 1 STATUS_ID null null null ", //$NON-NLS-1$ + "PartsSupplier PartsSupplier/PARTSSUPPLIER SUPPLIER 1 FK_SPLIER_STATS 3 1 SUPPLIER_STATUS null null null ", //$NON-NLS-1$ "PartsSupplier PartsSupplier/PARTSSUPPLIER SUPPLIER 0 PK_SUPPLIER 3 1 SUPPLIER_ID null null null ", //$NON-NLS-1$ - "PartsSupplier PartsSupplier/PARTSSUPPLIER SUPPLIER 1 FK_SPLIER_STATS 3 1 SUPPLIER_STATUS null null null ", //$NON-NLS-1$ - "PartsSupplier PartsSupplier/PARTSSUPPLIER SUPPLIER_PARTS 0 PK_SUPPLIER_PARTS 3 1 SUPPLIER_ID null null null ", //$NON-NLS-1$ + "PartsSupplier PartsSupplier/PARTSSUPPLIER SUPPLIER_PARTS 1 FK_SPLIER_PRTS_PRTS 3 1 PART_ID null null null ", //$NON-NLS-1$ "PartsSupplier PartsSupplier/PARTSSUPPLIER SUPPLIER_PARTS 1 FK_SPLY_PRTS_SPLY 3 1 SUPPLIER_ID null null null ", //$NON-NLS-1$ "PartsSupplier PartsSupplier/PARTSSUPPLIER SUPPLIER_PARTS 0 PK_SUPPLIER_PARTS 3 2 PART_ID null null null ", //$NON-NLS-1$ - "PartsSupplier PartsSupplier/PARTSSUPPLIER SUPPLIER_PARTS 1 FK_SPLIER_PRTS_PRTS 3 1 PART_ID null null null ", //$NON-NLS-1$ - "PartsSupplier PartsSupplier/PARTSSUPPLIER PARTS 0 PK_PARTS 3 1 PART_ID null null null", //$NON-NLS-1$ + "PartsSupplier PartsSupplier/PARTSSUPPLIER SUPPLIER_PARTS 0 PK_SUPPLIER_PARTS 3 1 SUPPLIER_ID null null null ", //$NON-NLS-1$ }; - executeAndAssertResults("select* FROM System.ODBC.OA_STATISTICS", //$NON-NLS-1$ + executeAndAssertResults("select* FROM System.ODBC.OA_STATISTICS order by TABLE_NAME, INDEX_NAME, COLUMN_NAME", //$NON-NLS-1$ expected); } @@ -1025,16 +1025,16 @@ " SHIP_VIA SHIPPER_ID null null 1 null null null PK_SHIP_VIA", //$NON-NLS-1$ " STATUS STATUS_ID SUPPLIER SUPPLIER_STATUS 1 null null FK_SPLIER_STATS PK_STATUS", //$NON-NLS-1$ " SUPPLIER SUPPLIER_ID SUPPLIER_PARTS SUPPLIER_ID 1 null null FK_SPLY_PRTS_SPLY PK_SUPPLIER", //$NON-NLS-1$ + " SUPPLIER SUPPLIER_STATUS null null 1 null null null FK_SPLIER_STATS", //$NON-NLS-1$ + " SUPPLIER_PARTS PART_ID null null 2 null null null PK_SUPPLIER_PARTS", //$NON-NLS-1$ " SUPPLIER_PARTS SUPPLIER_ID null null 1 null null null PK_SUPPLIER_PARTS", //$NON-NLS-1$ - " SUPPLIER_PARTS PART_ID null null 2 null null null PK_SUPPLIER_PARTS", //$NON-NLS-1$ - " SUPPLIER SUPPLIER_STATUS null null 1 null null null FK_SPLIER_STATS", //$NON-NLS-1$ " SUPPLIER_PARTS PART_ID null null 1 null null null FK_SPLIER_PRTS_PRTS", //$NON-NLS-1$ " SUPPLIER_PARTS SUPPLIER_ID null null 1 null null null FK_SPLY_PRTS_SPLY", //$NON-NLS-1$ }; executeAndAssertResults( - "select '' AS PKTABLE_QUALIFIER, '' AS PKTABLE_OWNER, PK.GroupName AS PKTABLE_NAME, PK.Name AS PKCOLUMN_NAME, '' AS FKTABLE_QUALIFIER, '' AS FKTABLE_OWNER, FK.GroupName AS FKTABLE_NAME, FK.Name AS FKCOLUMN_NAME, convert(PK.Position, short) AS KEY_SEQ, convert(null, short) AS UPDATE_RULE, convert(null, short) AS DELETE_RULE, FK.KeyName AS FK_NAME, PK.KeyName AS PK_NAME FROM System.KeyElements AS PK LEFT OUTER JOIN System.KeyElements AS FK ON FK.RefKeyUID = PK.UID", //$NON-NLS-1$ + "select '' AS PKTABLE_QUALIFIER, '' AS PKTABLE_OWNER, PK.GroupName AS PKTABLE_NAME, PK.Name AS PKCOLUMN_NAME, '' AS FKTABLE_QUALIFIER, '' AS FKTABLE_OWNER, FK.GroupName AS FKTABLE_NAME, FK.Name AS FKCOLUMN_NAME, convert(PK.Position, short) AS KEY_SEQ, convert(null, short) AS UPDATE_RULE, convert(null, short) AS DELETE_RULE, FK.KeyName AS FK_NAME, PK.KeyName AS PK_NAME FROM System.KeyElements AS PK LEFT OUTER JOIN System.KeyElements AS FK ON FK.RefKeyUID = PK.UID order by PKTABLE_NAME", //$NON-NLS-1$ expected); } @@ -1063,11 +1063,12 @@ @Test public void testReferenceKeyColumns() { String[] expected = { "PKTABLE_CAT[string] PKTABLE_SCHEM[string] PKTABLE_NAME[string] PKCOLUMN_NAME[string] FKTABLE_CAT[string] FKTABLE_SCHEM[string] FKTABLE_NAME[string] FKCOLUMN_NAME[string] KEY_SEQ[short] UPDATE_RULE[integer] DELETE_RULE[integer] FK_NAME[string] PK_NAME[string] DEFERRABILITY[integer]", //$NON-NLS-1$ + "null PartsSupplier PartsSupplier.PARTSSUPPLIER.PARTS PART_ID null PartsSupplier PartsSupplier.PARTSSUPPLIER.SUPPLIER_PARTS PART_ID 1 3 3 FK_SPLIER_PRTS_PRTS PK_PARTS 5", //$NON-NLS-1$ "null PartsSupplier PartsSupplier.PARTSSUPPLIER.STATUS STATUS_ID null PartsSupplier PartsSupplier.PARTSSUPPLIER.SUPPLIER SUPPLIER_STATUS 1 3 3 FK_SPLIER_STATS PK_STATUS 5", //$NON-NLS-1$ - "null PartsSupplier PartsSupplier.PARTSSUPPLIER.SUPPLIER SUPPLIER_ID null PartsSupplier PartsSupplier.PARTSSUPPLIER.SUPPLIER_PARTS SUPPLIER_ID 1 3 3 FK_SPLY_PRTS_SPLY PK_SUPPLIER 5", //$NON-NLS-1$ - "null PartsSupplier PartsSupplier.PARTSSUPPLIER.PARTS PART_ID null PartsSupplier PartsSupplier.PARTSSUPPLIER.SUPPLIER_PARTS PART_ID 1 3 3 FK_SPLIER_PRTS_PRTS PK_PARTS 5", }; //$NON-NLS-1$ + "null PartsSupplier PartsSupplier.PARTSSUPPLIER.SUPPLIER SUPPLIER_ID null PartsSupplier PartsSupplier.PARTSSUPPLIER.SUPPLIER_PARTS SUPPLIER_ID 1 3 3 FK_SPLY_PRTS_SPLY PK_SUPPLIER 5" //$NON-NLS-1$ + }; - executeAndAssertResults("select* FROM System.JDBC.ReferenceKeyColumns", //$NON-NLS-1$ + executeAndAssertResults("select* FROM System.JDBC.ReferenceKeyColumns order by PKTABLE_NAME", //$NON-NLS-1$ expected); } Modified: trunk/test-integration/src/test/resources/partssupplier/expected/CrossReference.txt =================================================================== --- trunk/test-integration/src/test/resources/partssupplier/expected/CrossReference.txt 2009-05-02 00:13:45 UTC (rev 856) +++ trunk/test-integration/src/test/resources/partssupplier/expected/CrossReference.txt 2009-05-04 13:21:45 UTC (rev 857) @@ -1,4 +1,4 @@ PKTABLE_CAT[string] PKTABLE_SCHEM[string] PKTABLE_NAME[string] PKCOLUMN_NAME[string] FKTABLE_CAT[string] FKTABLE_SCHEM[string] FKTABLE_NAME[string] FKCOLUMN_NAME[string] KEY_SEQ[short] UPDATE_RULE[integer] DELETE_RULE[integer] FK_NAME[string] PK_NAME[string] DEFERRABILITY[integer] null PartsSupplier PartsSupplier.PARTSSUPPLIER.STATUS STATUS_ID null PartsSupplier PartsSupplier.PARTSSUPPLIER.SUPPLIER SUPPLIER_STATUS 1 3 3 FK_SPLIER_STATS PK_STATUS 5 +null PartsSupplier PartsSupplier.PARTSSUPPLIER.PARTS PART_ID null PartsSupplier PartsSupplier.PARTSSUPPLIER.SUPPLIER_PARTS PART_ID 1 3 3 FK_SPLIER_PRTS_PRTS PK_PARTS 5 null PartsSupplier PartsSupplier.PARTSSUPPLIER.SUPPLIER SUPPLIER_ID null PartsSupplier PartsSupplier.PARTSSUPPLIER.SUPPLIER_PARTS SUPPLIER_ID 1 3 3 FK_SPLY_PRTS_SPLY PK_SUPPLIER 5 -null PartsSupplier PartsSupplier.PARTSSUPPLIER.PARTS PART_ID null PartsSupplier PartsSupplier.PARTSSUPPLIER.SUPPLIER_PARTS PART_ID 1 3 3 FK_SPLIER_PRTS_PRTS PK_PARTS 5 Modified: trunk/test-integration/src/test/resources/partssupplier/expected/ExportedKeys.txt =================================================================== --- trunk/test-integration/src/test/resources/partssupplier/expected/ExportedKeys.txt 2009-05-02 00:13:45 UTC (rev 856) +++ trunk/test-integration/src/test/resources/partssupplier/expected/ExportedKeys.txt 2009-05-04 13:21:45 UTC (rev 857) @@ -1,4 +1,4 @@ PKTABLE_CAT[string] PKTABLE_SCHEM[string] PKTABLE_NAME[string] PKCOLUMN_NAME[string] FKTABLE_CAT[string] FKTABLE_SCHEM[string] FKTABLE_NAME[string] FKCOLUMN_NAME[string] KEY_SEQ[short] UPDATE_RULE[integer] DELETE_RULE[integer] FK_NAME[string] PK_NAME[string] DEFERRABILITY[integer] null PartsSupplier PartsSupplier.PARTSSUPPLIER.STATUS STATUS_ID null PartsSupplier PartsSupplier.PARTSSUPPLIER.SUPPLIER SUPPLIER_STATUS 1 3 3 FK_SPLIER_STATS PK_STATUS 5 +null PartsSupplier PartsSupplier.PARTSSUPPLIER.PARTS PART_ID null PartsSupplier PartsSupplier.PARTSSUPPLIER.SUPPLIER_PARTS PART_ID 1 3 3 FK_SPLIER_PRTS_PRTS PK_PARTS 5 null PartsSupplier PartsSupplier.PARTSSUPPLIER.SUPPLIER SUPPLIER_ID null PartsSupplier PartsSupplier.PARTSSUPPLIER.SUPPLIER_PARTS SUPPLIER_ID 1 3 3 FK_SPLY_PRTS_SPLY PK_SUPPLIER 5 -null PartsSupplier PartsSupplier.PARTSSUPPLIER.PARTS PART_ID null PartsSupplier PartsSupplier.PARTSSUPPLIER.SUPPLIER_PARTS PART_ID 1 3 3 FK_SPLIER_PRTS_PRTS PK_PARTS 5