Author: shawkins Date: 2011-03-31 00:11:52 -0400 (Thu, 31 Mar 2011) New Revision: 3050 Modified: trunk/api/src/main/java/org/teiid/translator/ExecutionFactory.java trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/JDBCExecutionFactory.java trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/SimpleJDBCExecutionFactory.java trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/access/AccessExecutionFactory.java trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/sqlserver/SQLServerExecutionFactory.java trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/sybase/SybaseExecutionFactory.java trunk/connectors/translator-ldap/src/main/java/org/teiid/translator/ldap/LDAPExecutionFactory.java trunk/engine/src/main/java/org/teiid/dqp/internal/datamgr/CapabilitiesConverter.java trunk/engine/src/main/java/org/teiid/query/optimizer/capabilities/SourceCapabilities.java trunk/engine/src/main/java/org/teiid/query/optimizer/relational/PlanToProcessConverter.java trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/CapabilitiesUtil.java trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RulePushSelectCriteria.java trunk/engine/src/main/java/org/teiid/query/processor/relational/AccessNode.java trunk/engine/src/main/java/org/teiid/query/processor/relational/BatchedUpdateNode.java trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentAccessNode.java trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentCriteriaProcessor.java trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentProcedureCriteriaProcessor.java trunk/engine/src/main/java/org/teiid/query/rewriter/QueryRewriter.java trunk/engine/src/main/java/org/teiid/query/sql/lang/Create.java trunk/engine/src/main/java/org/teiid/query/tempdata/TempTable.java trunk/engine/src/test/java/org/teiid/query/optimizer/TestUnionPlanning.java trunk/engine/src/test/java/org/teiid/query/processor/TestDependentJoins.java trunk/engine/src/test/java/org/teiid/query/processor/relational/TestDependentCriteriaProcessor.java trunk/engine/src/test/java/org/teiid/query/processor/xml/TestXMLPlanningEnhancements.java Log: TEIID-1533 adding the ability to split dependent join values into multiple in predicates Modified: trunk/api/src/main/java/org/teiid/translator/ExecutionFactory.java =================================================================== --- trunk/api/src/main/java/org/teiid/translator/ExecutionFactory.java 2011-03-29 21:52:50 UTC (rev 3049) +++ trunk/api/src/main/java/org/teiid/translator/ExecutionFactory.java 2011-03-31 04:11:52 UTC (rev 3050) @@ -106,6 +106,7 @@ private boolean supportsFullOuterJoins; private boolean requiresCriteria; private int maxInSize = DEFAULT_MAX_IN_CRITERIA_SIZE; + private int maxDependentInPredicates = DEFAULT_MAX_IN_CRITERIA_SIZE; /** * Initialize the connector with supplied configuration @@ -667,8 +668,22 @@ public void setMaxInCriteriaSize(int maxInSize) { this.maxInSize = maxInSize; } - + /** + * Get the integer value representing the number of values allowed in an IN criteria + * in the WHERE clause of a query + * @since 5.0 + */ + @TranslatorProperty(display="Max number of dependent values across all IN predicates", advanced=true) + public int getMaxDependentInPredicates() { + return maxDependentInPredicates; + } + + public void setMaxDependentPredicates(int maxDependentInPredicates) { + this.maxDependentInPredicates = maxDependentInPredicates; + } + + /** * <p>Support indicates that the connector supports functions in GROUP BY, such as: * <code>SELECT dayofmonth(theDate), COUNT(*) FROM table GROUP BY dayofmonth(theDate)</code></p> * Modified: trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/JDBCExecutionFactory.java =================================================================== --- trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/JDBCExecutionFactory.java 2011-03-29 21:52:50 UTC (rev 3049) +++ trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/JDBCExecutionFactory.java 2011-03-31 04:11:52 UTC (rev 3050) @@ -81,6 +81,7 @@ public class JDBCExecutionFactory extends ExecutionFactory<DataSource, Connection> { public static final int DEFAULT_MAX_IN_CRITERIA = 1000; + public static final int DEFAULT_MAX_DEPENDENT_PREDICATES = 50; // Because the retrieveValue() method will be hit for every value of // every JDBC result set returned, we do lots of weird special stuff here @@ -149,6 +150,8 @@ setSupportsOuterJoins(true); setSupportsSelectDistinct(true); setSupportsInnerJoins(true); + setMaxInCriteriaSize(DEFAULT_MAX_IN_CRITERIA); + setMaxDependentPredicates(DEFAULT_MAX_DEPENDENT_PREDICATES); } @Override Modified: trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/SimpleJDBCExecutionFactory.java =================================================================== --- trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/SimpleJDBCExecutionFactory.java 2011-03-29 21:52:50 UTC (rev 3049) +++ trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/SimpleJDBCExecutionFactory.java 2011-03-31 04:11:52 UTC (rev 3050) @@ -41,6 +41,7 @@ setSupportsFullOuterJoins(false); setSupportsOrderBy(false); setMaxInCriteriaSize(250); + setMaxDependentPredicates(10); } @Override Modified: trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/access/AccessExecutionFactory.java =================================================================== --- trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/access/AccessExecutionFactory.java 2011-03-29 21:52:50 UTC (rev 3049) +++ trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/access/AccessExecutionFactory.java 2011-03-31 04:11:52 UTC (rev 3050) @@ -40,6 +40,7 @@ setSupportsOrderBy(false); setDatabaseVersion("2003"); //$NON-NLS-1$ setMaxInCriteriaSize(JDBCExecutionFactory.DEFAULT_MAX_IN_CRITERIA); + setMaxDependentPredicates(10); //sql length length is 64k } @Override Modified: trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/sqlserver/SQLServerExecutionFactory.java =================================================================== --- trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/sqlserver/SQLServerExecutionFactory.java 2011-03-29 21:52:50 UTC (rev 3049) +++ trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/sqlserver/SQLServerExecutionFactory.java 2011-03-31 04:11:52 UTC (rev 3050) @@ -51,6 +51,7 @@ public SQLServerExecutionFactory() { setDatabaseVersion(V_2005); setMaxInCriteriaSize(JDBCExecutionFactory.DEFAULT_MAX_IN_CRITERIA); + setMaxDependentPredicates(JDBCExecutionFactory.DEFAULT_MAX_DEPENDENT_PREDICATES); } @Override Modified: trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/sybase/SybaseExecutionFactory.java =================================================================== --- trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/sybase/SybaseExecutionFactory.java 2011-03-29 21:52:50 UTC (rev 3049) +++ trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/sybase/SybaseExecutionFactory.java 2011-03-31 04:11:52 UTC (rev 3050) @@ -65,6 +65,7 @@ setDatabaseVersion(TWELVE_5); setSupportsFullOuterJoins(false); setMaxInCriteriaSize(250); + setMaxDependentPredicates(10); } public void start() throws TranslatorException { Modified: trunk/connectors/translator-ldap/src/main/java/org/teiid/translator/ldap/LDAPExecutionFactory.java =================================================================== --- trunk/connectors/translator-ldap/src/main/java/org/teiid/translator/ldap/LDAPExecutionFactory.java 2011-03-29 21:52:50 UTC (rev 3049) +++ trunk/connectors/translator-ldap/src/main/java/org/teiid/translator/ldap/LDAPExecutionFactory.java 2011-03-31 04:11:52 UTC (rev 3050) @@ -56,6 +56,7 @@ public LDAPExecutionFactory() { this.setMaxInCriteriaSize(1000); + this.setMaxDependentPredicates(25); //no spec limit on query size, AD is 10MB for the query } @TranslatorProperty(display="Default Search Base DN", description="Default Base DN for LDAP Searches") Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/datamgr/CapabilitiesConverter.java =================================================================== --- trunk/engine/src/main/java/org/teiid/dqp/internal/datamgr/CapabilitiesConverter.java 2011-03-29 21:52:50 UTC (rev 3049) +++ trunk/engine/src/main/java/org/teiid/dqp/internal/datamgr/CapabilitiesConverter.java 2011-03-31 04:11:52 UTC (rev 3050) @@ -115,6 +115,7 @@ } tgtCaps.setSourceProperty(Capability.MAX_IN_CRITERIA_SIZE, new Integer(srcCaps.getMaxInCriteriaSize())); + tgtCaps.setSourceProperty(Capability.MAX_DEPENDENT_PREDICATES, new Integer(srcCaps.getMaxDependentInPredicates())); tgtCaps.setSourceProperty(Capability.CONNECTOR_ID, connectorID); tgtCaps.setSourceProperty(Capability.MAX_QUERY_FROM_GROUPS, new Integer(srcCaps.getMaxFromGroups())); tgtCaps.setSourceProperty(Capability.JOIN_CRITERIA_ALLOWED, srcCaps.getSupportedJoinCriteria()); Modified: trunk/engine/src/main/java/org/teiid/query/optimizer/capabilities/SourceCapabilities.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/optimizer/capabilities/SourceCapabilities.java 2011-03-29 21:52:50 UTC (rev 3049) +++ trunk/engine/src/main/java/org/teiid/query/optimizer/capabilities/SourceCapabilities.java 2011-03-31 04:11:52 UTC (rev 3050) @@ -319,7 +319,8 @@ REQUIRES_CRITERIA, INSERT_WITH_QUERYEXPRESSION, INSERT_WITH_ITERATOR, - COMMON_TABLE_EXPRESSIONS + COMMON_TABLE_EXPRESSIONS, + MAX_DEPENDENT_PREDICATES, } public enum Scope { Modified: trunk/engine/src/main/java/org/teiid/query/optimizer/relational/PlanToProcessConverter.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/optimizer/relational/PlanToProcessConverter.java 2011-03-29 21:52:50 UTC (rev 3049) +++ trunk/engine/src/main/java/org/teiid/query/optimizer/relational/PlanToProcessConverter.java 2011-03-31 04:11:52 UTC (rev 3050) @@ -152,7 +152,16 @@ // Call convertPlan recursively on children for (PlanNode childNode : planNode.getChildren()) { - nextParent.addChild(convertPlan(childNode)); + RelationalNode child = convertPlan(childNode); + if (planNode.getType() == NodeConstants.Types.SET_OP && childNode.getType() == NodeConstants.Types.SET_OP && childNode.hasBooleanProperty(Info.USE_ALL)) { + for (RelationalNode grandChild : child.getChildren()) { + if (grandChild != null) { + nextParent.addChild(grandChild); + } + } + } else { + nextParent.addChild(child); + } } // Return root of tree for top node @@ -297,7 +306,8 @@ DependentAccessNode depAccessNode = new DependentAccessNode(getID()); if(modelID != null){ - depAccessNode.setMaxSetSize(CapabilitiesUtil.getMaxInCriteriaSize(modelID, metadata, capFinder)); + depAccessNode.setMaxSetSize(CapabilitiesUtil.getMaxInCriteriaSize(modelID, metadata, capFinder)); + depAccessNode.setMaxPredicates(CapabilitiesUtil.getMaxDependentPredicatesSize(modelID, metadata, capFinder)); } processNode = depAccessNode; aNode = depAccessNode; Modified: trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/CapabilitiesUtil.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/CapabilitiesUtil.java 2011-03-29 21:52:50 UTC (rev 3049) +++ trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/CapabilitiesUtil.java 2011-03-31 04:11:52 UTC (rev 3050) @@ -296,6 +296,11 @@ public static int getMaxInCriteriaSize(Object modelID, QueryMetadataInterface metadata, CapabilitiesFinder capFinder) throws QueryMetadataException, TeiidComponentException { + return getProperty(Capability.MAX_IN_CRITERIA_SIZE, modelID, metadata, capFinder); + } + + private static int getProperty(Capability cap, Object modelID, QueryMetadataInterface metadata, CapabilitiesFinder capFinder) + throws QueryMetadataException, TeiidComponentException { if (metadata.isVirtualModel(modelID)){ return -1; @@ -303,7 +308,7 @@ // Find capabilities SourceCapabilities caps = getCapabilities(modelID, metadata, capFinder); - Object maxInCriteriaSize = caps.getSourceProperty(Capability.MAX_IN_CRITERIA_SIZE); + Object maxInCriteriaSize = caps.getSourceProperty(cap); int value = -1; if(maxInCriteriaSize != null) { value = ((Integer)maxInCriteriaSize).intValue(); @@ -316,6 +321,11 @@ return value; } + public static int getMaxDependentPredicatesSize(Object modelID, QueryMetadataInterface metadata, CapabilitiesFinder capFinder) + throws QueryMetadataException, TeiidComponentException { + return getProperty(Capability.MAX_DEPENDENT_PREDICATES, modelID, metadata, capFinder); + } + public static int getMaxFromGroups(Object modelID, QueryMetadataInterface metadata, CapabilitiesFinder capFinder) throws QueryMetadataException, TeiidComponentException { Modified: trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RulePushSelectCriteria.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RulePushSelectCriteria.java 2011-03-29 21:52:50 UTC (rev 3049) +++ trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RulePushSelectCriteria.java 2011-03-31 04:11:52 UTC (rev 3050) @@ -299,8 +299,7 @@ satisfyAccessPatterns(critNode, currentNode); } - if (critNode.hasBooleanProperty(NodeConstants.Info.IS_DEPENDENT_SET) - && CapabilitiesUtil.getMaxInCriteriaSize(RuleRaiseAccess.getModelIDFromAccess(currentNode, metadata), metadata, capFinder) > 0) { + if (critNode.hasBooleanProperty(NodeConstants.Info.IS_DEPENDENT_SET)) { //once a dependent crit node is pushed, don't bother pushing it further into the command //dependent access node will use this as an assumption for where dependent sets can appear in the command critNode.setProperty(NodeConstants.Info.IS_PUSHED, Boolean.TRUE); Modified: trunk/engine/src/main/java/org/teiid/query/processor/relational/AccessNode.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/processor/relational/AccessNode.java 2011-03-29 21:52:50 UTC (rev 3049) +++ trunk/engine/src/main/java/org/teiid/query/processor/relational/AccessNode.java 2011-03-31 04:11:52 UTC (rev 3050) @@ -28,7 +28,6 @@ import java.util.Collections; import java.util.List; -import org.teiid.api.exception.query.ExpressionEvaluationException; import org.teiid.api.exception.query.QueryValidatorException; import org.teiid.client.plan.PlanNode; import org.teiid.common.buffer.BlockedException; @@ -57,7 +56,7 @@ private ArrayList<TupleSource> tupleSources = new ArrayList<TupleSource>(); private boolean isUpdate = false; private boolean returnedRows = false; - private Command nextCommand; + protected Command nextCommand; protected AccessNode() { super(); @@ -101,9 +100,13 @@ // Copy command and resolve references if necessary Command atomicCommand = command; boolean needProcessing = true; + + if (shouldEvaluate) { + atomicCommand = initialCommand(); + } do { - if(shouldEvaluate) { - atomicCommand = nextCommand(); + atomicCommand = nextCommand(); + if(shouldEvaluate) { needProcessing = prepareNextCommand(atomicCommand); nextCommand = null; } else { @@ -122,7 +125,22 @@ } while (!processCommandsIndividually() && hasNextCommand() && this.tupleSources.size() < Math.min(MAX_CONCURRENT, this.getContext().getUserRequestSourceConcurrency())); } - private Command nextCommand() { + static void rewriteAndEvaluate(Command atomicCommand, Evaluator eval, CommandContext context, QueryMetadataInterface metadata) + throws TeiidProcessingException, TeiidComponentException { + try { + // Defect 16059 - Rewrite the command to replace references, etc. with values. + QueryRewriter.evaluateAndRewrite(atomicCommand, eval, context, metadata); + } catch (QueryValidatorException e) { + throw new TeiidProcessingException(e, QueryPlugin.Util.getString("AccessNode.rewrite_failed", atomicCommand)); //$NON-NLS-1$ + } + } + + @SuppressWarnings("unused") + protected Command initialCommand() throws TeiidProcessingException, TeiidComponentException { + return nextCommand(); + } + + protected Command nextCommand() { //it's important to save the next command //to ensure that the subquery ids remain stable if (nextCommand == null) { @@ -132,22 +150,10 @@ } protected boolean prepareNextCommand(Command atomicCommand) throws TeiidComponentException, TeiidProcessingException { - return prepareCommand(atomicCommand, getEvaluator(Collections.emptyMap()), this.getContext(), this.getContext().getMetadata()); + rewriteAndEvaluate(atomicCommand, getEvaluator(Collections.emptyMap()), this.getContext(), this.getContext().getMetadata()); + return RelationalNodeUtil.shouldExecute(atomicCommand, true); } - static boolean prepareCommand(Command atomicCommand, Evaluator eval, CommandContext context, QueryMetadataInterface metadata) - throws ExpressionEvaluationException, TeiidComponentException, - TeiidProcessingException { - try { - // Defect 16059 - Rewrite the command once the references have been replaced with values. - QueryRewriter.evaluateAndRewrite(atomicCommand, eval, context, metadata); - } catch (QueryValidatorException e) { - throw new TeiidProcessingException(e, QueryPlugin.Util.getString("AccessNode.rewrite_failed", atomicCommand)); //$NON-NLS-1$ - } - - return RelationalNodeUtil.shouldExecute(atomicCommand, true); - } - public TupleBatch nextBatchDirect() throws BlockedException, TeiidComponentException, TeiidProcessingException { Modified: trunk/engine/src/main/java/org/teiid/query/processor/relational/BatchedUpdateNode.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/processor/relational/BatchedUpdateNode.java 2011-03-29 21:52:50 UTC (rev 3049) +++ trunk/engine/src/main/java/org/teiid/query/processor/relational/BatchedUpdateNode.java 2011-03-31 04:11:52 UTC (rev 3050) @@ -102,10 +102,9 @@ updateCommand = (Command) updateCommand.clone(); Evaluator eval = getEvaluator(Collections.emptyMap()); eval.initialize(context, getDataManager()); - needProcessing = AccessNode.prepareCommand(updateCommand, eval, context, context.getMetadata()); - } else { - needProcessing = RelationalNodeUtil.shouldExecute(updateCommand, true); + AccessNode.rewriteAndEvaluate(updateCommand, eval, context, context.getMetadata()); } + needProcessing = RelationalNodeUtil.shouldExecute(updateCommand, true); if (needProcessing) { commandsToExecute.add(updateCommand); } else { Modified: trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentAccessNode.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentAccessNode.java 2011-03-29 21:52:50 UTC (rev 3049) +++ trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentAccessNode.java 2011-03-31 04:11:52 UTC (rev 3050) @@ -22,6 +22,8 @@ package org.teiid.query.processor.relational; +import java.util.Collections; + import org.teiid.core.TeiidComponentException; import org.teiid.core.TeiidProcessingException; import org.teiid.core.util.Assertion; @@ -40,12 +42,17 @@ //plan state private int maxSetSize; + private int maxPredicates; //processing state private DependentCriteriaProcessor criteriaProcessor; private Criteria dependentCrit; private boolean sort = true; - + /** + * Cached rewritten command to be used as the base for all dependent queries. + */ + private Command rewrittenCommand; + public DependentAccessNode(int nodeID) { super(nodeID); } @@ -66,11 +73,32 @@ criteriaProcessor = null; dependentCrit = null; sort = true; + rewrittenCommand = null; } + + @Override + protected Command initialCommand() throws TeiidProcessingException, TeiidComponentException { + if (rewrittenCommand == null) { + Command atomicCommand = nextCommand(); + rewriteAndEvaluate(atomicCommand, getEvaluator(Collections.emptyMap()), this.getContext(), this.getContext().getMetadata()); + rewrittenCommand = atomicCommand; + nextCommand = null; + } + return rewrittenCommand; + } + + @Override + protected Command nextCommand() { + if (nextCommand == null && rewrittenCommand != null) { + nextCommand = (Command)rewrittenCommand.clone(); + } + return super.nextCommand(); + } public Object clone() { DependentAccessNode clonedNode = new DependentAccessNode(super.getID()); clonedNode.maxSetSize = this.maxSetSize; + clonedNode.maxPredicates = this.maxPredicates; super.copy(this, clonedNode); return clonedNode; } @@ -81,6 +109,14 @@ public int getMaxSetSize() { return this.maxSetSize; } + + public int getMaxPredicates() { + return maxPredicates; + } + + public void setMaxPredicates(int maxPredicates) { + this.maxPredicates = maxPredicates; + } /** * @param maxSize @@ -100,7 +136,7 @@ Query query = (Query)atomicCommand; if (this.criteriaProcessor == null) { - this.criteriaProcessor = new DependentCriteriaProcessor(this.maxSetSize, this, query.getCriteria()); + this.criteriaProcessor = new DependentCriteriaProcessor(this.maxSetSize, this.maxPredicates, this, query.getCriteria()); } if (this.dependentCrit == null) { @@ -127,7 +163,7 @@ query.setOrderBy(null); } - boolean result = super.prepareNextCommand(query); + boolean result = RelationalNodeUtil.shouldExecute(atomicCommand, true); dependentCrit = null; Modified: trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentCriteriaProcessor.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentCriteriaProcessor.java 2011-03-29 21:52:50 UTC (rev 3049) +++ trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentCriteriaProcessor.java 2011-03-31 04:11:52 UTC (rev 3050) @@ -27,6 +27,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.LinkedList; @@ -81,6 +82,7 @@ public void sort() throws BlockedException, TeiidComponentException, TeiidProcessingException { if (dvs == null) { + //TODO: detect if we're already distinct if (sortUtility == null) { List<Expression> sortSymbols = new ArrayList<Expression>(dependentSetStates.size()); List<Boolean> sortDirection = new ArrayList<Boolean>(sortSymbols.size()); @@ -117,6 +119,7 @@ //constructor state private int maxSetSize; + private int maxPredicates; private RelationalNode dependentNode; private Criteria dependentCrit; @@ -133,8 +136,9 @@ private boolean hasNextCommand; protected SubqueryAwareEvaluator eval; - public DependentCriteriaProcessor(int maxSetSize, RelationalNode dependentNode, Criteria dependentCriteria) throws ExpressionEvaluationException, TeiidComponentException { + public DependentCriteriaProcessor(int maxSetSize, int maxPredicates, RelationalNode dependentNode, Criteria dependentCriteria) throws ExpressionEvaluationException, TeiidComponentException { this.maxSetSize = maxSetSize; + this.maxPredicates = maxPredicates; this.dependentNode = dependentNode; this.dependentCrit = dependentCriteria; this.eval = new SubqueryAwareEvaluator(Collections.emptyMap(), dependentNode.getDataManager(), dependentNode.getContext(), dependentNode.getBufferManager()); @@ -250,55 +254,82 @@ * @throws TeiidComponentException */ private void replaceDependentValueIterators() throws TeiidComponentException { + int totalPredicates = sources.size(); + if (this.maxPredicates > 0) { + //We have a bin packing problem if totalPredicates < sources - We'll address that case later. + //TODO: better handling for the correlated composite case + totalPredicates = Math.max(totalPredicates, this.maxPredicates); + } + long maxSize = Integer.MAX_VALUE; + if (this.maxSetSize > 0) { + maxSize = this.maxSetSize; + } + int currentPredicates = 0; + for (int run = 0; currentPredicates < totalPredicates; run++) { + currentPredicates = 0; + if (!restartIndexes.isEmpty()) { + currentIndex = restartIndexes.removeLast().intValue(); + } + for (int i = 0; i < sources.size(); i++) { - for (; currentIndex < sources.size(); currentIndex++) { + List<SetState> source = sources.get(i); - List<SetState> source = sources.get(currentIndex); - - boolean done = false; - - while (!done) { - - boolean isNull = false; - boolean lessThanMax = true; - - for (SetState state : source) { - if (state.nextValue == null && !state.isNull) { - if (state.valueIterator.hasNext()) { - state.nextValue = state.valueIterator.next(); - state.isNull = state.nextValue == null; - } else { - state.valueIterator.reset(); - done = true; // should be true for each iterator from this source - continue; - } - } - - isNull |= state.isNull; - lessThanMax &= state.replacement.size() < maxSetSize; - } - - if (done) { - if (!restartIndexes.isEmpty() && restartIndexes.getLast().intValue() == currentIndex) { - restartIndexes.removeLast(); - } - break; - } - - if (lessThanMax || isNull) { - for (SetState state : source) { - if (!isNull) { - state.replacement.add(state.nextValue); - } - state.nextValue = null; - state.isNull = false; - } - } else { - restartIndexes.add(currentIndex); - done = true; - } + if (i == currentIndex++) { + + boolean done = false; + + while (!done) { + + boolean isNull = false; + boolean lessThanMax = true; + + for (SetState state : source) { + if (state.nextValue == null && !state.isNull) { + if (state.valueIterator.hasNext()) { + state.nextValue = state.valueIterator.next(); + state.isNull = state.nextValue == null; + } else { + state.valueIterator.reset(); + done = true; // should be true for each iterator from this source + continue; + } + } + + isNull |= state.isNull; + lessThanMax &= state.replacement.size() < maxSize * (run + 1); + } + + if (done) { + if (!restartIndexes.isEmpty() && restartIndexes.getLast().intValue() == i) { + restartIndexes.removeLast(); + } + break; + } + + if (lessThanMax || isNull) { + for (SetState state : source) { + if (!isNull) { + state.replacement.add(state.nextValue); + } + state.nextValue = null; + state.isNull = false; + } + } else { + restartIndexes.add(i); + done = true; + } + } + } + + for (SetState setState : source) { + currentPredicates += setState.replacement.size()/maxSize+(setState.replacement.size()%maxSize!=0?1:0); + } + } + + if (restartIndexes.isEmpty()) { + break; } - } + } hasNextCommand = !restartIndexes.isEmpty(); } @@ -312,18 +343,36 @@ // No values - return criteria that is always false return QueryRewriter.FALSE_CRITERIA; } - if (state.replacement.size() == 1) { - return new CompareCriteria(crit.getExpression(), CompareCriteria.EQ, new Constant(state.replacement.iterator().next())); + int numberOfSets = 1; + int maxSize = Integer.MAX_VALUE; + if (this.maxSetSize > 0) { + maxSize = this.maxSetSize; + numberOfSets = state.replacement.size()/maxSize + (state.replacement.size()%maxSize!=0?1:0); } - List vals = new ArrayList(state.replacement.size()); - for (Object val : state.replacement) { - vals.add(new Constant(val)); - } - - SetCriteria sc = new SetCriteria(); - sc.setExpression(crit.getExpression()); - sc.setValues(vals); - return sc; + Iterator<Object> iter = state.replacement.iterator(); + ArrayList<Criteria> orCrits = new ArrayList<Criteria>(numberOfSets); + + for (int i = 0; i < numberOfSets; i++) { + if (maxSize == 1 || i + 1 == state.replacement.size()) { + orCrits.add(new CompareCriteria(crit.getExpression(), CompareCriteria.EQ, new Constant(iter.next()))); + } else { + List<Constant> vals = new ArrayList<Constant>(Math.min(state.replacement.size(), maxSize)); + + for (int j = 0; j < maxSize && iter.hasNext(); j++) { + Object val = iter.next(); + vals.add(new Constant(val)); + } + + SetCriteria sc = new SetCriteria(); + sc.setExpression(crit.getExpression()); + sc.setValues(vals); + orCrits.add(sc); + } + } + if (orCrits.size() == 1) { + return orCrits.get(0); + } + return new CompoundCriteria(CompoundCriteria.OR, orCrits); } } Modified: trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentProcedureCriteriaProcessor.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentProcedureCriteriaProcessor.java 2011-03-29 21:52:50 UTC (rev 3049) +++ trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentProcedureCriteriaProcessor.java 2011-03-31 04:11:52 UTC (rev 3050) @@ -50,7 +50,7 @@ Criteria dependentCriteria, List references, List defaults) throws ExpressionEvaluationException, TeiidComponentException { - super(1, dependentNode, dependentCriteria); + super(1, -1, dependentNode, dependentCriteria); this.inputDefaults = defaults; this.inputReferences = references; } Modified: trunk/engine/src/main/java/org/teiid/query/rewriter/QueryRewriter.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/rewriter/QueryRewriter.java 2011-03-29 21:52:50 UTC (rev 3049) +++ trunk/engine/src/main/java/org/teiid/query/rewriter/QueryRewriter.java 2011-03-31 04:11:52 UTC (rev 3050) @@ -67,7 +67,6 @@ import org.teiid.query.optimizer.relational.rules.RuleMergeCriteria; import org.teiid.query.optimizer.relational.rules.RulePlaceAccess; import org.teiid.query.optimizer.relational.rules.RuleMergeCriteria.PlannedResult; -import org.teiid.query.processor.relational.DependentValueSource; import org.teiid.query.processor.relational.RelationalNodeUtil; import org.teiid.query.resolver.ProcedureContainerResolver; import org.teiid.query.resolver.QueryResolver; @@ -152,7 +151,6 @@ import org.teiid.query.sql.symbol.SingleElementSymbol; import org.teiid.query.sql.symbol.AggregateSymbol.Type; import org.teiid.query.sql.util.SymbolMap; -import org.teiid.query.sql.util.ValueIterator; import org.teiid.query.sql.visitor.AggregateSymbolCollectorVisitor; import org.teiid.query.sql.visitor.CorrelatedReferenceCollectorVisitor; import org.teiid.query.sql.visitor.CriteriaTranslatorVisitor; @@ -1222,26 +1220,8 @@ if (rewriteLeftExpression(dsc)) { return UNKNOWN_CRITERIA; } - return dsc; } - SetCriteria setCrit = new SetCriteria(); - setCrit.setExpression(dsc.getExpression()); - HashSet<Object> values = new HashSet<Object>(); - try { - DependentValueSource dvs = (DependentValueSource)this.context.getVariableContext().getGlobalValue(dsc.getContextSymbol()); - ValueIterator iter = dvs.getValueIterator(dsc.getValueExpression()); - while (iter.hasNext()) { - values.add(iter.next()); - } - } catch (TeiidComponentException e) { - throw new TeiidRuntimeException(e); - } - List<Constant> constants = new ArrayList<Constant>(values.size()); - for (Object value : values) { - constants.add(new Constant(value, setCrit.getExpression().getType())); - } - setCrit.setValues(constants); - return rewriteCriteria(setCrit); + return dsc; } /** Modified: trunk/engine/src/main/java/org/teiid/query/sql/lang/Create.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/sql/lang/Create.java 2011-03-29 21:52:50 UTC (rev 3049) +++ trunk/engine/src/main/java/org/teiid/query/sql/lang/Create.java 2011-03-31 04:11:52 UTC (rev 3050) @@ -72,6 +72,7 @@ for (Column column : columns) { ElementSymbol es = new ElementSymbol(column.getName()); es.setType(DataTypeManager.getDataTypeClass(column.getRuntimeType())); + es.setGroupSymbol(table); columnSymbols.add(es); } } Modified: trunk/engine/src/main/java/org/teiid/query/tempdata/TempTable.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/tempdata/TempTable.java 2011-03-29 21:52:50 UTC (rev 3049) +++ trunk/engine/src/main/java/org/teiid/query/tempdata/TempTable.java 2011-03-31 04:11:52 UTC (rev 3050) @@ -194,7 +194,6 @@ private abstract class UpdateProcessor { private TupleSource ts; - protected final Map lookup; protected final Evaluator eval; private final Criteria crit; protected int updateCount = 0; @@ -204,8 +203,7 @@ UpdateProcessor(Criteria crit, TupleSource ts) throws TeiidComponentException { this.ts = ts; - this.lookup = RelationalNode.createLookupMap(columns); - this.eval = new Evaluator(lookup, null, null); + this.eval = new Evaluator(columnMap, null, null); this.crit = crit; this.undoLog = bm.createTupleBuffer(columns, sessionID, TupleSourceType.PROCESSOR); } @@ -530,7 +528,7 @@ BlockedException, TeiidComponentException { List<Object> newTuple = new ArrayList<Object>(tuple); for (Map.Entry<ElementSymbol, Expression> entry : update.getClauseMap().entrySet()) { - newTuple.set((Integer)lookup.get(entry.getKey()), eval.evaluate(entry.getValue(), tuple)); + newTuple.set((Integer)columnMap.get(entry.getKey()), eval.evaluate(entry.getValue(), tuple)); } if (primaryKeyChangePossible) { browser.removed(); Modified: trunk/engine/src/test/java/org/teiid/query/optimizer/TestUnionPlanning.java =================================================================== --- trunk/engine/src/test/java/org/teiid/query/optimizer/TestUnionPlanning.java 2011-03-29 21:52:50 UTC (rev 3049) +++ trunk/engine/src/test/java/org/teiid/query/optimizer/TestUnionPlanning.java 2011-03-31 04:11:52 UTC (rev 3050) @@ -121,7 +121,7 @@ 0, // Project 0, // Select 0, // Sort - 2 // UnionAll + 1 // UnionAll }); } Modified: trunk/engine/src/test/java/org/teiid/query/processor/TestDependentJoins.java =================================================================== --- trunk/engine/src/test/java/org/teiid/query/processor/TestDependentJoins.java 2011-03-29 21:52:50 UTC (rev 3049) +++ trunk/engine/src/test/java/org/teiid/query/processor/TestDependentJoins.java 2011-03-31 04:11:52 UTC (rev 3050) @@ -43,7 +43,7 @@ import org.teiid.query.unittest.FakeMetadataObject; import org.teiid.query.util.CommandContext; -@SuppressWarnings({"unchecked"}) +@SuppressWarnings({"unchecked", "nls"}) public class TestDependentJoins { /** @@ -529,8 +529,8 @@ new String[] {"SELECT g_0.stringkey, g_0.intkey FROM bqt1.smalla AS g_0 WHERE g_0.intkey IN (<dependent values>)", "SELECT g_0.stringkey, g_0.intkey FROM bqt2.smallb AS g_0"}, TestOptimizer.ComparisonMode.EXACT_COMMAND_STRING); //$NON-NLS-1$ //$NON-NLS-2$ TestOptimizer.checkNodeTypes(plan, new int[] { - unlimitIn?2:1, // Access - unlimitIn?0:1, // DependentAccess + 1, // Access + 1, // DependentAccess 0, // DependentSelect 0, // DependentProject 0, // DupRemove @@ -633,8 +633,30 @@ } @Test public void testLargeSetInDepAccessMultiJoinCriteria() throws Exception { + helpTestLargeSetInDepAccessMultiJoinCriteria(1, -1, 1, 2); + } + + @Test public void testLargeSetInDepAccessMultiJoinCriteriaConcurrent() throws Exception { + //allows concurrent + helpTestLargeSetInDepAccessMultiJoinCriteria(1, -1, 4, 4); + } + + @Test public void testLargeSetInDepAccessMultiJoinCriteriaCompound() throws Exception { + //max predicates forces multiple queries + helpTestLargeSetInDepAccessMultiJoinCriteria(1, 4, 3, 3); + } + + @Test public void testLargeSetInDepAccessMultiJoinCriteriaCompoundAll() throws Exception { + //max predicates allows a one shot + helpTestLargeSetInDepAccessMultiJoinCriteria(1, 10, 2, 2); + } + + /** + * concurrentOpen will be minimum of 2 to gather the pm1 results. + */ + public void helpTestLargeSetInDepAccessMultiJoinCriteria(int maxInSize, int maxPredicates, int maxConcurrency, int concurrentOpen) throws Exception { // Create query - String sql = "SELECT pm1.g1.e1 FROM pm1.g1, pm2.g1 WHERE pm1.g1.e1=pm2.g1.e1 AND pm1.g1.e2=pm2.g1.e2 order by e1 OPTION MAKEDEP pm2.g1"; //$NON-NLS-1$ + String sql = "SELECT pm1.g1.e1 FROM (pm1.g2 cross join pm1.g1) inner join pm2.g1 makedep ON pm1.g1.e1=pm2.g1.e1 AND pm1.g1.e2=pm2.g1.e2 AND pm1.g2.e4 = pm2.g1.e4 order by e1"; //$NON-NLS-1$ // Construct data manager with data FakeDataManager dataManager = new FakeDataManager(); TestProcessor.sampleData1(dataManager); @@ -647,7 +669,8 @@ FakeCapabilitiesFinder capFinder = new FakeCapabilitiesFinder(); BasicSourceCapabilities depcaps = new BasicSourceCapabilities(); depcaps.setCapabilitySupport(Capability.CRITERIA_IN, true); - depcaps.setSourceProperty(Capability.MAX_IN_CRITERIA_SIZE, new Integer(1)); + depcaps.setSourceProperty(Capability.MAX_IN_CRITERIA_SIZE, maxInSize); + depcaps.setSourceProperty(Capability.MAX_DEPENDENT_PREDICATES, maxPredicates); BasicSourceCapabilities caps = new BasicSourceCapabilities(); caps.setCapabilitySupport(Capability.CRITERIA_IN, true); @@ -656,30 +679,32 @@ capFinder.addCapabilities("pm2", depcaps); //$NON-NLS-1$ List[] expected = new List[] { - Arrays.asList(new Object[] { - new String("a")}), //$NON-NLS-1$ - Arrays.asList(new Object[] { - new String("a")}), //$NON-NLS-1$ - Arrays.asList(new Object[] { - new String("a")}), //$NON-NLS-1$ - Arrays.asList(new Object[] { - new String("a")}), //$NON-NLS-1$ - Arrays.asList(new Object[] { - new String("a")}), //$NON-NLS-1$ - Arrays.asList(new Object[] { - new String("b")}), //$NON-NLS-1$ - Arrays.asList(new Object[] { - new String("c")})}; //$NON-NLS-1$ + Arrays.asList("a"), //$NON-NLS-1$ + Arrays.asList("a"), //$NON-NLS-1$ + Arrays.asList("a"), //$NON-NLS-1$ + Arrays.asList("a"), //$NON-NLS-1$ + Arrays.asList("a"), //$NON-NLS-1$ + Arrays.asList("a"), //$NON-NLS-1$ + Arrays.asList("a"), //$NON-NLS-1$ + Arrays.asList("a"), //$NON-NLS-1$ + Arrays.asList("a"), //$NON-NLS-1$ + Arrays.asList("b"), //$NON-NLS-1$ + }; Command command = TestProcessor.helpParse(sql); ProcessorPlan plan = TestProcessor.helpGetPlan(command, fakeMetadata, capFinder); + TestOptimizer.checkAtomicQueries(new String[] { + "SELECT pm1.g2.e4 FROM pm1.g2", + "SELECT pm2.g1.e1, pm2.g1.e2, pm2.g1.e4 FROM pm2.g1 WHERE (pm2.g1.e1 IN (<dependent values>)) AND (pm2.g1.e2 IN (<dependent values>)) AND (pm2.g1.e4 IN (<dependent values>))", + "SELECT pm1.g1.e1, pm1.g1.e2 FROM pm1.g1" + }, plan); CommandContext cc = TestProcessor.createCommandContext(); - cc.setUserRequestSourceConcurrency(5); + cc.setUserRequestSourceConcurrency(maxConcurrency); FakeTupleSource.resetStats(); // Run query TestProcessor.helpProcess(plan, cc, dataManager, expected); - assertEquals(4, FakeTupleSource.maxOpen); + assertEquals("Wrong number of concurrent source queries", concurrentOpen, FakeTupleSource.maxOpen); } @Test public void testLargeSetInDepAccessWithAccessPattern() { Modified: trunk/engine/src/test/java/org/teiid/query/processor/relational/TestDependentCriteriaProcessor.java =================================================================== --- trunk/engine/src/test/java/org/teiid/query/processor/relational/TestDependentCriteriaProcessor.java 2011-03-29 21:52:50 UTC (rev 3049) +++ trunk/engine/src/test/java/org/teiid/query/processor/relational/TestDependentCriteriaProcessor.java 2011-03-31 04:11:52 UTC (rev 3050) @@ -22,13 +22,12 @@ package org.teiid.query.processor.relational; +import static org.junit.Assert.*; + import java.util.Arrays; import java.util.List; -import static org.junit.Assert.*; import org.junit.Test; -import org.teiid.query.processor.relational.DependentAccessNode; -import org.teiid.query.processor.relational.DependentCriteriaProcessor; import org.teiid.query.sql.lang.CompareCriteria; import org.teiid.query.sql.lang.Criteria; import org.teiid.query.sql.lang.SetCriteria; @@ -43,7 +42,7 @@ @Test public void testNegatedSetCriteria() throws Exception { DependentAccessNode dan = new DependentAccessNode(0); SetCriteria sc = new SetCriteria(new ElementSymbol("e1"), Arrays.asList(new Constant(1), new Constant(2))); //$NON-NLS-1$ - DependentCriteriaProcessor dcp = new DependentCriteriaProcessor(1, dan, sc); + DependentCriteriaProcessor dcp = new DependentCriteriaProcessor(1, -1, dan, sc); Criteria result = dcp.prepareCriteria(); assertEquals(new CompareCriteria(new ElementSymbol("e1"), CompareCriteria.EQ, new Constant(1)), result); //$NON-NLS-1$ assertTrue(dcp.hasNextCommand()); @@ -58,7 +57,7 @@ cc.getVariableContext().setGlobalValue(reference.getContextSymbol(), 1); } SetCriteria sc = new SetCriteria(new ElementSymbol("e1"), references); //$NON-NLS-1$ - DependentCriteriaProcessor dcp = new DependentCriteriaProcessor(1, dan, sc); + DependentCriteriaProcessor dcp = new DependentCriteriaProcessor(1, -1, dan, sc); Criteria result = dcp.prepareCriteria(); assertEquals(new CompareCriteria(new ElementSymbol("e1"), CompareCriteria.EQ, new Constant(1)), result); //$NON-NLS-1$ assertFalse(dcp.hasNextCommand()); Modified: trunk/engine/src/test/java/org/teiid/query/processor/xml/TestXMLPlanningEnhancements.java =================================================================== --- trunk/engine/src/test/java/org/teiid/query/processor/xml/TestXMLPlanningEnhancements.java 2011-03-29 21:52:50 UTC (rev 3049) +++ trunk/engine/src/test/java/org/teiid/query/processor/xml/TestXMLPlanningEnhancements.java 2011-03-31 04:11:52 UTC (rev 3050) @@ -496,8 +496,8 @@ ProcessorPlan plan = instr.info.getPlan(); TestOptimizer.checkNodeTypes(plan, new int[] { - 2, // Access - 0, // DependentAccess + 1, // Access + 1, // DependentAccess 0, // DependentSelect 0, // DependentProject 0, // DupRemove