[teiid-commits] teiid SVN: r3050 - in trunk: connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc and 16 other directories.
teiid-commits at lists.jboss.org
teiid-commits at lists.jboss.org
Thu Mar 31 00:11:52 EDT 2011
Author: shawkins
Date: 2011-03-31 00:11:52 -0400 (Thu, 31 Mar 2011)
New Revision: 3050
Modified:
trunk/api/src/main/java/org/teiid/translator/ExecutionFactory.java
trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/JDBCExecutionFactory.java
trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/SimpleJDBCExecutionFactory.java
trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/access/AccessExecutionFactory.java
trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/sqlserver/SQLServerExecutionFactory.java
trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/sybase/SybaseExecutionFactory.java
trunk/connectors/translator-ldap/src/main/java/org/teiid/translator/ldap/LDAPExecutionFactory.java
trunk/engine/src/main/java/org/teiid/dqp/internal/datamgr/CapabilitiesConverter.java
trunk/engine/src/main/java/org/teiid/query/optimizer/capabilities/SourceCapabilities.java
trunk/engine/src/main/java/org/teiid/query/optimizer/relational/PlanToProcessConverter.java
trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/CapabilitiesUtil.java
trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RulePushSelectCriteria.java
trunk/engine/src/main/java/org/teiid/query/processor/relational/AccessNode.java
trunk/engine/src/main/java/org/teiid/query/processor/relational/BatchedUpdateNode.java
trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentAccessNode.java
trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentCriteriaProcessor.java
trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentProcedureCriteriaProcessor.java
trunk/engine/src/main/java/org/teiid/query/rewriter/QueryRewriter.java
trunk/engine/src/main/java/org/teiid/query/sql/lang/Create.java
trunk/engine/src/main/java/org/teiid/query/tempdata/TempTable.java
trunk/engine/src/test/java/org/teiid/query/optimizer/TestUnionPlanning.java
trunk/engine/src/test/java/org/teiid/query/processor/TestDependentJoins.java
trunk/engine/src/test/java/org/teiid/query/processor/relational/TestDependentCriteriaProcessor.java
trunk/engine/src/test/java/org/teiid/query/processor/xml/TestXMLPlanningEnhancements.java
Log:
TEIID-1533 adding the ability to split dependent join values into multiple in predicates
Modified: trunk/api/src/main/java/org/teiid/translator/ExecutionFactory.java
===================================================================
--- trunk/api/src/main/java/org/teiid/translator/ExecutionFactory.java 2011-03-29 21:52:50 UTC (rev 3049)
+++ trunk/api/src/main/java/org/teiid/translator/ExecutionFactory.java 2011-03-31 04:11:52 UTC (rev 3050)
@@ -106,6 +106,7 @@
private boolean supportsFullOuterJoins;
private boolean requiresCriteria;
private int maxInSize = DEFAULT_MAX_IN_CRITERIA_SIZE;
+ private int maxDependentInPredicates = DEFAULT_MAX_IN_CRITERIA_SIZE;
/**
* Initialize the connector with supplied configuration
@@ -667,8 +668,22 @@
public void setMaxInCriteriaSize(int maxInSize) {
this.maxInSize = maxInSize;
}
-
+
/**
+ * Get the integer value representing the number of values allowed in an IN criteria
+ * in the WHERE clause of a query
+ * @since 5.0
+ */
+ @TranslatorProperty(display="Max number of dependent values across all IN predicates", advanced=true)
+ public int getMaxDependentInPredicates() {
+ return maxDependentInPredicates;
+ }
+
+ public void setMaxDependentPredicates(int maxDependentInPredicates) {
+ this.maxDependentInPredicates = maxDependentInPredicates;
+ }
+
+ /**
* <p>Support indicates that the connector supports functions in GROUP BY, such as:
* <code>SELECT dayofmonth(theDate), COUNT(*) FROM table GROUP BY dayofmonth(theDate)</code></p>
*
Modified: trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/JDBCExecutionFactory.java
===================================================================
--- trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/JDBCExecutionFactory.java 2011-03-29 21:52:50 UTC (rev 3049)
+++ trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/JDBCExecutionFactory.java 2011-03-31 04:11:52 UTC (rev 3050)
@@ -81,6 +81,7 @@
public class JDBCExecutionFactory extends ExecutionFactory<DataSource, Connection> {
public static final int DEFAULT_MAX_IN_CRITERIA = 1000;
+ public static final int DEFAULT_MAX_DEPENDENT_PREDICATES = 50;
// Because the retrieveValue() method will be hit for every value of
// every JDBC result set returned, we do lots of weird special stuff here
@@ -149,6 +150,8 @@
setSupportsOuterJoins(true);
setSupportsSelectDistinct(true);
setSupportsInnerJoins(true);
+ setMaxInCriteriaSize(DEFAULT_MAX_IN_CRITERIA);
+ setMaxDependentPredicates(DEFAULT_MAX_DEPENDENT_PREDICATES);
}
@Override
Modified: trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/SimpleJDBCExecutionFactory.java
===================================================================
--- trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/SimpleJDBCExecutionFactory.java 2011-03-29 21:52:50 UTC (rev 3049)
+++ trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/SimpleJDBCExecutionFactory.java 2011-03-31 04:11:52 UTC (rev 3050)
@@ -41,6 +41,7 @@
setSupportsFullOuterJoins(false);
setSupportsOrderBy(false);
setMaxInCriteriaSize(250);
+ setMaxDependentPredicates(10);
}
@Override
Modified: trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/access/AccessExecutionFactory.java
===================================================================
--- trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/access/AccessExecutionFactory.java 2011-03-29 21:52:50 UTC (rev 3049)
+++ trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/access/AccessExecutionFactory.java 2011-03-31 04:11:52 UTC (rev 3050)
@@ -40,6 +40,7 @@
setSupportsOrderBy(false);
setDatabaseVersion("2003"); //$NON-NLS-1$
setMaxInCriteriaSize(JDBCExecutionFactory.DEFAULT_MAX_IN_CRITERIA);
+ setMaxDependentPredicates(10); //sql length length is 64k
}
@Override
Modified: trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/sqlserver/SQLServerExecutionFactory.java
===================================================================
--- trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/sqlserver/SQLServerExecutionFactory.java 2011-03-29 21:52:50 UTC (rev 3049)
+++ trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/sqlserver/SQLServerExecutionFactory.java 2011-03-31 04:11:52 UTC (rev 3050)
@@ -51,6 +51,7 @@
public SQLServerExecutionFactory() {
setDatabaseVersion(V_2005);
setMaxInCriteriaSize(JDBCExecutionFactory.DEFAULT_MAX_IN_CRITERIA);
+ setMaxDependentPredicates(JDBCExecutionFactory.DEFAULT_MAX_DEPENDENT_PREDICATES);
}
@Override
Modified: trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/sybase/SybaseExecutionFactory.java
===================================================================
--- trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/sybase/SybaseExecutionFactory.java 2011-03-29 21:52:50 UTC (rev 3049)
+++ trunk/connectors/translator-jdbc/src/main/java/org/teiid/translator/jdbc/sybase/SybaseExecutionFactory.java 2011-03-31 04:11:52 UTC (rev 3050)
@@ -65,6 +65,7 @@
setDatabaseVersion(TWELVE_5);
setSupportsFullOuterJoins(false);
setMaxInCriteriaSize(250);
+ setMaxDependentPredicates(10);
}
public void start() throws TranslatorException {
Modified: trunk/connectors/translator-ldap/src/main/java/org/teiid/translator/ldap/LDAPExecutionFactory.java
===================================================================
--- trunk/connectors/translator-ldap/src/main/java/org/teiid/translator/ldap/LDAPExecutionFactory.java 2011-03-29 21:52:50 UTC (rev 3049)
+++ trunk/connectors/translator-ldap/src/main/java/org/teiid/translator/ldap/LDAPExecutionFactory.java 2011-03-31 04:11:52 UTC (rev 3050)
@@ -56,6 +56,7 @@
public LDAPExecutionFactory() {
this.setMaxInCriteriaSize(1000);
+ this.setMaxDependentPredicates(25); //no spec limit on query size, AD is 10MB for the query
}
@TranslatorProperty(display="Default Search Base DN", description="Default Base DN for LDAP Searches")
Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/datamgr/CapabilitiesConverter.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/dqp/internal/datamgr/CapabilitiesConverter.java 2011-03-29 21:52:50 UTC (rev 3049)
+++ trunk/engine/src/main/java/org/teiid/dqp/internal/datamgr/CapabilitiesConverter.java 2011-03-31 04:11:52 UTC (rev 3050)
@@ -115,6 +115,7 @@
}
tgtCaps.setSourceProperty(Capability.MAX_IN_CRITERIA_SIZE, new Integer(srcCaps.getMaxInCriteriaSize()));
+ tgtCaps.setSourceProperty(Capability.MAX_DEPENDENT_PREDICATES, new Integer(srcCaps.getMaxDependentInPredicates()));
tgtCaps.setSourceProperty(Capability.CONNECTOR_ID, connectorID);
tgtCaps.setSourceProperty(Capability.MAX_QUERY_FROM_GROUPS, new Integer(srcCaps.getMaxFromGroups()));
tgtCaps.setSourceProperty(Capability.JOIN_CRITERIA_ALLOWED, srcCaps.getSupportedJoinCriteria());
Modified: trunk/engine/src/main/java/org/teiid/query/optimizer/capabilities/SourceCapabilities.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/query/optimizer/capabilities/SourceCapabilities.java 2011-03-29 21:52:50 UTC (rev 3049)
+++ trunk/engine/src/main/java/org/teiid/query/optimizer/capabilities/SourceCapabilities.java 2011-03-31 04:11:52 UTC (rev 3050)
@@ -319,7 +319,8 @@
REQUIRES_CRITERIA,
INSERT_WITH_QUERYEXPRESSION,
INSERT_WITH_ITERATOR,
- COMMON_TABLE_EXPRESSIONS
+ COMMON_TABLE_EXPRESSIONS,
+ MAX_DEPENDENT_PREDICATES,
}
public enum Scope {
Modified: trunk/engine/src/main/java/org/teiid/query/optimizer/relational/PlanToProcessConverter.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/query/optimizer/relational/PlanToProcessConverter.java 2011-03-29 21:52:50 UTC (rev 3049)
+++ trunk/engine/src/main/java/org/teiid/query/optimizer/relational/PlanToProcessConverter.java 2011-03-31 04:11:52 UTC (rev 3050)
@@ -152,7 +152,16 @@
// Call convertPlan recursively on children
for (PlanNode childNode : planNode.getChildren()) {
- nextParent.addChild(convertPlan(childNode));
+ RelationalNode child = convertPlan(childNode);
+ if (planNode.getType() == NodeConstants.Types.SET_OP && childNode.getType() == NodeConstants.Types.SET_OP && childNode.hasBooleanProperty(Info.USE_ALL)) {
+ for (RelationalNode grandChild : child.getChildren()) {
+ if (grandChild != null) {
+ nextParent.addChild(grandChild);
+ }
+ }
+ } else {
+ nextParent.addChild(child);
+ }
}
// Return root of tree for top node
@@ -297,7 +306,8 @@
DependentAccessNode depAccessNode = new DependentAccessNode(getID());
if(modelID != null){
- depAccessNode.setMaxSetSize(CapabilitiesUtil.getMaxInCriteriaSize(modelID, metadata, capFinder));
+ depAccessNode.setMaxSetSize(CapabilitiesUtil.getMaxInCriteriaSize(modelID, metadata, capFinder));
+ depAccessNode.setMaxPredicates(CapabilitiesUtil.getMaxDependentPredicatesSize(modelID, metadata, capFinder));
}
processNode = depAccessNode;
aNode = depAccessNode;
Modified: trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/CapabilitiesUtil.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/CapabilitiesUtil.java 2011-03-29 21:52:50 UTC (rev 3049)
+++ trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/CapabilitiesUtil.java 2011-03-31 04:11:52 UTC (rev 3050)
@@ -296,6 +296,11 @@
public static int getMaxInCriteriaSize(Object modelID, QueryMetadataInterface metadata, CapabilitiesFinder capFinder)
throws QueryMetadataException, TeiidComponentException {
+ return getProperty(Capability.MAX_IN_CRITERIA_SIZE, modelID, metadata, capFinder);
+ }
+
+ private static int getProperty(Capability cap, Object modelID, QueryMetadataInterface metadata, CapabilitiesFinder capFinder)
+ throws QueryMetadataException, TeiidComponentException {
if (metadata.isVirtualModel(modelID)){
return -1;
@@ -303,7 +308,7 @@
// Find capabilities
SourceCapabilities caps = getCapabilities(modelID, metadata, capFinder);
- Object maxInCriteriaSize = caps.getSourceProperty(Capability.MAX_IN_CRITERIA_SIZE);
+ Object maxInCriteriaSize = caps.getSourceProperty(cap);
int value = -1;
if(maxInCriteriaSize != null) {
value = ((Integer)maxInCriteriaSize).intValue();
@@ -316,6 +321,11 @@
return value;
}
+ public static int getMaxDependentPredicatesSize(Object modelID, QueryMetadataInterface metadata, CapabilitiesFinder capFinder)
+ throws QueryMetadataException, TeiidComponentException {
+ return getProperty(Capability.MAX_DEPENDENT_PREDICATES, modelID, metadata, capFinder);
+ }
+
public static int getMaxFromGroups(Object modelID, QueryMetadataInterface metadata, CapabilitiesFinder capFinder)
throws QueryMetadataException, TeiidComponentException {
Modified: trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RulePushSelectCriteria.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RulePushSelectCriteria.java 2011-03-29 21:52:50 UTC (rev 3049)
+++ trunk/engine/src/main/java/org/teiid/query/optimizer/relational/rules/RulePushSelectCriteria.java 2011-03-31 04:11:52 UTC (rev 3050)
@@ -299,8 +299,7 @@
satisfyAccessPatterns(critNode, currentNode);
}
- if (critNode.hasBooleanProperty(NodeConstants.Info.IS_DEPENDENT_SET)
- && CapabilitiesUtil.getMaxInCriteriaSize(RuleRaiseAccess.getModelIDFromAccess(currentNode, metadata), metadata, capFinder) > 0) {
+ if (critNode.hasBooleanProperty(NodeConstants.Info.IS_DEPENDENT_SET)) {
//once a dependent crit node is pushed, don't bother pushing it further into the command
//dependent access node will use this as an assumption for where dependent sets can appear in the command
critNode.setProperty(NodeConstants.Info.IS_PUSHED, Boolean.TRUE);
Modified: trunk/engine/src/main/java/org/teiid/query/processor/relational/AccessNode.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/query/processor/relational/AccessNode.java 2011-03-29 21:52:50 UTC (rev 3049)
+++ trunk/engine/src/main/java/org/teiid/query/processor/relational/AccessNode.java 2011-03-31 04:11:52 UTC (rev 3050)
@@ -28,7 +28,6 @@
import java.util.Collections;
import java.util.List;
-import org.teiid.api.exception.query.ExpressionEvaluationException;
import org.teiid.api.exception.query.QueryValidatorException;
import org.teiid.client.plan.PlanNode;
import org.teiid.common.buffer.BlockedException;
@@ -57,7 +56,7 @@
private ArrayList<TupleSource> tupleSources = new ArrayList<TupleSource>();
private boolean isUpdate = false;
private boolean returnedRows = false;
- private Command nextCommand;
+ protected Command nextCommand;
protected AccessNode() {
super();
@@ -101,9 +100,13 @@
// Copy command and resolve references if necessary
Command atomicCommand = command;
boolean needProcessing = true;
+
+ if (shouldEvaluate) {
+ atomicCommand = initialCommand();
+ }
do {
- if(shouldEvaluate) {
- atomicCommand = nextCommand();
+ atomicCommand = nextCommand();
+ if(shouldEvaluate) {
needProcessing = prepareNextCommand(atomicCommand);
nextCommand = null;
} else {
@@ -122,7 +125,22 @@
} while (!processCommandsIndividually() && hasNextCommand() && this.tupleSources.size() < Math.min(MAX_CONCURRENT, this.getContext().getUserRequestSourceConcurrency()));
}
- private Command nextCommand() {
+ static void rewriteAndEvaluate(Command atomicCommand, Evaluator eval, CommandContext context, QueryMetadataInterface metadata)
+ throws TeiidProcessingException, TeiidComponentException {
+ try {
+ // Defect 16059 - Rewrite the command to replace references, etc. with values.
+ QueryRewriter.evaluateAndRewrite(atomicCommand, eval, context, metadata);
+ } catch (QueryValidatorException e) {
+ throw new TeiidProcessingException(e, QueryPlugin.Util.getString("AccessNode.rewrite_failed", atomicCommand)); //$NON-NLS-1$
+ }
+ }
+
+ @SuppressWarnings("unused")
+ protected Command initialCommand() throws TeiidProcessingException, TeiidComponentException {
+ return nextCommand();
+ }
+
+ protected Command nextCommand() {
//it's important to save the next command
//to ensure that the subquery ids remain stable
if (nextCommand == null) {
@@ -132,22 +150,10 @@
}
protected boolean prepareNextCommand(Command atomicCommand) throws TeiidComponentException, TeiidProcessingException {
- return prepareCommand(atomicCommand, getEvaluator(Collections.emptyMap()), this.getContext(), this.getContext().getMetadata());
+ rewriteAndEvaluate(atomicCommand, getEvaluator(Collections.emptyMap()), this.getContext(), this.getContext().getMetadata());
+ return RelationalNodeUtil.shouldExecute(atomicCommand, true);
}
- static boolean prepareCommand(Command atomicCommand, Evaluator eval, CommandContext context, QueryMetadataInterface metadata)
- throws ExpressionEvaluationException, TeiidComponentException,
- TeiidProcessingException {
- try {
- // Defect 16059 - Rewrite the command once the references have been replaced with values.
- QueryRewriter.evaluateAndRewrite(atomicCommand, eval, context, metadata);
- } catch (QueryValidatorException e) {
- throw new TeiidProcessingException(e, QueryPlugin.Util.getString("AccessNode.rewrite_failed", atomicCommand)); //$NON-NLS-1$
- }
-
- return RelationalNodeUtil.shouldExecute(atomicCommand, true);
- }
-
public TupleBatch nextBatchDirect()
throws BlockedException, TeiidComponentException, TeiidProcessingException {
Modified: trunk/engine/src/main/java/org/teiid/query/processor/relational/BatchedUpdateNode.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/query/processor/relational/BatchedUpdateNode.java 2011-03-29 21:52:50 UTC (rev 3049)
+++ trunk/engine/src/main/java/org/teiid/query/processor/relational/BatchedUpdateNode.java 2011-03-31 04:11:52 UTC (rev 3050)
@@ -102,10 +102,9 @@
updateCommand = (Command) updateCommand.clone();
Evaluator eval = getEvaluator(Collections.emptyMap());
eval.initialize(context, getDataManager());
- needProcessing = AccessNode.prepareCommand(updateCommand, eval, context, context.getMetadata());
- } else {
- needProcessing = RelationalNodeUtil.shouldExecute(updateCommand, true);
+ AccessNode.rewriteAndEvaluate(updateCommand, eval, context, context.getMetadata());
}
+ needProcessing = RelationalNodeUtil.shouldExecute(updateCommand, true);
if (needProcessing) {
commandsToExecute.add(updateCommand);
} else {
Modified: trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentAccessNode.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentAccessNode.java 2011-03-29 21:52:50 UTC (rev 3049)
+++ trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentAccessNode.java 2011-03-31 04:11:52 UTC (rev 3050)
@@ -22,6 +22,8 @@
package org.teiid.query.processor.relational;
+import java.util.Collections;
+
import org.teiid.core.TeiidComponentException;
import org.teiid.core.TeiidProcessingException;
import org.teiid.core.util.Assertion;
@@ -40,12 +42,17 @@
//plan state
private int maxSetSize;
+ private int maxPredicates;
//processing state
private DependentCriteriaProcessor criteriaProcessor;
private Criteria dependentCrit;
private boolean sort = true;
-
+ /**
+ * Cached rewritten command to be used as the base for all dependent queries.
+ */
+ private Command rewrittenCommand;
+
public DependentAccessNode(int nodeID) {
super(nodeID);
}
@@ -66,11 +73,32 @@
criteriaProcessor = null;
dependentCrit = null;
sort = true;
+ rewrittenCommand = null;
}
+
+ @Override
+ protected Command initialCommand() throws TeiidProcessingException, TeiidComponentException {
+ if (rewrittenCommand == null) {
+ Command atomicCommand = nextCommand();
+ rewriteAndEvaluate(atomicCommand, getEvaluator(Collections.emptyMap()), this.getContext(), this.getContext().getMetadata());
+ rewrittenCommand = atomicCommand;
+ nextCommand = null;
+ }
+ return rewrittenCommand;
+ }
+
+ @Override
+ protected Command nextCommand() {
+ if (nextCommand == null && rewrittenCommand != null) {
+ nextCommand = (Command)rewrittenCommand.clone();
+ }
+ return super.nextCommand();
+ }
public Object clone() {
DependentAccessNode clonedNode = new DependentAccessNode(super.getID());
clonedNode.maxSetSize = this.maxSetSize;
+ clonedNode.maxPredicates = this.maxPredicates;
super.copy(this, clonedNode);
return clonedNode;
}
@@ -81,6 +109,14 @@
public int getMaxSetSize() {
return this.maxSetSize;
}
+
+ public int getMaxPredicates() {
+ return maxPredicates;
+ }
+
+ public void setMaxPredicates(int maxPredicates) {
+ this.maxPredicates = maxPredicates;
+ }
/**
* @param maxSize
@@ -100,7 +136,7 @@
Query query = (Query)atomicCommand;
if (this.criteriaProcessor == null) {
- this.criteriaProcessor = new DependentCriteriaProcessor(this.maxSetSize, this, query.getCriteria());
+ this.criteriaProcessor = new DependentCriteriaProcessor(this.maxSetSize, this.maxPredicates, this, query.getCriteria());
}
if (this.dependentCrit == null) {
@@ -127,7 +163,7 @@
query.setOrderBy(null);
}
- boolean result = super.prepareNextCommand(query);
+ boolean result = RelationalNodeUtil.shouldExecute(atomicCommand, true);
dependentCrit = null;
Modified: trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentCriteriaProcessor.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentCriteriaProcessor.java 2011-03-29 21:52:50 UTC (rev 3049)
+++ trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentCriteriaProcessor.java 2011-03-31 04:11:52 UTC (rev 3050)
@@ -27,6 +27,7 @@
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.LinkedList;
@@ -81,6 +82,7 @@
public void sort() throws BlockedException,
TeiidComponentException, TeiidProcessingException {
if (dvs == null) {
+ //TODO: detect if we're already distinct
if (sortUtility == null) {
List<Expression> sortSymbols = new ArrayList<Expression>(dependentSetStates.size());
List<Boolean> sortDirection = new ArrayList<Boolean>(sortSymbols.size());
@@ -117,6 +119,7 @@
//constructor state
private int maxSetSize;
+ private int maxPredicates;
private RelationalNode dependentNode;
private Criteria dependentCrit;
@@ -133,8 +136,9 @@
private boolean hasNextCommand;
protected SubqueryAwareEvaluator eval;
- public DependentCriteriaProcessor(int maxSetSize, RelationalNode dependentNode, Criteria dependentCriteria) throws ExpressionEvaluationException, TeiidComponentException {
+ public DependentCriteriaProcessor(int maxSetSize, int maxPredicates, RelationalNode dependentNode, Criteria dependentCriteria) throws ExpressionEvaluationException, TeiidComponentException {
this.maxSetSize = maxSetSize;
+ this.maxPredicates = maxPredicates;
this.dependentNode = dependentNode;
this.dependentCrit = dependentCriteria;
this.eval = new SubqueryAwareEvaluator(Collections.emptyMap(), dependentNode.getDataManager(), dependentNode.getContext(), dependentNode.getBufferManager());
@@ -250,55 +254,82 @@
* @throws TeiidComponentException
*/
private void replaceDependentValueIterators() throws TeiidComponentException {
+ int totalPredicates = sources.size();
+ if (this.maxPredicates > 0) {
+ //We have a bin packing problem if totalPredicates < sources - We'll address that case later.
+ //TODO: better handling for the correlated composite case
+ totalPredicates = Math.max(totalPredicates, this.maxPredicates);
+ }
+ long maxSize = Integer.MAX_VALUE;
+ if (this.maxSetSize > 0) {
+ maxSize = this.maxSetSize;
+ }
+ int currentPredicates = 0;
+ for (int run = 0; currentPredicates < totalPredicates; run++) {
+ currentPredicates = 0;
+ if (!restartIndexes.isEmpty()) {
+ currentIndex = restartIndexes.removeLast().intValue();
+ }
+ for (int i = 0; i < sources.size(); i++) {
- for (; currentIndex < sources.size(); currentIndex++) {
+ List<SetState> source = sources.get(i);
- List<SetState> source = sources.get(currentIndex);
-
- boolean done = false;
-
- while (!done) {
-
- boolean isNull = false;
- boolean lessThanMax = true;
-
- for (SetState state : source) {
- if (state.nextValue == null && !state.isNull) {
- if (state.valueIterator.hasNext()) {
- state.nextValue = state.valueIterator.next();
- state.isNull = state.nextValue == null;
- } else {
- state.valueIterator.reset();
- done = true; // should be true for each iterator from this source
- continue;
- }
- }
-
- isNull |= state.isNull;
- lessThanMax &= state.replacement.size() < maxSetSize;
- }
-
- if (done) {
- if (!restartIndexes.isEmpty() && restartIndexes.getLast().intValue() == currentIndex) {
- restartIndexes.removeLast();
- }
- break;
- }
-
- if (lessThanMax || isNull) {
- for (SetState state : source) {
- if (!isNull) {
- state.replacement.add(state.nextValue);
- }
- state.nextValue = null;
- state.isNull = false;
- }
- } else {
- restartIndexes.add(currentIndex);
- done = true;
- }
+ if (i == currentIndex++) {
+
+ boolean done = false;
+
+ while (!done) {
+
+ boolean isNull = false;
+ boolean lessThanMax = true;
+
+ for (SetState state : source) {
+ if (state.nextValue == null && !state.isNull) {
+ if (state.valueIterator.hasNext()) {
+ state.nextValue = state.valueIterator.next();
+ state.isNull = state.nextValue == null;
+ } else {
+ state.valueIterator.reset();
+ done = true; // should be true for each iterator from this source
+ continue;
+ }
+ }
+
+ isNull |= state.isNull;
+ lessThanMax &= state.replacement.size() < maxSize * (run + 1);
+ }
+
+ if (done) {
+ if (!restartIndexes.isEmpty() && restartIndexes.getLast().intValue() == i) {
+ restartIndexes.removeLast();
+ }
+ break;
+ }
+
+ if (lessThanMax || isNull) {
+ for (SetState state : source) {
+ if (!isNull) {
+ state.replacement.add(state.nextValue);
+ }
+ state.nextValue = null;
+ state.isNull = false;
+ }
+ } else {
+ restartIndexes.add(i);
+ done = true;
+ }
+ }
+ }
+
+ for (SetState setState : source) {
+ currentPredicates += setState.replacement.size()/maxSize+(setState.replacement.size()%maxSize!=0?1:0);
+ }
+ }
+
+ if (restartIndexes.isEmpty()) {
+ break;
}
- }
+ }
hasNextCommand = !restartIndexes.isEmpty();
}
@@ -312,18 +343,36 @@
// No values - return criteria that is always false
return QueryRewriter.FALSE_CRITERIA;
}
- if (state.replacement.size() == 1) {
- return new CompareCriteria(crit.getExpression(), CompareCriteria.EQ, new Constant(state.replacement.iterator().next()));
+ int numberOfSets = 1;
+ int maxSize = Integer.MAX_VALUE;
+ if (this.maxSetSize > 0) {
+ maxSize = this.maxSetSize;
+ numberOfSets = state.replacement.size()/maxSize + (state.replacement.size()%maxSize!=0?1:0);
}
- List vals = new ArrayList(state.replacement.size());
- for (Object val : state.replacement) {
- vals.add(new Constant(val));
- }
-
- SetCriteria sc = new SetCriteria();
- sc.setExpression(crit.getExpression());
- sc.setValues(vals);
- return sc;
+ Iterator<Object> iter = state.replacement.iterator();
+ ArrayList<Criteria> orCrits = new ArrayList<Criteria>(numberOfSets);
+
+ for (int i = 0; i < numberOfSets; i++) {
+ if (maxSize == 1 || i + 1 == state.replacement.size()) {
+ orCrits.add(new CompareCriteria(crit.getExpression(), CompareCriteria.EQ, new Constant(iter.next())));
+ } else {
+ List<Constant> vals = new ArrayList<Constant>(Math.min(state.replacement.size(), maxSize));
+
+ for (int j = 0; j < maxSize && iter.hasNext(); j++) {
+ Object val = iter.next();
+ vals.add(new Constant(val));
+ }
+
+ SetCriteria sc = new SetCriteria();
+ sc.setExpression(crit.getExpression());
+ sc.setValues(vals);
+ orCrits.add(sc);
+ }
+ }
+ if (orCrits.size() == 1) {
+ return orCrits.get(0);
+ }
+ return new CompoundCriteria(CompoundCriteria.OR, orCrits);
}
}
Modified: trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentProcedureCriteriaProcessor.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentProcedureCriteriaProcessor.java 2011-03-29 21:52:50 UTC (rev 3049)
+++ trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentProcedureCriteriaProcessor.java 2011-03-31 04:11:52 UTC (rev 3050)
@@ -50,7 +50,7 @@
Criteria dependentCriteria,
List references,
List defaults) throws ExpressionEvaluationException, TeiidComponentException {
- super(1, dependentNode, dependentCriteria);
+ super(1, -1, dependentNode, dependentCriteria);
this.inputDefaults = defaults;
this.inputReferences = references;
}
Modified: trunk/engine/src/main/java/org/teiid/query/rewriter/QueryRewriter.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/query/rewriter/QueryRewriter.java 2011-03-29 21:52:50 UTC (rev 3049)
+++ trunk/engine/src/main/java/org/teiid/query/rewriter/QueryRewriter.java 2011-03-31 04:11:52 UTC (rev 3050)
@@ -67,7 +67,6 @@
import org.teiid.query.optimizer.relational.rules.RuleMergeCriteria;
import org.teiid.query.optimizer.relational.rules.RulePlaceAccess;
import org.teiid.query.optimizer.relational.rules.RuleMergeCriteria.PlannedResult;
-import org.teiid.query.processor.relational.DependentValueSource;
import org.teiid.query.processor.relational.RelationalNodeUtil;
import org.teiid.query.resolver.ProcedureContainerResolver;
import org.teiid.query.resolver.QueryResolver;
@@ -152,7 +151,6 @@
import org.teiid.query.sql.symbol.SingleElementSymbol;
import org.teiid.query.sql.symbol.AggregateSymbol.Type;
import org.teiid.query.sql.util.SymbolMap;
-import org.teiid.query.sql.util.ValueIterator;
import org.teiid.query.sql.visitor.AggregateSymbolCollectorVisitor;
import org.teiid.query.sql.visitor.CorrelatedReferenceCollectorVisitor;
import org.teiid.query.sql.visitor.CriteriaTranslatorVisitor;
@@ -1222,26 +1220,8 @@
if (rewriteLeftExpression(dsc)) {
return UNKNOWN_CRITERIA;
}
- return dsc;
}
- SetCriteria setCrit = new SetCriteria();
- setCrit.setExpression(dsc.getExpression());
- HashSet<Object> values = new HashSet<Object>();
- try {
- DependentValueSource dvs = (DependentValueSource)this.context.getVariableContext().getGlobalValue(dsc.getContextSymbol());
- ValueIterator iter = dvs.getValueIterator(dsc.getValueExpression());
- while (iter.hasNext()) {
- values.add(iter.next());
- }
- } catch (TeiidComponentException e) {
- throw new TeiidRuntimeException(e);
- }
- List<Constant> constants = new ArrayList<Constant>(values.size());
- for (Object value : values) {
- constants.add(new Constant(value, setCrit.getExpression().getType()));
- }
- setCrit.setValues(constants);
- return rewriteCriteria(setCrit);
+ return dsc;
}
/**
Modified: trunk/engine/src/main/java/org/teiid/query/sql/lang/Create.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/query/sql/lang/Create.java 2011-03-29 21:52:50 UTC (rev 3049)
+++ trunk/engine/src/main/java/org/teiid/query/sql/lang/Create.java 2011-03-31 04:11:52 UTC (rev 3050)
@@ -72,6 +72,7 @@
for (Column column : columns) {
ElementSymbol es = new ElementSymbol(column.getName());
es.setType(DataTypeManager.getDataTypeClass(column.getRuntimeType()));
+ es.setGroupSymbol(table);
columnSymbols.add(es);
}
}
Modified: trunk/engine/src/main/java/org/teiid/query/tempdata/TempTable.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/query/tempdata/TempTable.java 2011-03-29 21:52:50 UTC (rev 3049)
+++ trunk/engine/src/main/java/org/teiid/query/tempdata/TempTable.java 2011-03-31 04:11:52 UTC (rev 3050)
@@ -194,7 +194,6 @@
private abstract class UpdateProcessor {
private TupleSource ts;
- protected final Map lookup;
protected final Evaluator eval;
private final Criteria crit;
protected int updateCount = 0;
@@ -204,8 +203,7 @@
UpdateProcessor(Criteria crit, TupleSource ts) throws TeiidComponentException {
this.ts = ts;
- this.lookup = RelationalNode.createLookupMap(columns);
- this.eval = new Evaluator(lookup, null, null);
+ this.eval = new Evaluator(columnMap, null, null);
this.crit = crit;
this.undoLog = bm.createTupleBuffer(columns, sessionID, TupleSourceType.PROCESSOR);
}
@@ -530,7 +528,7 @@
BlockedException, TeiidComponentException {
List<Object> newTuple = new ArrayList<Object>(tuple);
for (Map.Entry<ElementSymbol, Expression> entry : update.getClauseMap().entrySet()) {
- newTuple.set((Integer)lookup.get(entry.getKey()), eval.evaluate(entry.getValue(), tuple));
+ newTuple.set((Integer)columnMap.get(entry.getKey()), eval.evaluate(entry.getValue(), tuple));
}
if (primaryKeyChangePossible) {
browser.removed();
Modified: trunk/engine/src/test/java/org/teiid/query/optimizer/TestUnionPlanning.java
===================================================================
--- trunk/engine/src/test/java/org/teiid/query/optimizer/TestUnionPlanning.java 2011-03-29 21:52:50 UTC (rev 3049)
+++ trunk/engine/src/test/java/org/teiid/query/optimizer/TestUnionPlanning.java 2011-03-31 04:11:52 UTC (rev 3050)
@@ -121,7 +121,7 @@
0, // Project
0, // Select
0, // Sort
- 2 // UnionAll
+ 1 // UnionAll
});
}
Modified: trunk/engine/src/test/java/org/teiid/query/processor/TestDependentJoins.java
===================================================================
--- trunk/engine/src/test/java/org/teiid/query/processor/TestDependentJoins.java 2011-03-29 21:52:50 UTC (rev 3049)
+++ trunk/engine/src/test/java/org/teiid/query/processor/TestDependentJoins.java 2011-03-31 04:11:52 UTC (rev 3050)
@@ -43,7 +43,7 @@
import org.teiid.query.unittest.FakeMetadataObject;
import org.teiid.query.util.CommandContext;
- at SuppressWarnings({"unchecked"})
+ at SuppressWarnings({"unchecked", "nls"})
public class TestDependentJoins {
/**
@@ -529,8 +529,8 @@
new String[] {"SELECT g_0.stringkey, g_0.intkey FROM bqt1.smalla AS g_0 WHERE g_0.intkey IN (<dependent values>)", "SELECT g_0.stringkey, g_0.intkey FROM bqt2.smallb AS g_0"}, TestOptimizer.ComparisonMode.EXACT_COMMAND_STRING); //$NON-NLS-1$ //$NON-NLS-2$
TestOptimizer.checkNodeTypes(plan, new int[] {
- unlimitIn?2:1, // Access
- unlimitIn?0:1, // DependentAccess
+ 1, // Access
+ 1, // DependentAccess
0, // DependentSelect
0, // DependentProject
0, // DupRemove
@@ -633,8 +633,30 @@
}
@Test public void testLargeSetInDepAccessMultiJoinCriteria() throws Exception {
+ helpTestLargeSetInDepAccessMultiJoinCriteria(1, -1, 1, 2);
+ }
+
+ @Test public void testLargeSetInDepAccessMultiJoinCriteriaConcurrent() throws Exception {
+ //allows concurrent
+ helpTestLargeSetInDepAccessMultiJoinCriteria(1, -1, 4, 4);
+ }
+
+ @Test public void testLargeSetInDepAccessMultiJoinCriteriaCompound() throws Exception {
+ //max predicates forces multiple queries
+ helpTestLargeSetInDepAccessMultiJoinCriteria(1, 4, 3, 3);
+ }
+
+ @Test public void testLargeSetInDepAccessMultiJoinCriteriaCompoundAll() throws Exception {
+ //max predicates allows a one shot
+ helpTestLargeSetInDepAccessMultiJoinCriteria(1, 10, 2, 2);
+ }
+
+ /**
+ * concurrentOpen will be minimum of 2 to gather the pm1 results.
+ */
+ public void helpTestLargeSetInDepAccessMultiJoinCriteria(int maxInSize, int maxPredicates, int maxConcurrency, int concurrentOpen) throws Exception {
// Create query
- String sql = "SELECT pm1.g1.e1 FROM pm1.g1, pm2.g1 WHERE pm1.g1.e1=pm2.g1.e1 AND pm1.g1.e2=pm2.g1.e2 order by e1 OPTION MAKEDEP pm2.g1"; //$NON-NLS-1$
+ String sql = "SELECT pm1.g1.e1 FROM (pm1.g2 cross join pm1.g1) inner join pm2.g1 makedep ON pm1.g1.e1=pm2.g1.e1 AND pm1.g1.e2=pm2.g1.e2 AND pm1.g2.e4 = pm2.g1.e4 order by e1"; //$NON-NLS-1$
// Construct data manager with data
FakeDataManager dataManager = new FakeDataManager();
TestProcessor.sampleData1(dataManager);
@@ -647,7 +669,8 @@
FakeCapabilitiesFinder capFinder = new FakeCapabilitiesFinder();
BasicSourceCapabilities depcaps = new BasicSourceCapabilities();
depcaps.setCapabilitySupport(Capability.CRITERIA_IN, true);
- depcaps.setSourceProperty(Capability.MAX_IN_CRITERIA_SIZE, new Integer(1));
+ depcaps.setSourceProperty(Capability.MAX_IN_CRITERIA_SIZE, maxInSize);
+ depcaps.setSourceProperty(Capability.MAX_DEPENDENT_PREDICATES, maxPredicates);
BasicSourceCapabilities caps = new BasicSourceCapabilities();
caps.setCapabilitySupport(Capability.CRITERIA_IN, true);
@@ -656,30 +679,32 @@
capFinder.addCapabilities("pm2", depcaps); //$NON-NLS-1$
List[] expected = new List[] {
- Arrays.asList(new Object[] {
- new String("a")}), //$NON-NLS-1$
- Arrays.asList(new Object[] {
- new String("a")}), //$NON-NLS-1$
- Arrays.asList(new Object[] {
- new String("a")}), //$NON-NLS-1$
- Arrays.asList(new Object[] {
- new String("a")}), //$NON-NLS-1$
- Arrays.asList(new Object[] {
- new String("a")}), //$NON-NLS-1$
- Arrays.asList(new Object[] {
- new String("b")}), //$NON-NLS-1$
- Arrays.asList(new Object[] {
- new String("c")})}; //$NON-NLS-1$
+ Arrays.asList("a"), //$NON-NLS-1$
+ Arrays.asList("a"), //$NON-NLS-1$
+ Arrays.asList("a"), //$NON-NLS-1$
+ Arrays.asList("a"), //$NON-NLS-1$
+ Arrays.asList("a"), //$NON-NLS-1$
+ Arrays.asList("a"), //$NON-NLS-1$
+ Arrays.asList("a"), //$NON-NLS-1$
+ Arrays.asList("a"), //$NON-NLS-1$
+ Arrays.asList("a"), //$NON-NLS-1$
+ Arrays.asList("b"), //$NON-NLS-1$
+ };
Command command = TestProcessor.helpParse(sql);
ProcessorPlan plan = TestProcessor.helpGetPlan(command, fakeMetadata, capFinder);
+ TestOptimizer.checkAtomicQueries(new String[] {
+ "SELECT pm1.g2.e4 FROM pm1.g2",
+ "SELECT pm2.g1.e1, pm2.g1.e2, pm2.g1.e4 FROM pm2.g1 WHERE (pm2.g1.e1 IN (<dependent values>)) AND (pm2.g1.e2 IN (<dependent values>)) AND (pm2.g1.e4 IN (<dependent values>))",
+ "SELECT pm1.g1.e1, pm1.g1.e2 FROM pm1.g1"
+ }, plan);
CommandContext cc = TestProcessor.createCommandContext();
- cc.setUserRequestSourceConcurrency(5);
+ cc.setUserRequestSourceConcurrency(maxConcurrency);
FakeTupleSource.resetStats();
// Run query
TestProcessor.helpProcess(plan, cc, dataManager, expected);
- assertEquals(4, FakeTupleSource.maxOpen);
+ assertEquals("Wrong number of concurrent source queries", concurrentOpen, FakeTupleSource.maxOpen);
}
@Test public void testLargeSetInDepAccessWithAccessPattern() {
Modified: trunk/engine/src/test/java/org/teiid/query/processor/relational/TestDependentCriteriaProcessor.java
===================================================================
--- trunk/engine/src/test/java/org/teiid/query/processor/relational/TestDependentCriteriaProcessor.java 2011-03-29 21:52:50 UTC (rev 3049)
+++ trunk/engine/src/test/java/org/teiid/query/processor/relational/TestDependentCriteriaProcessor.java 2011-03-31 04:11:52 UTC (rev 3050)
@@ -22,13 +22,12 @@
package org.teiid.query.processor.relational;
+import static org.junit.Assert.*;
+
import java.util.Arrays;
import java.util.List;
-import static org.junit.Assert.*;
import org.junit.Test;
-import org.teiid.query.processor.relational.DependentAccessNode;
-import org.teiid.query.processor.relational.DependentCriteriaProcessor;
import org.teiid.query.sql.lang.CompareCriteria;
import org.teiid.query.sql.lang.Criteria;
import org.teiid.query.sql.lang.SetCriteria;
@@ -43,7 +42,7 @@
@Test public void testNegatedSetCriteria() throws Exception {
DependentAccessNode dan = new DependentAccessNode(0);
SetCriteria sc = new SetCriteria(new ElementSymbol("e1"), Arrays.asList(new Constant(1), new Constant(2))); //$NON-NLS-1$
- DependentCriteriaProcessor dcp = new DependentCriteriaProcessor(1, dan, sc);
+ DependentCriteriaProcessor dcp = new DependentCriteriaProcessor(1, -1, dan, sc);
Criteria result = dcp.prepareCriteria();
assertEquals(new CompareCriteria(new ElementSymbol("e1"), CompareCriteria.EQ, new Constant(1)), result); //$NON-NLS-1$
assertTrue(dcp.hasNextCommand());
@@ -58,7 +57,7 @@
cc.getVariableContext().setGlobalValue(reference.getContextSymbol(), 1);
}
SetCriteria sc = new SetCriteria(new ElementSymbol("e1"), references); //$NON-NLS-1$
- DependentCriteriaProcessor dcp = new DependentCriteriaProcessor(1, dan, sc);
+ DependentCriteriaProcessor dcp = new DependentCriteriaProcessor(1, -1, dan, sc);
Criteria result = dcp.prepareCriteria();
assertEquals(new CompareCriteria(new ElementSymbol("e1"), CompareCriteria.EQ, new Constant(1)), result); //$NON-NLS-1$
assertFalse(dcp.hasNextCommand());
Modified: trunk/engine/src/test/java/org/teiid/query/processor/xml/TestXMLPlanningEnhancements.java
===================================================================
--- trunk/engine/src/test/java/org/teiid/query/processor/xml/TestXMLPlanningEnhancements.java 2011-03-29 21:52:50 UTC (rev 3049)
+++ trunk/engine/src/test/java/org/teiid/query/processor/xml/TestXMLPlanningEnhancements.java 2011-03-31 04:11:52 UTC (rev 3050)
@@ -496,8 +496,8 @@
ProcessorPlan plan = instr.info.getPlan();
TestOptimizer.checkNodeTypes(plan, new int[] {
- 2, // Access
- 0, // DependentAccess
+ 1, // Access
+ 1, // DependentAccess
0, // DependentSelect
0, // DependentProject
0, // DupRemove
More information about the teiid-commits
mailing list