[teiid-commits] teiid SVN: r3783 - in trunk: api/src/main/java/org/teiid/translator and 8 other directories.
teiid-commits at lists.jboss.org
teiid-commits at lists.jboss.org
Tue Jan 10 11:07:12 EST 2012
Author: shawkins
Date: 2012-01-10 11:07:11 -0500 (Tue, 10 Jan 2012)
New Revision: 3783
Modified:
trunk/api/src/main/java/org/teiid/language/Parameter.java
trunk/api/src/main/java/org/teiid/language/Select.java
trunk/api/src/main/java/org/teiid/translator/ExecutionFactory.java
trunk/build/kits/jboss-as7/docs/teiid/teiid-releasenotes.html
trunk/documentation/developer-guide/src/main/docbook/en-US/content/translator-api.xml
trunk/engine/src/main/java/org/teiid/dqp/internal/datamgr/CapabilitiesConverter.java
trunk/engine/src/main/java/org/teiid/dqp/internal/datamgr/LanguageBridgeFactory.java
trunk/engine/src/main/java/org/teiid/query/optimizer/capabilities/SourceCapabilities.java
trunk/engine/src/main/java/org/teiid/query/optimizer/relational/PlanToProcessConverter.java
trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentAccessNode.java
trunk/engine/src/main/java/org/teiid/query/sql/lang/DependentSetCriteria.java
trunk/engine/src/test/java/org/teiid/query/processor/TestDependentJoins.java
Log:
TEIID-1540 adding support for dependent join pushdown
Modified: trunk/api/src/main/java/org/teiid/language/Parameter.java
===================================================================
--- trunk/api/src/main/java/org/teiid/language/Parameter.java 2012-01-09 19:33:46 UTC (rev 3782)
+++ trunk/api/src/main/java/org/teiid/language/Parameter.java 2012-01-10 16:07:11 UTC (rev 3783)
@@ -28,6 +28,7 @@
private Class<?> type;
private int valueIndex;
+ private String dependentSet;
@Override
public Class<?> getType() {
@@ -55,4 +56,17 @@
return valueIndex;
}
+ /**
+ * The name of the dependent set this parameter references. Dependent sets are available via {@link Select#getDependentSets()}
+ * Will only be set for dependent join pushdown.
+ * @return
+ */
+ public String getDependentSet() {
+ return dependentSet;
+ }
+
+ public void setDependentSet(String dependentSet) {
+ this.dependentSet = dependentSet;
+ }
+
}
Modified: trunk/api/src/main/java/org/teiid/language/Select.java
===================================================================
--- trunk/api/src/main/java/org/teiid/language/Select.java 2012-01-09 19:33:46 UTC (rev 3782)
+++ trunk/api/src/main/java/org/teiid/language/Select.java 2012-01-10 16:07:11 UTC (rev 3783)
@@ -23,6 +23,7 @@
package org.teiid.language;
import java.util.List;
+import java.util.Map;
import org.teiid.language.visitor.LanguageObjectVisitor;
@@ -37,6 +38,7 @@
private Condition where;
private GroupBy groupBy;
private Condition having;
+ private Map<String, Iterable<List<?>>> dependentSets;
public Select(List<DerivedColumn> derivedColumns, boolean distinct, List<TableReference> from, Condition where,
GroupBy groupBy, Condition having, OrderBy orderBy) {
@@ -136,4 +138,15 @@
public Select getProjectedQuery() {
return this;
}
+
+ /**
+ * @return the dependent sets or null if this is not a dependent join pushdown
+ */
+ public Map<String, Iterable<List<?>>> getDependentSets() {
+ return dependentSets;
+ }
+
+ public void setDependentSets(Map<String, Iterable<List<?>>> dependentSets) {
+ this.dependentSets = dependentSets;
+ }
}
Modified: trunk/api/src/main/java/org/teiid/translator/ExecutionFactory.java
===================================================================
--- trunk/api/src/main/java/org/teiid/translator/ExecutionFactory.java 2012-01-09 19:33:46 UTC (rev 3782)
+++ trunk/api/src/main/java/org/teiid/translator/ExecutionFactory.java 2012-01-10 16:07:11 UTC (rev 3783)
@@ -930,5 +930,13 @@
public boolean supportsOnlyLiteralComparison() {
return false;
}
+
+ /**
+ * @return true if dependent join pushdown is supported
+ * @since 8.0
+ */
+ public boolean supportsDependentJoins() {
+ return false;
+ }
}
Modified: trunk/build/kits/jboss-as7/docs/teiid/teiid-releasenotes.html
===================================================================
--- trunk/build/kits/jboss-as7/docs/teiid/teiid-releasenotes.html 2012-01-09 19:33:46 UTC (rev 3782)
+++ trunk/build/kits/jboss-as7/docs/teiid/teiid-releasenotes.html 2012-01-10 16:07:11 UTC (rev 3783)
@@ -27,13 +27,12 @@
<H2><A NAME="Highlights"></A>Highlights</H2>
<UL>
<LI><B>CallableStatement Named Parameters</B> - you can now use CallableStatement named parameter get/set methods.
- <LI><B>New Translator capabilities</B> - translators may indicate which convert functions they support and restrict non-join comparisons
- to only literals.
<LI><B>New Translator capabilities</B>
<UL>
<LI>translators may indicate which convert functions they support
<LI>restrict non-join comparisons to only literals.
- <LI>return ReusableExecution instances for processing nodes that issue multiple queries.
+ <LI>return ReusableExecution instances for processing nodes that issue multiple queries.
+ <LI>translators may indicate support for dependent join handling
</UL>
<LI><B>Continuous Asynch Queries</B> to process plans in a streamed window fashion the TeiidStatement/TeiidPreparedStatement methods now take a RequestOptions object to specify continuous mode. See the Client and Developers Guides for more.
</UL>
Modified: trunk/documentation/developer-guide/src/main/docbook/en-US/content/translator-api.xml
===================================================================
--- trunk/documentation/developer-guide/src/main/docbook/en-US/content/translator-api.xml 2012-01-09 19:33:46 UTC (rev 3782)
+++ trunk/documentation/developer-guide/src/main/docbook/en-US/content/translator-api.xml 2012-01-10 16:07:11 UTC (rev 3783)
@@ -915,6 +915,18 @@
</row>
<row>
<entry>
+ <para>DependentJoins</para>
+ </entry>
+ <entry>
+ <para>Base join and criteria support</para>
+ </entry>
+ <entry>
+ <para>Translator supports dependent join pushdown. See <xref linkend="dependent_joins"/>. When set the MaxDependentInPredicates and MaxInCriteriaSize values are not used by the engine, rather all
+ independent values are made available to the pushdown command.</para>
+ </entry>
+ </row>
+ <row>
+ <entry>
<para>InlineViews</para>
</entry>
<entry>
@@ -1617,6 +1629,20 @@
</section>
+ <section id="dependent_joins">
+ <title>Dependent Join Pushdown</title>
+ <para>Dependent joins are a technique used in federation to reduce the cost of cross source joins. Join values from one side of a join are made available
+ to the other side which reduces the number of tuples needed to preform the join. Translators may indicate support for dependent join pushdown via the supportsDependentJoin capability. The
+ handling of pushdown dependent join queries can be quite complicated. The ordering (if present) and all of the non-dependent set constructs on the pushdown command must be honored, but if needed
+ the dependent criteria, which will be a <code>Comparison</code> with a <code>Parameter</code>, may be ignored in part or in total. Pushdown dependent join queries will be instances of
+ <code>Select</code> with the relevant dependent sets available via <code>Select.getDependentSets()</code>. The dependent set is associated to Parameters by name via the <code>Parameter.getDepenentSet()</code>
+ identifier. The dependent set tuple iterators provide rows that are referenced by the column positions (available via <code>Parameter.getValueIndex()</code>) on the dependent join Comparison
+ criteria right expression. Care should be taken with the tuple values as they may guareenteed to be unique or ordered.
+ </para>
+ <note><para>There is no reference implementation of this functionality as all built-in translators
+ rely on the engine to handle breaking up dependent joins into simplier queries.</para></note>
+ </section>
+
<section id="delegating_translator">
<title>Delegating Translator</title>
<para>In some instances you may wish to extend several differnt kinds of translators with the same functionality.
Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/datamgr/CapabilitiesConverter.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/dqp/internal/datamgr/CapabilitiesConverter.java 2012-01-09 19:33:46 UTC (rev 3782)
+++ trunk/engine/src/main/java/org/teiid/dqp/internal/datamgr/CapabilitiesConverter.java 2012-01-10 16:07:11 UTC (rev 3783)
@@ -111,6 +111,7 @@
tgtCaps.setCapabilitySupport(Capability.CRITERIA_LIKE_REGEX, srcCaps.supportsLikeRegex());
setSupports(connectorID, tgtCaps, Capability.WINDOW_FUNCTION_DISTINCT_AGGREGATES, srcCaps.supportsWindowDistinctAggregates(), Capability.ELEMENTARY_OLAP, Capability.QUERY_AGGREGATES_DISTINCT);
tgtCaps.setCapabilitySupport(Capability.CRITERIA_ONLY_LITERAL_COMPARE, srcCaps.supportsOnlyLiteralComparison());
+ tgtCaps.setCapabilitySupport(Capability.DEPENDENT_JOIN, srcCaps.supportsDependentJoins());
//TODO: as more types are added it may make more sense to just delegate calls to the executionfactory
for (int i = 0; i <= DataTypeManager.MAX_TYPE_CODE; i++) {
Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/datamgr/LanguageBridgeFactory.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/dqp/internal/datamgr/LanguageBridgeFactory.java 2012-01-09 19:33:46 UTC (rev 3782)
+++ trunk/engine/src/main/java/org/teiid/dqp/internal/datamgr/LanguageBridgeFactory.java 2012-01-10 16:07:11 UTC (rev 3783)
@@ -24,13 +24,16 @@
import java.util.ArrayList;
import java.util.Collection;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
+import java.util.Map;
import java.util.NoSuchElementException;
import org.teiid.api.exception.query.QueryMetadataException;
import org.teiid.client.metadata.ParameterInfo;
+import org.teiid.common.buffer.TupleBuffer;
import org.teiid.common.buffer.TupleSource;
import org.teiid.core.CoreConstants;
import org.teiid.core.TeiidComponentException;
@@ -67,9 +70,48 @@
public class LanguageBridgeFactory {
- private RuntimeMetadataImpl metadataFactory = null;
+ private final class TupleSourceIterator implements Iterator<List<?>> {
+ private final TupleSource ts;
+ List<?> nextRow;
+
+ private TupleSourceIterator(TupleSource ts) {
+ this.ts = ts;
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (nextRow == null) {
+ try {
+ nextRow = ts.nextTuple();
+ } catch (TeiidComponentException e) {
+ throw new TeiidRuntimeException(e);
+ } catch (TeiidProcessingException e) {
+ throw new TeiidRuntimeException(e);
+ }
+ }
+ return nextRow != null;
+ }
+
+ @Override
+ public List<?> next() {
+ if (nextRow == null && !hasNext()) {
+ throw new NoSuchElementException();
+ }
+ List<?> result = nextRow;
+ nextRow = null;
+ return result;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ private RuntimeMetadataImpl metadataFactory = null;
private int valueIndex = 0;
private List<List<?>> allValues = new LinkedList<List<?>>();
+ private Map<String, Iterable<List<?>>> dependentSets;
public LanguageBridgeFactory(QueryMetadataInterface metadata) {
if (metadata != null) {
@@ -80,7 +122,9 @@
public org.teiid.language.Command translate(Command command) {
if (command == null) return null;
if (command instanceof Query) {
- return translate((Query)command);
+ Select result = translate((Query)command);
+ result.setDependentSets(this.dependentSets);
+ return result;
} else if (command instanceof SetQuery) {
return translate((SetQuery)command);
} else if (command instanceof Insert) {
@@ -250,10 +294,34 @@
return translate((SubqueryCompareCriteria)criteria);
} else if (criteria instanceof SubquerySetCriteria) {
return translate((SubquerySetCriteria)criteria);
+ } else if (criteria instanceof DependentSetCriteria) {
+ return translate((DependentSetCriteria)criteria);
}
throw new AssertionError();
}
+
+ org.teiid.language.Comparison translate(DependentSetCriteria criteria) {
+ Operator operator = Operator.EQ;
+ Parameter p = new Parameter();
+ p.setType(criteria.getExpression().getType());
+ final TupleBuffer tb = criteria.getDependentValueSource().getTupleBuffer();
+ p.setValueIndex(tb.getSchema().indexOf(criteria.getValueExpression()));
+ p.setDependentSet(criteria.getContextSymbol());
+ if (this.dependentSets == null) {
+ this.dependentSets = new HashMap<String, Iterable<List<?>>>();
+ }
+ this.dependentSets.put(criteria.getContextSymbol(), new Iterable<List<?>>() {
+ @Override
+ public Iterator<List<?>> iterator() {
+ return new TupleSourceIterator(tb.createIndexedTupleSource());
+ }
+ });
+ Comparison result = new org.teiid.language.Comparison(translate(criteria.getExpression()),
+ p, operator);
+ return result;
+ }
+
org.teiid.language.Comparison translate(CompareCriteria criteria) {
Operator operator = Operator.EQ;
switch(criteria.getOperator()) {
@@ -558,39 +626,7 @@
valueSource = translate(insert.getQueryExpression());
} else if (insert.getTupleSource() != null) {
final TupleSource ts = insert.getTupleSource();
- parameterValues = new Iterator<List<?>>() {
- List<?> nextRow;
-
- @Override
- public boolean hasNext() {
- if (nextRow == null) {
- try {
- nextRow = ts.nextTuple();
- } catch (TeiidComponentException e) {
- throw new TeiidRuntimeException(e);
- } catch (TeiidProcessingException e) {
- throw new TeiidRuntimeException(e);
- }
- }
- return nextRow != null;
- }
-
- @Override
- public List<?> next() {
- if (nextRow == null && !hasNext()) {
- throw new NoSuchElementException();
- }
- List<?> result = nextRow;
- nextRow = null;
- return result;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- };
+ parameterValues = new TupleSourceIterator(ts);
List<org.teiid.language.Expression> translatedValues = new ArrayList<org.teiid.language.Expression>();
for (int i = 0; i < insert.getVariables().size(); i++) {
ElementSymbol es = insert.getVariables().get(i);
Modified: trunk/engine/src/main/java/org/teiid/query/optimizer/capabilities/SourceCapabilities.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/query/optimizer/capabilities/SourceCapabilities.java 2012-01-09 19:33:46 UTC (rev 3782)
+++ trunk/engine/src/main/java/org/teiid/query/optimizer/capabilities/SourceCapabilities.java 2012-01-10 16:07:11 UTC (rev 3783)
@@ -329,7 +329,8 @@
ELEMENTARY_OLAP("ElementaryOLAP"), //$NON-NLS-1$
WINDOW_FUNCTION_ORDER_BY_AGGREGATES("WindowOrderByAggregates"), //$NON-NLS-1$
CRITERIA_SIMILAR,
- CRITERIA_LIKE_REGEX,
+ CRITERIA_LIKE_REGEX,
+ DEPENDENT_JOIN,
WINDOW_FUNCTION_DISTINCT_AGGREGATES("WindowDistinctAggregates"); //$NON-NLS-1$
Modified: trunk/engine/src/main/java/org/teiid/query/optimizer/relational/PlanToProcessConverter.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/query/optimizer/relational/PlanToProcessConverter.java 2012-01-09 19:33:46 UTC (rev 3782)
+++ trunk/engine/src/main/java/org/teiid/query/optimizer/relational/PlanToProcessConverter.java 2012-01-10 16:07:11 UTC (rev 3783)
@@ -288,6 +288,7 @@
DependentAccessNode depAccessNode = new DependentAccessNode(getID());
if(modelID != null){
+ depAccessNode.setPushdown(CapabilitiesUtil.supports(Capability.DEPENDENT_JOIN, modelID, metadata, capFinder));
depAccessNode.setMaxSetSize(CapabilitiesUtil.getMaxInCriteriaSize(modelID, metadata, capFinder));
depAccessNode.setMaxPredicates(CapabilitiesUtil.getMaxDependentPredicates(modelID, metadata, capFinder));
}
Modified: trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentAccessNode.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentAccessNode.java 2012-01-09 19:33:46 UTC (rev 3782)
+++ trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentAccessNode.java 2012-01-10 16:07:11 UTC (rev 3783)
@@ -22,13 +22,16 @@
package org.teiid.query.processor.relational;
+import java.util.ArrayList;
import java.util.Collections;
+import java.util.List;
import org.teiid.core.TeiidComponentException;
import org.teiid.core.TeiidProcessingException;
import org.teiid.core.util.Assertion;
import org.teiid.query.sql.lang.Command;
import org.teiid.query.sql.lang.Criteria;
+import org.teiid.query.sql.lang.DependentSetCriteria;
import org.teiid.query.sql.lang.Query;
@@ -43,6 +46,7 @@
//plan state
private int maxSetSize;
private int maxPredicates;
+ private boolean pushdown;
//processing state
private DependentCriteriaProcessor criteriaProcessor;
@@ -99,6 +103,7 @@
DependentAccessNode clonedNode = new DependentAccessNode(super.getID());
clonedNode.maxSetSize = this.maxSetSize;
clonedNode.maxPredicates = this.maxPredicates;
+ clonedNode.pushdown = this.pushdown;
super.copy(this, clonedNode);
return clonedNode;
}
@@ -134,6 +139,24 @@
Assertion.assertTrue(atomicCommand instanceof Query);
Query query = (Query)atomicCommand;
+
+ if (pushdown) {
+ List<Criteria> newCriteria = new ArrayList<Criteria>();
+ List<Criteria> queryCriteria = Criteria.separateCriteriaByAnd(query.getCriteria());
+ for (Criteria criteria : queryCriteria) {
+ if (!(criteria instanceof DependentSetCriteria)) {
+ newCriteria.add(criteria);
+ continue;
+ }
+ DependentSetCriteria dsc = (DependentSetCriteria)criteria;
+ dsc = dsc.clone();
+ DependentValueSource dvs = (DependentValueSource) getContext().getVariableContext().getGlobalValue(dsc.getContextSymbol());
+ dsc.setDependentValueSource(dvs);
+ newCriteria.add(dsc);
+ }
+ query.setCriteria(Criteria.combineCriteria(newCriteria));
+ return RelationalNodeUtil.shouldExecute(atomicCommand, true);
+ }
if (this.criteriaProcessor == null) {
this.criteriaProcessor = new DependentCriteriaProcessor(this.maxSetSize, this.maxPredicates, this, query.getCriteria());
@@ -176,7 +199,14 @@
* @see org.teiid.query.processor.relational.AccessNode#hasNextCommand()
*/
protected boolean hasNextCommand() {
+ if (pushdown) {
+ return false;
+ }
return criteriaProcessor.hasNextCommand();
}
+ public void setPushdown(boolean pushdown) {
+ this.pushdown = pushdown;
+ }
+
}
\ No newline at end of file
Modified: trunk/engine/src/main/java/org/teiid/query/sql/lang/DependentSetCriteria.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/query/sql/lang/DependentSetCriteria.java 2012-01-09 19:33:46 UTC (rev 3782)
+++ trunk/engine/src/main/java/org/teiid/query/sql/lang/DependentSetCriteria.java 2012-01-10 16:07:11 UTC (rev 3783)
@@ -25,6 +25,7 @@
import org.teiid.core.util.EquivalenceUtil;
import org.teiid.core.util.HashCodeUtil;
import org.teiid.query.optimizer.relational.rules.NewCalculateCostUtil;
+import org.teiid.query.processor.relational.DependentValueSource;
import org.teiid.query.sql.LanguageVisitor;
import org.teiid.query.sql.symbol.ContextReference;
import org.teiid.query.sql.symbol.Expression;
@@ -50,6 +51,11 @@
private float ndv = NewCalculateCostUtil.UNKNOWN_VALUE;
private float maxNdv = NewCalculateCostUtil.UNKNOWN_VALUE;
+ /**
+ * set only for dependent pushdown
+ */
+ private DependentValueSource dependentValueSource;
+
/**
* Construct with the left expression
*/
@@ -142,7 +148,7 @@
* the original object, just like Reference.
* @return Deep copy of object
*/
- public Object clone() {
+ public DependentSetCriteria clone() {
Expression copy = null;
if(getExpression() != null) {
copy = (Expression) getExpression().clone();
@@ -165,4 +171,13 @@
}
}
+ public DependentValueSource getDependentValueSource() {
+ return dependentValueSource;
+ }
+
+ public void setDependentValueSource(
+ DependentValueSource dependentValueSource) {
+ this.dependentValueSource = dependentValueSource;
+ }
+
}
Modified: trunk/engine/src/test/java/org/teiid/query/processor/TestDependentJoins.java
===================================================================
--- trunk/engine/src/test/java/org/teiid/query/processor/TestDependentJoins.java 2012-01-09 19:33:46 UTC (rev 3782)
+++ trunk/engine/src/test/java/org/teiid/query/processor/TestDependentJoins.java 2012-01-10 16:07:11 UTC (rev 3783)
@@ -35,6 +35,7 @@
import org.teiid.query.optimizer.TestOptimizer;
import org.teiid.query.optimizer.TestOptimizer.ComparisonMode;
import org.teiid.query.optimizer.capabilities.BasicSourceCapabilities;
+import org.teiid.query.optimizer.capabilities.DefaultCapabilitiesFinder;
import org.teiid.query.optimizer.capabilities.FakeCapabilitiesFinder;
import org.teiid.query.optimizer.capabilities.SourceCapabilities.Capability;
import org.teiid.query.processor.relational.JoinNode;
@@ -921,4 +922,28 @@
TestProcessor.helpProcess(plan, dataManager, expected);
}
+ @Test public void testMakeIndHintPushdown() {
+ // Create query
+ String sql = "SELECT pm1.g1.e1 FROM /*+ MAKEIND */ pm1.g1, pm2.g1 WHERE pm1.g1.e1 = pm2.g1.e1 AND pm1.g1.e2=pm2.g1.e2 order by pm1.g1.e1"; //$NON-NLS-1$
+
+ // Create expected results
+ List[] expected = new List[] {
+ Arrays.asList(new Object[] { "a" }), //$NON-NLS-1$
+ };
+
+ // Construct data manager with data
+ HardcodedDataManager dataManager = new HardcodedDataManager(RealMetadataFactory.example1Cached());
+ dataManager.addData("SELECT g_0.e1 AS c_0, g_0.e2 AS c_1 FROM g1 AS g_0 ORDER BY c_0, c_1", new List[] {Arrays.asList("a", 1)});
+ dataManager.addData("SELECT g_0.e1 AS c_0, g_0.e2 AS c_1 FROM g1 AS g_0 WHERE g_0.e1 = ? AND g_0.e2 = ? ORDER BY c_0, c_1", new List[] {Arrays.asList("a", 1)});
+ BasicSourceCapabilities bsc = TestOptimizer.getTypicalCapabilities();
+ bsc.setCapabilitySupport(Capability.DEPENDENT_JOIN, true);
+ DefaultCapabilitiesFinder dcf = new DefaultCapabilitiesFinder(bsc);
+ // Plan query
+ ProcessorPlan plan = TestProcessor.helpGetPlan(sql, RealMetadataFactory.example1Cached(), dcf);
+ TestOptimizer.checkDependentJoinCount(plan, 1);
+
+ // Run query
+ TestProcessor.helpProcess(plan, dataManager, expected);
+ }
+
}
More information about the teiid-commits
mailing list