[teiid-commits] teiid SVN: r3783 - in trunk: api/src/main/java/org/teiid/translator and 8 other directories.

Author: shawkins Date: 2012-01-10 11:07:11 -0500 (Tue, 10 Jan 2012) New Revision: 3783 Modified: trunk/api/src/main/java/org/teiid/language/Parameter.java trunk/api/src/main/java/org/teiid/language/Select.java trunk/api/src/main/java/org/teiid/translator/ExecutionFactory.java trunk/build/kits/jboss-as7/docs/teiid/teiid-releasenotes.html trunk/documentation/developer-guide/src/main/docbook/en-US/content/translator-api.xml trunk/engine/src/main/java/org/teiid/dqp/internal/datamgr/CapabilitiesConverter.java trunk/engine/src/main/java/org/teiid/dqp/internal/datamgr/LanguageBridgeFactory.java trunk/engine/src/main/java/org/teiid/query/optimizer/capabilities/SourceCapabilities.java trunk/engine/src/main/java/org/teiid/query/optimizer/relational/PlanToProcessConverter.java trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentAccessNode.java trunk/engine/src/main/java/org/teiid/query/sql/lang/DependentSetCriteria.java trunk/engine/src/test/java/org/teiid/query/processor/TestDependentJoins.java Log: TEIID-1540 adding support for dependent join pushdown Modified: trunk/api/src/main/java/org/teiid/language/Parameter.java =================================================================== --- trunk/api/src/main/java/org/teiid/language/Parameter.java 2012-01-09 19:33:46 UTC (rev 3782) +++ trunk/api/src/main/java/org/teiid/language/Parameter.java 2012-01-10 16:07:11 UTC (rev 3783) @@ -28,6 +28,7 @@ private Class<?> type; private int valueIndex; + private String dependentSet; @Override public Class<?> getType() { @@ -55,4 +56,17 @@ return valueIndex; } + /** + * The name of the dependent set this parameter references. Dependent sets are available via {@link Select#getDependentSets()} + * Will only be set for dependent join pushdown. + * @return + */ + public String getDependentSet() { + return dependentSet; + } + + public void setDependentSet(String dependentSet) { + this.dependentSet = dependentSet; + } + } Modified: trunk/api/src/main/java/org/teiid/language/Select.java =================================================================== --- trunk/api/src/main/java/org/teiid/language/Select.java 2012-01-09 19:33:46 UTC (rev 3782) +++ trunk/api/src/main/java/org/teiid/language/Select.java 2012-01-10 16:07:11 UTC (rev 3783) @@ -23,6 +23,7 @@ package org.teiid.language; import java.util.List; +import java.util.Map; import org.teiid.language.visitor.LanguageObjectVisitor; @@ -37,6 +38,7 @@ private Condition where; private GroupBy groupBy; private Condition having; + private Map<String, Iterable<List<?>>> dependentSets; public Select(List<DerivedColumn> derivedColumns, boolean distinct, List<TableReference> from, Condition where, GroupBy groupBy, Condition having, OrderBy orderBy) { @@ -136,4 +138,15 @@ public Select getProjectedQuery() { return this; } + + /** + * @return the dependent sets or null if this is not a dependent join pushdown + */ + public Map<String, Iterable<List<?>>> getDependentSets() { + return dependentSets; + } + + public void setDependentSets(Map<String, Iterable<List<?>>> dependentSets) { + this.dependentSets = dependentSets; + } } Modified: trunk/api/src/main/java/org/teiid/translator/ExecutionFactory.java =================================================================== --- trunk/api/src/main/java/org/teiid/translator/ExecutionFactory.java 2012-01-09 19:33:46 UTC (rev 3782) +++ trunk/api/src/main/java/org/teiid/translator/ExecutionFactory.java 2012-01-10 16:07:11 UTC (rev 3783) @@ -930,5 +930,13 @@ public boolean supportsOnlyLiteralComparison() { return false; } + + /** + * @return true if dependent join pushdown is supported + * @since 8.0 + */ + public boolean supportsDependentJoins() { + return false; + } } Modified: trunk/build/kits/jboss-as7/docs/teiid/teiid-releasenotes.html =================================================================== --- trunk/build/kits/jboss-as7/docs/teiid/teiid-releasenotes.html 2012-01-09 19:33:46 UTC (rev 3782) +++ trunk/build/kits/jboss-as7/docs/teiid/teiid-releasenotes.html 2012-01-10 16:07:11 UTC (rev 3783) @@ -27,13 +27,12 @@ <H2><A NAME="Highlights"></A>Highlights</H2> <UL> <LI><B>CallableStatement Named Parameters</B> - you can now use CallableStatement named parameter get/set methods. - <LI><B>New Translator capabilities</B> - translators may indicate which convert functions they support and restrict non-join comparisons - to only literals. <LI><B>New Translator capabilities</B> <UL> <LI>translators may indicate which convert functions they support <LI>restrict non-join comparisons to only literals. - <LI>return ReusableExecution instances for processing nodes that issue multiple queries. + <LI>return ReusableExecution instances for processing nodes that issue multiple queries. + <LI>translators may indicate support for dependent join handling </UL> <LI><B>Continuous Asynch Queries</B> to process plans in a streamed window fashion the TeiidStatement/TeiidPreparedStatement methods now take a RequestOptions object to specify continuous mode. See the Client and Developers Guides for more. </UL> Modified: trunk/documentation/developer-guide/src/main/docbook/en-US/content/translator-api.xml =================================================================== --- trunk/documentation/developer-guide/src/main/docbook/en-US/content/translator-api.xml 2012-01-09 19:33:46 UTC (rev 3782) +++ trunk/documentation/developer-guide/src/main/docbook/en-US/content/translator-api.xml 2012-01-10 16:07:11 UTC (rev 3783) @@ -915,6 +915,18 @@ </row> <row> <entry> + <para>DependentJoins</para> + </entry> + <entry> + <para>Base join and criteria support</para> + </entry> + <entry> + <para>Translator supports dependent join pushdown. See <xref linkend="dependent_joins"/>. When set the MaxDependentInPredicates and MaxInCriteriaSize values are not used by the engine, rather all + independent values are made available to the pushdown command.</para> + </entry> + </row> + <row> + <entry> <para>InlineViews</para> </entry> <entry> @@ -1617,6 +1629,20 @@ </section> + <section id="dependent_joins"> + <title>Dependent Join Pushdown</title> + <para>Dependent joins are a technique used in federation to reduce the cost of cross source joins. Join values from one side of a join are made available + to the other side which reduces the number of tuples needed to preform the join. Translators may indicate support for dependent join pushdown via the supportsDependentJoin capability. The + handling of pushdown dependent join queries can be quite complicated. The ordering (if present) and all of the non-dependent set constructs on the pushdown command must be honored, but if needed + the dependent criteria, which will be a <code>Comparison</code> with a <code>Parameter</code>, may be ignored in part or in total. Pushdown dependent join queries will be instances of + <code>Select</code> with the relevant dependent sets available via <code>Select.getDependentSets()</code>. The dependent set is associated to Parameters by name via the <code>Parameter.getDepenentSet()</code> + identifier. The dependent set tuple iterators provide rows that are referenced by the column positions (available via <code>Parameter.getValueIndex()</code>) on the dependent join Comparison + criteria right expression. Care should be taken with the tuple values as they may guareenteed to be unique or ordered. + </para> + <note><para>There is no reference implementation of this functionality as all built-in translators + rely on the engine to handle breaking up dependent joins into simplier queries.</para></note> + </section> + <section id="delegating_translator"> <title>Delegating Translator</title> <para>In some instances you may wish to extend several differnt kinds of translators with the same functionality. Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/datamgr/CapabilitiesConverter.java =================================================================== --- trunk/engine/src/main/java/org/teiid/dqp/internal/datamgr/CapabilitiesConverter.java 2012-01-09 19:33:46 UTC (rev 3782) +++ trunk/engine/src/main/java/org/teiid/dqp/internal/datamgr/CapabilitiesConverter.java 2012-01-10 16:07:11 UTC (rev 3783) @@ -111,6 +111,7 @@ tgtCaps.setCapabilitySupport(Capability.CRITERIA_LIKE_REGEX, srcCaps.supportsLikeRegex()); setSupports(connectorID, tgtCaps, Capability.WINDOW_FUNCTION_DISTINCT_AGGREGATES, srcCaps.supportsWindowDistinctAggregates(), Capability.ELEMENTARY_OLAP, Capability.QUERY_AGGREGATES_DISTINCT); tgtCaps.setCapabilitySupport(Capability.CRITERIA_ONLY_LITERAL_COMPARE, srcCaps.supportsOnlyLiteralComparison()); + tgtCaps.setCapabilitySupport(Capability.DEPENDENT_JOIN, srcCaps.supportsDependentJoins()); //TODO: as more types are added it may make more sense to just delegate calls to the executionfactory for (int i = 0; i <= DataTypeManager.MAX_TYPE_CODE; i++) { Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/datamgr/LanguageBridgeFactory.java =================================================================== --- trunk/engine/src/main/java/org/teiid/dqp/internal/datamgr/LanguageBridgeFactory.java 2012-01-09 19:33:46 UTC (rev 3782) +++ trunk/engine/src/main/java/org/teiid/dqp/internal/datamgr/LanguageBridgeFactory.java 2012-01-10 16:07:11 UTC (rev 3783) @@ -24,13 +24,16 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.NoSuchElementException; import org.teiid.api.exception.query.QueryMetadataException; import org.teiid.client.metadata.ParameterInfo; +import org.teiid.common.buffer.TupleBuffer; import org.teiid.common.buffer.TupleSource; import org.teiid.core.CoreConstants; import org.teiid.core.TeiidComponentException; @@ -67,9 +70,48 @@ public class LanguageBridgeFactory { - private RuntimeMetadataImpl metadataFactory = null; + private final class TupleSourceIterator implements Iterator<List<?>> { + private final TupleSource ts; + List<?> nextRow; + + private TupleSourceIterator(TupleSource ts) { + this.ts = ts; + } + + @Override + public boolean hasNext() { + if (nextRow == null) { + try { + nextRow = ts.nextTuple(); + } catch (TeiidComponentException e) { + throw new TeiidRuntimeException(e); + } catch (TeiidProcessingException e) { + throw new TeiidRuntimeException(e); + } + } + return nextRow != null; + } + + @Override + public List<?> next() { + if (nextRow == null && !hasNext()) { + throw new NoSuchElementException(); + } + List<?> result = nextRow; + nextRow = null; + return result; + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + } + + private RuntimeMetadataImpl metadataFactory = null; private int valueIndex = 0; private List<List<?>> allValues = new LinkedList<List<?>>(); + private Map<String, Iterable<List<?>>> dependentSets; public LanguageBridgeFactory(QueryMetadataInterface metadata) { if (metadata != null) { @@ -80,7 +122,9 @@ public org.teiid.language.Command translate(Command command) { if (command == null) return null; if (command instanceof Query) { - return translate((Query)command); + Select result = translate((Query)command); + result.setDependentSets(this.dependentSets); + return result; } else if (command instanceof SetQuery) { return translate((SetQuery)command); } else if (command instanceof Insert) { @@ -250,10 +294,34 @@ return translate((SubqueryCompareCriteria)criteria); } else if (criteria instanceof SubquerySetCriteria) { return translate((SubquerySetCriteria)criteria); + } else if (criteria instanceof DependentSetCriteria) { + return translate((DependentSetCriteria)criteria); } throw new AssertionError(); } + + org.teiid.language.Comparison translate(DependentSetCriteria criteria) { + Operator operator = Operator.EQ; + Parameter p = new Parameter(); + p.setType(criteria.getExpression().getType()); + final TupleBuffer tb = criteria.getDependentValueSource().getTupleBuffer(); + p.setValueIndex(tb.getSchema().indexOf(criteria.getValueExpression())); + p.setDependentSet(criteria.getContextSymbol()); + if (this.dependentSets == null) { + this.dependentSets = new HashMap<String, Iterable<List<?>>>(); + } + this.dependentSets.put(criteria.getContextSymbol(), new Iterable<List<?>>() { + @Override + public Iterator<List<?>> iterator() { + return new TupleSourceIterator(tb.createIndexedTupleSource()); + } + }); + Comparison result = new org.teiid.language.Comparison(translate(criteria.getExpression()), + p, operator); + return result; + } + org.teiid.language.Comparison translate(CompareCriteria criteria) { Operator operator = Operator.EQ; switch(criteria.getOperator()) { @@ -558,39 +626,7 @@ valueSource = translate(insert.getQueryExpression()); } else if (insert.getTupleSource() != null) { final TupleSource ts = insert.getTupleSource(); - parameterValues = new Iterator<List<?>>() { - List<?> nextRow; - - @Override - public boolean hasNext() { - if (nextRow == null) { - try { - nextRow = ts.nextTuple(); - } catch (TeiidComponentException e) { - throw new TeiidRuntimeException(e); - } catch (TeiidProcessingException e) { - throw new TeiidRuntimeException(e); - } - } - return nextRow != null; - } - - @Override - public List<?> next() { - if (nextRow == null && !hasNext()) { - throw new NoSuchElementException(); - } - List<?> result = nextRow; - nextRow = null; - return result; - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - - }; + parameterValues = new TupleSourceIterator(ts); List<org.teiid.language.Expression> translatedValues = new ArrayList<org.teiid.language.Expression>(); for (int i = 0; i < insert.getVariables().size(); i++) { ElementSymbol es = insert.getVariables().get(i); Modified: trunk/engine/src/main/java/org/teiid/query/optimizer/capabilities/SourceCapabilities.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/optimizer/capabilities/SourceCapabilities.java 2012-01-09 19:33:46 UTC (rev 3782) +++ trunk/engine/src/main/java/org/teiid/query/optimizer/capabilities/SourceCapabilities.java 2012-01-10 16:07:11 UTC (rev 3783) @@ -329,7 +329,8 @@ ELEMENTARY_OLAP("ElementaryOLAP"), //$NON-NLS-1$ WINDOW_FUNCTION_ORDER_BY_AGGREGATES("WindowOrderByAggregates"), //$NON-NLS-1$ CRITERIA_SIMILAR, - CRITERIA_LIKE_REGEX, + CRITERIA_LIKE_REGEX, + DEPENDENT_JOIN, WINDOW_FUNCTION_DISTINCT_AGGREGATES("WindowDistinctAggregates"); //$NON-NLS-1$ Modified: trunk/engine/src/main/java/org/teiid/query/optimizer/relational/PlanToProcessConverter.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/optimizer/relational/PlanToProcessConverter.java 2012-01-09 19:33:46 UTC (rev 3782) +++ trunk/engine/src/main/java/org/teiid/query/optimizer/relational/PlanToProcessConverter.java 2012-01-10 16:07:11 UTC (rev 3783) @@ -288,6 +288,7 @@ DependentAccessNode depAccessNode = new DependentAccessNode(getID()); if(modelID != null){ + depAccessNode.setPushdown(CapabilitiesUtil.supports(Capability.DEPENDENT_JOIN, modelID, metadata, capFinder)); depAccessNode.setMaxSetSize(CapabilitiesUtil.getMaxInCriteriaSize(modelID, metadata, capFinder)); depAccessNode.setMaxPredicates(CapabilitiesUtil.getMaxDependentPredicates(modelID, metadata, capFinder)); } Modified: trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentAccessNode.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentAccessNode.java 2012-01-09 19:33:46 UTC (rev 3782) +++ trunk/engine/src/main/java/org/teiid/query/processor/relational/DependentAccessNode.java 2012-01-10 16:07:11 UTC (rev 3783) @@ -22,13 +22,16 @@ package org.teiid.query.processor.relational; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; import org.teiid.core.TeiidComponentException; import org.teiid.core.TeiidProcessingException; import org.teiid.core.util.Assertion; import org.teiid.query.sql.lang.Command; import org.teiid.query.sql.lang.Criteria; +import org.teiid.query.sql.lang.DependentSetCriteria; import org.teiid.query.sql.lang.Query; @@ -43,6 +46,7 @@ //plan state private int maxSetSize; private int maxPredicates; + private boolean pushdown; //processing state private DependentCriteriaProcessor criteriaProcessor; @@ -99,6 +103,7 @@ DependentAccessNode clonedNode = new DependentAccessNode(super.getID()); clonedNode.maxSetSize = this.maxSetSize; clonedNode.maxPredicates = this.maxPredicates; + clonedNode.pushdown = this.pushdown; super.copy(this, clonedNode); return clonedNode; } @@ -134,6 +139,24 @@ Assertion.assertTrue(atomicCommand instanceof Query); Query query = (Query)atomicCommand; + + if (pushdown) { + List<Criteria> newCriteria = new ArrayList<Criteria>(); + List<Criteria> queryCriteria = Criteria.separateCriteriaByAnd(query.getCriteria()); + for (Criteria criteria : queryCriteria) { + if (!(criteria instanceof DependentSetCriteria)) { + newCriteria.add(criteria); + continue; + } + DependentSetCriteria dsc = (DependentSetCriteria)criteria; + dsc = dsc.clone(); + DependentValueSource dvs = (DependentValueSource) getContext().getVariableContext().getGlobalValue(dsc.getContextSymbol()); + dsc.setDependentValueSource(dvs); + newCriteria.add(dsc); + } + query.setCriteria(Criteria.combineCriteria(newCriteria)); + return RelationalNodeUtil.shouldExecute(atomicCommand, true); + } if (this.criteriaProcessor == null) { this.criteriaProcessor = new DependentCriteriaProcessor(this.maxSetSize, this.maxPredicates, this, query.getCriteria()); @@ -176,7 +199,14 @@ * @see org.teiid.query.processor.relational.AccessNode#hasNextCommand() */ protected boolean hasNextCommand() { + if (pushdown) { + return false; + } return criteriaProcessor.hasNextCommand(); } + public void setPushdown(boolean pushdown) { + this.pushdown = pushdown; + } + } \ No newline at end of file Modified: trunk/engine/src/main/java/org/teiid/query/sql/lang/DependentSetCriteria.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/sql/lang/DependentSetCriteria.java 2012-01-09 19:33:46 UTC (rev 3782) +++ trunk/engine/src/main/java/org/teiid/query/sql/lang/DependentSetCriteria.java 2012-01-10 16:07:11 UTC (rev 3783) @@ -25,6 +25,7 @@ import org.teiid.core.util.EquivalenceUtil; import org.teiid.core.util.HashCodeUtil; import org.teiid.query.optimizer.relational.rules.NewCalculateCostUtil; +import org.teiid.query.processor.relational.DependentValueSource; import org.teiid.query.sql.LanguageVisitor; import org.teiid.query.sql.symbol.ContextReference; import org.teiid.query.sql.symbol.Expression; @@ -50,6 +51,11 @@ private float ndv = NewCalculateCostUtil.UNKNOWN_VALUE; private float maxNdv = NewCalculateCostUtil.UNKNOWN_VALUE; + /** + * set only for dependent pushdown + */ + private DependentValueSource dependentValueSource; + /** * Construct with the left expression */ @@ -142,7 +148,7 @@ * the original object, just like Reference. * @return Deep copy of object */ - public Object clone() { + public DependentSetCriteria clone() { Expression copy = null; if(getExpression() != null) { copy = (Expression) getExpression().clone(); @@ -165,4 +171,13 @@ } } + public DependentValueSource getDependentValueSource() { + return dependentValueSource; + } + + public void setDependentValueSource( + DependentValueSource dependentValueSource) { + this.dependentValueSource = dependentValueSource; + } + } Modified: trunk/engine/src/test/java/org/teiid/query/processor/TestDependentJoins.java =================================================================== --- trunk/engine/src/test/java/org/teiid/query/processor/TestDependentJoins.java 2012-01-09 19:33:46 UTC (rev 3782) +++ trunk/engine/src/test/java/org/teiid/query/processor/TestDependentJoins.java 2012-01-10 16:07:11 UTC (rev 3783) @@ -35,6 +35,7 @@ import org.teiid.query.optimizer.TestOptimizer; import org.teiid.query.optimizer.TestOptimizer.ComparisonMode; import org.teiid.query.optimizer.capabilities.BasicSourceCapabilities; +import org.teiid.query.optimizer.capabilities.DefaultCapabilitiesFinder; import org.teiid.query.optimizer.capabilities.FakeCapabilitiesFinder; import org.teiid.query.optimizer.capabilities.SourceCapabilities.Capability; import org.teiid.query.processor.relational.JoinNode; @@ -921,4 +922,28 @@ TestProcessor.helpProcess(plan, dataManager, expected); } + @Test public void testMakeIndHintPushdown() { + // Create query + String sql = "SELECT pm1.g1.e1 FROM /*+ MAKEIND */ pm1.g1, pm2.g1 WHERE pm1.g1.e1 = pm2.g1.e1 AND pm1.g1.e2=pm2.g1.e2 order by pm1.g1.e1"; //$NON-NLS-1$ + + // Create expected results + List[] expected = new List[] { + Arrays.asList(new Object[] { "a" }), //$NON-NLS-1$ + }; + + // Construct data manager with data + HardcodedDataManager dataManager = new HardcodedDataManager(RealMetadataFactory.example1Cached()); + dataManager.addData("SELECT g_0.e1 AS c_0, g_0.e2 AS c_1 FROM g1 AS g_0 ORDER BY c_0, c_1", new List[] {Arrays.asList("a", 1)}); + dataManager.addData("SELECT g_0.e1 AS c_0, g_0.e2 AS c_1 FROM g1 AS g_0 WHERE g_0.e1 = ? AND g_0.e2 = ? ORDER BY c_0, c_1", new List[] {Arrays.asList("a", 1)}); + BasicSourceCapabilities bsc = TestOptimizer.getTypicalCapabilities(); + bsc.setCapabilitySupport(Capability.DEPENDENT_JOIN, true); + DefaultCapabilitiesFinder dcf = new DefaultCapabilitiesFinder(bsc); + // Plan query + ProcessorPlan plan = TestProcessor.helpGetPlan(sql, RealMetadataFactory.example1Cached(), dcf); + TestOptimizer.checkDependentJoinCount(plan, 1); + + // Run query + TestProcessor.helpProcess(plan, dataManager, expected); + } + }