Author: shawkins Date: 2012-01-10 12:28:44 -0500 (Tue, 10 Jan 2012) New Revision: 3784 Modified: trunk/api/src/main/java/org/teiid/language/Parameter.java trunk/api/src/main/java/org/teiid/language/Select.java trunk/documentation/developer-guide/src/main/docbook/en-US/content/translator-api.xml trunk/engine/src/main/java/org/teiid/dqp/internal/datamgr/LanguageBridgeFactory.java trunk/engine/src/test/java/org/teiid/query/processor/HardcodedDataManager.java trunk/engine/src/test/java/org/teiid/query/processor/TestDependentJoins.java Log: TEIID-1540 refining support for dependent join pushdown, with a list rather than an iterator Modified: trunk/api/src/main/java/org/teiid/language/Parameter.java =================================================================== --- trunk/api/src/main/java/org/teiid/language/Parameter.java 2012-01-10 16:07:11 UTC (rev 3783) +++ trunk/api/src/main/java/org/teiid/language/Parameter.java 2012-01-10 17:28:44 UTC (rev 3784) @@ -28,7 +28,7 @@ private Class<?> type; private int valueIndex; - private String dependentSet; + private String dependentValueId; @Override public Class<?> getType() { @@ -57,16 +57,16 @@ } /** - * The name of the dependent set this parameter references. Dependent sets are available via {@link Select#getDependentSets()} + * The id of the dependent values this parameter references. Dependent values are available via {@link Select#getDependentValues()} * Will only be set for dependent join pushdown. * @return */ - public String getDependentSet() { - return dependentSet; + public String getDependentValueId() { + return dependentValueId; } - public void setDependentSet(String dependentSet) { - this.dependentSet = dependentSet; + public void setDependentValueId(String dependentValueId) { + this.dependentValueId = dependentValueId; } } Modified: trunk/api/src/main/java/org/teiid/language/Select.java =================================================================== --- trunk/api/src/main/java/org/teiid/language/Select.java 2012-01-10 16:07:11 UTC (rev 3783) +++ trunk/api/src/main/java/org/teiid/language/Select.java 2012-01-10 17:28:44 UTC (rev 3784) @@ -38,7 +38,7 @@ private Condition where; private GroupBy groupBy; private Condition having; - private Map<String, Iterable<List<?>>> dependentSets; + private Map<String, List<? extends List<?>>> dependentValues; public Select(List<DerivedColumn> derivedColumns, boolean distinct, List<TableReference> from, Condition where, GroupBy groupBy, Condition having, OrderBy orderBy) { @@ -140,13 +140,16 @@ } /** - * @return the dependent sets or null if this is not a dependent join pushdown + * Gets the dependent value lists. The lists are memory-safe. Caution should be used + * to not access large lists in a non-memory safe manner. The lists are invalid + * after returning results from the pushdown query. + * @return the map of dependent values or null if this is not a dependent join pushdown */ - public Map<String, Iterable<List<?>>> getDependentSets() { - return dependentSets; + public Map<String, List<? extends List<?>>> getDependentValues() { + return dependentValues; } - public void setDependentSets(Map<String, Iterable<List<?>>> dependentSets) { - this.dependentSets = dependentSets; + public void setDependentValues(Map<String, List<? extends List<?>>> dependentValues) { + this.dependentValues = dependentValues; } } Modified: trunk/documentation/developer-guide/src/main/docbook/en-US/content/translator-api.xml =================================================================== --- trunk/documentation/developer-guide/src/main/docbook/en-US/content/translator-api.xml 2012-01-10 16:07:11 UTC (rev 3783) +++ trunk/documentation/developer-guide/src/main/docbook/en-US/content/translator-api.xml 2012-01-10 17:28:44 UTC (rev 3784) @@ -1633,9 +1633,9 @@ <title>Dependent Join Pushdown</title> <para>Dependent joins are a technique used in federation to reduce the cost of cross source joins. Join values from one side of a join are made available to the other side which reduces the number of tuples needed to preform the join. Translators may indicate support for dependent join pushdown via the supportsDependentJoin capability. The - handling of pushdown dependent join queries can be quite complicated. The ordering (if present) and all of the non-dependent set constructs on the pushdown command must be honored, but if needed + handling of pushdown dependent join queries can be quite complicated. The ordering (if present) and all of the non-dependent criteria constructs on the pushdown command must be honored, but if needed the dependent criteria, which will be a <code>Comparison</code> with a <code>Parameter</code>, may be ignored in part or in total. Pushdown dependent join queries will be instances of - <code>Select</code> with the relevant dependent sets available via <code>Select.getDependentSets()</code>. The dependent set is associated to Parameters by name via the <code>Parameter.getDepenentSet()</code> + <code>Select</code> with the relevant dependent sets available via <code>Select.getDependentSets()</code>. The dependent set is associated to Parameters by id via the <code>Parameter.getDepenentValueId()</code> identifier. The dependent set tuple iterators provide rows that are referenced by the column positions (available via <code>Parameter.getValueIndex()</code>) on the dependent join Comparison criteria right expression. Care should be taken with the tuple values as they may guareenteed to be unique or ordered. </para> Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/datamgr/LanguageBridgeFactory.java =================================================================== --- trunk/engine/src/main/java/org/teiid/dqp/internal/datamgr/LanguageBridgeFactory.java 2012-01-10 16:07:11 UTC (rev 3783) +++ trunk/engine/src/main/java/org/teiid/dqp/internal/datamgr/LanguageBridgeFactory.java 2012-01-10 17:28:44 UTC (rev 3784) @@ -22,14 +22,7 @@ package org.teiid.dqp.internal.datamgr; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; +import java.util.*; import org.teiid.api.exception.query.QueryMetadataException; import org.teiid.client.metadata.ParameterInfo; @@ -70,7 +63,32 @@ public class LanguageBridgeFactory { - private final class TupleSourceIterator implements Iterator<List<?>> { + private final class TupleBufferList extends AbstractList<List<?>> implements RandomAccess { + private final TupleBuffer tb; + + private TupleBufferList(TupleBuffer tb) { + this.tb = tb; + } + + @Override + public List<?> get(int index) { + if (index < 0 || index >= size()) { + throw new IndexOutOfBoundsException(String.valueOf(index)); + } + try { + return tb.getBatch(index+1).getTuple(index+1); + } catch (TeiidComponentException e) { + throw new TeiidRuntimeException(e); + } + } + + @Override + public int size() { + return tb.getRowCount(); + } + } + + private final class TupleSourceIterator implements Iterator<List<?>> { private final TupleSource ts; List<?> nextRow; @@ -111,7 +129,7 @@ private RuntimeMetadataImpl metadataFactory = null; private int valueIndex = 0; private List<List<?>> allValues = new LinkedList<List<?>>(); - private Map<String, Iterable<List<?>>> dependentSets; + private Map<String, List<? extends List<?>>> dependentSets; public LanguageBridgeFactory(QueryMetadataInterface metadata) { if (metadata != null) { @@ -120,25 +138,31 @@ } public org.teiid.language.Command translate(Command command) { - if (command == null) return null; - if (command instanceof Query) { - Select result = translate((Query)command); - result.setDependentSets(this.dependentSets); - return result; - } else if (command instanceof SetQuery) { - return translate((SetQuery)command); - } else if (command instanceof Insert) { - return translate((Insert)command); - } else if (command instanceof Update) { - return translate((Update)command); - } else if (command instanceof Delete) { - return translate((Delete)command); - } else if (command instanceof StoredProcedure) { - return translate((StoredProcedure)command); - } else if (command instanceof BatchedUpdateCommand) { - return translate((BatchedUpdateCommand)command); - } - throw new AssertionError(); + try { + if (command == null) return null; + if (command instanceof Query) { + Select result = translate((Query)command); + result.setDependentValues(this.dependentSets); + return result; + } else if (command instanceof SetQuery) { + return translate((SetQuery)command); + } else if (command instanceof Insert) { + return translate((Insert)command); + } else if (command instanceof Update) { + return translate((Update)command); + } else if (command instanceof Delete) { + return translate((Delete)command); + } else if (command instanceof StoredProcedure) { + return translate((StoredProcedure)command); + } else if (command instanceof BatchedUpdateCommand) { + return translate((BatchedUpdateCommand)command); + } + throw new AssertionError(); + } finally { + this.allValues.clear(); + this.dependentSets = null; + this.valueIndex = 0; + } } QueryExpression translate(QueryCommand command) { @@ -306,17 +330,11 @@ p.setType(criteria.getExpression().getType()); final TupleBuffer tb = criteria.getDependentValueSource().getTupleBuffer(); p.setValueIndex(tb.getSchema().indexOf(criteria.getValueExpression())); - p.setDependentSet(criteria.getContextSymbol()); + p.setDependentValueId(criteria.getContextSymbol()); if (this.dependentSets == null) { - this.dependentSets = new HashMap<String, Iterable<List<?>>>(); + this.dependentSets = new HashMap<String, List<? extends List<?>>>(); } - this.dependentSets.put(criteria.getContextSymbol(), new Iterable<List<?>>() { - - @Override - public Iterator<List<?>> iterator() { - return new TupleSourceIterator(tb.createIndexedTupleSource()); - } - }); + this.dependentSets.put(criteria.getContextSymbol(), new TupleBufferList(tb)); Comparison result = new org.teiid.language.Comparison(translate(criteria.getExpression()), p, operator); return result; Modified: trunk/engine/src/test/java/org/teiid/query/processor/HardcodedDataManager.java =================================================================== --- trunk/engine/src/test/java/org/teiid/query/processor/HardcodedDataManager.java 2012-01-10 16:07:11 UTC (rev 3783) +++ trunk/engine/src/test/java/org/teiid/query/processor/HardcodedDataManager.java 2012-01-10 17:28:44 UTC (rev 3784) @@ -36,6 +36,7 @@ import org.teiid.metadata.MetadataRepository; import org.teiid.query.metadata.QueryMetadataInterface; import org.teiid.query.sql.lang.Command; +import org.teiid.query.sql.symbol.Expression; import org.teiid.query.util.CommandContext; @@ -47,10 +48,10 @@ ProcessorDataManager { // sql string to data - private Map<String, List[]> data = new HashMap<String, List[]>(); + private Map<String, List<?>[]> data = new HashMap<String, List<?>[]>(); // valid models - if null, any is assumed valid - private Set validModels; + private Set<String> validModels; private boolean mustRegisterCommands = true; @@ -58,6 +59,7 @@ // Collect all commands run against this class private List<Command> commandHistory = new ArrayList<Command>(); // Commands + private List<org.teiid.language.Command> pushdownCommands = new ArrayList<org.teiid.language.Command>(); // Commands private LanguageBridgeFactory lbf; @@ -74,7 +76,7 @@ this.mustRegisterCommands = mustRegisterCommands; } - public void addData(String sql, List[] rows) { + public void addData(String sql, List<?>[] rows) { data.put(sql, rows); } @@ -92,7 +94,7 @@ * @param models * @since 4.2 */ - public void setValidModels(Set models) { + public void setValidModels(Set<String> models) { this.validModels = models; } @@ -101,10 +103,14 @@ * @return * @since 4.2 */ - public List getCommandHistory() { + public List<Command> getCommandHistory() { return this.commandHistory; } + public List<org.teiid.language.Command> getPushdownCommands() { + return pushdownCommands; + } + /** * @see org.teiid.query.processor.ProcessorDataManager#lookupCodeValue(org.teiid.query.util.CommandContext, java.lang.String, java.lang.String, java.lang.String, java.lang.Object) * @since 4.2 @@ -132,16 +138,18 @@ } this.commandHistory.add(command); - List projectedSymbols = command.getProjectedSymbols(); + List<Expression> projectedSymbols = command.getProjectedSymbols(); String commandString = null; if (lbf == null) { commandString = command.toString(); } else { - commandString = lbf.translate(command).toString(); + org.teiid.language.Command cmd = lbf.translate(command); + this.pushdownCommands.add(cmd); + commandString = cmd.toString(); } - List[] rows = data.get(commandString); + List<?>[] rows = data.get(commandString); if(rows == null) { if (mustRegisterCommands) { throw new TeiidComponentException("Unknown command: " + commandString); //$NON-NLS-1$ @@ -172,13 +180,11 @@ @Override public EventDistributor getEventDistributor() { - // TODO Auto-generated method stub return null; } @Override public MetadataRepository getMetadataRepository() { - // TODO Auto-generated method stub return null; } Modified: trunk/engine/src/test/java/org/teiid/query/processor/TestDependentJoins.java =================================================================== --- trunk/engine/src/test/java/org/teiid/query/processor/TestDependentJoins.java 2012-01-10 16:07:11 UTC (rev 3783) +++ trunk/engine/src/test/java/org/teiid/query/processor/TestDependentJoins.java 2012-01-10 17:28:44 UTC (rev 3784) @@ -31,6 +31,7 @@ import org.junit.Test; import org.teiid.core.TeiidComponentException; import org.teiid.core.TeiidProcessingException; +import org.teiid.language.Select; import org.teiid.query.metadata.QueryMetadataInterface; import org.teiid.query.optimizer.TestOptimizer; import org.teiid.query.optimizer.TestOptimizer.ComparisonMode; @@ -944,6 +945,11 @@ // Run query TestProcessor.helpProcess(plan, dataManager, expected); + + Select s = (Select)dataManager.getPushdownCommands().get(1); + assertEquals(1, s.getDependentValues().size()); + List<? extends List<?>> vals = s.getDependentValues().values().iterator().next(); + assertEquals(1, vals.size()); } }