teiid SVN: r4562 - in branches/7.7.x/engine/src: main/java/org/teiid/query/processor/relational and 2 other directories.
by teiid-commits＠lists.jboss.org
Author: jolee
Date: 2013-05-06 10:30:47 -0400 (Mon, 06 May 2013)
New Revision: 4562
Modified:
branches/7.7.x/engine/src/main/java/org/teiid/query/optimizer/relational/AliasGenerator.java
branches/7.7.x/engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java
branches/7.7.x/engine/src/main/java/org/teiid/query/processor/relational/RelationalPlan.java
branches/7.7.x/engine/src/main/java/org/teiid/query/tempdata/TempTableDataManager.java
branches/7.7.x/engine/src/main/java/org/teiid/query/tempdata/TempTableStore.java
branches/7.7.x/engine/src/test/java/org/teiid/query/processor/TestWithClauseProcessing.java
Log:
TEIID-2478 backport: incomplete with pushdown. (required supporting changes)
Modified: branches/7.7.x/engine/src/main/java/org/teiid/query/optimizer/relational/AliasGenerator.java
===================================================================
--- branches/7.7.x/engine/src/main/java/org/teiid/query/optimizer/relational/AliasGenerator.java 2013-04-17 18:59:47 UTC (rev 4561)
+++ branches/7.7.x/engine/src/main/java/org/teiid/query/optimizer/relational/AliasGenerator.java 2013-05-06 14:30:47 UTC (rev 4562)
@@ -75,10 +75,14 @@
public SQLNamingContext(SQLNamingContext parent) {
this.parent = parent;
}
+
+ public String getElementName(SingleElementSymbol symbol, boolean renameGroup) {
+ return getElementName(symbol, renameGroup, false);
+ }
- public String getElementName(SingleElementSymbol symbol, boolean renameGroup) {
+ public String getElementName(SingleElementSymbol symbol, boolean renameGroup, boolean ignoreCurrent) {
String name = null;
- if (currentSymbols != null) {
+ if (currentSymbols != null && !ignoreCurrent) {
name = currentSymbols.get(symbol);
if (name != null) {
if (renameGroup && symbol instanceof ElementSymbol) {
@@ -255,7 +259,7 @@
private boolean needsAlias(String newAlias,
ElementSymbol symbol) {
- return !(symbol.getMetadataID() instanceof TempMetadataID) || !newAlias.equalsIgnoreCase(visitor.namingContext.getElementName(symbol, false));
+ return !(symbol.getMetadataID() instanceof TempMetadataID) || !newAlias.equalsIgnoreCase(visitor.namingContext.getElementName(symbol, false, true));
}
/**
Modified: branches/7.7.x/engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java
===================================================================
--- branches/7.7.x/engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java 2013-04-17 18:59:47 UTC (rev 4561)
+++ branches/7.7.x/engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java 2013-05-06 14:30:47 UTC (rev 4562)
@@ -48,10 +48,10 @@
import org.teiid.query.optimizer.capabilities.CapabilitiesFinder;
import org.teiid.query.optimizer.capabilities.SourceCapabilities.Capability;
import org.teiid.query.optimizer.relational.plantree.NodeConstants;
+import org.teiid.query.optimizer.relational.plantree.NodeConstants.Info;
import org.teiid.query.optimizer.relational.plantree.NodeEditor;
import org.teiid.query.optimizer.relational.plantree.NodeFactory;
import org.teiid.query.optimizer.relational.plantree.PlanNode;
-import org.teiid.query.optimizer.relational.plantree.NodeConstants.Info;
import org.teiid.query.optimizer.relational.rules.CapabilitiesUtil;
import org.teiid.query.optimizer.relational.rules.CriteriaCapabilityValidatorVisitor;
import org.teiid.query.optimizer.relational.rules.RuleAssignOutputElements;
@@ -62,15 +62,16 @@
import org.teiid.query.optimizer.relational.rules.RulePushAggregates;
import org.teiid.query.processor.ProcessorPlan;
import org.teiid.query.processor.relational.AccessNode;
+import org.teiid.query.processor.relational.JoinNode.JoinStrategyType;
+import org.teiid.query.processor.relational.RelationalNode;
import org.teiid.query.processor.relational.RelationalPlan;
-import org.teiid.query.processor.relational.JoinNode.JoinStrategyType;
import org.teiid.query.resolver.ProcedureContainerResolver;
import org.teiid.query.resolver.QueryResolver;
import org.teiid.query.resolver.util.ResolverUtil;
import org.teiid.query.rewriter.QueryRewriter;
import org.teiid.query.sql.LanguageObject;
+import org.teiid.query.sql.LanguageObject.Util;
import org.teiid.query.sql.LanguageVisitor;
-import org.teiid.query.sql.LanguageObject.Util;
import org.teiid.query.sql.lang.*;
import org.teiid.query.sql.navigator.PreOrPostOrderNavigator;
import org.teiid.query.sql.proc.CreateUpdateProcedureCommand;
@@ -81,11 +82,12 @@
import org.teiid.query.sql.visitor.CorrelatedReferenceCollectorVisitor;
import org.teiid.query.sql.visitor.ElementCollectorVisitor;
import org.teiid.query.sql.visitor.ExpressionMappingVisitor;
+import org.teiid.query.sql.visitor.GroupCollectorVisitor;
import org.teiid.query.sql.visitor.GroupsUsedByElementsVisitor;
import org.teiid.query.sql.visitor.ValueIteratorProviderCollectorVisitor;
import org.teiid.query.util.CommandContext;
+import org.teiid.query.validator.UpdateValidator.UpdateInfo;
import org.teiid.query.validator.ValidationVisitor;
-import org.teiid.query.validator.UpdateValidator.UpdateInfo;
/**
@@ -115,6 +117,13 @@
}
};
+ private static final Comparator<GroupSymbol> nonCorrelatedComparator = new Comparator<GroupSymbol>() {
+ @Override
+ public int compare(GroupSymbol arg0, GroupSymbol arg1) {
+ return arg0.getNonCorrelationName().compareTo(arg1.getNonCorrelationName());
+ }
+ };
+
public ProcessorPlan optimize(
Command command)
throws
@@ -138,40 +147,44 @@
//plan with
List<WithQueryCommand> withList = null;
- Object modelID = null;
- boolean supportsWithPushdown = true;
- List<WithQueryCommand> pushDownWith = null;
+ Map<String, WithQueryCommand> pushdownWith = null;
+ Set<GroupSymbol> withGroups = null;
if (command instanceof QueryCommand) {
QueryCommand queryCommand = (QueryCommand)command;
- final HashSet<String> names = new HashSet<String>();
- if (queryCommand.getWith() != null) {
- withList = queryCommand.getWith();
- for (WithQueryCommand with : queryCommand.getWith()) {
- Command subCommand = with.getCommand();
- ProcessorPlan procPlan = QueryOptimizer.optimizePlan(subCommand, metadata, idGenerator, capFinder, analysisRecord, context);
+ withList = queryCommand.getWith();
+ if (withList != null) {
+ for (WithQueryCommand with : withList) {
+ QueryCommand subCommand = with.getCommand();
+ RelationalPlan procPlan = (RelationalPlan)QueryOptimizer.optimizePlan(subCommand, metadata, idGenerator, capFinder, analysisRecord, context);
+ RelationalNode root = procPlan.getRootNode();
+ Number planCardinality = root.getEstimateNodeCardinality();
+ if (planCardinality != null) {
+ ((TempMetadataID)with.getGroupSymbol().getMetadataID()).setCardinality(planCardinality.intValue());
+ }
subCommand.setProcessorPlan(procPlan);
AccessNode aNode = CriteriaCapabilityValidatorVisitor.getAccessNode(procPlan);
- if (aNode != null && supportsWithPushdown) {
- modelID = CriteriaCapabilityValidatorVisitor.validateCommandPushdown(modelID, metadata, capFinder, aNode);
- }
+ if (aNode == null) {
+ continue;
+ }
+ Object modelID = CriteriaCapabilityValidatorVisitor.validateCommandPushdown(null, metadata, capFinder, aNode);
QueryCommand withCommand = CriteriaCapabilityValidatorVisitor.getQueryCommand(aNode);
if (modelID == null || withCommand == null) {
- supportsWithPushdown = false;
- } else {
- if (pushDownWith == null) {
- pushDownWith = new ArrayList<WithQueryCommand>();
- }
- WithQueryCommand wqc = new WithQueryCommand(with.getGroupSymbol(), with.getColumns(), withCommand);
- pushDownWith.add(wqc);
+ continue;
}
- names.add(with.getGroupSymbol().getCanonicalName());
+ if (!CapabilitiesUtil.supports(Capability.COMMON_TABLE_EXPRESSIONS, modelID, metadata, capFinder)) {
+ continue;
+ }
+ WithQueryCommand wqc = new WithQueryCommand(with.getGroupSymbol(), with.getColumns(), withCommand);
+ with.getGroupSymbol().setModelMetadataId(modelID);
+ if (pushdownWith == null) {
+ pushdownWith = new LinkedHashMap<String, WithQueryCommand>();
+ withGroups = new TreeSet<GroupSymbol>(nonCorrelatedComparator);
+ }
+ pushdownWith.put(with.getGroupSymbol().getName(), wqc);
}
- if (modelID != null && supportsWithPushdown) {
- supportsWithPushdown = CapabilitiesUtil.supports(Capability.COMMON_TABLE_EXPRESSIONS, modelID, metadata, capFinder);
+ if (pushdownWith != null) {
+ addModelIds(command, pushdownWith);
}
- if (supportsWithPushdown) {
- addModelIds(command, modelID, names);
- }
}
}
@@ -187,7 +200,7 @@
}
// Connect ProcessorPlan to SubqueryContainer (if any) of SELECT or PROJECT nodes
- connectSubqueryContainers(plan); //TODO: merge with node creation
+ connectSubqueryContainers(plan, pushdownWith, withGroups); //TODO: merge with node creation
// Set top column information on top node
List<SingleElementSymbol> topCols = Util.deepClone(command.getProjectedSymbols(), SingleElementSymbol.class);
@@ -199,27 +212,62 @@
plan = executeRules(rules, plan);
RelationalPlan result = planToProcessConverter.convert(plan);
- if (withList != null && supportsWithPushdown) {
- AccessNode aNode = CriteriaCapabilityValidatorVisitor.getAccessNode(result);
- if (aNode != null) {
- QueryCommand queryCommand = CriteriaCapabilityValidatorVisitor.getQueryCommand(aNode);
- if (queryCommand == null || CriteriaCapabilityValidatorVisitor.validateCommandPushdown(modelID, metadata, capFinder, aNode) == null) {
- supportsWithPushdown = false;
- } else {
- //TODO: do this prior to alias generation, or rerun alias generation here
- queryCommand.setWith(pushDownWith);
- }
- } else {
- supportsWithPushdown = false;
- }
+ boolean fullPushdown = false;
+ if (pushdownWith != null) {
+ AccessNode aNode = CriteriaCapabilityValidatorVisitor.getAccessNode(result);
+ if (aNode != null) {
+ QueryCommand queryCommand = CriteriaCapabilityValidatorVisitor.getQueryCommand(aNode);
+ if (queryCommand != null) {
+ fullPushdown = true;
+ }
+ }
+ //distribute the appropriate clauses to the pushdowns
+ assignWithClause(result.getRootNode(), pushdownWith, withGroups);
}
- if (!supportsWithPushdown) {
+ if (!fullPushdown) {
+ //generally any with item associated with a pushdown will not be needed as we're converting to a source query
result.setWith(withList);
}
result.setOutputElements(topCols);
result.setSourceHint(sourceHint);
return result;
}
+
+ private static void assignWithClause(RelationalNode node, Map<String, WithQueryCommand> pushdownWith, Set<GroupSymbol> groups) {
+ if(node instanceof AccessNode) {
+ AccessNode accessNode = (AccessNode) node;
+ Command command = accessNode.getCommand();
+ if (command instanceof QueryCommand) {
+ groups.clear();
+ GroupCollectorVisitor.getGroupsIgnoreInlineViews(command, groups);
+ List<WithQueryCommand> with = new ArrayList<WithQueryCommand>();
+ for (GroupSymbol groupSymbol : groups) {
+ WithQueryCommand clause = pushdownWith.get(groupSymbol.getNonCorrelationName());
+ if (clause != null) {
+ with.add(clause.clone());
+ }
+ }
+ if (!with.isEmpty()) {
+ QueryCommand query = (QueryCommand)command;
+ if (query.getWith() != null) {
+ //we need to accumulate as a with clause could have been used at a lower scope
+ query.getWith().addAll(with);
+ } else {
+ query.setWith(with);
+ }
+ }
+ }
+ }
+
+ // Recurse through children
+ RelationalNode[] children = node.getChildren();
+ for(int i=0; i<children.length; i++) {
+ if (children[i] == null) {
+ break;
+ }
+ assignWithClause(children[i], pushdownWith, groups);
+ }
+ }
/**
* mark all relevant group symbols as being from the modelid
@@ -227,14 +275,14 @@
* @param modelID
* @param names
*/
- private void addModelIds(Command command, final Object modelID,
- final HashSet<String> names) {
+ private void addModelIds(Command command, final Map<String, WithQueryCommand> with) {
PreOrPostOrderNavigator.doVisit(command, new LanguageVisitor() {
@Override
public void visit(UnaryFromClause obj) {
GroupSymbol group = obj.getGroup();
- if (names.contains(group.getNonCorrelationName().toUpperCase())) {
- group.setModelMetadataId(modelID);
+ WithQueryCommand wqc = with.get(group.getNonCorrelationName());
+ if (wqc != null) {
+ group.setModelMetadataId(wqc.getGroupSymbol().getModelMetadataId());
}
}
}, PreOrPostOrderNavigator.POST_ORDER, true);
@@ -251,7 +299,7 @@
this.context = context;
}
- private void connectSubqueryContainers(PlanNode plan) throws QueryPlannerException, QueryMetadataException, TeiidComponentException {
+ private void connectSubqueryContainers(PlanNode plan, Map<String, WithQueryCommand> pushdownWith, Set<GroupSymbol> groups) throws QueryPlannerException, QueryMetadataException, TeiidComponentException {
Set<GroupSymbol> groupSymbols = getGroupSymbols(plan);
for (PlanNode node : NodeEditor.findAllNodes(plan, NodeConstants.Types.PROJECT | NodeConstants.Types.SELECT | NodeConstants.Types.JOIN | NodeConstants.Types.SOURCE)) {
@@ -285,6 +333,20 @@
}
}
ProcessorPlan procPlan = QueryOptimizer.optimizePlan(subCommand, metadata, idGenerator, capFinder, analysisRecord, context);
+ if (procPlan instanceof RelationalPlan && pushdownWith != null) {
+ Map<String, WithQueryCommand> parentPushdownWith = pushdownWith;
+ if (subCommand instanceof QueryCommand) {
+ QueryCommand query = (QueryCommand)subCommand;
+ List<WithQueryCommand> with = query.getWith();
+ if (with != null && !with.isEmpty()) {
+ parentPushdownWith = new HashMap<String, WithQueryCommand>(parentPushdownWith);
+ for (WithQueryCommand withQueryCommand : with) {
+ parentPushdownWith.remove(withQueryCommand.getGroupSymbol().getNonCorrelationName());
+ }
+ }
+ }
+ assignWithClause(((RelationalPlan) procPlan).getRootNode(), parentPushdownWith, groups);
+ }
container.getCommand().setProcessorPlan(procPlan);
setCorrelatedReferences(container, correlatedReferences);
}
Modified: branches/7.7.x/engine/src/main/java/org/teiid/query/processor/relational/RelationalPlan.java
===================================================================
--- branches/7.7.x/engine/src/main/java/org/teiid/query/processor/relational/RelationalPlan.java 2013-04-17 18:59:47 UTC (rev 4561)
+++ branches/7.7.x/engine/src/main/java/org/teiid/query/processor/relational/RelationalPlan.java 2013-05-06 14:30:47 UTC (rev 4562)
@@ -22,8 +22,8 @@
package org.teiid.query.processor.relational;
-import java.util.LinkedList;
import java.util.List;
+import java.util.TreeMap;
import org.teiid.client.plan.PlanNode;
import org.teiid.common.buffer.BlockedException;
@@ -34,19 +34,16 @@
import org.teiid.core.TeiidProcessingException;
import org.teiid.language.SQLConstants;
import org.teiid.query.analysis.AnalysisRecord;
-import org.teiid.query.metadata.TempMetadataAdapter;
-import org.teiid.query.processor.CollectionTupleSource;
import org.teiid.query.processor.ProcessorDataManager;
import org.teiid.query.processor.ProcessorPlan;
import org.teiid.query.processor.QueryProcessor;
import org.teiid.query.processor.relational.ProjectIntoNode.Mode;
import org.teiid.query.sql.LanguageObject;
-import org.teiid.query.sql.lang.Create;
-import org.teiid.query.sql.lang.Insert;
import org.teiid.query.sql.lang.QueryCommand;
import org.teiid.query.sql.lang.SourceHint;
import org.teiid.query.sql.lang.WithQueryCommand;
import org.teiid.query.tempdata.TempTableStore;
+import org.teiid.query.tempdata.TempTableStore.TableProcessor;
import org.teiid.query.tempdata.TempTableStore.TransactionMode;
import org.teiid.query.util.CommandContext;
@@ -59,8 +56,6 @@
private List outputCols;
private List<WithQueryCommand> with;
- private List<WithQueryCommand> withToProcess;
- private QueryProcessor withProcessor;
private TempTableStore tempTableStore;
private boolean multisourceUpdate;
private SourceHint sourceHint;
@@ -104,9 +99,6 @@
tempTableStore = new TempTableStore(context.getConnectionID(), TransactionMode.NONE);
tempTableStore.setParentTempTableStore(context.getTempTableStore());
context.setTempTableStore(tempTableStore);
- for (WithQueryCommand withCommand : this.with) {
- withCommand.getCommand().getProcessorPlan().initialize(context, dataMgr, bufferMgr);
- }
}
setContext(context);
connectExternal(this.root, context, dataMgr, bufferMgr);
@@ -136,34 +128,15 @@
public void open()
throws TeiidComponentException, TeiidProcessingException {
- if (this.with != null) {
- if (withToProcess == null) {
- withToProcess = new LinkedList<WithQueryCommand>(with);
- }
- while (!withToProcess.isEmpty()) {
- WithQueryCommand withCommand = withToProcess.get(0);
- if (withProcessor == null) {
- ProcessorPlan plan = withCommand.getCommand().getProcessorPlan();
- withProcessor = new QueryProcessor(plan, getContext(), this.root.getBufferManager(), this.root.getDataManager());
- Create create = new Create();
- create.setElementSymbolsAsColumns(withCommand.getColumns());
- create.setTable(withCommand.getGroupSymbol());
- this.root.getDataManager().registerRequest(getContext(), create, TempMetadataAdapter.TEMP_MODEL.getID(), null, 0, -1);
- }
- while (true) {
- TupleBatch batch = withProcessor.nextBatch();
- Insert insert = new Insert(withCommand.getGroupSymbol(), withCommand.getColumns(), null);
- insert.setTupleSource(new CollectionTupleSource(batch.getTuples().iterator()));
- this.root.getDataManager().registerRequest(getContext(), insert, TempMetadataAdapter.TEMP_MODEL.getID(), null, 0, -1);
- if (batch.getTerminationFlag()) {
- break;
- }
- }
- this.tempTableStore.setUpdatable(withCommand.getGroupSymbol().getCanonicalName(), false);
- withToProcess.remove(0);
- withProcessor = null;
+ if (with != null && tempTableStore.getProcessors() == null) {
+ TreeMap<String, TableProcessor> processors = new TreeMap<String, TableProcessor>(String.CASE_INSENSITIVE_ORDER);
+ tempTableStore.setProcessors(processors);
+ for (WithQueryCommand withCommand : this.with) {
+ ProcessorPlan plan = withCommand.getCommand().getProcessorPlan();
+ QueryProcessor withProcessor = new QueryProcessor(plan, getContext(), root.getBufferManager(), root.getDataManager());
+ processors.put(withCommand.getGroupSymbol().getName(), new TableProcessor(withProcessor, withCommand.getColumns()));
}
- }
+ }
this.root.open();
}
@@ -178,14 +151,15 @@
public void close()
throws TeiidComponentException {
- if (this.with != null) {
- for (WithQueryCommand withCommand : this.with) {
- withCommand.getCommand().getProcessorPlan().close();
+ if (this.tempTableStore != null) {
+ this.tempTableStore.removeTempTables();
+ if (this.tempTableStore.getProcessors() != null) {
+ for (TableProcessor proc : this.tempTableStore.getProcessors().values()) {
+ proc.getQueryProcessor().closeProcessing();
+ }
+ this.tempTableStore.setProcessors(null);
}
- if (this.tempTableStore != null) {
- this.tempTableStore.removeTempTables();
- }
- }
+ }
this.root.close();
}
@@ -197,8 +171,6 @@
this.root.reset();
if (this.with != null) {
- withToProcess = null;
- withProcessor = null;
for (WithQueryCommand withCommand : this.with) {
withCommand.getCommand().getProcessorPlan().reset();
}
Modified: branches/7.7.x/engine/src/main/java/org/teiid/query/tempdata/TempTableDataManager.java
===================================================================
--- branches/7.7.x/engine/src/main/java/org/teiid/query/tempdata/TempTableDataManager.java 2013-04-17 18:59:47 UTC (rev 4561)
+++ branches/7.7.x/engine/src/main/java/org/teiid/query/tempdata/TempTableDataManager.java 2013-05-06 14:30:47 UTC (rev 4562)
@@ -53,8 +53,8 @@
import org.teiid.language.SQLConstants.Reserved;
import org.teiid.logging.LogConstants;
import org.teiid.logging.LogManager;
+import org.teiid.metadata.FunctionMethod.Determinism;
import org.teiid.metadata.MetadataRepository;
-import org.teiid.metadata.FunctionMethod.Determinism;
import org.teiid.query.QueryPlugin;
import org.teiid.query.eval.Evaluator;
import org.teiid.query.metadata.QueryMetadataInterface;
@@ -67,21 +67,7 @@
import org.teiid.query.processor.ProcessorDataManager;
import org.teiid.query.processor.QueryProcessor;
import org.teiid.query.resolver.util.ResolverUtil;
-import org.teiid.query.sql.lang.CacheHint;
-import org.teiid.query.sql.lang.Command;
-import org.teiid.query.sql.lang.CompareCriteria;
-import org.teiid.query.sql.lang.Create;
-import org.teiid.query.sql.lang.Criteria;
-import org.teiid.query.sql.lang.Delete;
-import org.teiid.query.sql.lang.Drop;
-import org.teiid.query.sql.lang.Insert;
-import org.teiid.query.sql.lang.Option;
-import org.teiid.query.sql.lang.ProcedureContainer;
-import org.teiid.query.sql.lang.Query;
-import org.teiid.query.sql.lang.SPParameter;
-import org.teiid.query.sql.lang.StoredProcedure;
-import org.teiid.query.sql.lang.UnaryFromClause;
-import org.teiid.query.sql.lang.Update;
+import org.teiid.query.sql.lang.*;
import org.teiid.query.sql.navigator.PostOrderNavigator;
import org.teiid.query.sql.symbol.Constant;
import org.teiid.query.sql.symbol.ElementSymbol;
@@ -354,7 +340,7 @@
}
private TupleSource registerQuery(final CommandContext context,
- TempTableStore contextStore, Query query)
+ final TempTableStore contextStore, final Query query)
throws TeiidComponentException, QueryMetadataException,
TeiidProcessingException, ExpressionEvaluationException,
QueryProcessingException {
@@ -416,16 +402,36 @@
}
table = globalStore.getTempTableStore().getOrCreateTempTable(tableName, query, bufferManager, false, false, context);
context.accessedDataObject(group.getMetadataID());
- } else {
- table = contextStore.getOrCreateTempTable(tableName, query, bufferManager, true, false, context);
- if (context.getDataObjects() != null) {
- Object id = RelationalPlanner.getTrackableGroup(group, context.getMetadata());
- if (id != null) {
- context.accessedDataObject(group.getMetadataID());
+ return table.createTupleSource(query.getProjectedSymbols(), query.getCriteria(), query.getOrderBy());
+ }
+ final GroupSymbol g = group;
+ //it's not expected for a blocked exception to bubble up from here, so return a tuplesource to perform getOrCreateTempTable
+ return new TupleSource() {
+ TupleSource ts = null;
+
+ @Override
+ public List<?> nextTuple() throws TeiidComponentException,
+ TeiidProcessingException {
+ if (ts == null) {
+ TempTable tt = contextStore.getOrCreateTempTable(tableName, query, bufferManager, true, false, context);
+ if (context.getDataObjects() != null) {
+ Object id = RelationalPlanner.getTrackableGroup(g, context.getMetadata());
+ if (id != null) {
+ context.accessedDataObject(g.getMetadataID());
+ }
+ }
+ ts = tt.createTupleSource(query.getProjectedSymbols(), query.getCriteria(), query.getOrderBy());
}
+ return ts.nextTuple();
}
- }
- return table.createTupleSource(query.getProjectedSymbols(), query.getCriteria(), query.getOrderBy());
+
+ @Override
+ public void closeSource() {
+ if (ts != null) {
+ ts.closeSource();
+ }
+ }
+ };
}
private void loadAsynch(final CommandContext context,
Modified: branches/7.7.x/engine/src/main/java/org/teiid/query/tempdata/TempTableStore.java
===================================================================
--- branches/7.7.x/engine/src/main/java/org/teiid/query/tempdata/TempTableStore.java 2013-04-17 18:59:47 UTC (rev 4561)
+++ branches/7.7.x/engine/src/main/java/org/teiid/query/tempdata/TempTableStore.java 2013-05-06 14:30:47 UTC (rev 4562)
@@ -36,6 +36,7 @@
import javax.transaction.Synchronization;
import javax.transaction.SystemException;
+import org.teiid.api.exception.query.ExpressionEvaluationException;
import org.teiid.api.exception.query.QueryProcessingException;
import org.teiid.common.buffer.BufferManager;
import org.teiid.core.TeiidComponentException;
@@ -47,6 +48,8 @@
import org.teiid.query.QueryPlugin;
import org.teiid.query.metadata.TempMetadataID;
import org.teiid.query.metadata.TempMetadataStore;
+import org.teiid.query.processor.BatchIterator;
+import org.teiid.query.processor.QueryProcessor;
import org.teiid.query.resolver.command.TempTableResolver;
import org.teiid.query.sql.lang.Command;
import org.teiid.query.sql.lang.Create;
@@ -79,6 +82,21 @@
NONE
}
+ public static class TableProcessor {
+ QueryProcessor queryProcessor;
+ List<ElementSymbol> columns;
+
+ public TableProcessor(QueryProcessor queryProcessor,
+ List<ElementSymbol> columns) {
+ this.queryProcessor = queryProcessor;
+ this.columns = columns;
+ }
+
+ public QueryProcessor getQueryProcessor() {
+ return queryProcessor;
+ }
+ }
+
public class TempTableSynchronization implements Synchronization {
private String id;
@@ -180,6 +198,8 @@
private String sessionID;
private TempTableStore parentTempTableStore;
+ private Map<String, TableProcessor> processors;
+
public TempTableStore(String sessionID, TransactionMode transactionMode) {
this.sessionID = sessionID;
this.transactionMode = transactionMode;
@@ -192,6 +212,10 @@
public boolean hasTempTable(String tempTableName) {
return tempTables.containsKey(tempTableName);
}
+
+ public void setProcessors(Map<String, TableProcessor> plans) {
+ this.processors = plans;
+ }
TempTable addTempTable(final String tempTableName, Create create, BufferManager buffer, boolean add, CommandContext context) throws TeiidProcessingException {
List<ElementSymbol> columns = create.getColumnSymbols();
@@ -284,9 +308,19 @@
return this.tempTables.get(tempTableID);
}
- TempTable getOrCreateTempTable(String tempTableID, Command command, BufferManager buffer, boolean delegate, boolean forUpdate, CommandContext context) throws TeiidProcessingException{
+ public Map<String, TableProcessor> getProcessors() {
+ return processors;
+ }
+
+ TempTable getOrCreateTempTable(String tempTableID, Command command, BufferManager buffer, boolean delegate, boolean forUpdate, CommandContext context) throws TeiidProcessingException, TeiidComponentException{
TempTable tempTable = getTempTable(tempTableID, command, buffer, delegate, forUpdate, context);
if (tempTable != null) {
+ if (processors != null) {
+ TableProcessor withProcessor = processors.get(tempTableID);
+ if (withProcessor != null) {
+ buildWithTable(tempTableID, withProcessor, tempTable);
+ }
+ }
return tempTable;
}
//allow implicit temp group definition
@@ -299,6 +333,18 @@
}
}
if (columns == null) {
+ if (processors != null) {
+ TableProcessor withProcessor = processors.get(tempTableID);
+ if (withProcessor != null) {
+ LogManager.logDetail(LogConstants.CTX_DQP, "Creating temporary table for with clause", tempTableID); //$NON-NLS-1$
+ Create create = new Create();
+ create.setTable(new GroupSymbol(tempTableID));
+ create.setElementSymbolsAsColumns(withProcessor.columns);
+ tempTable = addTempTable(tempTableID, create, buffer, true, context);
+ buildWithTable(tempTableID, withProcessor, tempTable);
+ return tempTable;
+ }
+ }
throw new QueryProcessingException(QueryPlugin.Util.getString("TempTableStore.table_doesnt_exist_error", tempTableID)); //$NON-NLS-1$
}
LogManager.logDetail(LogConstants.CTX_DQP, "Creating temporary table", tempTableID); //$NON-NLS-1$
@@ -307,6 +353,15 @@
create.setElementSymbolsAsColumns(columns);
return addTempTable(tempTableID, create, buffer, true, context);
}
+
+ private void buildWithTable(String tempTableID,
+ TableProcessor withProcessor, TempTable tempTable)
+ throws TeiidComponentException, ExpressionEvaluationException,
+ TeiidProcessingException {
+ tempTable.insert(new BatchIterator(withProcessor.queryProcessor), withProcessor.columns, false);
+ tempTable.setUpdatable(false);
+ processors.remove(tempTableID);
+ }
private TempTable getTempTable(String tempTableID, Command command,
BufferManager buffer, boolean delegate, boolean forUpdate, CommandContext context)
Modified: branches/7.7.x/engine/src/test/java/org/teiid/query/processor/TestWithClauseProcessing.java
===================================================================
--- branches/7.7.x/engine/src/test/java/org/teiid/query/processor/TestWithClauseProcessing.java 2013-04-17 18:59:47 UTC (rev 4561)
+++ branches/7.7.x/engine/src/test/java/org/teiid/query/processor/TestWithClauseProcessing.java 2013-05-06 14:30:47 UTC (rev 4562)
@@ -28,6 +28,7 @@
};
FakeDataManager dataManager = new FakeDataManager();
+ dataManager.setBlockOnce();
sampleData1(dataManager);
ProcessorPlan plan = helpGetPlan(helpParse(sql), RealMetadataFactory.example1Cached());
@@ -98,19 +99,98 @@
helpProcess(plan, dataManager, expected);
}
- @Test public void testWithPushdownWithConstants() throws TeiidException {
- FakeCapabilitiesFinder capFinder = new FakeCapabilitiesFinder();
+ /**
+ * This tests both an intervening parent plan construct (count) and a reference to a parent with in a subquery
+ */
+ @Test public void testWithPushdownNotFullyPushed() throws TeiidException {
+ FakeCapabilitiesFinder capFinder = new FakeCapabilitiesFinder();
BasicSourceCapabilities caps = TestOptimizer.getTypicalCapabilities();
caps.setCapabilitySupport(Capability.COMMON_TABLE_EXPRESSIONS, true);
caps.setCapabilitySupport(Capability.QUERY_FROM_JOIN_SELFJOIN, true);
+ caps.setCapabilitySupport(Capability.QUERY_AGGREGATES_COUNT, false);
capFinder.addCapabilities("pm1", caps); //$NON-NLS-1$
+ String sql = "with a as (select x, y, z from (select e1 as x, e2 as y, e3 as z from pm1.g1) v), b as (select e4 from pm1.g3) SELECT count(a.x), max(a.y) from a, a z group by z.x having max(a.y) < (with b as (select e1 from pm1.g1) select a.y from a, b where a.x = z.x)"; //$NON-NLS-1$
+
+ HardcodedDataManager dataManager = new HardcodedDataManager(RealMetadataFactory.example1Cached());
+ List<?>[] expected = new List[] {
+ Arrays.asList("a", 1, "a"),
+ };
+
+ dataManager.addData("WITH a (x, y, z) AS (SELECT g_0.e1, g_0.e2, g_0.e3 FROM g1 AS g_0) SELECT g_1.x, g_0.y, g_0.x FROM a AS g_0, a AS g_1", expected);
+ dataManager.addData("WITH b (e1) AS (SELECT g_0.e1 FROM g1 AS g_0), a (x, y, z) AS (SELECT g_0.e1, g_0.e2, g_0.e3 FROM g1 AS g_0) SELECT g_0.y FROM a AS g_0, b AS g_1 WHERE g_0.x = 'a'",
+ new List[] {Arrays.asList(2)});
+
+ ProcessorPlan plan = TestOptimizer.helpPlan(sql, RealMetadataFactory.example1Cached(), null, capFinder, new String[] {"WITH a (x, y, z) AS (SELECT g_0.e1, g_0.e2, g_0.e3 FROM pm1.g1 AS g_0) SELECT g_1.x, g_0.y, g_0.x FROM a AS g_0, a AS g_1"}, ComparisonMode.EXACT_COMMAND_STRING);
+
+ helpProcess(plan, dataManager, new List[] {
+ Arrays.asList(1, 1),
+ });
+ }
+
+ /**
+ * Tests source affinity
+ */
+ @Test public void testWithPushdownNotFullyPushed1() throws TeiidException {
+ FakeCapabilitiesFinder capFinder = new FakeCapabilitiesFinder();
+ BasicSourceCapabilities caps = TestOptimizer.getTypicalCapabilities();
+ caps.setCapabilitySupport(Capability.COMMON_TABLE_EXPRESSIONS, true);
+ caps.setCapabilitySupport(Capability.QUERY_FROM_JOIN_SELFJOIN, true);
+ caps.setCapabilitySupport(Capability.QUERY_AGGREGATES_COUNT, false);
+ capFinder.addCapabilities("pm1", caps); //$NON-NLS-1$
+ capFinder.addCapabilities("pm2", caps); //$NON-NLS-1$
+
+ String sql = "with a as (select e1 from pm1.g1), b as (select e1 from pm2.g2), c as (select count(*) as x from pm1.g1) SELECT a.e1, (select max(x) from c), pm1.g1.e2 from pm1.g1, a, b"; //$NON-NLS-1$
+
+ HardcodedDataManager dataManager = new HardcodedDataManager(RealMetadataFactory.example1Cached());
+
+ dataManager.addData("WITH a (e1) AS (SELECT g_0.e1 FROM g1 AS g_0) SELECT g_1.e1, g_0.e2 FROM g1 AS g_0, a AS g_1", new List[] {
+ Arrays.asList("a", 1),
+ Arrays.asList("a", 2)
+ });
+ dataManager.addData("WITH b (e1) AS (SELECT g_0.e1 FROM g2 AS g_0) SELECT 1 FROM b AS g_0", new List[] {
+ Arrays.asList("b"),
+ });
+ dataManager.addData("SELECT 1 FROM g1 AS g_0", new List[] {
+ Arrays.asList(1), Arrays.asList(1), Arrays.asList(1)
+ });
+
+ ProcessorPlan plan = TestOptimizer.helpPlan(sql, RealMetadataFactory.example1Cached(), null, capFinder, new String[] {
+ "WITH a (e1) AS (SELECT g_0.e1 FROM pm1.g1 AS g_0) SELECT g_1.e1, g_0.e2 FROM pm1.g1 AS g_0, a AS g_1",
+ "WITH b (e1) AS (SELECT g_0.e1 FROM pm2.g2 AS g_0) SELECT 1 FROM b AS g_0"}, ComparisonMode.EXACT_COMMAND_STRING);
+
+ helpProcess(plan, dataManager, new List[] {
+ Arrays.asList("a", 3, 1),
+ Arrays.asList("a", 3, 2),
+ });
+ }
+
+ @Test public void testWithPushdownWithConstants() throws TeiidException {
+ FakeCapabilitiesFinder capFinder = new FakeCapabilitiesFinder();
+ BasicSourceCapabilities caps = TestOptimizer.getTypicalCapabilities();
+ caps.setCapabilitySupport(Capability.COMMON_TABLE_EXPRESSIONS, true);
+ caps.setCapabilitySupport(Capability.QUERY_FROM_JOIN_SELFJOIN, true);
+ capFinder.addCapabilities("pm1", caps); //$NON-NLS-1$
+
String sql = "with a (x, y) as (select 1, 2 from pm1.g1) SELECT a.x from a, a z"; //$NON-NLS-1$
- FakeDataManager dataManager = new FakeDataManager();
- sampleData1(dataManager);
-
TestOptimizer.helpPlan(sql, RealMetadataFactory.example1Cached(), null, capFinder, new String[] {"WITH a (x, y) AS (SELECT 1, 2 FROM pm1.g1 AS g_0) SELECT g_0.x FROM a AS g_0, a AS g_1"}, ComparisonMode.EXACT_COMMAND_STRING);
}
+
+ @Test public void testWithOrderBy() throws TeiidException {
+ FakeCapabilitiesFinder capFinder = new FakeCapabilitiesFinder();
+ BasicSourceCapabilities caps = TestOptimizer.getTypicalCapabilities();
+ caps.setCapabilitySupport(Capability.COMMON_TABLE_EXPRESSIONS, true);
+ caps.setCapabilitySupport(Capability.QUERY_FROM_JOIN_SELFJOIN, true);
+ capFinder.addCapabilities("pm1", caps); //$NON-NLS-1$
+
+ String sql = "with a (x, y) as (select 1, 2 from pm1.g1) SELECT a.x from a, a z order by x"; //$NON-NLS-1$
+
+ HardcodedDataManager dataManager = new HardcodedDataManager(RealMetadataFactory.example1Cached());
+ dataManager.addData("WITH a (x, y) AS (SELECT 1, 2 FROM g1 AS g_0) SELECT g_0.x AS c_0 FROM a AS g_0, a AS g_1 ORDER BY c_0", new List[0]);
+ ProcessorPlan plan = TestOptimizer.helpPlan(sql, RealMetadataFactory.example1Cached(), null, capFinder, new String[] {"WITH a (x, y) AS (SELECT 1, 2 FROM pm1.g1 AS g_0) SELECT g_0.x AS c_0 FROM a AS g_0, a AS g_1 ORDER BY c_0"}, ComparisonMode.EXACT_COMMAND_STRING);
+ //check the full pushdown command
+ helpProcess(plan, dataManager, new List[0]);
+ }
}
11 years, 7 months
[teiid/teiid] 3a2a0b: TEIID-2327 initial commit of column masking
by shawkins
Branch: refs/heads/master
Home: https://github.com/teiid/teiid
Commit: 3a2a0bce73abf49f66a59b7768087b2532a11a1b
https://github.com/teiid/teiid/commit/3a2a0bce73abf49f66a59b7768087b2532a...
Author: shawkins <shawkins(a)redhat.com>
Date: 2013-05-03 (Fri, 03 May 2013)
Changed paths:
M admin/src/main/java/org/teiid/adminapi/AdminPlugin.java
M admin/src/main/java/org/teiid/adminapi/DataPolicy.java
M admin/src/main/java/org/teiid/adminapi/impl/DataPolicyMetadata.java
M admin/src/main/java/org/teiid/adminapi/impl/VDBMetadataMapper.java
M admin/src/main/java/org/teiid/adminapi/impl/VDBMetadataParser.java
M admin/src/main/resources/org/teiid/adminapi/i18n.properties
M admin/src/test/java/org/teiid/adminapi/impl/TestVDBMetaData.java
M admin/src/test/resources/parser-test-vdb.xml
M api/src/main/java/org/teiid/translator/ExecutionFactory.java
M client/src/main/resources/vdb-deployer.xsd
M engine/src/main/java/org/teiid/query/QueryPlugin.java
A engine/src/main/java/org/teiid/query/optimizer/relational/ColumnMaskingHelper.java
M engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java
M engine/src/main/java/org/teiid/query/optimizer/relational/RowBasedSecurityHelper.java
A engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleApplyColumnMasks.java
M engine/src/main/java/org/teiid/query/optimizer/relational/rules/RuleConstants.java
M engine/src/main/java/org/teiid/query/sql/lang/StoredProcedure.java
M engine/src/main/resources/org/teiid/query/i18n.properties
A engine/src/test/java/org/teiid/query/processor/TestColumMasking.java
M engine/src/test/java/org/teiid/query/processor/TestRowBasedSecurity.java
M engine/src/test/java/org/teiid/query/unittest/RealMetadataFactory.java
Log Message:
-----------
TEIID-2327 initial commit of column masking
11 years, 7 months
