Author: shawkins Date: 2009-12-22 14:36:54 -0500 (Tue, 22 Dec 2009) New Revision: 1698 Added: trunk/engine/src/main/java/com/metamatrix/common/buffer/ManagedBatch.java trunk/engine/src/main/java/com/metamatrix/common/buffer/TupleBuffer.java trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/FileStorageManager.java trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/MemoryStorageManager.java trunk/engine/src/main/java/com/metamatrix/dqp/internal/ trunk/engine/src/main/java/com/metamatrix/dqp/internal/datamgr/ trunk/engine/src/main/java/com/metamatrix/query/processor/BatchCollector.java trunk/engine/src/main/java/com/metamatrix/query/processor/BatchIterator.java trunk/engine/src/main/java/com/metamatrix/query/processor/CollectionTupleSource.java trunk/engine/src/main/java/org/teiid/dqp/internal/process/CacheResults.java trunk/engine/src/test/java/com/metamatrix/query/processor/TestCollectionTupleSource.java Removed: trunk/client/src/main/java/com/metamatrix/dqp/internal/datamgr/ trunk/engine/src/main/java/com/metamatrix/common/buffer/BlockedOnMemoryException.java trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManagerLobChunkStream.java trunk/engine/src/main/java/com/metamatrix/common/buffer/MemoryNotAvailableException.java trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferStats.java trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/ManagedBatch.java trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/SizeUtility.java trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleGroupInfo.java trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleSourceImpl.java trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleSourceInfo.java trunk/engine/src/main/java/com/metamatrix/common/buffer/storage/file/FileStorageManager.java trunk/engine/src/main/java/com/metamatrix/connector/metadata/internal/ trunk/engine/src/main/java/com/metamatrix/query/processor/NullTupleSource.java trunk/engine/src/main/java/com/metamatrix/query/processor/proc/ExecSqlInstruction.java trunk/engine/src/main/java/com/metamatrix/query/processor/proc/UpdateCountTupleSource.java trunk/engine/src/main/java/com/metamatrix/query/processor/relational/BatchCollector.java trunk/engine/src/main/java/com/metamatrix/query/processor/relational/BatchIterator.java trunk/engine/src/main/java/com/metamatrix/query/processor/relational/TupleCollector.java trunk/engine/src/main/java/org/teiid/dqp/internal/cache/CacheID.java trunk/engine/src/main/java/org/teiid/dqp/internal/cache/CacheResults.java trunk/engine/src/main/java/org/teiid/dqp/internal/cache/ResultSetCache.java trunk/engine/src/main/java/org/teiid/dqp/internal/process/CachedRequestWorkItem.java trunk/engine/src/main/java/org/teiid/dqp/internal/process/LobChunkStream.java trunk/engine/src/test/java/com/metamatrix/common/buffer/impl/ trunk/engine/src/test/java/com/metamatrix/query/processor/TestNullTupleSource.java trunk/engine/src/test/java/org/teiid/dqp/internal/cache/TestCacheID.java trunk/engine/src/test/java/org/teiid/dqp/internal/cache/TestResultSetCache.java Modified: trunk/cache-jbosscache/src/main/java/com/metamatrix/cache/jboss/JBossCacheFactory.java trunk/client/src/main/java/com/metamatrix/common/lob/ByteLobChunkStream.java trunk/engine/src/main/java/com/metamatrix/cache/CacheConfiguration.java trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManager.java trunk/engine/src/main/java/com/metamatrix/common/buffer/IndexedTupleSource.java trunk/engine/src/main/java/com/metamatrix/common/buffer/StorageManager.java trunk/engine/src/main/java/com/metamatrix/common/buffer/TupleBatch.java trunk/engine/src/main/java/com/metamatrix/common/buffer/TupleSourceID.java trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferManagerImpl.java trunk/engine/src/main/java/com/metamatrix/query/eval/Evaluator.java trunk/engine/src/main/java/com/metamatrix/query/optimizer/proc/ProcedurePlanner.java trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/RelationalPlanner.java trunk/engine/src/main/java/com/metamatrix/query/processor/ProcessorPlan.java trunk/engine/src/main/java/com/metamatrix/query/processor/QueryProcessor.java trunk/engine/src/main/java/com/metamatrix/query/processor/dynamic/SqlEval.java trunk/engine/src/main/java/com/metamatrix/query/processor/proc/CreateCursorResultSetInstruction.java trunk/engine/src/main/java/com/metamatrix/query/processor/proc/ExecDynamicSqlInstruction.java trunk/engine/src/main/java/com/metamatrix/query/processor/proc/ProcedurePlan.java trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentCriteriaProcessor.java trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentProcedureExecutionNode.java trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentValueSource.java trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DuplicateFilter.java trunk/engine/src/main/java/com/metamatrix/query/processor/relational/GroupingNode.java trunk/engine/src/main/java/com/metamatrix/query/processor/relational/JoinNode.java trunk/engine/src/main/java/com/metamatrix/query/processor/relational/JoinStrategy.java trunk/engine/src/main/java/com/metamatrix/query/processor/relational/MergeJoinStrategy.java trunk/engine/src/main/java/com/metamatrix/query/processor/relational/NestedLoopJoinStrategy.java trunk/engine/src/main/java/com/metamatrix/query/processor/relational/PartitionedSortJoin.java trunk/engine/src/main/java/com/metamatrix/query/processor/relational/RelationalNode.java trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SortNode.java trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SortUtility.java trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SourceState.java trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SubqueryAwareEvaluator.java trunk/engine/src/main/java/com/metamatrix/query/processor/relational/UnionAllNode.java trunk/engine/src/main/java/com/metamatrix/query/processor/xml/RelationalPlanExecutor.java trunk/engine/src/main/java/com/metamatrix/query/processor/xml/XMLPlan.java trunk/engine/src/main/java/com/metamatrix/query/processor/xml/XMLUtil.java trunk/engine/src/main/java/com/metamatrix/query/processor/xquery/XQueryPlan.java trunk/engine/src/main/java/com/metamatrix/query/sql/lang/Command.java trunk/engine/src/main/java/com/metamatrix/query/sql/lang/Query.java trunk/engine/src/main/java/com/metamatrix/query/sql/lang/StoredProcedure.java trunk/engine/src/main/java/com/metamatrix/query/sql/lang/XQuery.java trunk/engine/src/main/java/com/metamatrix/query/sql/proc/CreateUpdateProcedureCommand.java trunk/engine/src/main/java/com/metamatrix/query/sql/util/ValueIteratorSource.java trunk/engine/src/main/java/com/metamatrix/query/tempdata/TempTableStore.java trunk/engine/src/main/java/com/metamatrix/query/tempdata/TempTableStoreImpl.java trunk/engine/src/main/java/com/metamatrix/query/util/CommandContext.java trunk/engine/src/main/java/org/teiid/dqp/internal/process/DQPCore.java trunk/engine/src/main/java/org/teiid/dqp/internal/process/DataTierManagerImpl.java trunk/engine/src/main/java/org/teiid/dqp/internal/process/LobWorkItem.java trunk/engine/src/main/java/org/teiid/dqp/internal/process/PreparedPlanCache.java trunk/engine/src/main/java/org/teiid/dqp/internal/process/PreparedStatementRequest.java trunk/engine/src/main/java/org/teiid/dqp/internal/process/Request.java trunk/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java trunk/engine/src/test/java/com/metamatrix/common/buffer/BufferManagerFactory.java trunk/engine/src/test/java/com/metamatrix/common/buffer/storage/file/TestFileStorageManager.java trunk/engine/src/test/java/com/metamatrix/query/optimizer/relational/rules/TestRuleAccessPatternValidation.java trunk/engine/src/test/java/com/metamatrix/query/processor/TestQueryProcessor.java trunk/engine/src/test/java/com/metamatrix/query/processor/TestVirtualDepJoin.java trunk/engine/src/test/java/com/metamatrix/query/processor/dynamic/TestSqlEval.java trunk/engine/src/test/java/com/metamatrix/query/processor/proc/TestProcedureProcessor.java trunk/engine/src/test/java/com/metamatrix/query/processor/relational/NodeTestUtil.java trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestDuplicateFilter.java trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestSelectNode.java trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestSortNode.java trunk/engine/src/test/java/com/metamatrix/query/processor/xml/TestXMLProcessor.java trunk/engine/src/test/java/org/teiid/dqp/internal/process/TestPreparedPlanCache.java trunk/engine/src/test/java/org/teiid/dqp/internal/process/TestPreparedStatement.java trunk/engine/src/test/java/org/teiid/dqp/internal/process/validator/TestAuthorizationValidationVisitor.java trunk/runtime/src/main/java/com/metamatrix/dqp/embedded/services/EmbeddedBufferService.java trunk/test-integration/common/src/test/java/com/metamatrix/server/integration/BaseQueryTest.java trunk/test-integration/common/src/test/java/com/metamatrix/server/integration/TestTPCR.java trunk/test-integration/common/src/test/java/com/metamatrix/server/integration/TestXMLTypeTranslations.java Log: TEIID-168 TEIID-913 intermediate commit in removing old implementation of result set caching and addressing buffermanager shortcomings. Modified: trunk/cache-jbosscache/src/main/java/com/metamatrix/cache/jboss/JBossCacheFactory.java =================================================================== --- trunk/cache-jbosscache/src/main/java/com/metamatrix/cache/jboss/JBossCacheFactory.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/cache-jbosscache/src/main/java/com/metamatrix/cache/jboss/JBossCacheFactory.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -30,7 +30,6 @@ import org.jboss.cache.eviction.FIFOAlgorithmConfig; import org.jboss.cache.eviction.LFUAlgorithmConfig; import org.jboss.cache.eviction.LRUAlgorithmConfig; -import org.jboss.cache.eviction.MRUAlgorithmConfig; import org.jboss.cache.jmx.JmxRegistrationManager; import com.google.inject.Inject; @@ -91,11 +90,6 @@ lru.setTimeToLive(-1); // -1 no limit evictionConfig = lru; } - else if (config.getPolicy() == Policy.MRU) { - MRUAlgorithmConfig mru = new MRUAlgorithmConfig(); - mru.setMaxNodes(config.getMaxNodes()); - evictionConfig = mru; - } else if (config.getPolicy() == Policy.FIFO) { FIFOAlgorithmConfig fifo = new FIFOAlgorithmConfig(); fifo.setMaxNodes(config.getMaxNodes()); Modified: trunk/client/src/main/java/com/metamatrix/common/lob/ByteLobChunkStream.java =================================================================== --- trunk/client/src/main/java/com/metamatrix/common/lob/ByteLobChunkStream.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/client/src/main/java/com/metamatrix/common/lob/ByteLobChunkStream.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -31,9 +31,6 @@ /** * A wrapper class, given a InputStream object can convert a underlying * stream into sequence of ByteLobChunk objects of given chunk size. - * Opposite of the ByteLobChunkInputStream. - * - * @see ByteLobChunkInputStream */ public class ByteLobChunkStream implements LobChunkProducer { private PushbackInputStream stream; Modified: trunk/engine/src/main/java/com/metamatrix/cache/CacheConfiguration.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/cache/CacheConfiguration.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/cache/CacheConfiguration.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -24,11 +24,10 @@ public class CacheConfiguration { - public static CacheConfiguration DEFAULT = new CacheConfiguration(Policy.MRU, 60*60, 100); // 1 hours with 100 nodes. + public static CacheConfiguration DEFAULT = new CacheConfiguration(Policy.LRU, 60*60, 100); // 1 hours with 100 nodes. public enum Policy { LRU, // Least Recently Used - MRU, // Most Recently Used FIFO, // First in First Out LFU; // Least frequently Used } Deleted: trunk/engine/src/main/java/com/metamatrix/common/buffer/BlockedOnMemoryException.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/BlockedOnMemoryException.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/BlockedOnMemoryException.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -1,80 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.common.buffer; - -/** - * This exception is thrown by the plan if it receives a MemoryNotAvailableException while - * attempting to pin a batch in the BufferManager. - */ -public class BlockedOnMemoryException extends BlockedException { - - public static final BlockedOnMemoryException INSTANCE = new BlockedOnMemoryException(""); //$NON-NLS-1$ - - /** - * No-arg costructor required by Externalizable semantics - */ - public BlockedOnMemoryException() { - super(); - } - - /** - * Construct an instance with the message specified. - * - * @param message A message describing the exception - */ - public BlockedOnMemoryException( String message ) { - super( message ); - } - - /** - * Construct an instance with the message and error code specified. - * - * @param message A message describing the exception - * @param code The error code - */ - public BlockedOnMemoryException( String code, String message ) { - super( code, message ); - } - - /** - * Construct an instance from a message and an exception to chain to this one. - * - * @param message A message describing the exception - * @param e An exception to nest within this one - */ - public BlockedOnMemoryException( Throwable e, String message ) { - super( e, message ); - } - - /** - * Construct an instance from a message and a code and an exception to - * chain to this one. - * - * @param e An exception to nest within this one - * @param message A message describing the exception - * @param code A code denoting the exception - */ - public BlockedOnMemoryException( Throwable e, String code, String message ) { - super( e, code, message ); - } -} Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManager.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManager.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManager.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -25,8 +25,6 @@ import java.util.List; import com.metamatrix.api.exception.MetaMatrixComponentException; -import com.metamatrix.common.lob.LobChunk; -import com.metamatrix.common.types.Streamable; /** * The buffer manager controls how memory is used and how data flows through @@ -38,23 +36,6 @@ */ public interface BufferManager { - public enum TupleSourceStatus { - /** - * Indicates the status of a {@link TupleSource} is active; the - * TupleSource is itself currently still receiving data. - * @see #getStatus - * @see #setStatus - */ - ACTIVE, - /** - * Indicates the status of a {@link TupleSource} is full; the - * TupleSource has loaded all of its tuples. - * @see #getStatus - * @see #setStatus - */ - FULL - } - public enum TupleSourceType { /** * Indicates that a tuple source is use during query processing as a @@ -79,29 +60,9 @@ */ int getConnectorBatchSize(); - /** - * Creates a tuple source based on a schema and properties describing - * hints about the source - * @param elements Elements of the tuple source - * @param groupName Tuple source group name - * @param tupleSourceType Type of tuple source - * @return Identifier for tuple source - * @throws MetaMatrixComponentException indicating a non-business-related - * exception (such as a communication exception) - */ - TupleSourceID createTupleSource(List elements, String groupName, TupleSourceType tupleSourceType) + TupleBuffer createTupleBuffer(List elements, String groupName, TupleSourceType tupleSourceType) throws MetaMatrixComponentException; - /** - * Removes a tuple source by ID - * @param tupleSourceID Tuple source identifier - * @throws TupleSourceNotFoundException if tuple source could not be found - * @throws MetaMatrixComponentException indicating a non-business-related - * exception (such as a communication exception) - */ - void removeTupleSource(TupleSourceID tupleSourceID) - throws TupleSourceNotFoundException, MetaMatrixComponentException; - /** * Removes all tuple sources by group name * @param groupName Tuple source group name @@ -109,164 +70,7 @@ * @throws MetaMatrixComponentException indicating a non-business-related * exception (such as a communication exception) */ - void removeTupleSources(String groupName) + void removeTupleBuffers(String groupName) throws MetaMatrixComponentException; - - /** - * Gets a tuple source by ID - * @param tupleSourceID Tuple source identifier - * @return Tuple source to get tuples from the specified source - * @throws TupleSourceNotFoundException if tuple source could not be found - * @throws MetaMatrixComponentException indicating a non-business-related - * exception (such as a communication exception) - */ - IndexedTupleSource getTupleSource(TupleSourceID tupleSourceID) - throws TupleSourceNotFoundException, MetaMatrixComponentException; - - /** - * Gets a tuple batch by ID and indexes. Pins this tuple batch in memory until - * it is unpinned or tuple source is removed. If memory does not exist to pin the - * batch, a MemoryNotAvailableException is thrown. - * - * @param tupleSourceID Tuple source identifier - * @param beginRow First row index to return - * @return Batch of rows starting from beginRow and not past maxEndRow - * @throws TupleSourceNotFoundException if tuple source could not be found - * @throws MetaMatrixComponentException indicating a non-business-related - * exception (such as a communication exception) - * @throws MemoryNotAvailableException If memory was not available for the pin - */ - TupleBatch pinTupleBatch(TupleSourceID tupleSourceID, int beginRow) - throws TupleSourceNotFoundException, MemoryNotAvailableException, MetaMatrixComponentException; - - /** - * Unpins a range of rows from the given tuple source - * @param tupleSourceID Tuple source identifier - * @param firstRow First row to unpin - * @throws TupleSourceNotFoundException if tuple source could not be found - * @throws MetaMatrixComponentException indicating a non-business-related - * exception (such as a communication exception) - */ - void unpinTupleBatch(TupleSourceID tupleSourceID, int firstRow) - throws TupleSourceNotFoundException, MetaMatrixComponentException; - - /** - * Gets a tuple source schema by ID - * @param tupleSourceID Tuple source identifier - * @return List of ElementSymbol describing elements of tuple source - * @throws TupleSourceNotFoundException if tuple source could not be found - * @throws MetaMatrixComponentException indicating a non-business-related - * exception (such as a communication exception) - */ - List getTupleSchema(TupleSourceID tupleSourceID) - throws TupleSourceNotFoundException, MetaMatrixComponentException; - - - /** - * Adds a batch of tuples for the specified tuple source - * @param tupleSourceID Tuple source identifier - * @param tupleBatch Batch of rows to add - * @throws TupleSourceNotFoundException if tuple source could not be found - * @throws MetaMatrixComponentException indicating a non-business-related - * exception (such as a communication exception) - */ - void addTupleBatch(TupleSourceID tupleSourceID, TupleBatch tupleBatch) - throws TupleSourceNotFoundException, MetaMatrixComponentException; - - /** - * Gets the current row count - * @param tupleSourceID Tuple source identifier - * @return Current known number of rows in tuple source - * @throws TupleSourceNotFoundException if tuple source could not be found - * @throws MetaMatrixComponentException indicating a non-business-related - * exception (such as a communication exception) - */ - int getRowCount(TupleSourceID tupleSourceID) - throws TupleSourceNotFoundException, MetaMatrixComponentException; - - /** - * Sets the status of the tuple source - * @param tupleSourceID Tuple source identifier - * @param status New status - * @throws TupleSourceNotFoundException if tuple source could not be found - * @throws MetaMatrixComponentException indicating a non-business-related - * exception (such as a communication exception) - * @see TupleSourceStatus#ACTIVE - * @see TupleSourceStatus#FULL - */ - void setStatus(TupleSourceID tupleSourceID, TupleSourceStatus status) - throws TupleSourceNotFoundException, MetaMatrixComponentException; - - /** - * Gets the status of the tuple source - * @param tupleSourceID Tuple source identifier - * @return Status of tuple source - * @throws TupleSourceNotFoundException if tuple source could not be found - * @throws MetaMatrixComponentException indicating a non-business-related - * exception (such as a communication exception) - * @see TupleSourceStatus#ACTIVE - * @see TupleSourceStatus#FULL - */ - TupleSourceStatus getStatus(TupleSourceID tupleSourceID) - throws TupleSourceNotFoundException, MetaMatrixComponentException; - - /** - * Gets the final row count if tuple source is FULL, otherwise returns -1. - * @param tupleSourceID Tuple source identifier - * @return Final row count if status == FULL, -1 otherwise - */ - int getFinalRowCount(TupleSourceID tupleSourceID) - throws TupleSourceNotFoundException, MetaMatrixComponentException; - /** - * Add a streamable object to the persistent store. The distinction is made between - * regular tuple sources and streamable object beacuse, streamable objects are fairly - * large in size can not be loaded totally into memory, as other tuple sources are done - * this mechanism allows to stream these objects chunk by chunk. - * @param tupleSourceID - * @param streamGlob part of the stream - * @param stream - * @throws TupleSourceNotFoundException - * @throws MetaMatrixComponentException - */ - void addStreamablePart(TupleSourceID tupleSourceID, LobChunk streamGlob, int beginRow) - throws TupleSourceNotFoundException, MetaMatrixComponentException; - - /** - * Returns the streamable batch object's part stored with specified identifier - * @param tupleSourceID - identifier - * @return LobChunk a part of the Streamable object stored. - * @throws TupleSourceNotFoundException - * @throws MetaMatrixComponentException - */ - LobChunk getStreamablePart(TupleSourceID tupleSourceID, int beginRow) - throws TupleSourceNotFoundException, MetaMatrixComponentException; - - /** - * Release batches that have been pinned by this thread. - * This method should be called when processing is terminated - * to ensure that the memory can be freed. - */ - void releasePinnedBatches() throws MetaMatrixComponentException; - - /** - * Return the LOB associated with the referenceId - * @param id - * @param referenceId - * @return - * @throws TupleSourceNotFoundException - * @throws MetaMatrixComponentException - */ - Streamable<?> getStreamable(TupleSourceID id, String referenceId) - throws TupleSourceNotFoundException, MetaMatrixComponentException; - - /** - * Assign the {@link TupleSource} as the persistent stream for the {@link Streamable} - * @param id - * @param s - * @throws TupleSourceNotFoundException - */ - void setPersistentTupleSource(TupleSourceID id, Streamable<? extends Object> s) - throws TupleSourceNotFoundException; - } Deleted: trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManagerLobChunkStream.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManagerLobChunkStream.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManagerLobChunkStream.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -1,67 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.common.buffer; - - -import java.io.IOException; - -import com.metamatrix.api.exception.MetaMatrixComponentException; -import com.metamatrix.common.lob.LobChunk; -import com.metamatrix.common.lob.LobChunkProducer; -import com.metamatrix.common.log.LogManager; -import com.metamatrix.dqp.DQPPlugin; -import com.metamatrix.dqp.util.LogConstants; - -public class BufferManagerLobChunkStream implements LobChunkProducer { - private TupleSourceID sourceId; - private BufferManager bufferMgr; - private int position = 0; - - public BufferManagerLobChunkStream(String persitentId, BufferManager bufferMgr) { - this.sourceId = new TupleSourceID(persitentId); - this.bufferMgr = bufferMgr; - } - - public LobChunk getNextChunk() throws IOException { - try { - this.position++; - return bufferMgr.getStreamablePart(sourceId, position); - } catch (TupleSourceNotFoundException e) { - String msg = DQPPlugin.Util.getString("BufferManagerLobChunkStream.no_tuple_source", new Object[] {sourceId}); //$NON-NLS-1$ - LogManager.logWarning(LogConstants.CTX_BUFFER_MGR, e, msg); - throw new IOException(msg); - } catch (MetaMatrixComponentException e) { - String msg = DQPPlugin.Util.getString("BufferManagerLobChunkStream.error_processing", new Object[] {sourceId}); //$NON-NLS-1$ - LogManager.logWarning(LogConstants.CTX_BUFFER_MGR, e, msg); - throw new IOException(msg); - } - } - - /** - * @see com.metamatrix.common.lob.LobChunkProducer#close() - */ - public void close() throws IOException { - // we could remove the buffer tuple here but, this is just a stream, so we need to delete - // that when we close th eplan. - } -} \ No newline at end of file Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/IndexedTupleSource.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/IndexedTupleSource.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/IndexedTupleSource.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -57,6 +57,4 @@ */ int getCurrentIndex(); - TupleBatch getBatch() - throws MetaMatrixComponentException; } Added: trunk/engine/src/main/java/com/metamatrix/common/buffer/ManagedBatch.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/ManagedBatch.java (rev 0) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/ManagedBatch.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -0,0 +1,70 @@ +/* + * JBoss, Home of Professional Open Source. + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA. + */ + +package com.metamatrix.common.buffer; + +import java.lang.ref.SoftReference; + +class ManagedBatch { + + private int beginRow; + private int endRow; + private SoftReference<TupleBatch> batchReference; + + /** + * Constructor for ManagedBatch. + */ + public ManagedBatch(TupleBatch batch) { + this.beginRow = batch.getBeginRow(); + this.endRow = batch.getEndRow(); + this.batchReference = new SoftReference<TupleBatch>(batch); + } + + /** + * Get the begin row, must be >= 1 + * @return Begin row + */ + public int getBeginRow() { + return this.beginRow; + } + + /** + * Get the end row, inclusive + * @return End row + */ + public int getEndRow() { + return this.endRow; + } + + public String toString() { + return "ManagedBatch[" + beginRow + ", " + endRow + "]"; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + } + + public TupleBatch getBatch() { + return this.batchReference.get(); + } + + public void setBatchReference(TupleBatch batch) { + this.batchReference = new SoftReference<TupleBatch>(batch); + } + +} Property changes on: trunk/engine/src/main/java/com/metamatrix/common/buffer/ManagedBatch.java ___________________________________________________________________ Name: svn:mime-type + text/plain Deleted: trunk/engine/src/main/java/com/metamatrix/common/buffer/MemoryNotAvailableException.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/MemoryNotAvailableException.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/MemoryNotAvailableException.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -1,75 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.common.buffer; - -import com.metamatrix.api.exception.MetaMatrixException; - -/** - * Indicates memory was not available for the requested operation. - */ -public class MemoryNotAvailableException extends MetaMatrixException { - - /** - * No-arg costructor required by Externalizable semantics - */ - public MemoryNotAvailableException() { - super(); - } - - /** - * Constructor for MemoryNotAvailableException. - * @param message - */ - public MemoryNotAvailableException(String message) { - super(message); - } - - /** - * Constructor for MemoryNotAvailableException. - * @param code - * @param message - */ - public MemoryNotAvailableException(String code, String message) { - super(code, message); - } - - /** - * Constructor for MemoryNotAvailableException. - * @param e - * @param message - */ - public MemoryNotAvailableException(Throwable e, String message) { - super(e, message); - } - - /** - * Constructor for MemoryNotAvailableException. - * @param e - * @param code - * @param message - */ - public MemoryNotAvailableException(Throwable e, String code, String message) { - super(e, code, message); - } - -} Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/StorageManager.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/StorageManager.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/StorageManager.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -89,8 +89,7 @@ * @throws MetaMatrixComponentException indicating a non-business-related * exception (such as a communication exception) */ - void removeBatches(TupleSourceID sourceID) - throws MetaMatrixComponentException; + void removeBatches(TupleSourceID sourceID); /** * Shut down the Storage Manager. Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/TupleBatch.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/TupleBatch.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/TupleBatch.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -41,14 +41,11 @@ */ public class TupleBatch implements Externalizable { - public static final long UNKNOWN_SIZE = -1; - private int rowOffset; private List[] tuples; // Optional state private boolean terminationFlag = false; - private long size = UNKNOWN_SIZE; /** * Contains ordered data types of each of the columns in the batch. Although it is not serialized, @@ -143,22 +140,6 @@ this.terminationFlag = terminationFlag; } - /** - * Get the size of this batch - may be constant indicating unknown. - * @return Size of batch in bytes or UNKNOWN_SIZE - */ - public long getSize() { - return this.size; - } - - /** - * Set the size of this batch. - * @param size Size in bytes or UNKNOWN_SIZE - */ - public void setSize(long size) { - this.size = size; - } - public void setDataTypes(String[] types) { this.types = types; } @@ -179,13 +160,11 @@ } public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { - size = in.readLong(); rowOffset = in.readInt(); terminationFlag = in.readBoolean(); tuples = BatchSerializer.readBatch(in, types); } public void writeExternal(ObjectOutput out) throws IOException { - out.writeLong(size); out.writeInt(rowOffset); out.writeBoolean(terminationFlag); BatchSerializer.writeBatch(out, types, tuples); Added: trunk/engine/src/main/java/com/metamatrix/common/buffer/TupleBuffer.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/TupleBuffer.java (rev 0) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/TupleBuffer.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -0,0 +1,390 @@ +/* + * JBoss, Home of Professional Open Source. + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA. + */ + +package com.metamatrix.common.buffer; + +import java.lang.ref.PhantomReference; +import java.lang.ref.ReferenceQueue; +import java.lang.ref.SoftReference; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicLong; + +import com.metamatrix.api.exception.MetaMatrixComponentException; +import com.metamatrix.common.types.DataTypeManager; +import com.metamatrix.common.types.Streamable; +import com.metamatrix.core.util.Assertion; +import com.metamatrix.dqp.DQPPlugin; + +public class TupleBuffer { + + class TupleSourceImpl implements IndexedTupleSource { + private SoftReference<TupleBatch> currentBatch; + private int currentRow = 1; + private int mark = 1; + private List<?> currentTuple; + + @Override + public int getCurrentIndex() { + return this.currentRow; + } + + @Override + public List getSchema(){ + return schema; + } + + @Override + public List<?> nextTuple() + throws MetaMatrixComponentException{ + List<?> result = null; + if (currentTuple != null){ + result = currentTuple; + currentTuple = null; + } else { + result = getCurrentTuple(); + } + if (result != null) { + currentRow++; + } + return result; + } + + private List<?> getCurrentTuple() throws MetaMatrixComponentException, + BlockedException { + TupleBatch batch = getBatch(); + if(batch.getRowCount() == 0) { + // Check if last + if(isFinal) { + currentBatch = null; + return null; + } + throw BlockedException.INSTANCE; + } + + return batch.getTuple(currentRow); + } + + @Override + public void closeSource() + throws MetaMatrixComponentException{ + currentBatch = null; + mark = 1; + reset(); + } + + // Retrieves the necessary batch based on the currentRow + TupleBatch getBatch() + throws MetaMatrixComponentException{ + TupleBatch batch = null; + if (currentBatch != null) { + batch = currentBatch.get(); + } + if (batch != null) { + if (currentRow <= batch.getEndRow() && currentRow >= batch.getBeginRow()) { + return batch; + } + currentBatch = null; + } + + batch = TupleBuffer.this.getBatch(currentRow); + if (batch != null) { + currentBatch = new SoftReference<TupleBatch>(batch); + } + return batch; + } + + @Override + public boolean hasNext() throws MetaMatrixComponentException { + if (this.currentTuple != null) { + return true; + } + + this.currentTuple = getCurrentTuple(); + return this.currentTuple != null; + } + + @Override + public void reset() { + this.setPosition(mark); + this.mark = 1; + } + + @Override + public void mark() { + this.mark = currentRow; + } + + @Override + public void setPosition(int position) { + if (this.currentRow != position) { + this.currentRow = position; + this.currentTuple = null; + } + } + } + + private static class LobReference extends PhantomReference<Streamable<?>> { + + String persistentStreamId; + + public LobReference(Streamable<?> lob) { + super(lob, LOB_QUEUE); + this.persistentStreamId = lob.getPersistenceStreamId(); + } + } + + private static final AtomicLong LOB_ID = new AtomicLong(); + private static final ReferenceQueue<Streamable<?>> LOB_QUEUE = new ReferenceQueue<Streamable<?>>(); + + //construction state + private StorageManager manager; + private String groupName; + private TupleSourceID tupleSourceID; + private List<?> schema; + private String[] types; + private int batchSize; + + private int rowCount; + private boolean isFinal; + private TreeMap<Integer, ManagedBatch> batches = new TreeMap<Integer, ManagedBatch>(); + private ArrayList<List<?>> batchBuffer; + + //lob management + private Map<String, Streamable<?>> lobReferences; //references to contained lobs + private boolean lobs = true; + private LobReference containingLobReference; //reference to containing lob + + public TupleBuffer(StorageManager manager, String groupName, TupleSourceID id, List<?> schema, String[] types, int batchSize) { + this.manager = manager; + this.tupleSourceID = id; + this.schema = schema; + this.types = types; + this.batchSize = batchSize; + if (types != null) { + int i = 0; + for (i = 0; i < types.length; i++) { + if (DataTypeManager.isLOB(types[i]) || types[i] == DataTypeManager.DefaultDataTypes.OBJECT) { + break; + } + } + if (i == types.length) { + lobs = false; + } + } + this.groupName = groupName; + } + + public String getGroupName() { + return groupName; + } + + public boolean isBatchBufferEmpty() { + return this.batchBuffer == null || this.batchBuffer.isEmpty(); + } + + public void addTuple(List<?> tuple) throws MetaMatrixComponentException { + this.rowCount++; + if (batchBuffer == null) { + batchBuffer = new ArrayList<List<?>>(batchSize/4); + } + batchBuffer.add(tuple); + if (batchBuffer.size() == batchSize) { + saveBatch(false); + } + } + + void saveBatch(boolean finalBatch) throws MetaMatrixComponentException { + if ((batchBuffer == null || batchBuffer.isEmpty()) && !finalBatch) { + return; + } + List rows = batchBuffer==null?Collections.emptyList():batchBuffer; + TupleBatch writeBatch = new TupleBatch(rowCount - rows.size() + 1, rows); + if (finalBatch) { + writeBatch.setTerminationFlag(true); + } + if (writeBatch.getTerminationFlag()) { + this.isFinal = true; + } + correctLobReferences(writeBatch); + ManagedBatch mbatch = new ManagedBatch(writeBatch); + this.batches.put(mbatch.getBeginRow(), mbatch); + manager.addBatch(tupleSourceID, writeBatch, types); + batchBuffer = null; + } + + public void close() throws MetaMatrixComponentException { + saveBatch(true); + } + + public TupleBatch getBatch(int row) throws MetaMatrixComponentException { + if (row > rowCount) { + TupleBatch batch = new TupleBatch(rowCount + 1, new List[] {}); + if (isFinal) { + batch.setTerminationFlag(true); + } + return batch; + } + if (this.batchBuffer != null && row > rowCount - this.batchBuffer.size()) { + return new TupleBatch(rowCount - this.batchBuffer.size() + 1, batchBuffer); + } + Map.Entry<Integer, ManagedBatch> entry = batches.floorEntry(row); + ManagedBatch batch = null; + if (entry != null && entry.getValue().getEndRow() >= row) { + batch = entry.getValue(); + TupleBatch result = batch.getBatch(); + if (result != null) { + return result; + } + row = batch.getBeginRow(); + } + try { + TupleBatch result = manager.getBatch(tupleSourceID, row, types); + correctLobReferences(result); + if (batch != null) { + batch.setBatchReference(result); + } + return result; + } catch (TupleSourceNotFoundException e) { + throw new MetaMatrixComponentException(e); + } + } + + public void remove() { + manager.removeBatches(tupleSourceID); + } + + public int getRowCount() { + return rowCount; + } + + public boolean isFinal() { + return isFinal; + } + + public void setFinal(boolean isFinal) { + this.isFinal = isFinal; + } + + public List<?> getSchema() { + return schema; + } + + public int getBatchSize() { + return batchSize; + } + + public void setBatchSize(int batchSize) { + this.batchSize = batchSize; + } + + public void setContainingLobReference(Streamable<?> s) { + cleanLobTupleSource(); + s.setPersistenceStreamId(tupleSourceID.getStringID()); + Assertion.isNull(this.containingLobReference); + this.containingLobReference = new LobReference(s); + } + + /** + * Attempt to clean up lingering tuplebuffers to lobs that are no longer referenced + */ + private void cleanLobTupleSource() { + for (int i = 0; i < 10; i++) { + String tupleSourceId = null; + LobReference ref = (LobReference)LOB_QUEUE.poll(); + if (ref != null) { + tupleSourceId = ref.persistentStreamId; + } + if (tupleSourceId == null) { + break; + } + this.manager.removeBatches(new TupleSourceID(tupleSourceId)); + } + } + + public Streamable<?> getLobReference(String id) throws MetaMatrixComponentException { + Streamable<?> lob = null; + if (this.lobReferences != null) { + lob = this.lobReferences.get(id); + } + if (lob == null) { + throw new MetaMatrixComponentException(DQPPlugin.Util.getString("ProcessWorker.wrongdata")); //$NON-NLS-1$ + } + return lob; + } + + /** + * If a tuple batch is being added with Lobs, then references to + * the lobs will be held on the {@link TupleSourceInfo} + * @param batch + * @throws MetaMatrixComponentException + */ + @SuppressWarnings("unchecked") + private void correctLobReferences(TupleBatch batch) throws MetaMatrixComponentException { + if (!lobs) { + return; + } + List[] rows = batch.getAllTuples(); + int columns = schema.size(); + // walk through the results and find all the lobs + for (int row = 0; row < rows.length; row++) { + for (int col = 0; col < columns; col++) { + Object anObj = rows[row].get(col); + + if (!(anObj instanceof Streamable<?>)) { + continue; + } + Streamable lob = (Streamable)anObj; + String id = lob.getReferenceStreamId(); + if (id == null) { + id = String.valueOf(LOB_ID.getAndIncrement()); + lob.setReferenceStreamId(id); + } + if (this.lobReferences == null) { + this.lobReferences = Collections.synchronizedMap(new HashMap<String, Streamable<?>>()); + } + this.lobReferences.put(id, lob); + if (lob.getReference() == null) { + lob.setReference(getLobReference(lob.getReferenceStreamId()).getReference()); + } + } + } + } + + /** + * Create a new iterator for this buffer + * @return + */ + public IndexedTupleSource createIndexedTupleSource() { + return new TupleSourceImpl(); + } + + @Override + public String toString() { + return this.tupleSourceID.toString(); + } + +} Property changes on: trunk/engine/src/main/java/com/metamatrix/common/buffer/TupleBuffer.java ___________________________________________________________________ Name: svn:mime-type + text/plain Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/TupleSourceID.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/TupleSourceID.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/TupleSourceID.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -28,9 +28,7 @@ * Identifier for a tuple source. The tuple source ID contains two * pieces of information: a unique ID (unique across locations) and * a location. Both must be represented by a string but different - * systems may use different location descriptions. The - * {@link com.metamatrix.common.buffer.BufferManagerLookup} - * is the only party responsible for providing and decoding the location string. + * systems may use different location descriptions. */ public class TupleSourceID implements Serializable { Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferManagerImpl.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferManagerImpl.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferManagerImpl.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -26,130 +26,47 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedHashSet; import java.util.List; import java.util.ListIterator; import java.util.Map; import java.util.Properties; import java.util.Set; -import java.util.Timer; -import java.util.TimerTask; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import com.metamatrix.api.exception.MetaMatrixComponentException; import com.metamatrix.common.buffer.BufferManager; -import com.metamatrix.common.buffer.IndexedTupleSource; -import com.metamatrix.common.buffer.MemoryNotAvailableException; import com.metamatrix.common.buffer.StorageManager; import com.metamatrix.common.buffer.TupleBatch; +import com.metamatrix.common.buffer.TupleBuffer; import com.metamatrix.common.buffer.TupleSourceID; import com.metamatrix.common.buffer.TupleSourceNotFoundException; -import com.metamatrix.common.lob.LobChunk; import com.metamatrix.common.log.LogManager; import com.metamatrix.common.types.DataTypeManager; -import com.metamatrix.common.types.Streamable; import com.metamatrix.core.log.MessageLevel; import com.metamatrix.core.util.Assertion; -import com.metamatrix.dqp.DQPPlugin; import com.metamatrix.dqp.util.LogConstants; import com.metamatrix.query.execution.QueryExecPlugin; import com.metamatrix.query.sql.symbol.Expression; /** - * <p>Default implementation of BufferManager. This buffer manager implementation - * assumes the usage of a StorageManager of type memory and optionally (preferred) - * an additional StorageManager of type FILE.</p> + * <p>Default implementation of BufferManager.</p> + * Responsible for creating/tracking TupleBuffers and providing access to the StorageManager */ -public class BufferManagerImpl implements BufferManager { +public class BufferManagerImpl implements BufferManager, StorageManager { - //memory availability when reserveMemory() is called - static final int MEMORY_AVAILABLE = 1; - static final int MEMORY_EXCEED_MAX = 2; //exceed buffer manager max memory - static final int MEMORY_EXCEED_SESSION_MAX = 3; //exceed session max memory - - private static ThreadLocal<Set<ManagedBatch>> PINNED_BY_THREAD = new ThreadLocal<Set<ManagedBatch>>() { - protected Set<ManagedBatch> initialValue() { - return new HashSet<ManagedBatch>(); - }; - }; - // Initialized stuff private String lookup; private BufferConfig config; - // Cache tuple source info in memory - private Map<TupleSourceID, TupleSourceInfo> tupleSourceMap = new ConcurrentHashMap<TupleSourceID, TupleSourceInfo>(); - // groupName (String) -> TupleGroupInfo map - private Map<String, TupleGroupInfo> groupInfos = new HashMap<String, TupleGroupInfo>(); + private Map<TupleSourceID, TupleBuffer> tupleSourceMap = new ConcurrentHashMap<TupleSourceID, TupleBuffer>(); + private Map<String, Set<TupleSourceID>> groupInfos = new HashMap<String, Set<TupleSourceID>>(); - // Storage manager private StorageManager diskMgr; - // ID creator private AtomicLong currentTuple = new AtomicLong(0); - // Keep track of how memory usage - private volatile long memoryUsed = 0; - - // Track the currently unpinned stuff in a sorted set - private Set<ManagedBatch> unpinned = Collections.synchronizedSet(new LinkedHashSet<ManagedBatch>()); - - // Trigger to handle management and stats logging - private Timer timer; - - // Collected stats - private AtomicInteger pinRequests = new AtomicInteger(0); - private AtomicInteger pinFailures = new AtomicInteger(0); - private AtomicInteger pinnedFromDisk = new AtomicInteger(0); - private AtomicInteger cleanings = new AtomicInteger(0); - private AtomicLong totalCleaned = new AtomicLong(0); - private AtomicInteger pinned = new AtomicInteger(); - /** - * See {@link com.metamatrix.common.buffer.BufferManagerPropertyNames} for a - * description of all the properties. - * @param lookup An object telling the buffer manager what his location is and - * how to find other buffer managers of different locations - * @param properties Properties to configure the buffer manager - */ - public synchronized void initialize(String lookup, Properties properties) throws MetaMatrixComponentException { - this.lookup = lookup; - - // Set up config based on properties - this.config = new BufferConfig(properties); - - // Set up alarms based on config - if(this.config.getManagementInterval() > 0) { - TimerTask mgmtTask = new TimerTask() { - public void run() { - clean(0, null); - } - - }; - getTimer().schedule(mgmtTask, 0, this.config.getManagementInterval()); - } - if(this.config.getLogStatInterval() > 0) { - TimerTask statTask = new TimerTask() { - public void run() { - BufferStats stats = getStats(); - stats.log(); - } - }; - getTimer().schedule(statTask, 0, this.config.getLogStatInterval()); - } - } - - private Timer getTimer() { - if (timer == null) { - timer = new Timer("BufferManagementThread", true); //$NON-NLS-1$ - } - return timer; - } - - /** * Get the configuration of the buffer manager * @return Configuration */ @@ -158,66 +75,6 @@ } /** - * Construct a BufferStats object by looking at all the state. This should - * be multi-thread safe but calling this method clears several counters and - * resets that state. - * @return Buffer statistics - */ - public BufferStats getStats() { - BufferStats stats = new BufferStats(); - - // Get memory info - stats.memoryUsed = this.memoryUsed; - stats.memoryFree = config.getTotalAvailableMemory() - memoryUsed; - - // Get picture of what's happening - Set<TupleSourceID> copyKeys = tupleSourceMap.keySet(); - Iterator<TupleSourceID> infoIter = copyKeys.iterator(); - stats.numTupleSources = copyKeys.size(); - - // Walk through each info and count where all the batches are - - // lock only a single info at a time to minimize locking - while(infoIter.hasNext()) { - TupleSourceID key = infoIter.next(); - TupleSourceInfo info = tupleSourceMap.get(key); - if(info == null) { - continue; - } - - synchronized(info) { - Iterator<ManagedBatch> batchIter = info.getBatchIterator(); - while(batchIter.hasNext()) { - ManagedBatch batch = batchIter.next(); - switch(batch.getLocation()) { - case ManagedBatch.PERSISTENT: - stats.numPersistentBatches++; - break; - case ManagedBatch.PINNED: - stats.numPinnedBatches++; - - if ( LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.TRACE )) { - stats.pinnedManagedBatches.add(batch); - } - - break; - case ManagedBatch.UNPINNED: - stats.numUnpinnedBatches++; - break; - } - } - } - } - - stats.pinRequests = this.pinRequests.get(); - stats.pinSuccesses = this.pinRequests.get() - this.pinFailures.get(); - stats.pinnedFromMemory = this.pinRequests.get() - this.pinnedFromDisk.get(); - stats.numCleanings = this.cleanings.get(); - stats.totalCleaned = this.totalCleaned.get(); - - return stats; - } - - /** * Get processor batch size * @return Number of rows in a processor batch */ @@ -242,28 +99,20 @@ Assertion.isNull(diskMgr); this.diskMgr = storageManager; } - - /** - * Register a new tuple source and return a unique ID for it. - * @param groupName Group name - * @param tupleSourceType Type of tuple source as defined in BufferManager constants - * @param schema List of ElementSymbol - * @return New unique ID for this tuple source - * @throws MetaMatrixComponentException If internal server error occurs - */ - public TupleSourceID createTupleSource(List schema, String groupName, TupleSourceType tupleSourceType) - throws MetaMatrixComponentException { - + + @Override + public TupleBuffer createTupleBuffer(List elements, String groupName, + TupleSourceType tupleSourceType) + throws MetaMatrixComponentException { TupleSourceID newID = new TupleSourceID(String.valueOf(this.currentTuple.getAndIncrement()), this.lookup); - TupleGroupInfo tupleGroupInfo = getGroupInfo(groupName); - TupleSourceInfo info = new TupleSourceInfo(newID, schema, getTypeNames(schema), tupleGroupInfo, tupleSourceType); - tupleGroupInfo.getTupleSourceIDs().add(newID); - tupleSourceMap.put(newID, info); + Set<TupleSourceID> tupleGroupInfo = getGroupInfo(groupName, true); + TupleBuffer tupleBuffer = new TupleBuffer(this, groupName, newID, elements, getTypeNames(elements), getProcessorBatchSize()); + tupleGroupInfo.add(newID); + tupleSourceMap.put(newID, tupleBuffer); if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.DETAIL)) { - LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, new Object[]{"Creating TupleSource:", newID, "of type "+tupleSourceType}); //$NON-NLS-1$ //$NON-NLS-2$ + LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, new Object[]{"Creating TupleBuffer:", newID, "of type "+tupleSourceType}); //$NON-NLS-1$ //$NON-NLS-2$ } - - return newID; + return tupleBuffer; } /** @@ -272,7 +121,7 @@ * @return * @since 4.2 */ - public static String[] getTypeNames(List expressions) { + private static String[] getTypeNames(List expressions) { if (expressions == null) { return null; } @@ -285,73 +134,20 @@ } /** - * Remove a tuple source based on ID - * @throws TupleSourceNotFoundException If tuple source not found - * @throws MetaMatrixComponentException If internal server error occurs - */ - public void removeTupleSource(TupleSourceID tupleSourceID) - throws TupleSourceNotFoundException, MetaMatrixComponentException { - - if(tupleSourceID == null || tupleSourceID.getLocation() == null) { - // this is a bogus tuple source ID - return; - } - if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.DETAIL)) { - LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, new Object[]{"Removing TupleSource:", tupleSourceID}); //$NON-NLS-1$ - } - - // Remove from main map first - TupleSourceInfo info = this.tupleSourceMap.remove(tupleSourceID); - if(info == null) { - throw new TupleSourceNotFoundException(QueryExecPlugin.Util.getString("BufferManagerImpl.tuple_source_not_found", tupleSourceID)); //$NON-NLS-1$ - } - - // Walk through batches and determine whether memory or disk cleanup needs to occur - synchronized(info) { - if(! info.isRemoved()) { - info.setRemoved(); - - Iterator<ManagedBatch> iter = info.getBatchIterator(); - while(iter.hasNext()) { - ManagedBatch batch = iter.next(); - switch(batch.getLocation()) { - case ManagedBatch.UNPINNED: - this.unpinned.remove(batch); - releaseMemory(batch.getSize(), info.getGroupInfo()); - break; - case ManagedBatch.PINNED: - PINNED_BY_THREAD.get().remove(batch); - this.pinned.getAndDecrement(); - releaseMemory(batch.getSize(), info.getGroupInfo()); - break; - } - } - } - } - TupleGroupInfo tupleGroupInfo = info.getGroupInfo(); - tupleGroupInfo.getTupleSourceIDs().remove(tupleSourceID); - - // Remove disk storage - if (this.diskMgr != null){ - this.diskMgr.removeBatches(tupleSourceID); - } - } - - /** * Remove all the tuple sources with the specified group name. Typically the group * name is really a session identifier and this is called to remove all tuple sources * used by a particular session. * @param groupName Name of the group * @throws MetaMatrixComponentException If internal server error occurs */ - public void removeTupleSources(String groupName) + public void removeTupleBuffers(String groupName) throws MetaMatrixComponentException { if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.DETAIL)) { - LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, new Object[]{"Removing TupleSources for group", groupName}); //$NON-NLS-1$ + LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, new Object[]{"Removing TupleBuffers for group", groupName}); //$NON-NLS-1$ } - TupleGroupInfo tupleGroupInfo = null; + Set<TupleSourceID> tupleGroupInfo = null; synchronized(groupInfos) { tupleGroupInfo = groupInfos.remove(groupName); } @@ -359,613 +155,97 @@ return; } List<TupleSourceID> tupleSourceIDs = null; - synchronized (tupleGroupInfo.getTupleSourceIDs()) { - tupleSourceIDs = new ArrayList<TupleSourceID>(tupleGroupInfo.getTupleSourceIDs()); + synchronized (tupleGroupInfo) { + tupleSourceIDs = new ArrayList<TupleSourceID>(tupleGroupInfo); } - MetaMatrixComponentException ex = null; - for (TupleSourceID tsID : tupleSourceIDs) { try { - this.removeTupleSource(tsID); + TupleBuffer tupleBuffer = getTupleBuffer(tsID); + tupleBuffer.remove(); } catch(TupleSourceNotFoundException e) { // ignore and go on - } catch(MetaMatrixComponentException e) { - if(ex == null) { - ex = e; - } } } - - if(ex != null) { - throw ex; - } } /** - * Get a tuple source to walk through the rows for a particular - * tupleSourceID. - * @param tupleSourceID Tuple source identifier - * @return TupleSource - * @throws TupleSourceNotFoundException If tuple source not found - * @throws MetaMatrixComponentException If an internal server error occurred - */ - public IndexedTupleSource getTupleSource(TupleSourceID tupleSourceID) - throws TupleSourceNotFoundException, MetaMatrixComponentException { - - TupleSourceInfo info = getTupleSourceInfo(tupleSourceID, true); - int batchSize = this.config.getProcessorBatchSize(); - return new TupleSourceImpl(this, tupleSourceID, info.getTupleSchema()); - } - - /** - * Get the tuple schema for a particular tuple source - * @param tupleSourceID Tuple source identifier - * @return List of elements describing tuple source - * @throws TupleSourceNotFoundException If tuple source not found - * @throws MetaMatrixComponentException If an internal server error occurred - */ - public List getTupleSchema(TupleSourceID tupleSourceID) - throws TupleSourceNotFoundException, MetaMatrixComponentException { - - TupleSourceInfo info = getTupleSourceInfo(tupleSourceID, true); - return info.getTupleSchema(); - } - - /** - * Set the status for a particular tuple source - * @param tupleSourceID Tuple source identifier - * @param status New status - * @throws TupleSourceNotFoundException If tuple source not found - * @throws MetaMatrixComponentException If an internal server error occurred - */ - public void setStatus(TupleSourceID tupleSourceID, TupleSourceStatus status) - throws TupleSourceNotFoundException, MetaMatrixComponentException { - - if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.DETAIL)) { - LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, new Object[]{"Setting status for", tupleSourceID, "to", status}); //$NON-NLS-1$ //$NON-NLS-2$ - } - - TupleSourceInfo info = getTupleSourceInfo(tupleSourceID, true); - synchronized(info) { - info.setStatus(status); - } - } - - /** - * Get the status for a particular tuple source - * @param tupleSourceID Tuple source identifier - * @return Status - * @throws TupleSourceNotFoundException If tuple source not found - * @throws MetaMatrixComponentException If an internal server error occurred - */ - public TupleSourceStatus getStatus(TupleSourceID tupleSourceID) - throws TupleSourceNotFoundException, MetaMatrixComponentException { - - TupleSourceInfo info = getTupleSourceInfo(tupleSourceID, true); - return info.getStatus(); - } - - /** - * Get the row count for a particular tuple source - * @param tupleSourceID Tuple source identifier - * @return Row count - * @throws TupleSourceNotFoundException If tuple source not found - * @throws MetaMatrixComponentException If an internal server error occurred - */ - public int getRowCount(TupleSourceID tupleSourceID) - throws TupleSourceNotFoundException, MetaMatrixComponentException { - - TupleSourceInfo info = getTupleSourceInfo(tupleSourceID, true); - return info.getRowCount(); - } - - /** - * Add a batch to the given tuple source. It is assumed that batches are - * added in order, so the row count is reset to the end row of this batch. - * @param tupleSourceID Tuple source identifier - * @param tupleBatch New batch - * @throws TupleSourceNotFoundException If tuple source not found - * @throws MetaMatrixComponentException If an internal server error occurred - */ - public void addTupleBatch(TupleSourceID tupleSourceID, TupleBatch tupleBatch) - throws TupleSourceNotFoundException, MetaMatrixComponentException { - - if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.TRACE)) { - LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, new Object[]{"AddTupleBatch for", tupleSourceID, "with " + tupleBatch.getRowCount() + " rows"}); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ - } - - if (tupleBatch.getRowCount() == 0 && !tupleBatch.getTerminationFlag()) { - return; - } - - // Look up info - TupleSourceInfo info = getTupleSourceInfo(tupleSourceID, true); - - if (info.lobsInSource()) { - correctLobReferences(info, tupleBatch); - } - - // Determine where to store - long bytes = SizeUtility.getBatchSize(info.getTypes(), tupleBatch.getAllTuples()); - tupleBatch.setSize(bytes); - short location = ManagedBatch.PERSISTENT; - - if(reserveMemory(bytes, info.getGroupInfo()) == BufferManagerImpl.MEMORY_AVAILABLE) { - location = ManagedBatch.UNPINNED; - } - - synchronized(info) { - if(info.isRemoved()) { - if(location != ManagedBatch.PERSISTENT) { - releaseMemory(bytes, info.getGroupInfo()); - } - throw new TupleSourceNotFoundException(QueryExecPlugin.Util.getString("BufferManagerImpl.tuple_source_not_found", tupleSourceID)); //$NON-NLS-1$ - } - - // Update tuple source state - ManagedBatch managedBatch = new ManagedBatch(tupleSourceID, tupleBatch.getBeginRow(), tupleBatch.getEndRow(), bytes); - managedBatch.setLocation(location); - - // Store into storage manager - try { - if(location == ManagedBatch.PERSISTENT) { - this.diskMgr.addBatch(tupleSourceID, tupleBatch, info.getTypes()); - } else { - managedBatch.setBatch(tupleBatch); - } - } catch(MetaMatrixComponentException e) { - // If we were storing to memory, clean up memory we reserved - if(location != ManagedBatch.PERSISTENT) { - releaseMemory(bytes, info.getGroupInfo()); - } - throw e; - } - - // Add to memory state if in memory - if(location == ManagedBatch.UNPINNED) { - this.unpinned.add(managedBatch); - } - - // Update info with new rows - info.addBatch(managedBatch); - info.setRowCount(tupleBatch.getEndRow()); - } - } - - /** - * Pin a tuple batch in memory and return it. This batch must be unpinned by - * passed the identical tuple source ID and beginning row. - * NOTE: the returned {@link TupleBatch} will have the exact same bounds as the {@link ManagedBatch} - * @param tupleSourceID Tuple source identifier - * @param beginRow Beginning row - * @throws TupleSourceNotFoundException If tuple source not found - * @throws MetaMatrixComponentException If an internal server error occurred - * @throws MemoryNotAvailableException If memory was not available for the pin - */ - public TupleBatch pinTupleBatch(TupleSourceID tupleSourceID, int beginRow) - throws TupleSourceNotFoundException, MemoryNotAvailableException, MetaMatrixComponentException { - - if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.TRACE)) { - LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, new Object[]{"Pinning tupleBatch for", tupleSourceID, "beginRow:", beginRow}); //$NON-NLS-1$ //$NON-NLS-2$ - } - - this.pinRequests.incrementAndGet(); - - TupleSourceInfo info = getTupleSourceInfo(tupleSourceID, true); - int memoryAvailability = BufferManagerImpl.MEMORY_AVAILABLE; - - ManagedBatch mbatch = null; - synchronized (info) { - mbatch = info.getBatch(beginRow); - } - if (mbatch == null || beginRow == mbatch.getEndRow() + 1) { - return new TupleBatch(beginRow, Collections.EMPTY_LIST); - } - long memoryRequiredByBatch = mbatch.getSize(); - - TupleBatch memoryBatch = null; - - for (int pass = 0; pass < 2; pass++) { - if (memoryAvailability == BufferManagerImpl.MEMORY_EXCEED_MAX) { - clean(memoryRequiredByBatch, null); - } else if (memoryAvailability == BufferManagerImpl.MEMORY_EXCEED_SESSION_MAX) { - clean(memoryRequiredByBatch, info.getGroupInfo()); - } - - synchronized(info) { - if(mbatch.getLocation() == ManagedBatch.PINNED) { - // Load batch from memory - already pinned - memoryBatch = mbatch.getBatch(); - break; - } - if(mbatch.getLocation() == ManagedBatch.UNPINNED) { - // Already in memory - just move from unpinned to pinned - this.unpinned.remove(mbatch); - pin(mbatch); - - // Load batch from memory - memoryBatch = mbatch.getBatch(); - break; - } - memoryRequiredByBatch = mbatch.getSize(); - - // Try to reserve some memory - if((memoryAvailability = reserveMemory(memoryRequiredByBatch, info.getGroupInfo())) != BufferManagerImpl.MEMORY_AVAILABLE) { - continue; - } - - this.pinnedFromDisk.incrementAndGet(); - - // Memory was reserved, so move from persistent to memory and pin - int internalBeginRow = mbatch.getBeginRow(); - memoryBatch = diskMgr.getBatch(tupleSourceID, internalBeginRow, info.getTypes()); - - mbatch.setBatch(memoryBatch); - if (info.lobsInSource()) { - correctLobReferences(info, memoryBatch); - } - pin(mbatch); - } - } - - if (memoryBatch == null) { - // Failed to reserve the memory even after a clean, so record the failure and fail the pin - this.pinFailures.incrementAndGet(); - throw new MemoryNotAvailableException(QueryExecPlugin.Util.getString("BufferManagerImpl.no_memory_available")); //$NON-NLS-1$ - } - - return memoryBatch; - } - - private void pin(ManagedBatch mbatch) { - mbatch.setLocation(ManagedBatch.PINNED); - PINNED_BY_THREAD.get().add(mbatch); - if (!mbatch.hasPinnedRows()) { - pinned.getAndIncrement(); - } - mbatch.pin(); - } - - /** - * Unpin a tuple source batch. - * @param tupleSourceID Tuple source identifier - * @param beginRow Beginning row - * @throws TupleSourceNotFoundException If tuple source not found - * @throws MetaMatrixComponentException If an internal server error occurred - */ - public void unpinTupleBatch(TupleSourceID tupleSourceID, int beginRow) - throws TupleSourceNotFoundException, MetaMatrixComponentException { - - if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.TRACE)) { - LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, new Object[]{"Unpinning tupleBatch for", tupleSourceID, "beginRow: " + beginRow}); //$NON-NLS-1$ //$NON-NLS-2$ - } - - TupleSourceInfo info = getTupleSourceInfo(tupleSourceID, true); - synchronized(info) { - ManagedBatch mbatch = info.getBatch(beginRow); - if(mbatch == null || mbatch.getLocation() != ManagedBatch.PINNED) { - return; - } - - mbatch.unpin(); - - // Determine whether batch itself should be unpinned - if(! mbatch.hasPinnedRows()) { - mbatch.setLocation(ManagedBatch.UNPINNED); - PINNED_BY_THREAD.get().remove(mbatch); - this.unpinned.add(mbatch); - pinned.getAndDecrement(); - } - } - } - - /** - * Gets the final row count if tuple source is FULL, otherwise returns -1. - * @param tupleSourceID Tuple source identifier - * @return Final row count if status == FULL, -1 otherwise - * @throws TupleSourceNotFoundException If tuple source not found - * @throws MetaMatrixComponentException If an internal server error occurred - */ - public int getFinalRowCount(TupleSourceID tupleSourceID) - throws TupleSourceNotFoundException, MetaMatrixComponentException { - - TupleSourceInfo info = getTupleSourceInfo(tupleSourceID, true); - synchronized(info) { - if(info.getStatus() == TupleSourceStatus.FULL) { - return info.getRowCount(); - } - return -1; - } - } - - /** * Get the tuple source info for the specified tuple source ID * @param tupleSourceID Tuple source to get - * @param throwException Whether to throw exception if the TupleSourceInfo is not found * @return Tuple source info * @throws TupleSourceNotFoundException If not found */ - TupleSourceInfo getTupleSourceInfo(TupleSourceID tupleSourceID, boolean throwException) - throws TupleSourceNotFoundException { + public TupleBuffer getTupleBuffer(TupleSourceID tupleSourceID) throws TupleSourceNotFoundException { + TupleBuffer info = this.tupleSourceMap.get(tupleSourceID); - // Look up info - TupleSourceInfo info = this.tupleSourceMap.get(tupleSourceID); - - if(info == null && throwException) { + if(info == null) { throw new TupleSourceNotFoundException(QueryExecPlugin.Util.getString("BufferManagerImpl.tuple_source_not_found", tupleSourceID)); //$NON-NLS-1$ } return info; } /** - * This can be done actively if someone wants memory and none is free. - */ - protected void clean(long memoryRequired, TupleGroupInfo targetGroupInfo) { - cleanLobTupleSource(); - - long released = 0; - - long targetLevel = config.getActiveMemoryLevel(); - long totalMemory = config.getTotalAvailableMemory(); - - boolean generalCleaningDone = false; - List<ManagedBatch> toClean = null; - synchronized (unpinned) { - //TODO: re-implement without having to scan and compete - toClean = new ArrayList<ManagedBatch>(unpinned); - } - for (ManagedBatch batch : toClean) { - TupleSourceID tsID = batch.getTupleSourceID(); - TupleSourceInfo tsInfo = this.tupleSourceMap.get(tsID); - if(tsInfo == null) { - //may be removed by another thread - continue; - } - - long currentMemoryUsed = memoryUsed; - if (!generalCleaningDone && (memoryRequired <= totalMemory - currentMemoryUsed && // if the memory needed is more than what's available, or - currentMemoryUsed <= targetLevel)) { // if we've crossed the active memory threshold, then cleanup - generalCleaningDone = true; - } - if (generalCleaningDone) { - if (targetGroupInfo == null) { - break; - } - if (targetGroupInfo == tsInfo.getGroupInfo()) { - //if the memory needed is more than what is available for the session, then cleanup. Otherwise, break the loop. - if(memoryRequired <= config.getMaxAvailableSession() - targetGroupInfo.getGroupMemoryUsed()) { - break; - } - } - } - - released += releaseMemory(batch, tsInfo); - } - - if(released > 0) { - this.cleanings.incrementAndGet(); - this.totalCleaned.addAndGet(released); - } - } - - private void cleanLobTupleSource() { - String tupleSourceId = TupleSourceInfo.getStaleLobTupleSource(); - if (tupleSourceId != null) { - try { - removeTupleSource(new TupleSourceID(tupleSourceId)); - } catch (TupleSourceNotFoundException e) { - } catch (MetaMatrixComponentException e) { - LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, e, "Exception removing stale lob tuple source"); //$NON-NLS-1$ - } - } - } - - /** - * Release the memory for the given unpinned batch. - * @param batch Batch to be released from memory - * @param tsID ID of the tuple source to be released from memory - * @return The size of memory released in bytes - * @since 4.3 - */ - private long releaseMemory(ManagedBatch batch, TupleSourceInfo info) { - // Find info and lock on it - synchronized(info) { - if(info.isRemoved() || batch.getLocation() != ManagedBatch.UNPINNED) { - return 0; - } - - // This batch is still unpinned - move to persistent storage - TupleBatch dataBatch = batch.getBatch(); - - try { - diskMgr.addBatch(info.getTupleSourceID(), dataBatch, info.getTypes()); - } catch(MetaMatrixComponentException e) { - // Can't move - return 0; - } - - batch.setBatch(null); - - // Update memory - batch.setLocation(ManagedBatch.PERSISTENT); - this.unpinned.remove(batch); - releaseMemory(batch.getSize(), info.getGroupInfo()); - - return batch.getSize(); - } - } - - /** * Gets a TupleGroupInfo representing the group * @param groupName Logical grouping name for tuple sources. Can be null. * @return TupleGroupInfo for the group. Never null. * @since 4.3 */ - private TupleGroupInfo getGroupInfo(String groupName) { - TupleGroupInfo info = null; + private Set<TupleSourceID> getGroupInfo(String groupName, boolean create) { + Set<TupleSourceID> info = null; synchronized(groupInfos) { info = groupInfos.get(groupName); - if (info == null) { + if (info == null && create) { // If it doesn't exist, create a new one - info = new TupleGroupInfo(groupName); + info = Collections.synchronizedSet(new HashSet<TupleSourceID>()); groupInfos.put(groupName, info); } } return info; } - /** - * @see com.metamatrix.common.buffer.BufferManager#stop() - */ - public synchronized void stop() { - if (timer != null) { - timer.cancel(); - } - } - - /** - * If a tuple batch is being added with Lobs, then references to - * the lobs will be held on the {@link TupleSourceInfo} - * @param batch - */ - @SuppressWarnings("unchecked") - private void correctLobReferences(TupleSourceInfo info, TupleBatch batch) { - List parentSchema = info.getTupleSchema(); - List[] rows = batch.getAllTuples(); - int columns = parentSchema.size(); - // walk through the results and find all the lobs - for (int row = 0; row < rows.length; row++) { - for (int col = 0; col < columns; col++) { - Object anObj = rows[row].get(col); - - if (!(anObj instanceof Streamable<?>)) { - continue; - } - Streamable lob = (Streamable)anObj; - info.addLobReference(lob); - if (lob.getReference() == null) { - lob.setReference(info.getLobReference(lob.getReferenceStreamId()).getReference()); - } - } + @Override + public void addBatch(TupleSourceID sourceID, TupleBatch batch, + String[] types) throws MetaMatrixComponentException { + if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.TRACE)) { + LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, new Object[]{"AddTupleBatch for", sourceID, "with " + batch.getRowCount() + " rows"}); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ } - } - - @Override - public Streamable<?> getStreamable(TupleSourceID id, String referenceId) throws TupleSourceNotFoundException, MetaMatrixComponentException { - TupleSourceInfo tsInfo = getTupleSourceInfo(id, true); - Streamable<?> s = tsInfo.getLobReference(referenceId); - if (s == null) { - throw new MetaMatrixComponentException(DQPPlugin.Util.getString("ProcessWorker.wrongdata")); //$NON-NLS-1$ - } - return s; - } - - @Override - public void setPersistentTupleSource(TupleSourceID id, Streamable<?> s) throws TupleSourceNotFoundException { - cleanLobTupleSource(); - TupleSourceInfo tsInfo = getTupleSourceInfo(id, true); - s.setPersistenceStreamId(id.getStringID()); - tsInfo.setContainingLobReference(s); - } + this.diskMgr.addBatch(sourceID, batch, types); + } - /** - * @see com.metamatrix.common.buffer.BufferManager#addStreamablePart(com.metamatrix.common.buffer.TupleSourceID, com.metamatrix.common.lob.LobChunk, int) - */ - public void addStreamablePart(TupleSourceID tupleSourceID, LobChunk streamChunk, int beginRow) - throws TupleSourceNotFoundException, MetaMatrixComponentException { - - // Look up info - TupleSourceInfo info = getTupleSourceInfo(tupleSourceID, true); - - synchronized(info) { - List<LobChunk> data = new ArrayList<LobChunk>(); - data.add(streamChunk); - TupleBatch batch = new TupleBatch(beginRow, new List[] {data}); - this.diskMgr.addBatch(tupleSourceID, batch, info.getTypes()); - - // Update tuple source state (we could calculate the size of stream if need to) - ManagedBatch managedBatch = new ManagedBatch(tupleSourceID, beginRow, batch.getEndRow(), 0); - managedBatch.setLocation(ManagedBatch.PERSISTENT); + @Override + public TupleBatch getBatch(TupleSourceID sourceID, int beginRow, + String[] types) throws TupleSourceNotFoundException, + MetaMatrixComponentException { + if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.TRACE)) { + LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, new Object[]{"Getting tupleBatch for", sourceID, "beginRow:", beginRow}); //$NON-NLS-1$ //$NON-NLS-2$ + } + return this.diskMgr.getBatch(sourceID, beginRow, types); + } - // Update info with new rows - info.addBatch(managedBatch); - info.setRowCount(batch.getEndRow()); - } - } + @Override + public void initialize(Properties props) + throws MetaMatrixComponentException { + this.config = new BufferConfig(props); + this.lookup = "local"; //$NON-NLS-1$ + } - /** - * @see com.metamatrix.common.buffer.BufferManager#getStreamable(com.metamatrix.common.buffer.TupleSourceID) - */ - public LobChunk getStreamablePart(final TupleSourceID tupleSourceID, int beginRow) - throws TupleSourceNotFoundException, MetaMatrixComponentException { - - final TupleSourceInfo info = getTupleSourceInfo(tupleSourceID, true); - TupleBatch batch = diskMgr.getBatch(tupleSourceID, beginRow, info.getTypes()); - LobChunk chunk = (LobChunk)batch.getAllTuples()[0].get(0); - - return chunk; - } - - /** - * @see com.metamatrix.common.buffer.BufferManager#releasePinnedBatches() - */ - public void releasePinnedBatches() throws MetaMatrixComponentException { - MetaMatrixComponentException e = null; - List<ManagedBatch> pinnedByThread = new ArrayList<ManagedBatch>(PINNED_BY_THREAD.get()); - for (ManagedBatch managedBatch : pinnedByThread) { - try { - //TODO: add trace logging about the batch that is being unpinned - unpinTupleBatch(managedBatch.getTupleSourceID(), managedBatch.getBeginRow()); - } catch (TupleSourceNotFoundException err) { - } catch (MetaMatrixComponentException err) { - e = err; - } - } - if (e != null) { - throw e; - } - } - - /** - * Check for whether the specified amount of memory can be reserved, - * and if so reserve it. This is done in the same method so that the - * memory is not taken away by a different thread between checking and - * reserving - standard "test and set" behavior. - * @param bytes Bytes requested - * @return One of MEMORY_AVAILABLE, MEMORY_EXCEED_MAX, or MEMORY_EXCEED_SESSION_MAX - */ - private synchronized int reserveMemory(long bytes, TupleGroupInfo groupInfo) { - //check session limit first - long sessionMax = config.getMaxAvailableSession(); - if(sessionMax - groupInfo.getGroupMemoryUsed() < bytes) { - return BufferManagerImpl.MEMORY_EXCEED_SESSION_MAX; + @Override + public void removeBatches(TupleSourceID sourceID) { + if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.DETAIL)) { + LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, new Object[]{"Removing TupleBuffer:", sourceID}); //$NON-NLS-1$ } - - //then check the total memory limit - long max = config.getTotalAvailableMemory(); - if(max - memoryUsed < bytes) { - return BufferManagerImpl.MEMORY_EXCEED_MAX; - } - - groupInfo.reserveMemory(bytes); - memoryUsed += bytes; - - return BufferManagerImpl.MEMORY_AVAILABLE; - } + TupleBuffer tupleBuffer = this.tupleSourceMap.remove(sourceID); + if (tupleBuffer != null) { + Set<TupleSourceID> ids = getGroupInfo(tupleBuffer.getGroupName(), false); + if (ids != null) { + ids.remove(sourceID); + } + } + this.diskMgr.removeBatches(sourceID); + } - /** - * Release memory - * @param bytes Bytes to release - */ - private synchronized void releaseMemory(long bytes, TupleGroupInfo groupInfo) { - groupInfo.releaseMemory(bytes); - memoryUsed -= bytes; - } + @Override + public void shutdown() { + this.diskMgr.shutdown(); + } - /** - * for testing purposes - */ - public int getPinnedCount() { - return pinned.get(); - } } Deleted: trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferStats.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferStats.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferStats.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -1,128 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.common.buffer.impl; - -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; - -import com.metamatrix.common.log.LogManager; -import com.metamatrix.core.log.MessageLevel; -import com.metamatrix.dqp.util.LogConstants; - -/** - * This encapsulates all the statistics for the BufferManagerImpl. The attribute - * values are set by direct attribute manipulation (no getter/setters). - */ -public class BufferStats { - - // Basic stats - public long memoryUsed; - public long memoryFree; - public int numTupleSources; - public int numPersistentBatches; - public int numPinnedBatches; - public int numUnpinnedBatches; - public String sLocalDrive; - - // Accumulated stats - public int pinRequests; - public int pinSuccesses; - public int pinnedFromMemory; - public int numCleanings; - public long totalCleaned; - - // Pinned batch details - public List<ManagedBatch> pinnedManagedBatches = new LinkedList<ManagedBatch>(); - - /** - * Constructor for BufferStats. - */ - BufferStats() { - } - - /** - * Log this set of stats. - */ - void log() { - LogManager.logInfo(LogConstants.CTX_BUFFER_MGR, "\n"); //$NON-NLS-1$ - LogManager.logInfo(LogConstants.CTX_BUFFER_MGR, "BUFFER MANAGER STATS"); //$NON-NLS-1$ - LogManager.logInfo(LogConstants.CTX_BUFFER_MGR, " memoryFree = " + memoryFree); //$NON-NLS-1$ - LogManager.logInfo(LogConstants.CTX_BUFFER_MGR, " memoryUsed = " + memoryUsed); //$NON-NLS-1$ - LogManager.logInfo(LogConstants.CTX_BUFFER_MGR, " numTupleSources = " + numTupleSources); //$NON-NLS-1$ - LogManager.logInfo(LogConstants.CTX_BUFFER_MGR, " numPersistentBatches = " + numPersistentBatches); //$NON-NLS-1$ - LogManager.logInfo(LogConstants.CTX_BUFFER_MGR, " numPinnedBatches = " + numPinnedBatches); //$NON-NLS-1$ - LogManager.logInfo(LogConstants.CTX_BUFFER_MGR, " numUnpinnedBatches = " + numUnpinnedBatches); //$NON-NLS-1$ - LogManager.logInfo(LogConstants.CTX_BUFFER_MGR, " pinRequests = " + pinRequests); //$NON-NLS-1$ - - // pinSuccesses / pinRequests - double pinSuccessRate = (pinRequests > 0) ? (((double)pinSuccesses / (double)pinRequests) * 100) : 100; - LogManager.logInfo(LogConstants.CTX_BUFFER_MGR, " pinSuccessRate = " + pinSuccessRate); //$NON-NLS-1$ - - // pinnedFromMemory / pinRequests - double memoryHitRate = (pinRequests > 0) ? (((double)pinnedFromMemory / (double)pinRequests) * 100): 100; - LogManager.logInfo(LogConstants.CTX_BUFFER_MGR, " memoryHitRate = " + memoryHitRate); //$NON-NLS-1$ - LogManager.logInfo(LogConstants.CTX_BUFFER_MGR, " numCleanings = " + numCleanings); //$NON-NLS-1$ - - // totalCleaned / numCleanings - long avgCleaned = (numCleanings > 0) ? (totalCleaned / numCleanings) : 0; - LogManager.logInfo(LogConstants.CTX_BUFFER_MGR, " avgCleaned = " + avgCleaned); //$NON-NLS-1$ - - if ( LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.TRACE) ) { - HashMap<List<String>, Integer> stackTraces = new HashMap<List<String>, Integer>(); - - if ( pinnedManagedBatches.isEmpty() ) { - return; - } - - LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, "\n"); //$NON-NLS-1$ - LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, " =========== Pinned Batch Details: ==========="); //$NON-NLS-1$ - - int stackNumber = 1; - - // pinned batch details - for (ManagedBatch batch : pinnedManagedBatches) { - LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, " TupleSourceID: " + batch.getTupleSourceID() + " Begin: " + batch.getBeginRow() + " End: " + batch.getEndRow()); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ - - Integer stackKey = stackTraces.get(batch.getCallStack()); - - boolean isFirst = false; - - if (stackKey == null) { - isFirst = true; - stackKey = new Integer(stackNumber++); - stackTraces.put(batch.getCallStack(), stackKey); - } - - LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, " Pinned at: " + batch.getCallStackTimeStamp() + " by call# " + stackKey); //$NON-NLS-1$ //$NON-NLS-2$ - if (isFirst) { - for (Iterator<String> j = batch.getCallStack().iterator(); j.hasNext();) { - LogManager.logTrace( LogConstants.CTX_BUFFER_MGR, " " + j.next() ); //$NON-NLS-1$ - } - } - } - } - } - -} Copied: trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/FileStorageManager.java (from rev 1678, trunk/engine/src/main/java/com/metamatrix/common/buffer/storage/file/FileStorageManager.java) =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/FileStorageManager.java (rev 0) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/FileStorageManager.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -0,0 +1,425 @@ +/* + * JBoss, Home of Professional Open Source. + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA. + */ + +package com.metamatrix.common.buffer.impl; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.RandomAccessFile; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import com.metamatrix.api.exception.MetaMatrixComponentException; +import com.metamatrix.common.buffer.BufferManagerPropertyNames; +import com.metamatrix.common.buffer.StorageManager; +import com.metamatrix.common.buffer.TupleBatch; +import com.metamatrix.common.buffer.TupleSourceID; +import com.metamatrix.common.buffer.TupleSourceNotFoundException; +import com.metamatrix.common.log.LogManager; +import com.metamatrix.common.util.PropertiesUtils; +import com.metamatrix.core.log.MessageLevel; +import com.metamatrix.core.util.Assertion; +import com.metamatrix.dqp.util.LogConstants; +import com.metamatrix.query.execution.QueryExecPlugin; +/** + * This class stores batches in files on disk in a specified directory. Every + * tuple source gets it's own set of files, named b_<i>id</i>_<i>num</i> where <i>id</i> is the + * unique id value of the TupleSourceID, and <i>num</i> is a file counter for the files + * associated with the TupleSourceID. Batches are stored random access into the + * file, typically but not necessarily in order. An in memory data structure stores + * info about all open files and where the batches are in the file (file pointers and + * lengths). + */ +public class FileStorageManager implements StorageManager { + + private static final String FILE_PREFIX = "b_"; //$NON-NLS-1$ + + // Initialization + private int maxOpenFiles = 10; + private long maxFileSize = 2L * 1024L * 1024L * 1024L; // 2GB + private String directory; + private File dirFile; + + // State + private Map<TupleSourceID, TupleSourceInfo> tupleSourceMap = new HashMap<TupleSourceID, TupleSourceInfo>(); + private Map<File, RandomAccessFile> fileCache = Collections.synchronizedMap(new LinkedHashMap<File, RandomAccessFile>() { + @Override + protected boolean removeEldestEntry( + java.util.Map.Entry<File, RandomAccessFile> eldest) { + if (this.size() > maxOpenFiles) { + try { + eldest.getValue().close(); + } catch (IOException e) { + } + return true; + } + return false; + } + }); + + /** + * Initialize with properties + * @param props Initialization properties + * @see com.metamatrix.common.buffer.BufferManagerPropertyNames#BUFFER_STORAGE_DIRECTORY + * @see com.metamatrix.common.buffer.BufferManagerPropertyNames#MAX_OPEN_FILES + * @see com.metamatrix.common.buffer.BufferManagerPropertyNames#MAX_FILE_SIZE + */ + public void initialize(Properties props) throws MetaMatrixComponentException { + this.directory = props.getProperty(BufferManagerPropertyNames.BUFFER_STORAGE_DIRECTORY); + if(this.directory == null) { + throw new MetaMatrixComponentException(QueryExecPlugin.Util.getString("FileStoreageManager.no_directory")); //$NON-NLS-1$ + } + + dirFile = new File(this.directory); + if(dirFile.exists()) { + if(! dirFile.isDirectory()) { + throw new MetaMatrixComponentException(QueryExecPlugin.Util.getString("FileStoreageManager.not_a_directory", dirFile.getAbsoluteFile())); //$NON-NLS-1$ + + } + } else if(! dirFile.mkdirs()) { + throw new MetaMatrixComponentException(QueryExecPlugin.Util.getString("FileStoreageManager.error_creating", dirFile.getAbsoluteFile())); //$NON-NLS-1$ + } + + // Set up max number of open file descriptors + maxOpenFiles = PropertiesUtils.getIntProperty(props, BufferManagerPropertyNames.MAX_OPEN_FILES, 10); + + // Set the max file size + maxFileSize = PropertiesUtils.getIntProperty(props, BufferManagerPropertyNames.MAX_FILE_SIZE, 2048) * 1024L * 1024L; // Multiply by 1MB + } + + /** + * Look up tuple source info and possibly create. First the file map is used to find an + * existing file info. If the info is found it is returned. If not, then + * a TupleSourceInfo is created according to shouldCreate flag + * @param sourceID Source identifier + * @param shouldCreate true if this method should create info related to the tuple source ID if it does not exist. + * @return All the tuple source info or null if shouldCreate == false and no tuple source info was found + */ + private TupleSourceInfo getTupleSourceInfo(TupleSourceID sourceID, boolean shouldCreate) { + + // Try to find in cache + synchronized(tupleSourceMap) { + TupleSourceInfo info = tupleSourceMap.get(sourceID); + if(info == null && shouldCreate) { + info = new TupleSourceInfo(); + tupleSourceMap.put(sourceID, info); + } + return info; + } + } + + /** + * Creates a new file to the specified TupleSourceID + * @param sourceID The TupleSourceID + * @param fileNumber a number uniquely identifying this file within the set of tuple source files. + * @return the newly created file with the name b_<i>sourceID</i>_<i>fileNumber</i> + * @throws MetaMatrixComponentException if a file with this name already exists in the directory, or if it cannot be created + * @since 4.2 + */ + private File createFile(TupleSourceID sourceID, int fileNumber) throws MetaMatrixComponentException { + File storageFile = new File(this.directory, FILE_PREFIX + sourceID.getIDValue() + "_" + fileNumber); //$NON-NLS-1$ + if (LogManager.isMessageToBeRecorded(LogConstants.CTX_STORAGE_MGR, MessageLevel.DETAIL)) { + LogManager.logDetail(LogConstants.CTX_STORAGE_MGR, "Creating temporary storage area file " + storageFile.getAbsoluteFile()); //$NON-NLS-1$ + } + try { + boolean created = storageFile.createNewFile(); + if(!created) { + throw new MetaMatrixComponentException(QueryExecPlugin.Util.getString("FileStoreageManager.file_exists", storageFile.getAbsoluteFile())); //$NON-NLS-1$ + } + } catch(IOException e) { + throw new MetaMatrixComponentException(e, QueryExecPlugin.Util.getString("FileStoreageManager.error_creating", storageFile.getAbsoluteFile())); //$NON-NLS-1$ + } + return storageFile; + } + + /** + * Add a batch to the storage manager. Because we don't implement remove, it's + * possible that this batch already exists, in which case nothing is done. + * @param sourceID Source identifier + * @param batch Batch to add + */ + public void addBatch(TupleSourceID sourceID, TupleBatch batch, String[] types) + throws MetaMatrixComponentException { + + // Defect 13342 - addBatch method now creates spill files if the total bytes exceeds the max file size limit + TupleSourceInfo tsInfo = getTupleSourceInfo(sourceID, true); + synchronized (tsInfo) { + if (tsInfo.isRemoved) { + return; + } + Integer batchKey = new Integer(batch.getBeginRow()); + if (tsInfo.tupleBatchPointers.containsKey(batchKey)) { + return; + } + byte[] bytes = convertToBytes(batch, types); + if (bytes.length > maxFileSize) { + LogManager.logWarning(LogConstants.CTX_STORAGE_MGR, "Detected an attempt to save a batch (" + sourceID + ", begin=" + batch.getBeginRow()+ ", size=" + bytes.length + ") larger than the buffer max file size setting of " + maxFileSize + " bytes. The buffer manager will ignore the max file size setting for this batch, and create a buffer file dedicated to this batch. It may be necessary to reduce the buffer batch setting or increase the buffer max file size setting."); //$NON-NLS-1$//$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$ + } + FileInfo fileInfo = tsInfo.getMostRecentlyCreatedFile(); + if (fileInfo == null || + (/*fileInfo.file.length() != 0 && */fileInfo.file.length() + bytes.length > maxFileSize)) { + // Create and add + fileInfo = new FileInfo(createFile(sourceID, tsInfo.storageFiles.size())); + tsInfo.storageFiles.add(fileInfo); + } + long pointer = 0; + + if (LogManager.isMessageToBeRecorded(LogConstants.CTX_STORAGE_MGR, MessageLevel.DETAIL)) { + LogManager.logDetail(LogConstants.CTX_STORAGE_MGR, "Adding batch to storage area file " + fileInfo.file.getAbsoluteFile() + " [ sourceID: " + sourceID + "batch: " + batch + " ]"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ + } + try { + // Get access to the file and remember whether we had to open it or not + fileInfo.open(); + RandomAccessFile fileAccess = fileInfo.getAccess(); + + // Store the batch in the file + pointer = fileAccess.length(); + fileAccess.setLength(pointer + bytes.length); + fileAccess.seek(pointer); + fileAccess.write(bytes); + } catch(IOException e) { + throw new MetaMatrixComponentException(e, QueryExecPlugin.Util.getString("FileStoreageManager.error_reading", fileInfo.file.getAbsoluteFile())); //$NON-NLS-1$ + } finally { + fileInfo.close(); + } + + // Update the pointers + tsInfo.tupleBatchPointers.put(batchKey, new PointerInfo(fileInfo, pointer, bytes.length)); + } + } + + /** + * Convert from an object to a byte array + * @param object Object to convert + * @return Byte array + */ + private byte[] convertToBytes(TupleBatch batch, String[] types) throws MetaMatrixComponentException { + ObjectOutputStream oos = null; + try { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + oos = new ObjectOutputStream(baos); + + batch.setDataTypes(types); + batch.writeExternal(oos); + oos.flush(); + return baos.toByteArray(); + + } catch(IOException e) { + throw new MetaMatrixComponentException(e, QueryExecPlugin.Util.getString("FileStorageManager.batch_error")); //$NON-NLS-1$ + } finally { + if(oos != null) { + try { + oos.close(); + } catch(IOException e) { + } + } + } + } + + /** + * Get a batch from the storage manager based on the beginRow. + * @param sourceID Source identifier + * @param beginRow Beginning row of batch to retrieve + * @return Batch retrieved + */ + public TupleBatch getBatch(TupleSourceID sourceID, int beginRow, String[] types) + throws TupleSourceNotFoundException, MetaMatrixComponentException { + + TupleSourceInfo info = getTupleSourceInfo(sourceID, false); + if(info == null) { + throw new TupleSourceNotFoundException(QueryExecPlugin.Util.getString("BufferManagerImpl.tuple_source_not_found", sourceID)); //$NON-NLS-1$ + } + + byte[] bytes = null; + synchronized(info) { + if(info.isRemoved) { + throw new TupleSourceNotFoundException(QueryExecPlugin.Util.getString("BufferManagerImpl.tuple_source_not_found", sourceID)); //$NON-NLS-1$ + } + // Find pointer + PointerInfo pointerInfo = info.tupleBatchPointers.get(new Integer(beginRow)); + Assertion.isNotNull(pointerInfo); + + FileInfo fileInfo = pointerInfo.fileInfo; + // Get access to the file + RandomAccessFile fileAccess = null; + try { + fileInfo.open(); + fileAccess = fileInfo.getAccess(); + fileAccess.seek(pointerInfo.pointer); + bytes = new byte[pointerInfo.length]; + fileAccess.readFully(bytes); + ByteArrayInputStream bais = new ByteArrayInputStream(bytes); + ObjectInputStream ois = new ObjectInputStream(bais); + + TupleBatch batch = new TupleBatch(); + batch.setDataTypes(types); + batch.readExternal(ois); + return batch; + } catch(IOException e) { + throw new MetaMatrixComponentException(e, QueryExecPlugin.Util.getString("FileStoreageManager.error_reading", fileInfo.file.getAbsoluteFile())); //$NON-NLS-1$ + } catch (ClassNotFoundException e) { + throw new MetaMatrixComponentException(e, QueryExecPlugin.Util.getString("FileStoreageManager.error_reading", fileInfo.file.getAbsoluteFile())); //$NON-NLS-1$ + } finally { + fileInfo.close(); + } + } + } + + /** + * Remove all batches for a sourceID. Before removal, the file is closed. + * @param sourceID Tuple source ID + */ + public void removeBatches(TupleSourceID sourceID) { + TupleSourceInfo info = null; + // Remove info from the file map + synchronized(tupleSourceMap) { + info = tupleSourceMap.remove(sourceID); + } + + // Didn't find a file + if(info == null) { + return; + } + + if (LogManager.isMessageToBeRecorded(LogConstants.CTX_STORAGE_MGR, MessageLevel.DETAIL)) { + LogManager.logDetail(LogConstants.CTX_STORAGE_MGR, "Removing storage for " + sourceID); //$NON-NLS-1$ + } + + synchronized(info) { + // If open, close the file and decrement the open file counter + for (int i = 0; i < info.storageFiles.size(); i++) { + FileInfo fileInfo = info.storageFiles.get(i); + fileInfo.delete(); + } + // Delete the file and mark info as being removed + info.isRemoved = true; + } + } + + /** + * This method removes all storage area files by walking through the file info + * map and closing and removing each file. + */ + public synchronized void shutdown() { + + LogManager.logDetail(LogConstants.CTX_STORAGE_MGR, "Removing all storage area files "); //$NON-NLS-1$ + + List<TupleSourceID> ids = null; + synchronized (tupleSourceMap) { + ids = new LinkedList<TupleSourceID>(tupleSourceMap.keySet()); + } + + for (TupleSourceID key : ids) { + removeBatches(key); + } + + tupleSourceMap = null; + } + + public int getOpenFiles() { + return this.fileCache.size(); + } + + private class FileInfo { + private File file; + private RandomAccessFile fileData; // may be null if not open + + public FileInfo(File file) { + this.file = file; + } + + public void open() throws FileNotFoundException { + if(this.fileData == null) { + this.fileData = fileCache.remove(this.file); + if (this.fileData == null) { + this.fileData = new RandomAccessFile(file, "rw"); //$NON-NLS-1$ + } + } + } + + public RandomAccessFile getAccess() { + return this.fileData; + } + + public void close() { + if(this.fileData != null) { + fileCache.put(file, this.fileData); + this.fileData = null; + } + } + + public void delete() { + if (this.fileData == null) { + this.fileData = fileCache.remove(this.file); + } + if (this.fileData != null) { + try { + this.fileData.close(); + } catch(Exception e) { + } + this.fileData = null; + } + file.delete(); + } + + public String toString() { + return "FileInfo<" + file.getName() + ", has fileData = " + (fileData != null) + ">"; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + } + } + + private static class PointerInfo { + FileInfo fileInfo; + public long pointer; + public int length; + + public PointerInfo(FileInfo fileInfo, long pointer, int length) { + this.fileInfo = fileInfo; + this.pointer = pointer; + this.length = length; + } + } + + private static class TupleSourceInfo { + Map<Integer, PointerInfo> tupleBatchPointers = new HashMap<Integer, PointerInfo>(); + List<FileInfo> storageFiles = new ArrayList<FileInfo>(2); // Stores all the FileInfos for this tupleSource + private boolean isRemoved = false; + + FileInfo getMostRecentlyCreatedFile() { + if (storageFiles.isEmpty()) { + return null; + } + return storageFiles.get(storageFiles.size() - 1); + } + } + +} Deleted: trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/ManagedBatch.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/ManagedBatch.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/ManagedBatch.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -1,217 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.common.buffer.impl; - -import java.sql.Timestamp; -import java.util.LinkedList; -import java.util.List; - -import com.metamatrix.common.buffer.TupleBatch; -import com.metamatrix.common.buffer.TupleSourceID; -import com.metamatrix.core.util.HashCodeUtil; - -/** - * A ManagedBatch basically describes a TupleBatch that is being managed and - * appends a bunch of additional state used within the BufferManagerImpl. The - * batch is described by it's beginRow and endRow and it's size is in bytes. - * The lastAccessed field records the last accessed time in memory. The location - * tells where the batch is currently stored, as defined in the TupleSourceInfo - * constants. The pinned intervals describe which rows within the batch are - * currently pinned. - */ -class ManagedBatch { - - /** Constant for the location of this batch - stored in persistent storage */ - public final static short PERSISTENT = 0; - - /** Constant for the location of this batch - stored in memory, but not currently in use */ - public final static short UNPINNED = 1; - - /** Constant for the location of this batch - stored in memory and currently in use */ - public final static short PINNED = 2; - - private TupleSourceID tupleSourceID; - private int beginRow; - private int endRow; - private long size; - private int location; - private int pinnedCount; - private int hashCode; - private TupleBatch batch; - - // logging - private static int STACK_LEVELS_TO_OMIT = 2; - private static int STACK_LEVELS_TO_CAPTURE = 5; - - private List<String> whoCalledUs; - private String sCallStackTimeStamp; - - /** - * Constructor for ManagedBatch. - */ - public ManagedBatch(TupleSourceID tupleSourceID, int begin, int end, long size) { - this.tupleSourceID = tupleSourceID; - this.beginRow = begin; - this.endRow = end; - this.size = size; - this.hashCode = HashCodeUtil.hashCode(tupleSourceID.hashCode(), beginRow); - } - - public TupleBatch getBatch() { - return batch; - } - - public void setBatch(TupleBatch batch) { - this.batch = batch; - } - - /** - * Get the tuple source ID - * @return Tuple sourceID - */ - public TupleSourceID getTupleSourceID() { - return this.tupleSourceID; - } - - /** - * Get the begin row, must be >= 1 - * @return Begin row - */ - public int getBeginRow() { - return this.beginRow; - } - - /** - * Get the end row, inclusive - * @return End row - */ - public int getEndRow() { - return this.endRow; - } - - /** - * Get the size of the batch in bytes - * @return Size, in bytes - */ - public long getSize() { - return this.size; - } - - /** - * Get the location of the batch, as defined in constants - * @return Location - */ - public int getLocation() { - return this.location; - } - - /** - * Set location of the batch, as defined in constants - */ - public void setLocation(int location) { - this.location = location; - } - - /** - * Check whether this managed batch has any pinned rows - * @return True if any are pinned - */ - public boolean hasPinnedRows() { - return pinnedCount > 0; - } - - public void pin() { - this.pinnedCount++; - } - - public void unpin() { - this.pinnedCount--; - } - - /** - * Capture partial stack trace to express who owns this batch - */ - public void captureCallStack() { - /* - * If more detail is needed StackTraceElement has several useful methods. - */ - StackTraceElement[] elements = new Exception().getStackTrace(); - - whoCalledUs = new LinkedList<String>(); - - for ( int i = STACK_LEVELS_TO_OMIT; i < elements.length && i < STACK_LEVELS_TO_OMIT + STACK_LEVELS_TO_CAPTURE; i++ ) { - whoCalledUs.add(elements[ i ].toString()); - } - sCallStackTimeStamp = new Timestamp(System.currentTimeMillis()).toString(); - } - - /** - * Returns call stack - */ - public List<String> getCallStack() { - - return whoCalledUs; - } - - /** - * Returns call stack timestamp - */ - public String getCallStackTimeStamp() { - - return this.sCallStackTimeStamp; - } - - /** - * Compare two managed batches for equality - this check - * is made based on TupleSourceID and beginRow. - * @param obj Other object - * @return True if equal - */ - public boolean equals(Object obj) { - if(obj == this) { - return true; - } - - if(! (obj instanceof ManagedBatch)) { - return false; - } - - ManagedBatch other = (ManagedBatch) obj; - return ( this.getTupleSourceID().equals(other.getTupleSourceID()) && - this.getBeginRow() == other.getBeginRow() ); - - } - - /** - * Get hash code, based on tupleSourceID and beginRow - * @return Hash code - */ - public int hashCode() { - return hashCode; - } - - public String toString() { - return "ManagedBatch[" + tupleSourceID + ", " + beginRow + ", " + endRow + "]"; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ - } - -} Added: trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/MemoryStorageManager.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/MemoryStorageManager.java (rev 0) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/MemoryStorageManager.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -0,0 +1,104 @@ +/* + * JBoss, Home of Professional Open Source. + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA. + */ + +package com.metamatrix.common.buffer.impl; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import com.metamatrix.api.exception.MetaMatrixComponentException; +import com.metamatrix.common.buffer.StorageManager; +import com.metamatrix.common.buffer.TupleBatch; +import com.metamatrix.common.buffer.TupleSourceID; +import com.metamatrix.common.buffer.TupleSourceNotFoundException; +import com.metamatrix.query.execution.QueryExecPlugin; + +public class MemoryStorageManager implements StorageManager { + + // TupleSourceID -> List<TupleBatch> (ordered by startRow) + private Map<TupleSourceID, Map<Integer, TupleBatch>> storage = Collections.synchronizedMap(new HashMap<TupleSourceID, Map<Integer, TupleBatch>>()); + + /** + * @see StorageManager#initialize(Properties) + */ + public void initialize(Properties props) throws MetaMatrixComponentException { + } + + /** + * @see StorageManager#addBatch(TupleSourceID, TupleBatch) + */ + public void addBatch(TupleSourceID storageID, TupleBatch batch, String[] types) + throws MetaMatrixComponentException { + + Map<Integer, TupleBatch> batches = null; + synchronized(this.storage) { + batches = storage.get(storageID); + if(batches == null) { + batches = new HashMap<Integer, TupleBatch>(); + this.storage.put(storageID, batches); + } + } + + synchronized(batches) { + batches.put(batch.getBeginRow(), batch); + } + } + + /** + * @see StorageManager#getBatch(TupleSourceID, int, int) + */ + public TupleBatch getBatch(TupleSourceID storageID, int beginRow, String[] types) + throws TupleSourceNotFoundException, MetaMatrixComponentException { + + Map<Integer, TupleBatch> batches = storage.get(storageID); + + if(batches == null) { + throw new TupleSourceNotFoundException(QueryExecPlugin.Util.getString("BufferManagerImpl.tuple_source_not_found", storageID)); //$NON-NLS-1$ + } + + synchronized(batches) { + TupleBatch batch = batches.get(beginRow); + if(batch == null) { + throw new MetaMatrixComponentException("unknown batch"); //$NON-NLS-1$ + } + return batch; + } + } + + /** + * @see StorageManager#removeStorageArea(TupleSourceID) + */ + public void removeBatches(TupleSourceID storageID) { + storage.remove(storageID); + } + + /** + * @see StorageManager#shutdown() + */ + public void shutdown() { + this.storage.clear(); + this.storage = null; + } + +} \ No newline at end of file Property changes on: trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/MemoryStorageManager.java ___________________________________________________________________ Name: svn:mime-type + text/plain Deleted: trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/SizeUtility.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/SizeUtility.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/SizeUtility.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -1,243 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.common.buffer.impl; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.ObjectOutputStream; -import java.io.Serializable; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Vector; - -import com.metamatrix.common.types.DataTypeManager; -import com.metamatrix.core.MetaMatrixRuntimeException; -import com.metamatrix.core.util.Assertion; - - -/** - * Utility methods to determine the size of Java objects, particularly with - * respect to the MetaMatrix runtime types. - */ -public final class SizeUtility { - private static final int DEFAULT_BUFFER_SIZE = 4096; - public static final boolean IS_64BIT = System.getProperty("sun.arch.data.model", "32").indexOf("64") != -1; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ -// public static final boolean IS_64BIT = true; - public static final int REFERENCE_SIZE = IS_64BIT?8:4; - - private static final int SAMPLE_RATE = 10; - - - /** - * Constructor for SizeUtility - utility class, can't construct - */ - private SizeUtility() { - } - - /** - * This method calculates the sizes of the tuples based on their list sizes - * Since we know the type and length, it jsut calculates size from that - * information, instead of walking whole object tree. - * @param types - Data type for the each column - * @param data - data - * @return size in total - */ - public static long getBatchSize(String[] types, List[] data) { - - // If the type information is not available then, use the long route - // and calculate each and every size. - if (types == null) { - return getSize(data); - } - - int colLength = types.length; - int rowLength = data.length; - - // Array overhead for row array - long size = 16 + alignMemory(rowLength * REFERENCE_SIZE); - // array overhead for all the columns ( 8 object overhead + 4 ref + 4 int) - size += (rowLength * (48 + alignMemory(colLength * REFERENCE_SIZE))); - for (int col = 0; col < colLength; col++) { - Class type = DataTypeManager.getDataTypeClass(types[col]); - - if (type == DataTypeManager.DefaultDataClasses.CHAR - || type == DataTypeManager.DefaultDataClasses.BOOLEAN - || type == DataTypeManager.DefaultDataClasses.BYTE - || type == DataTypeManager.DefaultDataClasses.SHORT - || type == DataTypeManager.DefaultDataClasses.INTEGER - || type == DataTypeManager.DefaultDataClasses.LONG - || type == DataTypeManager.DefaultDataClasses.FLOAT - || type == DataTypeManager.DefaultDataClasses.DOUBLE) { - size += (20*rowLength); - } else if (type == DataTypeManager.DefaultDataClasses.DATE - || type == DataTypeManager.DefaultDataClasses.TIME - || type == DataTypeManager.DefaultDataClasses.TIMESTAMP) { - // Even though Timestamp contains an extra int, these are - // the same size because of rounding - size += (32*rowLength); - } else if (type == DataTypeManager.DefaultDataClasses.NULL) { - //do nothing - } - else { - for (int row = 0; row < rowLength; row+=SAMPLE_RATE) { - size += getSize(data[row].get(col)) * Math.min(rowLength - row, SAMPLE_RATE); - } - } - } - return size; - } - - /** - * Get size of object - * @return Size in bytes - */ - public static long getSize(Object obj) { - if(obj == null) { - return 0; - } - - Class type = obj.getClass(); - // Defect 14530 - this code represents the size of the object in the heap, - // and not necessarily the number of bytes serialized. - if (type == DataTypeManager.DefaultDataClasses.CHAR || - type == DataTypeManager.DefaultDataClasses.BOOLEAN || - type == DataTypeManager.DefaultDataClasses.BYTE || - type == DataTypeManager.DefaultDataClasses.SHORT || - type == DataTypeManager.DefaultDataClasses.INTEGER || - type == DataTypeManager.DefaultDataClasses.LONG || - type == DataTypeManager.DefaultDataClasses.FLOAT || - type == DataTypeManager.DefaultDataClasses.DOUBLE) { - return 20; - } else if (type == DataTypeManager.DefaultDataClasses.DATE || - type == DataTypeManager.DefaultDataClasses.TIME || - type == DataTypeManager.DefaultDataClasses.TIMESTAMP) { // Even though Timestamp contains an extra int, these are the same size because of rounding - return 32; - } else if(type == DataTypeManager.DefaultDataClasses.STRING) { - int length = ((String)obj).length(); - if (length > 0) { - return alignMemory(40 + (2 * length)); - } - return 40; - } else if(obj instanceof List) { - int total = 16; // Object overhead + 4(int modcount), rounded - if (obj instanceof LinkedList) { - total += 8; // 4 (Entry ref) + 4 (int size) - for (Iterator i = ((List)obj).iterator(); i.hasNext();) { - total+= 16 + SizeUtility.getSize(i.next()); // rounded(4 (Object ref + 4 (next Entry ref) + 4 (previous Entry ref)) + actual object - } - } else if (obj instanceof ArrayList) { - List arrayList = (ArrayList)obj; - int arraySize = arrayList.size(); - int maxEnsuredSize = (arraySize < 10) ? 10 : arraySize * 3 / 2; // assuming default size - total += 24 /*4(char[] ref) + 4(int) + 16 (array overhead)*/ - +alignMemory(maxEnsuredSize * REFERENCE_SIZE); /*number of references held in the array*/ if (arraySize > 0) { - for (int i = 0; i < arraySize; i++) { - total += SizeUtility.getSize(arrayList.get(i)); - } - } - } else if (obj instanceof Vector) { - List arrayList = (List)obj; - int arraySize = arrayList.size(); - int maxEnsuredSize = (arraySize < 10) ? 10 : arraySize * 2; // assuming default size, default capacity growth - total += 16 // 4(Object[] ref) + 4 (int capacity) + 4 (int capIncrement), rounded - + alignMemory(maxEnsuredSize * REFERENCE_SIZE); // Array overhead - } else { // Assume java.util.Arrays.ArrayList - List list = (List)obj; - int arraySize = list.size(); - total += 16 + alignMemory(arraySize * REFERENCE_SIZE); // For the Object[] - if (arraySize > 0) { - for (int i = 0; i < arraySize; i++) { - total += SizeUtility.getSize(list.get(i)); - } - } - } - return total; - } else if(type == DataTypeManager.DefaultDataClasses.BIG_DECIMAL) { - int bitLength = ((BigDecimal)obj).unscaledValue().bitLength(); - return // 8 // Object overhead - // + 4 // BigInteger reference - // + 4 // int scale - // Not aligning the above since they are already a multiple of 8 - // The remaining should be the same as BigInteger - // + 8 // object overhead - // + 24 // 6 int fields - // + 4 // int[] ref - // + 16 + alignMemory(4 * ((bitLength/32) + 1)) // int[] overhead - 72 + alignMemory(4 + (bitLength >> 3)); // Same as above calculation - } else if(type == DataTypeManager.DefaultDataClasses.BIG_INTEGER) { - int bitLength = ((BigInteger)obj).bitLength(); - return // + 8 // object overhead - // + 24 // 6 int fields - // + 4 // int[] ref - // + 16 + alignMemory(4 * ((bitLength/32) + 1)) // int[] overhead. BigInteger represents all values in all bases as a concatenation of chunks of 32-bits. - 56 + alignMemory(4 + (bitLength >> 3)); // Same as above calculation - } else if(type.isArray() && !type.getComponentType().isPrimitive()) { - Object[] rows = (Object[]) obj; - long total = 16 + alignMemory(rows.length * REFERENCE_SIZE); // Array overhead - for(int i=0; i<rows.length; i++) { - total += SizeUtility.getSize(rows[i]); - } - return total; - } else { - // Unknown object - // 8 - for object overhead - return 8+alignMemory(4+getSerializedSize(obj)); // Assume some max value for unknown objects. - } - } - - /** - * Most current VMs have memory alignment that places objects into heap space that is a multiple of 8 Bytes. - * This utility method helps with calculating the aligned size of an object. - * @param numBytes - * @return - * @since 4.2 - */ - private static long alignMemory(long numBytes) { - long remainder = numBytes % 8; - if (remainder != 0) { - numBytes += (8 - remainder); - } - return numBytes; - } - - private static long getSerializedSize(Object anObj) { - Assertion.assertTrue(anObj instanceof Serializable); - try { - ByteArrayOutputStream bout = new ByteArrayOutputStream(DEFAULT_BUFFER_SIZE); - ObjectOutputStream out = new ObjectOutputStream(bout); - out.writeObject(anObj); - out.flush(); - long size = bout.size(); - out.close(); - bout.close(); - return size; - } catch (IOException e) { - // just return exception, this should not happen - throw new MetaMatrixRuntimeException(e); - } - } -} \ No newline at end of file Deleted: trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleGroupInfo.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleGroupInfo.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleGroupInfo.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -1,68 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.common.buffer.impl; - -import java.util.Collections; -import java.util.HashSet; -import java.util.Set; - -import com.metamatrix.common.buffer.TupleSourceID; - - - -/** - * Represents a logical grouping of tuple sources managed by the buffer manager. - * Tuple sources are typically grouped by session, and the groupName is typically a sessionID/connectionID. - * @since 4.3 - */ -class TupleGroupInfo { - - private String groupName; - /** The bytes of memory used by this tuple group*/ - private volatile long memoryUsed; - private Set<TupleSourceID> tupleSourceIDs = Collections.synchronizedSet(new HashSet<TupleSourceID>()); - - TupleGroupInfo(String groupName) { - this.groupName = groupName; - } - - public Set<TupleSourceID> getTupleSourceIDs() { - return tupleSourceIDs; - } - - String getGroupName() { - return groupName; - } - - long getGroupMemoryUsed() { - return memoryUsed; - } - - long reserveMemory(long bytes) { - return memoryUsed += bytes; - } - - long releaseMemory(long bytes) { - return memoryUsed -= bytes; - } -} Deleted: trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleSourceImpl.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleSourceImpl.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleSourceImpl.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -1,188 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.common.buffer.impl; - -import java.lang.ref.WeakReference; -import java.util.List; - -import com.metamatrix.api.exception.ComponentNotFoundException; -import com.metamatrix.api.exception.MetaMatrixComponentException; -import com.metamatrix.common.buffer.BlockedException; -import com.metamatrix.common.buffer.BlockedOnMemoryException; -import com.metamatrix.common.buffer.IndexedTupleSource; -import com.metamatrix.common.buffer.MemoryNotAvailableException; -import com.metamatrix.common.buffer.TupleBatch; -import com.metamatrix.common.buffer.TupleSourceID; -import com.metamatrix.common.buffer.TupleSourceNotFoundException; -import com.metamatrix.common.buffer.BufferManager.TupleSourceStatus; - -class TupleSourceImpl implements IndexedTupleSource { - private final BufferManagerImpl bufferManagerImpl; - private TupleSourceID tupleSourceID; - private List<?> schema; - private WeakReference<TupleBatch> currentBatch; - private int currentRow = 1; - private int mark = 1; - private List<?> currentTuple; - - TupleSourceImpl(BufferManagerImpl bufferManagerImpl, TupleSourceID tupleSourceID, List schema){ - this.bufferManagerImpl = bufferManagerImpl; - this.tupleSourceID = tupleSourceID; - this.schema = schema; - } - - @Override - public int getCurrentIndex() { - return this.currentRow; - } - - @Override - public List getSchema(){ - return this.schema; - } - - @Override - public List<?> nextTuple() - throws MetaMatrixComponentException{ - List<?> result = null; - if (currentTuple != null){ - result = currentTuple; - currentTuple = null; - } else { - result = getCurrentTuple(); - } - if (result != null) { - currentRow++; - } - return result; - } - - private List getCurrentTuple() throws MetaMatrixComponentException, - BlockedException, ComponentNotFoundException { - TupleBatch batch = getBatch(); - if(batch.getRowCount() == 0) { - // Check if last - try { - TupleSourceStatus status = this.bufferManagerImpl.getStatus(this.tupleSourceID); - if(status == TupleSourceStatus.FULL) { - unpinCurrentBatch(); - return null; - } - throw BlockedException.INSTANCE; - } catch(TupleSourceNotFoundException e) { - throw new ComponentNotFoundException(e, e.getMessage()); - } - } - - return batch.getTuple(currentRow); - } - - @Override - public void closeSource() - throws MetaMatrixComponentException{ - // Reset to same state as newly-instantiated TupleSourceImpl - unpinCurrentBatch(); - mark = 1; - reset(); - //TODO: this is not quite correct wrt the javadoc, close does - //not need to ensure that we are back at the beginning - } - - private TupleBatch getCurrentBatch() { - if (currentBatch != null) { - return currentBatch.get(); - } - return null; - } - - private void unpinCurrentBatch() - throws MetaMatrixComponentException { - TupleBatch batch = getCurrentBatch(); - if(batch != null ) { - try { - this.bufferManagerImpl.unpinTupleBatch(this.tupleSourceID, batch.getBeginRow()); - } catch (TupleSourceNotFoundException e) { - throw new MetaMatrixComponentException(e); - } finally { - currentBatch = null; - } - } - } - - // Retrieves the necessary batch based on the currentRow - public TupleBatch getBatch() - throws MetaMatrixComponentException{ - TupleBatch batch = getCurrentBatch(); - if (batch != null) { - if (currentRow <= batch.getEndRow() && currentRow >= batch.getBeginRow()) { - return batch; - } - unpinCurrentBatch(); - } - - try{ - batch = this.bufferManagerImpl.pinTupleBatch(this.tupleSourceID, currentRow); - currentBatch = new WeakReference<TupleBatch>(batch); - } catch (MemoryNotAvailableException e) { - /* Defect 18499 - ProcessWorker doesn't know how to handle MemoryNotAvailableException properly, - * and this should always be converted to a BlockedOnMemoryException during processing so that - * the work can be requeued. - */ - throw BlockedOnMemoryException.INSTANCE; - } catch(MetaMatrixComponentException e) { - throw e; - } catch(TupleSourceNotFoundException e){ - throw new MetaMatrixComponentException(e); - } - return batch; - } - - @Override - public boolean hasNext() throws MetaMatrixComponentException { - if (this.currentTuple != null) { - return true; - } - - this.currentTuple = getCurrentTuple(); - return this.currentTuple != null; - } - - @Override - public void reset() { - this.setPosition(mark); - this.mark = 1; - } - - @Override - public void mark() { - this.mark = currentRow; - } - - @Override - public void setPosition(int position) { - if (this.currentRow != position) { - this.currentRow = position; - this.currentTuple = null; - } - } -} \ No newline at end of file Deleted: trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleSourceInfo.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleSourceInfo.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleSourceInfo.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -1,259 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.common.buffer.impl; - -import java.lang.ref.PhantomReference; -import java.lang.ref.ReferenceQueue; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; -import java.util.concurrent.atomic.AtomicLong; - -import com.metamatrix.common.buffer.TupleSourceID; -import com.metamatrix.common.buffer.BufferManager.TupleSourceStatus; -import com.metamatrix.common.buffer.BufferManager.TupleSourceType; -import com.metamatrix.common.types.DataTypeManager; -import com.metamatrix.common.types.Streamable; - -/** - * Describe a TupleSource and all important information about it. - */ -public class TupleSourceInfo { - - private static final AtomicLong LOB_ID = new AtomicLong(); - private static final ReferenceQueue<Streamable<?>> LOB_QUEUE = new ReferenceQueue<Streamable<?>>(); - - private static class LobReference extends PhantomReference<Streamable<?>> { - - String persistentStreamId; - - public LobReference(Streamable<?> lob) { - super(lob, LOB_QUEUE); - this.persistentStreamId = lob.getPersistenceStreamId(); - } - } - - private TupleSourceType type; // Type of TupleSource, as defined in BufferManager constants - private TupleSourceID tsID; - private List schema; - private String[] types; - private int rowCount; - private TupleSourceStatus status = TupleSourceStatus.ACTIVE; - private TupleGroupInfo groupInfo; - private boolean removed = false; - private TreeMap<Integer, ManagedBatch> batches = new TreeMap<Integer, ManagedBatch>(); - private Map<String, Streamable<?>> lobReferences; //references to contained lobs - private boolean lobs; - - @SuppressWarnings("unused") - private LobReference containingLobReference; //reference to containing lob - - /** - * Construct a TupleSourceInfo given information about it. - * @param tsID Identifier - * @param schema Schema describing tuple source - * @param groupInfo The tuple gorup to which this tuple source belongs - * @param type Type of tuple source, as defined in BufferManager constants - */ - public TupleSourceInfo(TupleSourceID tsID, List schema, String[] types, TupleGroupInfo groupInfo, TupleSourceType type) { - this.tsID = tsID; - this.schema = schema; - this.types = types; - this.groupInfo = groupInfo; - this.type = type; - this.lobs = checkForLobs(); - } - - public void setContainingLobReference(Streamable<?> s) { - this.containingLobReference = new LobReference(s); - } - - public void addLobReference(Streamable<Object> lob) { - String id = lob.getReferenceStreamId(); - if (id == null) { - id = String.valueOf(LOB_ID.getAndIncrement()); - lob.setReferenceStreamId(id); - } - if (this.lobReferences == null) { - this.lobReferences = Collections.synchronizedMap(new HashMap<String, Streamable<?>>()); - } - this.lobReferences.put(id, lob); - } - - public static String getStaleLobTupleSource() { - LobReference ref = (LobReference)LOB_QUEUE.poll(); - if (ref == null) { - return null; - } - return ref.persistentStreamId; - } - - public Streamable<?> getLobReference(String id) { - if (this.lobReferences == null) { - return null; - } - return this.lobReferences.get(id); - } - - public void addBatch(ManagedBatch batch) { - batches.put(batch.getBeginRow(), batch); - } - - /** - * Returns the batch containing the begin row or null - * if it doesn't exist - * @param beginRow - * @return - */ - public ManagedBatch getBatch(int beginRow) { - Map.Entry<Integer, ManagedBatch> entry = batches.floorEntry(beginRow); - if (entry != null && entry.getValue().getEndRow() >= beginRow) { - return entry.getValue(); - } - return null; - } - - public void removeBatch(int beginRow) { - batches.remove(new Integer(beginRow)); - } - - public Iterator<ManagedBatch> getBatchIterator() { - return batches.values().iterator(); - } - - /** - * Get the tuple source identifier - * @return Tuple source identifier - */ - public TupleSourceID getTupleSourceID() { - return this.tsID; - } - - /** - * Get the tuple schema describing a tuple source - * @return Schema - */ - public List getTupleSchema() { - return this.schema; - } - - /** - * Get group this tuple source is in. Never null - * @return TupleGroupInfo instance representing the tuple group to which this tuple source belongs. - */ - public TupleGroupInfo getGroupInfo() { - return this.groupInfo; - } - - void setGroupInfo(TupleGroupInfo info) { - this.groupInfo = info; - } - - /** - * Get current row count - * @return Row count - */ - public int getRowCount() { - return this.rowCount; - } - - /** - * Set the current row count - * @param rows New row count - */ - public void setRowCount(int rows) { - this.rowCount = rows; - } - - /** - * Get status of this tuple source, as defined in BufferManager constants - * @return Status - */ - public TupleSourceStatus getStatus() { - return this.status; - } - - /** - * Set status of this tuple source - * @param status New status - */ - public void setStatus(TupleSourceStatus status) { - this.status = status; - } - - /** - * Get type of this tuple source - * @return Type of tuple source - */ - public TupleSourceType getType() { - return this.type; - } - - /** - * Check whether this tuple source has been removed. Due to - * synchronization, this tuple source may have been removed - * between the time it was retrieved and now. - */ - public boolean isRemoved() { - return this.removed; - } - - /** - * Set a flag that this tuple source has been removed - */ - public void setRemoved() { - this.removed = true; - } - - public boolean lobsInSource() { - return this.lobs; - } - - public String[] getTypes() { - return types; - } - - /** - * Get string representation - * @return String representation - */ - public String toString() { - return "TupleSourceInfo[" + this.tsID + "]"; //$NON-NLS-1$ //$NON-NLS-2$ - } - - private boolean checkForLobs() { - if (types == null) { - // assume the worst - return true; - } - for (int i = 0; i < types.length; i++) { - if (DataTypeManager.isLOB(types[i]) || types[i] == DataTypeManager.DefaultDataTypes.OBJECT) { - return true; - } - } - return false; - } -} Deleted: trunk/engine/src/main/java/com/metamatrix/common/buffer/storage/file/FileStorageManager.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/common/buffer/storage/file/FileStorageManager.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/common/buffer/storage/file/FileStorageManager.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -1,429 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.common.buffer.storage.file; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; -import java.io.RandomAccessFile; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Properties; - -import com.metamatrix.api.exception.MetaMatrixComponentException; -import com.metamatrix.common.buffer.BufferManagerPropertyNames; -import com.metamatrix.common.buffer.StorageManager; -import com.metamatrix.common.buffer.TupleBatch; -import com.metamatrix.common.buffer.TupleSourceID; -import com.metamatrix.common.buffer.TupleSourceNotFoundException; -import com.metamatrix.common.log.LogManager; -import com.metamatrix.common.util.PropertiesUtils; -import com.metamatrix.core.log.MessageLevel; -import com.metamatrix.core.util.Assertion; -import com.metamatrix.dqp.util.LogConstants; -import com.metamatrix.query.execution.QueryExecPlugin; -/** - * This class stores batches in files on disk in a specified directory. Every - * tuple source gets it's own set of files, named b_<i>id</i>_<i>num</i> where <i>id</i> is the - * unique id value of the TupleSourceID, and <i>num</i> is a file counter for the files - * associated with the TupleSourceID. Batches are stored random access into the - * file, typically but not necessarily in order. An in memory data structure stores - * info about all open files and where the batches are in the file (file pointers and - * lengths). - */ -public class FileStorageManager implements StorageManager { - - private static final String FILE_PREFIX = "b_"; //$NON-NLS-1$ - - // Initialization - private int maxOpenFiles = 10; - private long maxFileSize = 2L * 1024L * 1024L * 1024L; // 2GB - private String directory; - private File dirFile; - - // State - private Map<TupleSourceID, TupleSourceInfo> tupleSourceMap = new HashMap<TupleSourceID, TupleSourceInfo>(); - private Map<File, RandomAccessFile> fileCache = Collections.synchronizedMap(new LinkedHashMap<File, RandomAccessFile>() { - @Override - protected boolean removeEldestEntry( - java.util.Map.Entry<File, RandomAccessFile> eldest) { - if (this.size() > maxOpenFiles) { - try { - eldest.getValue().close(); - } catch (IOException e) { - } - return true; - } - return false; - } - }); - - /** - * Initialize with properties - * @param props Initialization properties - * @see com.metamatrix.common.buffer.BufferManagerPropertyNames#BUFFER_STORAGE_DIRECTORY - * @see com.metamatrix.common.buffer.BufferManagerPropertyNames#MAX_OPEN_FILES - * @see com.metamatrix.common.buffer.BufferManagerPropertyNames#MAX_FILE_SIZE - */ - public void initialize(Properties props) throws MetaMatrixComponentException { - this.directory = props.getProperty(BufferManagerPropertyNames.BUFFER_STORAGE_DIRECTORY); - if(this.directory == null) { - throw new MetaMatrixComponentException(QueryExecPlugin.Util.getString("FileStoreageManager.no_directory")); //$NON-NLS-1$ - } - - dirFile = new File(this.directory); - if(dirFile.exists()) { - if(! dirFile.isDirectory()) { - throw new MetaMatrixComponentException(QueryExecPlugin.Util.getString("FileStoreageManager.not_a_directory", dirFile.getAbsoluteFile())); //$NON-NLS-1$ - - } - } else if(! dirFile.mkdirs()) { - throw new MetaMatrixComponentException(QueryExecPlugin.Util.getString("FileStoreageManager.error_creating", dirFile.getAbsoluteFile())); //$NON-NLS-1$ - } - - // Set up max number of open file descriptors - maxOpenFiles = PropertiesUtils.getIntProperty(props, BufferManagerPropertyNames.MAX_OPEN_FILES, 10); - - // Set the max file size - maxFileSize = PropertiesUtils.getIntProperty(props, BufferManagerPropertyNames.MAX_FILE_SIZE, 2048) * 1024L * 1024L; // Multiply by 1MB - } - - /** - * Look up tuple source info and possibly create. First the file map is used to find an - * existing file info. If the info is found it is returned. If not, then - * a TupleSourceInfo is created according to shouldCreate flag - * @param sourceID Source identifier - * @param shouldCreate true if this method should create info related to the tuple source ID if it does not exist. - * @return All the tuple source info or null if shouldCreate == false and no tuple source info was found - */ - private TupleSourceInfo getTupleSourceInfo(TupleSourceID sourceID, boolean shouldCreate) { - - // Try to find in cache - synchronized(tupleSourceMap) { - TupleSourceInfo info = tupleSourceMap.get(sourceID); - if(info == null && shouldCreate) { - info = new TupleSourceInfo(); - tupleSourceMap.put(sourceID, info); - } - return info; - } - } - - /** - * Creates a new file to the specified TupleSourceID - * @param sourceID The TupleSourceID - * @param fileNumber a number uniquely identifying this file within the set of tuple source files. - * @return the newly created file with the name b_<i>sourceID</i>_<i>fileNumber</i> - * @throws MetaMatrixComponentException if a file with this name already exists in the directory, or if it cannot be created - * @since 4.2 - */ - private File createFile(TupleSourceID sourceID, int fileNumber) throws MetaMatrixComponentException { - File storageFile = new File(this.directory, FILE_PREFIX + sourceID.getIDValue() + "_" + fileNumber); //$NON-NLS-1$ - if (LogManager.isMessageToBeRecorded(LogConstants.CTX_STORAGE_MGR, MessageLevel.DETAIL)) { - LogManager.logDetail(LogConstants.CTX_STORAGE_MGR, "Creating temporary storage area file " + storageFile.getAbsoluteFile()); //$NON-NLS-1$ - } - try { - boolean created = storageFile.createNewFile(); - if(!created) { - throw new MetaMatrixComponentException(QueryExecPlugin.Util.getString("FileStoreageManager.file_exists", storageFile.getAbsoluteFile())); //$NON-NLS-1$ - } - } catch(IOException e) { - throw new MetaMatrixComponentException(e, QueryExecPlugin.Util.getString("FileStoreageManager.error_creating", storageFile.getAbsoluteFile())); //$NON-NLS-1$ - } - return storageFile; - } - - /** - * Add a batch to the storage manager. Because we don't implement remove, it's - * possible that this batch already exists, in which case nothing is done. - * @param sourceID Source identifier - * @param batch Batch to add - */ - public void addBatch(TupleSourceID sourceID, TupleBatch batch, String[] types) - throws MetaMatrixComponentException { - - // Defect 13342 - addBatch method now creates spill files if the total bytes exceeds the max file size limit - TupleSourceInfo tsInfo = getTupleSourceInfo(sourceID, true); - synchronized (tsInfo) { - if (tsInfo.isRemoved) { - return; - } - Integer batchKey = new Integer(batch.getBeginRow()); - if (tsInfo.tupleBatchPointers.containsKey(batchKey)) { - return; - } - byte[] bytes = convertToBytes(batch, types); - if (bytes.length > maxFileSize) { - LogManager.logWarning(LogConstants.CTX_STORAGE_MGR, "Detected an attempt to save a batch (" + sourceID + ", begin=" + batch.getBeginRow()+ ", size=" + bytes.length + ") larger than the buffer max file size setting of " + maxFileSize + " bytes. The buffer manager will ignore the max file size setting for this batch, and create a buffer file dedicated to this batch. It may be necessary to reduce the buffer batch setting or increase the buffer max file size setting."); //$NON-NLS-1$//$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$ - } - FileInfo fileInfo = tsInfo.getMostRecentlyCreatedFile(); - if (fileInfo == null || - (/*fileInfo.file.length() != 0 && */fileInfo.file.length() + bytes.length > maxFileSize)) { - // Create and add - fileInfo = new FileInfo(createFile(sourceID, tsInfo.storageFiles.size())); - tsInfo.storageFiles.add(fileInfo); - } - long pointer = 0; - - if (LogManager.isMessageToBeRecorded(LogConstants.CTX_STORAGE_MGR, MessageLevel.DETAIL)) { - LogManager.logDetail(LogConstants.CTX_STORAGE_MGR, "Adding batch to storage area file " + fileInfo.file.getAbsoluteFile() + " [ sourceID: " + sourceID + "batch: " + batch + " ]"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ - } - try { - // Get access to the file and remember whether we had to open it or not - fileInfo.open(); - RandomAccessFile fileAccess = fileInfo.getAccess(); - - // Store the batch in the file - pointer = fileAccess.length(); - fileAccess.setLength(pointer + bytes.length); - fileAccess.seek(pointer); - fileAccess.write(bytes); - } catch(IOException e) { - throw new MetaMatrixComponentException(e, QueryExecPlugin.Util.getString("FileStoreageManager.error_reading", fileInfo.file.getAbsoluteFile())); //$NON-NLS-1$ - } finally { - fileInfo.close(); - } - - // Update the pointers - tsInfo.tupleBatchPointers.put(batchKey, new PointerInfo(fileInfo, pointer, bytes.length)); - } - } - - /** - * Convert from an object to a byte array - * @param object Object to convert - * @return Byte array - */ - private byte[] convertToBytes(TupleBatch batch, String[] types) throws MetaMatrixComponentException { - ObjectOutputStream oos = null; - try { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - oos = new ObjectOutputStream(baos); - - batch.setDataTypes(types); - batch.writeExternal(oos); - oos.flush(); - return baos.toByteArray(); - - } catch(IOException e) { - throw new MetaMatrixComponentException(e, QueryExecPlugin.Util.getString("FileStorageManager.batch_error")); //$NON-NLS-1$ - } finally { - if(oos != null) { - try { - oos.close(); - } catch(IOException e) { - } - } - } - } - - /** - * Get a batch from the storage manager based on the beginRow. - * @param sourceID Source identifier - * @param beginRow Beginning row of batch to retrieve - * @return Batch retrieved - */ - public TupleBatch getBatch(TupleSourceID sourceID, int beginRow, String[] types) - throws TupleSourceNotFoundException, MetaMatrixComponentException { - - TupleSourceInfo info = getTupleSourceInfo(sourceID, false); - if(info == null) { - throw new TupleSourceNotFoundException(QueryExecPlugin.Util.getString("BufferManagerImpl.tuple_source_not_found", sourceID)); //$NON-NLS-1$ - } - - byte[] bytes = null; - synchronized(info) { - if(info.isRemoved) { - throw new TupleSourceNotFoundException(QueryExecPlugin.Util.getString("BufferManagerImpl.tuple_source_not_found", sourceID)); //$NON-NLS-1$ - } - // Find pointer - PointerInfo pointerInfo = info.tupleBatchPointers.get(new Integer(beginRow)); - Assertion.isNotNull(pointerInfo); - - FileInfo fileInfo = pointerInfo.fileInfo; - // Get access to the file - RandomAccessFile fileAccess = null; - try { - fileInfo.open(); - fileAccess = fileInfo.getAccess(); - fileAccess.seek(pointerInfo.pointer); - bytes = new byte[pointerInfo.length]; - fileAccess.readFully(bytes); - ByteArrayInputStream bais = new ByteArrayInputStream(bytes); - ObjectInputStream ois = new ObjectInputStream(bais); - - TupleBatch batch = new TupleBatch(); - batch.setDataTypes(types); - batch.readExternal(ois); - return batch; - } catch(IOException e) { - throw new MetaMatrixComponentException(e, QueryExecPlugin.Util.getString("FileStoreageManager.error_reading", fileInfo.file.getAbsoluteFile())); //$NON-NLS-1$ - } catch (ClassNotFoundException e) { - throw new MetaMatrixComponentException(e, QueryExecPlugin.Util.getString("FileStoreageManager.error_reading", fileInfo.file.getAbsoluteFile())); //$NON-NLS-1$ - } finally { - fileInfo.close(); - } - } - } - - /** - * Remove all batches for a sourceID. Before removal, the file is closed. - * @param sourceID Tuple source ID - */ - public void removeBatches(TupleSourceID sourceID) throws MetaMatrixComponentException { - TupleSourceInfo info = null; - // Remove info from the file map - synchronized(tupleSourceMap) { - info = tupleSourceMap.remove(sourceID); - } - - // Didn't find a file - if(info == null) { - return; - } - - if (LogManager.isMessageToBeRecorded(LogConstants.CTX_STORAGE_MGR, MessageLevel.DETAIL)) { - LogManager.logDetail(LogConstants.CTX_STORAGE_MGR, "Removing storage for " + sourceID); //$NON-NLS-1$ - } - - synchronized(info) { - // If open, close the file and decrement the open file counter - for (int i = 0; i < info.storageFiles.size(); i++) { - FileInfo fileInfo = info.storageFiles.get(i); - fileInfo.delete(); - } - // Delete the file and mark info as being removed - info.isRemoved = true; - } - } - - /** - * This method removes all storage area files by walking through the file info - * map and closing and removing each file. - */ - public synchronized void shutdown() { - - LogManager.logDetail(LogConstants.CTX_STORAGE_MGR, "Removing all storage area files "); //$NON-NLS-1$ - - List<TupleSourceID> ids = null; - synchronized (tupleSourceMap) { - ids = new LinkedList<TupleSourceID>(tupleSourceMap.keySet()); - } - - for (TupleSourceID key : ids) { - try { - removeBatches(key); - } catch (MetaMatrixComponentException e) { - LogManager.logWarning(LogConstants.CTX_STORAGE_MGR, e, "Shutdown failed while removing batches for tuple source: " + key); //$NON-NLS-1$ - } - } - - tupleSourceMap = null; - } - - public int getOpenFiles() { - return this.fileCache.size(); - } - - private class FileInfo { - private File file; - private RandomAccessFile fileData; // may be null if not open - - public FileInfo(File file) { - this.file = file; - } - - public void open() throws FileNotFoundException { - if(this.fileData == null) { - this.fileData = fileCache.remove(this.file); - if (this.fileData == null) { - this.fileData = new RandomAccessFile(file, "rw"); //$NON-NLS-1$ - } - } - } - - public RandomAccessFile getAccess() { - return this.fileData; - } - - public void close() { - if(this.fileData != null) { - fileCache.put(file, this.fileData); - this.fileData = null; - } - } - - public void delete() { - if (this.fileData == null) { - this.fileData = fileCache.remove(this.file); - } - if (this.fileData != null) { - try { - this.fileData.close(); - } catch(Exception e) { - } - this.fileData = null; - } - file.delete(); - } - - public String toString() { - return "FileInfo<" + file.getName() + ", has fileData = " + (fileData != null) + ">"; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ - } - } - - private static class PointerInfo { - FileInfo fileInfo; - public long pointer; - public int length; - - public PointerInfo(FileInfo fileInfo, long pointer, int length) { - this.fileInfo = fileInfo; - this.pointer = pointer; - this.length = length; - } - } - - private static class TupleSourceInfo { - Map<Integer, PointerInfo> tupleBatchPointers = new HashMap<Integer, PointerInfo>(); - List<FileInfo> storageFiles = new ArrayList<FileInfo>(2); // Stores all the FileInfos for this tupleSource - private boolean isRemoved = false; - - FileInfo getMostRecentlyCreatedFile() { - if (storageFiles.isEmpty()) { - return null; - } - return storageFiles.get(storageFiles.size() - 1); - } - } - -} Copied: trunk/engine/src/main/java/com/metamatrix/dqp/internal/datamgr (from rev 1659, trunk/client/src/main/java/com/metamatrix/dqp/internal/datamgr) Modified: trunk/engine/src/main/java/com/metamatrix/query/eval/Evaluator.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/eval/Evaluator.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/eval/Evaluator.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -22,13 +22,12 @@ package com.metamatrix.query.eval; -import java.math.BigDecimal; import java.sql.SQLException; import java.util.Collections; -import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; @@ -319,21 +318,13 @@ } else if (criteria instanceof DependentSetCriteria){ ContextReference ref = (ContextReference)criteria; ValueIteratorSource vis = (ValueIteratorSource)getContext(criteria).getVariableContext().getGlobalValue(ref.getContextSymbol()); - HashSet<Object> values; + Set<Object> values; try { values = vis.getCachedSet(ref.getValueExpression()); } catch (MetaMatrixProcessingException e) { throw new CriteriaEvaluationException(e, e.getMessage()); } if (values != null) { - if (leftValue instanceof BigDecimal) { - for (Object object : values) { - if (compareValues(leftValue, object) == 0) { - return true; - } - } - return false; - } return values.contains(leftValue); } //there are too many values to justify a linear search or holding Modified: trunk/engine/src/main/java/com/metamatrix/query/optimizer/proc/ProcedurePlanner.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/optimizer/proc/ProcedurePlanner.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/optimizer/proc/ProcedurePlanner.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -41,9 +41,9 @@ import com.metamatrix.query.processor.proc.AssignmentInstruction; import com.metamatrix.query.processor.proc.BreakInstruction; import com.metamatrix.query.processor.proc.ContinueInstruction; +import com.metamatrix.query.processor.proc.CreateCursorResultSetInstruction; import com.metamatrix.query.processor.proc.ErrorInstruction; import com.metamatrix.query.processor.proc.ExecDynamicSqlInstruction; -import com.metamatrix.query.processor.proc.ExecSqlInstruction; import com.metamatrix.query.processor.proc.IfInstruction; import com.metamatrix.query.processor.proc.LoopInstruction; import com.metamatrix.query.processor.proc.ProcedurePlan; @@ -52,9 +52,7 @@ import com.metamatrix.query.processor.program.ProgramInstruction; import com.metamatrix.query.sql.lang.Command; import com.metamatrix.query.sql.lang.DynamicCommand; -import com.metamatrix.query.sql.lang.Into; import com.metamatrix.query.sql.lang.ProcedureContainer; -import com.metamatrix.query.sql.lang.Query; import com.metamatrix.query.sql.lang.TranslatableProcedureContainer; import com.metamatrix.query.sql.proc.AssignmentStatement; import com.metamatrix.query.sql.proc.Block; @@ -65,7 +63,6 @@ import com.metamatrix.query.sql.proc.Statement; import com.metamatrix.query.sql.proc.WhileStatement; import com.metamatrix.query.sql.symbol.Expression; -import com.metamatrix.query.sql.symbol.GroupSymbol; import com.metamatrix.query.sql.visitor.CommandCollectorVisitor; import com.metamatrix.query.util.CommandContext; @@ -246,19 +243,12 @@ { CommandStatement cmdStmt = (CommandStatement) statement; Command command = cmdStmt.getCommand(); - GroupSymbol intoGroup = null; - if(command instanceof Query){ - Into into = ((Query)command).getInto(); - if(into != null){ - intoGroup = into.getGroup(); - } - } ProcessorPlan commandPlan = cmdStmt.getCommand().getProcessorPlan(); if (command.getType() == Command.TYPE_DYNAMIC){ instruction = new ExecDynamicSqlInstruction(parentProcCommand,((DynamicCommand)command), metadata, idGenerator, capFinder ); }else{ - instruction = new ExecSqlInstruction(commandPlan, intoGroup); + instruction = new CreateCursorResultSetInstruction(CreateCursorResultSetInstruction.RS_NAME, commandPlan); } if(debug) { Modified: trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/RelationalPlanner.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/RelationalPlanner.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/RelationalPlanner.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -470,7 +470,7 @@ if (c == null) { c = QueryResolver.expandCommand(container, metadata, analysisRecord); if (c != null) { - Request.validateWithVisitor(new ValidationVisitor(), metadata, c, true); + Request.validateWithVisitor(new ValidationVisitor(), metadata, c); metadata.addToMetadataCache(container.getGroup().getMetadataID(), cacheString, c.clone()); } } else { @@ -903,7 +903,7 @@ //parse, resolve, validate result = convertToSubquery(qnode, metadata); QueryResolver.resolveCommand(result, Collections.EMPTY_MAP, metadata, analysisRecord); - Request.validateWithVisitor(new ValidationVisitor(), metadata, result, true); + Request.validateWithVisitor(new ValidationVisitor(), metadata, result); metadata.addToMetadataCache(virtualGroup.getMetadataID(), "transformation/" + cacheString, result.clone()); //$NON-NLS-1$ } return QueryRewriter.rewrite(result, metadata, context); Copied: trunk/engine/src/main/java/com/metamatrix/query/processor/BatchCollector.java (from rev 1686, trunk/engine/src/main/java/com/metamatrix/query/processor/relational/BatchCollector.java) =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/BatchCollector.java (rev 0) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/BatchCollector.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -0,0 +1,124 @@ +/* + * JBoss, Home of Professional Open Source. + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA. + */ + +package com.metamatrix.query.processor; + +import java.util.List; + +import com.metamatrix.api.exception.MetaMatrixComponentException; +import com.metamatrix.api.exception.MetaMatrixProcessingException; +import com.metamatrix.common.buffer.BlockedException; +import com.metamatrix.common.buffer.TupleBatch; +import com.metamatrix.common.buffer.TupleBuffer; + +public class BatchCollector { + + public interface BatchProducer { + /** + * Get a batch of results or possibly an Exception. + * @return Batch of results + * @throws BlockedException indicating next batch is not available yet + * @throws MetaMatrixComponentException for non-business rule exception + * @throws MetaMatrixProcessingException for business rule exception, related + * to user input or modeling + */ + TupleBatch nextBatch() throws BlockedException, MetaMatrixComponentException, MetaMatrixProcessingException; + + /** + * Get list of resolved elements describing output columns for this plan. + * @return List of SingleElementSymbol + */ + List getOutputElements(); + } + + public interface BatchHandler { + void batchProduced(TupleBatch batch) throws MetaMatrixProcessingException, MetaMatrixComponentException; + } + + private BatchProducer sourceNode; + private BatchHandler batchHandler; + + private boolean done = false; + private TupleBuffer buffer; + private boolean collectedAny; + + public BatchCollector(BatchProducer sourceNode, TupleBuffer buffer) { + this.sourceNode = sourceNode; + this.buffer = buffer; + } + + public TupleBuffer collectTuples() throws MetaMatrixComponentException, MetaMatrixProcessingException { + TupleBatch batch = null; + while(!done) { + batch = sourceNode.nextBatch(); + + flushBatch(batch); + + // Check for termination condition + if(batch.getTerminationFlag()) { + done = true; + buffer.close(); + break; + } + } + return buffer; + } + + public TupleBuffer getTupleBuffer() { + return buffer; + } + + /** + * Flush the batch by giving it to the buffer manager. + */ + private void flushBatch(TupleBatch batch) throws MetaMatrixComponentException, MetaMatrixProcessingException { + // Add batch + if(batch.getRowCount() > 0) { + for (List tuple : batch.getAllTuples()) { + buffer.addTuple(tuple); + } + collectedAny = true; + } + if (this.batchHandler != null && (batch.getRowCount() > 0 || batch.getTerminationFlag())) { + this.batchHandler.batchProduced(batch); + } + } + + public void setBatchHandler(BatchHandler batchHandler) { + this.batchHandler = batchHandler; + } + + public boolean collectedAny() { + boolean result = collectedAny; + collectedAny = false; + return result; + } + + public int getRowCount() { + return buffer.getRowCount(); + } + + public boolean isDone() { + return done; + } + +} Copied: trunk/engine/src/main/java/com/metamatrix/query/processor/BatchIterator.java (from rev 1686, trunk/engine/src/main/java/com/metamatrix/query/processor/relational/BatchIterator.java) =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/BatchIterator.java (rev 0) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/BatchIterator.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -0,0 +1,119 @@ +/* + * JBoss, Home of Professional Open Source. + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA. + */ + +package com.metamatrix.query.processor; + +import java.util.List; + +import com.metamatrix.api.exception.MetaMatrixComponentException; +import com.metamatrix.api.exception.MetaMatrixProcessingException; +import com.metamatrix.common.buffer.IndexedTupleSource; +import com.metamatrix.common.buffer.TupleBatch; +import com.metamatrix.query.processor.BatchCollector.BatchProducer; +import com.metamatrix.query.sql.symbol.SingleElementSymbol; + +public class BatchIterator implements + IndexedTupleSource { + + private final BatchProducer source; + + public BatchIterator(BatchProducer source) { + this.source = source; + } + + private boolean done; + private int currentRow = 1; + private TupleBatch currentBatch; + private List currentTuple; + + public boolean hasNext() throws MetaMatrixComponentException, + MetaMatrixProcessingException { + if (done) { + return false; + } + while (currentTuple == null) { + if (currentBatch == null) { + currentBatch = this.source.nextBatch(); + } + + if (currentBatch.getEndRow() >= currentRow) { + this.currentTuple = currentBatch.getTuple(currentRow++); + } else { + done = currentBatch.getTerminationFlag(); + currentBatch = null; + if (done) { + return false; + } + } + } + return true; + } + + @Override + public void closeSource() throws MetaMatrixComponentException { + + } + + @Override + public List<SingleElementSymbol> getSchema() { + return source.getOutputElements(); + } + + @Override + public List<?> nextTuple() throws MetaMatrixComponentException, + MetaMatrixProcessingException { + if (currentTuple == null && !hasNext()) { + return null; + } + List result = currentTuple; + currentTuple = null; + return result; + } + + public void reset() { + throw new UnsupportedOperationException(); + } + + public void mark() { + //does nothing + } + + @Override + public int getCurrentIndex() { + return currentRow; + } + + public void setPosition(int position) { + if (position == this.currentRow) { + return; + } + if (position < this.currentRow && (this.currentBatch == null || position < this.currentBatch.getBeginRow())) { + throw new UnsupportedOperationException("Backwards positioning is not allowed"); //$NON-NLS-1$ + } + this.currentRow = position; + this.currentTuple = null; + if (currentBatch.getEndRow() < currentRow) { + this.currentBatch = null; + } + } + +} Added: trunk/engine/src/main/java/com/metamatrix/query/processor/CollectionTupleSource.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/CollectionTupleSource.java (rev 0) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/CollectionTupleSource.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -0,0 +1,70 @@ +/* + * JBoss, Home of Professional Open Source. + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA. + */ + +package com.metamatrix.query.processor; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; + +import com.metamatrix.common.buffer.TupleSource; +import com.metamatrix.query.sql.lang.Command; +import com.metamatrix.query.sql.symbol.SingleElementSymbol; + +public class CollectionTupleSource implements TupleSource { + + private Iterator<? extends List<?>> tuples; + private List<SingleElementSymbol> schema; + + public static CollectionTupleSource createUpdateCountTupleSource(int count) { + return new CollectionTupleSource(Arrays.asList(Arrays.asList(count)).iterator(), Command.getUpdateCommandSymbol()); + } + + public static CollectionTupleSource createNullTupleSource(List<SingleElementSymbol> schema) { + return new CollectionTupleSource(new ArrayList<List<Object>>(0).iterator(), schema); + } + + public CollectionTupleSource(Iterator<? extends List<?>> tuples, + List<SingleElementSymbol> schema) { + this.tuples = tuples; + this.schema = schema; + } + + @Override + public List<?> nextTuple() { + if (tuples.hasNext()) { + return tuples.next(); + } + return null; + } + + @Override + public List<SingleElementSymbol> getSchema() { + return schema; + } + + @Override + public void closeSource() { + + } +} \ No newline at end of file Property changes on: trunk/engine/src/main/java/com/metamatrix/query/processor/CollectionTupleSource.java ___________________________________________________________________ Name: svn:mime-type + text/plain Deleted: trunk/engine/src/main/java/com/metamatrix/query/processor/NullTupleSource.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/NullTupleSource.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/NullTupleSource.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -1,79 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.query.processor; - -import java.util.List; - -import com.metamatrix.common.buffer.TupleSource; - -/** - * <p>This class can be used to represent null tuples or no data for an - * atomic query execution. Example usage is in implementation of partial - * results feature where we have to return null tuples if there is an error - * in the execution of atomic query.</p> - */ - -public class NullTupleSource implements TupleSource { - - private List elements; // elementSymbols containing null tuples - - /** - * <p>Constructor that takes the list of elementSymbols - * that have null rows.</p> - * @param elements List of elementSymbols that this source contains. - */ - public NullTupleSource(List elements) { - this.elements = elements; - } - - /** - * @see TupleSource#getSchema() - */ - public List getSchema() { - return this.elements; - } - - /** - * <p>Open the tuple source to read tuples. This method does nothing.</p> - */ - public void openSource() { - // do nothing - } - - /** - * <p>This method returns a null indicating, that there are no - * rows in the source.</p> - * @return list containing the values of the next tuple. - */ - public List nextTuple() { - return null; - } - - /** - * <p>Close the tuple source after reading tuples. This method does nothing.</p> - */ - public void closeSource() { - // do nothing - } - -} // END CLASS Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/ProcessorPlan.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/ProcessorPlan.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/ProcessorPlan.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -28,6 +28,7 @@ import com.metamatrix.api.exception.MetaMatrixComponentException; import com.metamatrix.api.exception.MetaMatrixProcessingException; import com.metamatrix.common.buffer.*; +import com.metamatrix.query.processor.BatchCollector.BatchProducer; import com.metamatrix.query.util.CommandContext; /** @@ -43,7 +44,7 @@ * the call to {@link #close}. * </p> */ -public interface ProcessorPlan extends Cloneable, Describable { +public interface ProcessorPlan extends Cloneable, Describable, BatchProducer { /** * Initialize the plan with some required pieces of data for making Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/QueryProcessor.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/QueryProcessor.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/QueryProcessor.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -30,43 +30,33 @@ import com.metamatrix.common.buffer.BlockedException; import com.metamatrix.common.buffer.BufferManager; import com.metamatrix.common.buffer.TupleBatch; -import com.metamatrix.common.buffer.TupleSourceID; -import com.metamatrix.common.buffer.TupleSourceNotFoundException; -import com.metamatrix.common.buffer.BufferManager.TupleSourceStatus; import com.metamatrix.common.buffer.BufferManager.TupleSourceType; import com.metamatrix.common.log.LogManager; -import com.metamatrix.core.MetaMatrixCoreException; import com.metamatrix.core.log.MessageLevel; +import com.metamatrix.core.util.Assertion; import com.metamatrix.dqp.util.LogConstants; import com.metamatrix.query.execution.QueryExecPlugin; +import com.metamatrix.query.processor.BatchCollector.BatchProducer; import com.metamatrix.query.util.CommandContext; -/** - * TODO: combine overlapping responsibilities with RequestWorkItem - */ -public class QueryProcessor { +public class QueryProcessor implements BatchProducer { public interface ProcessorFactory { QueryProcessor createQueryProcessor(String query, String recursionGroup, CommandContext commandContext) throws MetaMatrixProcessingException, MetaMatrixComponentException; } - public interface BatchHandler { - void batchProduced(TupleBatch batch) throws MetaMatrixProcessingException, MetaMatrixComponentException; - } - - private TupleSourceID resultsID; private CommandContext context; private ProcessorDataManager dataMgr; - private BufferManager bufferMgr; + private BufferManager bufferMgr; private ProcessorPlan processPlan; private boolean initialized = false; /** Flag that marks whether the request has been canceled. */ private volatile boolean requestCanceled = false; - private volatile boolean requestClosed = false; private static final int DEFAULT_WAIT = 50; private boolean processorClosed = false; - private BatchHandler batchHandler; private volatile int highestRow; + + private boolean nonBlocking = false; /** * Construct a processor with all necessary information to process. @@ -79,12 +69,10 @@ public QueryProcessor(ProcessorPlan plan, CommandContext context, BufferManager bufferMgr, ProcessorDataManager dataMgr) throws MetaMatrixComponentException { this.context = context; this.dataMgr = dataMgr; - this.bufferMgr = bufferMgr; this.processPlan = plan; - + this.bufferMgr = bufferMgr; // Add data manager to all nodes in tree this.processPlan.initialize(context, this.dataMgr, bufferMgr); - this.resultsID = bufferMgr.createTupleSource(processPlan.getOutputElements(), context.getConnectionID(), TupleSourceType.PROCESSOR); } public CommandContext getContext() { @@ -95,123 +83,98 @@ return this.context.getProcessorID(); } - public TupleSourceID getResultsID() { - return resultsID; - } - public ProcessorPlan getProcessorPlan() { return this.processPlan; } - private void initialize() - throws BlockedException, MetaMatrixComponentException, MetaMatrixProcessingException { - - // initialize if necessary - if(! initialized) { - // Open the top node for reading - processPlan.open(); - - initialized = true; - } - } - - /** - * Process fully blocking - * @throws MetaMatrixCoreException - */ - public void process() + public TupleBatch nextBatch() throws BlockedException, MetaMatrixProcessingException, MetaMatrixComponentException { - - while(true) { - try { - if (process(Integer.MAX_VALUE)) { - break; - } - } catch(BlockedException e) { - try { - Thread.sleep(DEFAULT_WAIT); - } catch (InterruptedException err) { - throw new MetaMatrixComponentException(err); - } - } - } + + while (true) { + try { + return nextBatchDirect(); + } catch (BlockedException e) { + if (nonBlocking) { + try { + Thread.sleep(DEFAULT_WAIT); + } catch (InterruptedException err) { + throw new MetaMatrixComponentException(err); + } + continue; + } + throw e; + } + } } - - /** - * Process until time expires or a {@link BlockedException} occurs - * @param time - * @return - */ - public boolean process(long time) + + private TupleBatch nextBatchDirect() throws BlockedException, MetaMatrixProcessingException, MetaMatrixComponentException { boolean done = false; + TupleBatch result = null; try { - // Will only initialize the first time process() is called - initialize(); + // initialize if necessary + if(! initialized) { + // Open the top node for reading + processPlan.open(); - long start = System.currentTimeMillis(); - - while(System.currentTimeMillis() - start < time && !requestClosed && !processorClosed) { - checkState(); - - TupleBatch batch = processPlan.nextBatch(); - flushBatch(batch); - - if(batch.getTerminationFlag()) { - done = true; - break; - } - } - } catch (BlockedException e) { - throw e; - } catch (MetaMatrixException e) { - try { - closeProcessing(); - } catch (MetaMatrixException e1){ - LogManager.logDetail(LogConstants.CTX_DQP, e1, "Error closing processor"); //$NON-NLS-1$ - } - if (e instanceof MetaMatrixProcessingException) { - throw (MetaMatrixProcessingException)e; - } - if (e instanceof MetaMatrixComponentException) { - throw (MetaMatrixComponentException)e; - } - throw new MetaMatrixComponentException(e); - } finally { - //iff this is the root command context release any pinned (Unclosed tuplesources) - if (this.context.getParent() == null) { - bufferMgr.releasePinnedBatches(); - } - } - - if(done || requestClosed) { + initialized = true; + } + + long currentTime = System.currentTimeMillis(); + Assertion.assertTrue(!processorClosed); + while(currentTime < context.getTimeSliceEnd()) { + if (requestCanceled) { + throw new MetaMatrixProcessingException(QueryExecPlugin.Util.getString("QueryProcessor.request_cancelled", getProcessID())); //$NON-NLS-1$ + } + if (currentTime > context.getTimeoutEnd()) { + throw new MetaMatrixProcessingException("Query timed out"); //$NON-NLS-1$ + } + result = processPlan.nextBatch(); + this.highestRow = result.getEndRow(); + + if(result.getTerminationFlag()) { + done = true; + break; + } + + if (result.getRowCount() > 0) { + break; + } + + } + } catch (BlockedException e) { + throw e; + } catch (MetaMatrixException e) { + try { + closeProcessing(); + } catch (MetaMatrixException e1){ + LogManager.logDetail(LogConstants.CTX_DQP, e1, "Error closing processor"); //$NON-NLS-1$ + } + if (e instanceof MetaMatrixProcessingException) { + throw (MetaMatrixProcessingException)e; + } + if (e instanceof MetaMatrixComponentException) { + throw (MetaMatrixComponentException)e; + } + throw new MetaMatrixComponentException(e); + } + if(done) { closeProcessing(); - return true; } - return processorClosed; + if (result == null) { + throw BlockedException.INSTANCE; //expired timeslice + } + return result; } - + + /** - * Flush the batch by giving it to the buffer manager. - */ - private void flushBatch(TupleBatch batch) throws MetaMatrixComponentException, MetaMatrixProcessingException { - if(batch.getRowCount() > 0) { - this.bufferMgr.addTupleBatch(this.resultsID, batch); - this.highestRow = batch.getEndRow(); - } - if (this.batchHandler != null && (batch.getRowCount() > 0 || batch.getTerminationFlag())) { - this.batchHandler.batchProduced(batch); - } - } - - /** * Close processing and clean everything up. Should only be called by the same thread that called process. * @throws MetaMatrixComponentException - * @throws TupleSourceNotFoundException */ - public void closeProcessing() throws TupleSourceNotFoundException, MetaMatrixComponentException { + public void closeProcessing() throws MetaMatrixComponentException { if (processorClosed) { return; } @@ -220,23 +183,12 @@ } processorClosed = true; - try { - processPlan.close(); - } finally { - // Mark tuple source FULL - this.bufferMgr.setStatus(this.resultsID, TupleSourceStatus.FULL); - } + processPlan.close(); } - /** - * Checks whether the request has been canceled of an error has occurred, and throws an exception if either of these conditions is true. - * @throws MetaMatrixException if the request has been canceled or an error occurred during processing. - * @since 4.2 - */ - private void checkState() throws MetaMatrixException { - if (requestCanceled) { - throw new MetaMatrixProcessingException(QueryExecPlugin.Util.getString("QueryProcessor.request_cancelled", getProcessID())); //$NON-NLS-1$ - } + @Override + public List getOutputElements() { + return this.processPlan.getOutputElements(); } public List<Exception> getAndClearWarnings() { @@ -250,18 +202,15 @@ this.requestCanceled = true; } - /** - * Asynch graceful shutdown of the QueryProcessor - */ - public void requestClosed() { - this.requestClosed = true; - } - - public void setBatchHandler(BatchHandler batchHandler) { - this.batchHandler = batchHandler; - } - public int getHighestRow() { return highestRow; } + + public BatchCollector createBatchCollector() throws MetaMatrixComponentException { + return new BatchCollector(this, this.bufferMgr.createTupleBuffer(this.processPlan.getOutputElements(), context.getConnectionID(), TupleSourceType.PROCESSOR)); + } + + public void setNonBlocking(boolean nonBlocking) { + this.nonBlocking = nonBlocking; + } } Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/dynamic/SqlEval.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/dynamic/SqlEval.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/dynamic/SqlEval.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -25,7 +25,6 @@ import java.io.StringReader; import java.util.ArrayList; import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.Map; @@ -44,13 +43,11 @@ import com.metamatrix.api.exception.query.ExpressionEvaluationException; import com.metamatrix.api.exception.query.QueryParserException; import com.metamatrix.common.buffer.BlockedException; -import com.metamatrix.common.buffer.BufferManager; import com.metamatrix.common.buffer.TupleSource; -import com.metamatrix.common.buffer.TupleSourceID; -import com.metamatrix.common.buffer.TupleSourceNotFoundException; import com.metamatrix.common.types.DataTypeManager; import com.metamatrix.common.types.XMLType; import com.metamatrix.query.eval.Evaluator; +import com.metamatrix.query.processor.BatchIterator; import com.metamatrix.query.processor.ProcessorDataManager; import com.metamatrix.query.processor.QueryProcessor; import com.metamatrix.query.sql.symbol.ElementSymbol; @@ -69,9 +66,8 @@ private static final String NO_RESULTS_DOCUMENT = "<?xml version=\"1.0\" encoding=\"UTF-8\"?><results/>"; //$NON-NLS-1$ - private BufferManager bufferMgr; private CommandContext context; - private ArrayList<TupleSourceID> openTupleList; + private ArrayList<QueryProcessor> processors; private String parentGroup; private Map<String, Expression> params; private ProcessorDataManager dataManager; @@ -120,8 +116,7 @@ } } - public SqlEval(BufferManager bufferMgr, ProcessorDataManager dataManager, CommandContext context, String parentGroup, Map<String, Expression> params) { - this.bufferMgr = bufferMgr; + public SqlEval(ProcessorDataManager dataManager, CommandContext context, String parentGroup, Map<String, Expression> params) { this.dataManager = dataManager; this.context = context; this.parentGroup = parentGroup; @@ -136,17 +131,13 @@ throws QueryParserException, MetaMatrixProcessingException, MetaMatrixComponentException { QueryProcessor processor = context.getQueryProcessorFactory().createQueryProcessor(sql, parentGroup, context); - // keep track of all the tuple sources opened during the - // xquery process; - if (openTupleList == null) { - openTupleList = new ArrayList<TupleSourceID>(); + processor.setNonBlocking(true); + if (processors == null) { + processors = new ArrayList<QueryProcessor>(); } - openTupleList.add(processor.getResultsID()); + processors.add(processor); + TupleSource src = new BatchIterator(processor); - processor.process(); - - TupleSourceID tsID = processor.getResultsID(); - TupleSource src = this.bufferMgr.getTupleSource(tsID); String[] columns = elementNames(src.getSchema()); Class[] types= elementTypes(src.getSchema()); boolean xml = false; @@ -209,16 +200,10 @@ * Closes any resources opened during the evaluation */ public void close() throws MetaMatrixComponentException { - if (openTupleList != null && !openTupleList.isEmpty()) { - for (Iterator i = openTupleList.iterator(); i.hasNext();) { - TupleSourceID id = (TupleSourceID)i.next(); - try { - this.bufferMgr.removeTupleSource(id); - } catch (TupleSourceNotFoundException e) { - // ignore and go on.. - } - } + if (processors != null) { + for (QueryProcessor processor : processors) { + processor.closeProcessing(); + } } - } } Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/proc/CreateCursorResultSetInstruction.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/proc/CreateCursorResultSetInstruction.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/proc/CreateCursorResultSetInstruction.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -39,6 +39,9 @@ /** */ public class CreateCursorResultSetInstruction extends ProgramInstruction { + + public static final String RS_NAME = "EXECSQL_INSTRUCTION"; //$NON-NLS-1$ + protected String rsName; protected ProcessorPlan plan; Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/proc/ExecDynamicSqlInstruction.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/proc/ExecDynamicSqlInstruction.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/proc/ExecDynamicSqlInstruction.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -196,8 +196,7 @@ .createNonRecordingRecord(), procEnv .getContext()); - ExecSqlInstruction inst = new ExecSqlInstruction(commandPlan, - dynamicCommand.getIntoGroup()); + CreateCursorResultSetInstruction inst = new CreateCursorResultSetInstruction(CreateCursorResultSetInstruction.RS_NAME, commandPlan); dynamicProgram = new Program(); dynamicProgram.addInstruction(inst); @@ -332,7 +331,7 @@ public Map getDescriptionProperties() { Map props = new HashMap(); props.put(PROP_TYPE, "SQL"); //$NON-NLS-1$ - props.put(PROP_SQL, ExecSqlInstruction.RS_NAME); + props.put(PROP_SQL, CreateCursorResultSetInstruction.RS_NAME); if (dynamicCommand.getIntoGroup() != null) { props.put(PROP_GROUP, dynamicCommand.getIntoGroup().toString()); } Deleted: trunk/engine/src/main/java/com/metamatrix/query/processor/proc/ExecSqlInstruction.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/proc/ExecSqlInstruction.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/proc/ExecSqlInstruction.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -1,123 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.query.processor.proc; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import com.metamatrix.api.exception.MetaMatrixComponentException; -import com.metamatrix.api.exception.MetaMatrixProcessingException; -import com.metamatrix.common.buffer.BlockedException; -import com.metamatrix.query.processor.ProcessorPlan; -import com.metamatrix.query.processor.program.ProgramInstruction; -import com.metamatrix.query.sql.symbol.GroupSymbol; - -/** - * <p> Executes a SQL statement, and remove its results from the buffer manager. Executing this - * instruction does not modify the values of any of the variables, hence it's results are not - * important so they are removed immediately.</p> - */ -public class ExecSqlInstruction extends ProgramInstruction { - - public static final String RS_NAME = "EXECSQL_INSTRUCTION"; //$NON-NLS-1$ - - // processor plan for the command on the CommandStatement - private ProcessorPlan commandPlan; - //group for into if it is Select Into - private GroupSymbol intoGroup; - - /** - * Constructor for ExecSqlInstruction. - * @param command Object (such as a ProcessorPlan) that is executed when this instruction - * is processed. - */ - public ExecSqlInstruction(ProcessorPlan commandPlan, GroupSymbol intoGroup) { - this.commandPlan = commandPlan; - this.intoGroup = intoGroup; - } - - /** - * <p>Processing this instruction executes the ProcessorPlan for the command on the - * CommandStatement of the update procedure language. Executing this plan does not effect - * the values of any of the variables defined as part of the update procedure and hence - * the results of the ProcessPlan execution need not be stored for further processing. The - * results are removed from the buffer manager immediately after execution. The program - * counter is incremented after execution of the plan.</p> - * @throws BlockedException if this processing the plan throws a currentVarContext - */ - public void process(ProcedurePlan procEnv) - throws BlockedException, MetaMatrixComponentException, MetaMatrixProcessingException { - - if(intoGroup != null && intoGroup.isTempGroupSymbol()){ - procEnv.executePlan(commandPlan, intoGroup.getName()); - }else{ - procEnv.executePlan(commandPlan, RS_NAME); - } - - // We don't close the result here because this ExecSQLInstruction may be the last one in the procedure, - // in which case the containing ProcedurePlan.close will manage the lifecycle of the last tuple source. - // Those tuple sources that are not the last will be closed by the ProcedureEnvironment.executePlan method before the next - // instruction is executed. - } - - public ProcessorPlan getProcessorPlan(){ //Defect 13291 - made public to support changes to ProcedurePlan - return this.commandPlan; - } - - /** - * Returns a deep clone - */ - public Object clone(){ - ExecSqlInstruction clone = new ExecSqlInstruction((ProcessorPlan)commandPlan.clone(), intoGroup); - return clone; - } - - public String toString() { - return "ExecSQLInstruction"; //$NON-NLS-1$ - } - - public Map getDescriptionProperties() { - Map props = new HashMap(); - props.put(PROP_TYPE, "SQL"); //$NON-NLS-1$ - props.put(PROP_SQL, RS_NAME); - props.put(PROP_PROGRAM, this.commandPlan.getDescriptionProperties()); - if(intoGroup != null) { - props.put(PROP_GROUP, intoGroup.toString()); - } - return props; - } - - /** - * @see com.metamatrix.query.processor.program.ProgramInstruction#getChildPlans() - * @since 4.2 - */ - public Collection getChildPlans() { - List plans = new ArrayList(1); - plans.add(this.commandPlan); - return plans; - } - -} Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/proc/ProcedurePlan.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/proc/ProcedurePlan.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/proc/ProcedurePlan.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -39,18 +39,18 @@ import com.metamatrix.api.exception.query.QueryValidatorException; import com.metamatrix.common.buffer.BlockedException; import com.metamatrix.common.buffer.BufferManager; +import com.metamatrix.common.buffer.IndexedTupleSource; import com.metamatrix.common.buffer.TupleBatch; import com.metamatrix.common.buffer.TupleSource; -import com.metamatrix.common.buffer.TupleSourceID; -import com.metamatrix.common.buffer.TupleSourceNotFoundException; import com.metamatrix.common.log.LogManager; import com.metamatrix.dqp.util.LogConstants; import com.metamatrix.query.execution.QueryExecPlugin; import com.metamatrix.query.metadata.QueryMetadataInterface; import com.metamatrix.query.metadata.SupportConstants; import com.metamatrix.query.processor.BaseProcessorPlan; +import com.metamatrix.query.processor.BatchIterator; +import com.metamatrix.query.processor.CollectionTupleSource; import com.metamatrix.query.processor.DescribableUtil; -import com.metamatrix.query.processor.NullTupleSource; import com.metamatrix.query.processor.ProcessorDataManager; import com.metamatrix.query.processor.ProcessorPlan; import com.metamatrix.query.processor.QueryProcessor; @@ -73,10 +73,10 @@ */ public class ProcedurePlan extends BaseProcessorPlan { - private class CursorState { - TupleSourceID tsID; - TupleSource ts; - List<?> currentRow; + public static class CursorState { + private QueryProcessor processor; + private IndexedTupleSource ts; + private List<?> currentRow; } private Program originalProgram; @@ -88,8 +88,7 @@ private int batchSize; private boolean done = false; - private QueryProcessor internalProcessor; - private TupleSourceID internalResultID; + private CursorState currentState; // Temp state for final results private TupleSource finalTupleSource; @@ -157,8 +156,7 @@ lastTupleSource = null; done = false; - internalProcessor = null; - internalResultID = null; + currentState = null; finalTupleSource = null; beginBatch = 1; @@ -295,7 +293,7 @@ } if(lastTupleSource == null){ - return new NullTupleSource(null); + return CollectionTupleSource.createNullTupleSource(null); } return lastTupleSource; } @@ -305,14 +303,9 @@ if (!this.cursorStates.isEmpty()) { List<String> cursors = new ArrayList<String>(this.cursorStates.keySet()); for (String rsName : cursors) { - try { - removeResults(rsName); - } catch (MetaMatrixComponentException e) { - LogManager.logDetail(LogConstants.CTX_DQP, e, "Exception closing procedure plan cursor"); //$NON-NLS-1$ - } + removeResults(rsName); } } - internalResultID = null; if(getTempTableStore()!=null) { getTempTableStore().removeTempTables(); } @@ -409,7 +402,7 @@ if(rowCount == null) { rowCount = new Integer(NO_ROWS_UPDATED); } - return new UpdateCountTupleSource((Integer)rowCount); + return CollectionTupleSource.createUpdateCountTupleSource((Integer)rowCount); } /** @@ -423,43 +416,35 @@ return this.currentVarContext; } - public void executePlan(ProcessorPlan command, String rsName) + public CursorState executePlan(ProcessorPlan command, String rsName) throws MetaMatrixComponentException, MetaMatrixProcessingException { - boolean isExecSQLInstruction = rsName.equals(ExecSqlInstruction.RS_NAME); - // Defect 14544: Close all non-final ExecSqlInstruction tuple sources before creating a new source. - // This guarantees that the tuple source will be removed predictably from the buffer manager. - if (isExecSQLInstruction) { - removeResults(ExecSqlInstruction.RS_NAME); - } - - if(this.internalProcessor == null){ - //this may not be the first time the plan is being run - command.reset(); + + CursorState state = this.cursorStates.get(rsName.toUpperCase()); + if (state == null) { + if (this.currentState == null) { + //this may not be the first time the plan is being run + command.reset(); + + CommandContext subContext = (CommandContext) getContext().clone(); + subContext.setVariableContext(this.currentVarContext); + subContext.setTempTableStore(getTempTableStore()); + state = new CursorState(); + state.processor = new QueryProcessor(command, subContext, this.bufferMgr, this.dataMgr); + state.ts = new BatchIterator(state.processor); + + //keep a reference to the tuple source + //it may be the last one + this.lastTupleSource = state.ts; + + this.currentState = state; + } + //force execution to the first batch + this.currentState.ts.hasNext(); - // Run query processor on command - CommandContext subContext = (CommandContext) getContext().clone(); - subContext.setVariableContext(this.currentVarContext); - subContext.setTempTableStore(getTempTableStore()); - internalProcessor = new QueryProcessor(command, subContext, this.bufferMgr, this.dataMgr); - this.internalResultID = this.internalProcessor.getResultsID(); + this.cursorStates.put(rsName.toUpperCase(), this.currentState); + this.currentState = null; } - this.internalProcessor.process(Integer.MAX_VALUE); //TODO: put a better value here - - // didn't throw processor blocked, so must be done - TupleSource source = this.bufferMgr.getTupleSource(this.internalResultID); - - this.internalProcessor = null; - - CursorState cursorState = new CursorState(); - cursorState.tsID = this.internalResultID; - cursorState.ts = source; - this.cursorStates.put(rsName.toUpperCase(), cursorState); - - if(isExecSQLInstruction){ - //keep a reference to the tuple source - //it may be the last one - this.lastTupleSource = source; - } + return state; } /** @@ -561,15 +546,8 @@ public void removeResults(String rsName) throws MetaMatrixComponentException { String rsKey = rsName.toUpperCase(); CursorState state = this.cursorStates.remove(rsKey); - if(state != null) { - try { - this.bufferMgr.removeTupleSource(state.tsID); - } catch (TupleSourceNotFoundException e) { - throw new MetaMatrixComponentException(e, ErrorMessageKeys.PROCESSOR_0021, QueryExecPlugin.Util.getString(ErrorMessageKeys.PROCESSOR_0021, (String)null)); - } catch (MetaMatrixComponentException e) { - throw new MetaMatrixComponentException(e, ErrorMessageKeys.PROCESSOR_0022, QueryExecPlugin.Util.getString(ErrorMessageKeys.PROCESSOR_0022, (String) null)); - } - LogManager.logTrace(LogConstants.CTX_DQP, new Object[]{"removed tuple source", state.tsID, "for result set"}); //$NON-NLS-1$ //$NON-NLS-2$ + if (state != null) { + state.processor.closeProcessing(); } } Deleted: trunk/engine/src/main/java/com/metamatrix/query/processor/proc/UpdateCountTupleSource.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/proc/UpdateCountTupleSource.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/proc/UpdateCountTupleSource.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -1,61 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.query.processor.proc; - -import java.util.ArrayList; -import java.util.List; - -import com.metamatrix.api.exception.MetaMatrixComponentException; -import com.metamatrix.common.buffer.TupleSource; -import com.metamatrix.query.sql.lang.Command; - -public class UpdateCountTupleSource implements - TupleSource { - - private boolean hasNextTuple = true; - private List nextTuple; - - public UpdateCountTupleSource(List tuple) { - nextTuple = tuple; - } - - public UpdateCountTupleSource(int count) { - nextTuple = new ArrayList(1); - nextTuple.add(new Integer(count)); - } - - public List getSchema() { - return Command.getUpdateCommandSymbol(); - } - - public List nextTuple() throws MetaMatrixComponentException { - if (hasNextTuple) { - hasNextTuple = false; - return nextTuple; - } - return null; - } - - public void closeSource() throws MetaMatrixComponentException { - } -} \ No newline at end of file Deleted: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/BatchCollector.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/BatchCollector.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/BatchCollector.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -1,107 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.query.processor.relational; - -import java.util.List; - -import com.metamatrix.api.exception.MetaMatrixComponentException; -import com.metamatrix.api.exception.MetaMatrixProcessingException; -import com.metamatrix.common.buffer.TupleBatch; -import com.metamatrix.common.buffer.TupleSourceID; -import com.metamatrix.common.buffer.TupleSourceNotFoundException; -import com.metamatrix.common.buffer.BufferManager.TupleSourceStatus; -import com.metamatrix.common.buffer.BufferManager.TupleSourceType; - -public class BatchCollector { - - private RelationalNode sourceNode; - - private boolean done = false; - private TupleSourceID tsID; - private int rowCount = 0; - private boolean collectedAny; - - public BatchCollector(RelationalNode sourceNode) throws MetaMatrixComponentException { - this.sourceNode = sourceNode; - List sourceElements = sourceNode.getElements(); - tsID = sourceNode.getBufferManager().createTupleSource(sourceElements, sourceNode.getConnectionID(), TupleSourceType.PROCESSOR); - } - - public TupleSourceID collectTuples() throws MetaMatrixComponentException, MetaMatrixProcessingException { - return collectTuples(null); - } - - public TupleSourceID collectTuples(TupleBatch batch) throws MetaMatrixComponentException, MetaMatrixProcessingException { - while(!done) { - if (batch == null) { - batch = sourceNode.nextBatch(); - } - - // Add batch - if(batch.getRowCount() > 0) { - this.rowCount = batch.getEndRow(); - sourceNode.getBufferManager().addTupleBatch(tsID, batch); - collectedAny = true; - } - - // Check for termination condition - batch ending with null row - if(batch.getTerminationFlag()) { - break; - } - - batch = null; - } - if (!done) { - sourceNode.getBufferManager().setStatus(tsID, TupleSourceStatus.FULL); - done = true; - } - return tsID; - } - - public boolean collectedAny() { - boolean result = collectedAny; - collectedAny = false; - return result; - } - - public int getRowCount() { - return rowCount; - } - - public void close() throws MetaMatrixComponentException { - try { - this.sourceNode.getBufferManager().removeTupleSource(tsID); - } catch (TupleSourceNotFoundException err) { - //ignore - } - } - - public TupleSourceID getTupleSourceID() { - return this.tsID; - } - - public boolean isDone() { - return done; - } - -} Deleted: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/BatchIterator.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/BatchIterator.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/BatchIterator.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -1,123 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.query.processor.relational; - -import java.util.List; - -import com.metamatrix.api.exception.MetaMatrixComponentException; -import com.metamatrix.api.exception.MetaMatrixProcessingException; -import com.metamatrix.common.buffer.IndexedTupleSource; -import com.metamatrix.common.buffer.TupleBatch; -import com.metamatrix.query.sql.symbol.SingleElementSymbol; - -final class BatchIterator implements - IndexedTupleSource { - - private final RelationalNode source; - - BatchIterator(RelationalNode source) { - this.source = source; - } - - private boolean done; - private int currentRow = 1; - private TupleBatch currentBatch; - private List currentTuple; - - public boolean hasNext() throws MetaMatrixComponentException, - MetaMatrixProcessingException { - if (done) { - return false; - } - while (currentTuple == null) { - if (currentBatch == null) { - currentBatch = this.source.nextBatch(); - } - - if (currentBatch.getEndRow() >= currentRow) { - this.currentTuple = currentBatch.getTuple(currentRow++); - } else { - done = currentBatch.getTerminationFlag(); - currentBatch = null; - if (done) { - return false; - } - } - } - return true; - } - - @Override - public void closeSource() throws MetaMatrixComponentException { - - } - - @Override - public List<SingleElementSymbol> getSchema() { - return source.getElements(); - } - - @Override - public List<?> nextTuple() throws MetaMatrixComponentException, - MetaMatrixProcessingException { - if (currentTuple == null && !hasNext()) { - return null; - } - List result = currentTuple; - currentTuple = null; - return result; - } - - public void reset() { - throw new UnsupportedOperationException(); - } - - public void mark() { - //does nothing - } - - @Override - public int getCurrentIndex() { - return currentRow; - } - - public void setPosition(int position) { - if (position == this.currentRow) { - return; - } - if (position < this.currentRow && (this.currentBatch == null || position < this.currentBatch.getBeginRow())) { - throw new UnsupportedOperationException("Backwards positioning is not allowed"); //$NON-NLS-1$ - } - this.currentRow = position; - this.currentTuple = null; - if (currentBatch.getEndRow() < currentRow) { - this.currentBatch = null; - } - } - - @Override - public TupleBatch getBatch() throws MetaMatrixComponentException { - return currentBatch; - } - -} Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentCriteriaProcessor.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentCriteriaProcessor.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentCriteriaProcessor.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -34,9 +34,10 @@ import java.util.Map; import com.metamatrix.api.exception.MetaMatrixComponentException; +import com.metamatrix.api.exception.MetaMatrixProcessingException; import com.metamatrix.api.exception.query.ExpressionEvaluationException; import com.metamatrix.common.buffer.BlockedException; -import com.metamatrix.common.buffer.TupleSourceNotFoundException; +import com.metamatrix.query.processor.relational.SortUtility.Mode; import com.metamatrix.query.rewriter.QueryRewriter; import com.metamatrix.query.sql.lang.AbstractSetCriteria; import com.metamatrix.query.sql.lang.CollectionValueIterator; @@ -77,7 +78,7 @@ } public void sort() throws BlockedException, - MetaMatrixComponentException { + MetaMatrixComponentException, MetaMatrixProcessingException { if (dvs == null) { if (sortUtility == null) { List<Expression> sortSymbols = new ArrayList<Expression>(dependentSetStates.size()); @@ -87,9 +88,9 @@ sortSymbols.add(dependentSetStates.get(i).valueExpression); } DependentValueSource originalVs = (DependentValueSource)dependentNode.getContext().getVariableContext().getGlobalValue(valueSource); - this.sortUtility = new SortUtility(originalVs.getTupleSourceID(), sortSymbols, sortDirection, true, dependentNode.getBufferManager(), dependentNode.getConnectionID()); + this.sortUtility = new SortUtility(originalVs.getTupleBuffer().createIndexedTupleSource(), sortSymbols, sortDirection, Mode.DUP_REMOVE, dependentNode.getBufferManager(), dependentNode.getConnectionID()); } - dvs = new DependentValueSource(sortUtility.sort(), dependentNode.getBufferManager()); + dvs = new DependentValueSource(sortUtility.sort(), dependentNode.getBufferManager().getProcessorBatchSize() / 2); for (SetState setState : dependentSetStates) { setState.valueIterator = dvs.getValueIterator(setState.valueExpression); } @@ -99,10 +100,7 @@ public void close() throws MetaMatrixComponentException { if (dvs != null) { sortUtility = null; - try { - dependentNode.getBufferManager().removeTupleSource(dvs.getTupleSourceID()); - } catch (TupleSourceNotFoundException e) { - } + dvs.getTupleBuffer().remove(); dvs = null; } } @@ -189,7 +187,7 @@ } } - public Criteria prepareCriteria() throws MetaMatrixComponentException { + public Criteria prepareCriteria() throws MetaMatrixComponentException, MetaMatrixProcessingException { if (phase == SORT) { for (TupleState state : dependentState.values()) { Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentProcedureExecutionNode.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentProcedureExecutionNode.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentProcedureExecutionNode.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -89,7 +89,6 @@ } /** - * @throws TupleSourceNotFoundException * @see com.metamatrix.query.processor.relational.PlanExecutionNode#prepareNextCommand() */ protected boolean prepareNextCommand() throws BlockedException, Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentValueSource.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentValueSource.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentValueSource.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -25,15 +25,17 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.Set; +import java.util.TreeSet; import com.metamatrix.api.exception.MetaMatrixComponentException; import com.metamatrix.api.exception.MetaMatrixProcessingException; -import com.metamatrix.common.buffer.BufferManager; import com.metamatrix.common.buffer.IndexedTupleSource; -import com.metamatrix.common.buffer.TupleSourceID; -import com.metamatrix.common.buffer.TupleSourceNotFoundException; +import com.metamatrix.common.buffer.TupleBuffer; +import com.metamatrix.common.types.DataTypeManager; import com.metamatrix.core.util.Assertion; import com.metamatrix.query.sql.symbol.Expression; +import com.metamatrix.query.sql.symbol.SingleElementSymbol; import com.metamatrix.query.sql.util.ValueIterator; import com.metamatrix.query.sql.util.ValueIteratorSource; @@ -43,31 +45,25 @@ public class DependentValueSource implements ValueIteratorSource { - private TupleSourceID tupleSourceID; - private BufferManager bm; - private Map<Expression, HashSet<Object>> cachedSets; + private TupleBuffer tupleSourceID; + private int maxSetSize; + private Map<Expression, Set<Object>> cachedSets; - public DependentValueSource(TupleSourceID tupleSourceID, BufferManager bm) { + public DependentValueSource(TupleBuffer tupleSourceID, int maxSetSize) { this.tupleSourceID = tupleSourceID; - this.bm = bm; + this.maxSetSize = maxSetSize; } - public TupleSourceID getTupleSourceID() { + public TupleBuffer getTupleBuffer() { return tupleSourceID; } /** * @throws MetaMatrixComponentException - * @throws TupleSourceNotFoundException * @see com.metamatrix.query.sql.util.ValueIteratorSource#getValueIterator(com.metamatrix.query.sql.symbol.Expression) */ public ValueIterator getValueIterator(Expression valueExpression) throws MetaMatrixComponentException { - IndexedTupleSource its; - try { - its = bm.getTupleSource(tupleSourceID); - } catch (TupleSourceNotFoundException e) { - throw new MetaMatrixComponentException(e); - } + IndexedTupleSource its = tupleSourceID.createIndexedTupleSource(); int index = 0; if (valueExpression != null) { index = its.getSchema().indexOf(valueExpression); @@ -76,27 +72,26 @@ return new TupleSourceValueIterator(its, index); } - public HashSet<Object> getCachedSet(Expression valueExpression) throws MetaMatrixComponentException, MetaMatrixProcessingException { - HashSet<Object> result = null; + public Set<Object> getCachedSet(Expression valueExpression) throws MetaMatrixComponentException, MetaMatrixProcessingException { + Set<Object> result = null; if (cachedSets != null) { result = cachedSets.get(valueExpression); } if (result == null) { - IndexedTupleSource its; - try { - if (bm.getRowCount(tupleSourceID) > bm.getProcessorBatchSize() / 2) { - return null; - } - its = bm.getTupleSource(tupleSourceID); - } catch (TupleSourceNotFoundException e) { - throw new MetaMatrixComponentException(e); - } + if (tupleSourceID.getRowCount() > maxSetSize) { + return null; + } + IndexedTupleSource its = tupleSourceID.createIndexedTupleSource(); int index = 0; if (valueExpression != null) { index = its.getSchema().indexOf(valueExpression); } Assertion.assertTrue(index != -1); - result = new HashSet<Object>(); + if (((SingleElementSymbol)its.getSchema().get(index)).getType() == DataTypeManager.DefaultDataClasses.BIG_DECIMAL) { + result = new TreeSet<Object>(); + } else { + result = new HashSet<Object>(); + } while (its.hasNext()) { Object value = its.nextTuple().get(index); if (value != null) { @@ -105,7 +100,7 @@ } its.closeSource(); if (cachedSets == null) { - cachedSets = new HashMap<Expression, HashSet<Object>>(); + cachedSets = new HashMap<Expression, Set<Object>>(); } cachedSets.put(valueExpression, result); } Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DuplicateFilter.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DuplicateFilter.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DuplicateFilter.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -30,11 +30,11 @@ import com.metamatrix.api.exception.query.ExpressionEvaluationException; import com.metamatrix.api.exception.query.FunctionExecutionException; import com.metamatrix.common.buffer.BufferManager; +import com.metamatrix.common.buffer.TupleBuffer; import com.metamatrix.common.buffer.TupleSource; -import com.metamatrix.common.buffer.TupleSourceID; -import com.metamatrix.common.buffer.TupleSourceNotFoundException; import com.metamatrix.common.buffer.BufferManager.TupleSourceType; import com.metamatrix.query.function.aggregate.AggregateFunction; +import com.metamatrix.query.processor.relational.SortUtility.Mode; import com.metamatrix.query.sql.lang.OrderBy; import com.metamatrix.query.sql.symbol.ElementSymbol; @@ -52,15 +52,13 @@ private List sortTypes; // Temporary state - should be reset - private TupleSourceID collectionID = null; - private SortUtility sortUtility = null; - private TupleSourceID sortedID = null; - private TupleCollector tupleCollector; + private TupleBuffer collectionBuffer; + private SortUtility sortUtility; /** * Constructor for DuplicateFilter. */ - public DuplicateFilter(AggregateFunction proxy, BufferManager mgr, String groupName, int batchSize) { + public DuplicateFilter(AggregateFunction proxy, BufferManager mgr, String groupName) { super(); this.proxy = proxy; @@ -89,12 +87,16 @@ public void reset() { this.proxy.reset(); + close(); + } - this.collectionID = null; - this.tupleCollector = null; + private void close() { + if (this.collectionBuffer != null) { + collectionBuffer.remove(); + } + this.collectionBuffer = null; this.sortUtility = null; - this.sortedID = null; - } + } /** * @see com.metamatrix.query.function.aggregate.AggregateFunction#addInput(Object) @@ -102,18 +104,13 @@ public void addInput(Object input) throws FunctionExecutionException, ExpressionEvaluationException, MetaMatrixComponentException { - try { - if(collectionID == null) { - collectionID = mgr.createTupleSource(elements, groupName, TupleSourceType.PROCESSOR); - this.tupleCollector = new TupleCollector(collectionID, mgr); - } - - List row = new ArrayList(1); - row.add(input); - this.tupleCollector.addTuple(row); - } catch(TupleSourceNotFoundException e) { - throw new MetaMatrixComponentException(e, e.getMessage()); + if(collectionBuffer == null) { + collectionBuffer = mgr.createTupleBuffer(elements, groupName, TupleSourceType.PROCESSOR); } + + List row = new ArrayList(1); + row.add(input); + this.collectionBuffer.addTuple(row); } /** @@ -123,32 +120,27 @@ public Object getResult() throws MetaMatrixComponentException, MetaMatrixProcessingException { - try { - if(collectionID != null) { - this.tupleCollector.close(); + if(collectionBuffer != null) { + this.collectionBuffer.close(); - // Sort - sortUtility = new SortUtility(collectionID, elements, sortTypes, true, mgr, groupName); - this.sortedID = sortUtility.sort(); - - // Add all input to proxy - TupleSource sortedSource = mgr.getTupleSource(sortedID); - while(true) { - List tuple = sortedSource.nextTuple(); - if(tuple == null) { - break; - } - this.proxy.addInput(tuple.get(0)); - } - sortedSource.closeSource(); - - // Clean up - mgr.removeTupleSource(collectionID); - mgr.removeTupleSource(sortedID); + // Sort + sortUtility = new SortUtility(collectionBuffer.createIndexedTupleSource(), elements, sortTypes, Mode.DUP_REMOVE, mgr, groupName); + TupleBuffer sorted = sortUtility.sort(); + try { + // Add all input to proxy + TupleSource sortedSource = sorted.createIndexedTupleSource(); + while(true) { + List tuple = sortedSource.nextTuple(); + if(tuple == null) { + break; + } + this.proxy.addInput(tuple.get(0)); + } + } finally { + sorted.remove(); } - - } catch(TupleSourceNotFoundException e) { - throw new MetaMatrixComponentException(e, e.getMessage()); + + close(); } // Return Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/GroupingNode.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/GroupingNode.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/GroupingNode.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -32,12 +32,9 @@ import com.metamatrix.api.exception.MetaMatrixProcessingException; import com.metamatrix.api.exception.query.ExpressionEvaluationException; import com.metamatrix.common.buffer.BlockedException; -import com.metamatrix.common.buffer.IndexedTupleSource; import com.metamatrix.common.buffer.TupleBatch; -import com.metamatrix.common.buffer.TupleSourceID; -import com.metamatrix.common.buffer.TupleSourceNotFoundException; -import com.metamatrix.common.buffer.BufferManager.TupleSourceStatus; -import com.metamatrix.common.buffer.BufferManager.TupleSourceType; +import com.metamatrix.common.buffer.TupleBuffer; +import com.metamatrix.common.buffer.TupleSource; import com.metamatrix.query.eval.Evaluator; import com.metamatrix.query.function.aggregate.AggregateFunction; import com.metamatrix.query.function.aggregate.Avg; @@ -65,21 +62,15 @@ private int phase = COLLECTION; private Map elementMap; // Map of incoming symbol to index in source elements private List collectedExpressions; // Collected Expressions - private TupleBatch sourceBatch; // Current batch loaded from the source, if blocked - private int sourceRow; // Current row index in batch, if blocked - private List[] collectedRows; // Rows evaluated and collected - private TupleSourceID collectionID; // Tuple source collecting the endpoint of collection phase - private int rowCount; // Row count of incoming rows // Sort phase private SortUtility sortUtility; - private TupleSourceID sortedID; + private TupleBuffer sortedID; + private TupleSource groupTupleSource; // Group phase private Map expressionMap; // Index map for all collected expressions (Expression -> index in collectedExpressions) private AggregateFunction[] functions; - private IndexedTupleSource groupTupleSource; - private int groupBegin = 1; private List lastRow; private List currentGroupTuple; @@ -97,19 +88,12 @@ phase = COLLECTION; elementMap = null; collectedExpressions = null; - sourceBatch = null; - sourceRow = -1; - collectedRows = null; - collectionID = null; - rowCount = 0; sortUtility = null; sortedID = null; expressionMap = null; functions = null; - groupBegin = 1; - groupTupleSource = null; lastRow = null; currentGroupTuple = null; } @@ -147,8 +131,6 @@ // Determine expressions to build (all grouping expressions + expressions used by aggregates) collectExpressions(); - this.collectionID = getBufferManager().createTupleSource(collectedExpressions, getConnectionID(), TupleSourceType.PROCESSOR); - initializeFunctionAccumulators(); } @@ -213,7 +195,7 @@ } if(aggSymbol.isDistinct()) { - functions[i] = new DuplicateFilter(functions[i], getBufferManager(), getConnectionID(), getBatchSize()); + functions[i] = new DuplicateFilter(functions[i], getBufferManager(), getConnectionID()); } functions[i] = new NullFilter(functions[i]); @@ -233,107 +215,108 @@ public TupleBatch nextBatchDirect() throws BlockedException, MetaMatrixComponentException, MetaMatrixProcessingException { - try { - // Take inputs, evaluate expressions, and build initial tuple source - if(this.phase == COLLECTION) { - collectionPhase(); - } + // Take inputs, evaluate expressions, and build initial tuple source + if(this.phase == COLLECTION) { + collectionPhase(); + } - // If necessary, sort to determine groups (if no group cols, no need to sort) - if(this.phase == SORT) { - sortPhase(); - } - - // Walk through the sorted results and for each group, emit a row - if(this.phase == GROUP) { - return groupPhase(); - } - - TupleBatch terminationBatch = new TupleBatch(1, Collections.EMPTY_LIST); - terminationBatch.setTerminationFlag(true); - return terminationBatch; - } catch(TupleSourceNotFoundException e) { - throw new MetaMatrixComponentException(e, e.getMessage()); + // If necessary, sort to determine groups (if no group cols, no need to sort) + if(this.phase == SORT) { + sortPhase(); } + // Walk through the sorted results and for each group, emit a row + if(this.phase == GROUP) { + return groupPhase(); + } + + TupleBatch terminationBatch = new TupleBatch(1, Collections.EMPTY_LIST); + terminationBatch.setTerminationFlag(true); + return terminationBatch; } + + public TupleSource getCollectionTupleSource() { + + final RelationalNode sourceNode = this.getChildren()[0]; + + return new TupleSource() { + private TupleBatch sourceBatch; // Current batch loaded from the source, if blocked + private int sourceRow = 1; - private void collectionPhase() throws BlockedException, TupleSourceNotFoundException, MetaMatrixComponentException, MetaMatrixProcessingException { - RelationalNode sourceNode = this.getChildren()[0]; + @Override + public List<?> nextTuple() throws MetaMatrixComponentException, + MetaMatrixProcessingException { + while (true) { + if(sourceBatch == null) { + // Read next batch + sourceBatch = sourceNode.nextBatch(); + } + + if(sourceBatch.getRowCount() > 0 && sourceRow <= sourceBatch.getEndRow()) { + // Evaluate expressions needed for grouping + List tuple = sourceBatch.getTuple(sourceRow); + + int columns = collectedExpressions.size(); + List exprTuple = new ArrayList(columns); + for(int col = 0; col<columns; col++) { + // The following call may throw BlockedException, but all state to this point + // is saved in class variables so we can start over on building this tuple + Object value = new Evaluator(elementMap, getDataManager(), getContext()).evaluate((Expression) collectedExpressions.get(col), tuple); + exprTuple.add(value); + } + sourceRow++; + return exprTuple; + } + + // Check for termination condition + if(sourceBatch.getTerminationFlag()) { + sourceBatch = null; + return null; + } + sourceBatch = null; + } + } + + @Override + public List<SingleElementSymbol> getSchema() { + return collectedExpressions; + } + + @Override + public void closeSource() throws MetaMatrixComponentException { + + } + }; + + } - while(true) { - if(this.sourceBatch == null) { - // Read next batch - this.sourceBatch = sourceNode.nextBatch(); - } - - if(this.sourceBatch.getRowCount() > 0) { - this.rowCount = this.sourceBatch.getEndRow(); - this.sourceRow = this.sourceBatch.getBeginRow(); - this.collectedRows = new List[this.sourceBatch.getRowCount()]; - // Evaluate expressions needed for grouping - for(int row=this.sourceRow; row<=this.sourceBatch.getEndRow(); row++) { - List tuple = this.sourceBatch.getTuple(row); - - int columns = this.collectedExpressions.size(); - List exprTuple = new ArrayList(columns); - for(int col = 0; col<columns; col++) { - // The following call may throw BlockedException, but all state to this point - // is saved in class variables so we can start over on building this tuple - Object value = new Evaluator(this.elementMap, getDataManager(), getContext()).evaluate((Expression) this.collectedExpressions.get(col), tuple); - exprTuple.add(value); - } - - collectedRows[row-this.sourceBatch.getBeginRow()] = exprTuple; - } - - // Add collected batch - TupleBatch exprBatch = new TupleBatch(this.sourceBatch.getBeginRow(), collectedRows); - getBufferManager().addTupleBatch(collectionID, exprBatch); - } - // Clear and setup for next loop - this.sourceRow = -1; - this.collectedRows = null; - - // Check for termination condition - if(this.sourceBatch.getTerminationFlag()) { - this.sourceBatch = null; - break; - } - - this.sourceBatch = null; - } - - this.getBufferManager().setStatus(this.collectionID, TupleSourceStatus.FULL); - - if(this.sortElements == null || this.rowCount == 0) { + private void collectionPhase() { + if(this.sortElements == null) { // No need to sort - this.sortedID = this.collectionID; - this.collectionID = null; + this.groupTupleSource = getCollectionTupleSource(); this.phase = GROUP; } else { - this.sortUtility = new SortUtility(collectionID, sortElements, + this.sortUtility = new SortUtility(getCollectionTupleSource(), sortElements, sortTypes, removeDuplicates?Mode.DUP_REMOVE_SORT:Mode.SORT, getBufferManager(), - getConnectionID(), removeDuplicates); + getConnectionID()); this.phase = SORT; } } private void sortPhase() throws BlockedException, MetaMatrixComponentException, MetaMatrixProcessingException { this.sortedID = this.sortUtility.sort(); - this.rowCount = this.getBufferManager().getFinalRowCount(this.sortedID); + this.groupTupleSource = this.sortedID.createIndexedTupleSource(); this.phase = GROUP; } - private TupleBatch groupPhase() throws BlockedException, TupleSourceNotFoundException, MetaMatrixComponentException, MetaMatrixProcessingException { - if (this.groupTupleSource == null) { - this.groupTupleSource = this.getBufferManager().getTupleSource(sortedID); - } - - while(groupBegin <= rowCount) { + private TupleBatch groupPhase() throws BlockedException, MetaMatrixComponentException, MetaMatrixProcessingException { + while(true) { if (currentGroupTuple == null) { currentGroupTuple = this.groupTupleSource.nextTuple(); + if (currentGroupTuple == null) { + break; + } } if(lastRow == null) { @@ -365,10 +348,8 @@ // Update function accumulators with new row - can throw blocked exception updateAggregates(currentGroupTuple); currentGroupTuple = null; - groupBegin++; } - this.groupTupleSource.closeSource(); - if(rowCount != 0 || sortElements == null) { + if(lastRow != null || sortElements == null) { // Close last group List row = new ArrayList(functions.length); for(int i=0; i<functions.length; i++) { @@ -431,16 +412,6 @@ public void close() throws MetaMatrixComponentException { if (!isClosed()) { super.close(); - try { - if(this.collectionID != null) { - getBufferManager().removeTupleSource(collectionID); - } - if(this.sortedID != null) { - getBufferManager().removeTupleSource(sortedID); - } - } catch(TupleSourceNotFoundException e) { - throw new MetaMatrixComponentException(e, e.getMessage()); - } } } Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/JoinNode.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/JoinNode.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/JoinNode.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -35,8 +35,7 @@ import com.metamatrix.common.buffer.BlockedException; import com.metamatrix.common.buffer.BufferManager; import com.metamatrix.common.buffer.TupleBatch; -import com.metamatrix.common.buffer.TupleSourceID; -import com.metamatrix.common.buffer.TupleSourceNotFoundException; +import com.metamatrix.common.buffer.TupleBuffer; import com.metamatrix.query.processor.ProcessorDataManager; import com.metamatrix.query.sql.LanguageObject; import com.metamatrix.query.sql.lang.Criteria; @@ -54,7 +53,7 @@ NESTED_LOOP } - private enum State { LOAD_LEFT, LOAD_RIGHT, POST_LOAD_LEFT, POST_LOAD_RIGHT, EXECUTE } + private enum State { LOAD_LEFT, LOAD_RIGHT, EXECUTE } private State state = State.LOAD_LEFT; private boolean leftOpened; @@ -191,23 +190,20 @@ } if (state == State.LOAD_RIGHT) { if (isDependent() && !this.rightOpened) { - TupleSourceID tsID = this.joinStrategy.leftSource.getTupleSourceID(); - this.getContext().getVariableContext().setGlobalValue(this.dependentValueSource, new DependentValueSource(tsID, this.getBufferManager())); + TupleBuffer tsID = this.joinStrategy.leftSource.getTupleBuffer(); + this.getContext().getVariableContext().setGlobalValue(this.dependentValueSource, new DependentValueSource(tsID, this.getBufferManager().getProcessorBatchSize() / 2)); //open the right side now that the tuples have been collected this.getChildren()[1].open(); this.rightOpened = true; } this.joinStrategy.loadRight(); - state = State.POST_LOAD_LEFT; + //force buffering based upon join type - may have already happened in the strategy load methods + this.joinStrategy.rightSource.getTupleBuffer(); + if (joinType == JoinType.JOIN_FULL_OUTER) { + this.joinStrategy.leftSource.getTupleBuffer(); + } + state = State.EXECUTE; } - if (state == State.POST_LOAD_LEFT) { - this.joinStrategy.postLoadLeft(); - state = State.POST_LOAD_RIGHT; - } - if (state == State.POST_LOAD_RIGHT) { - this.joinStrategy.postLoadRight(); - state = State.EXECUTE; - } while(true) { if(super.isBatchFull()) { @@ -299,12 +295,7 @@ public void close() throws MetaMatrixComponentException { super.close(); - - try { - joinStrategy.close(); - } catch (TupleSourceNotFoundException err) { - //ignore - } + joinStrategy.close(); if (this.isDependent()) { this.getContext().getVariableContext().setGlobalValue(this.dependentValueSource, null); } Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/JoinStrategy.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/JoinStrategy.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/JoinStrategy.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -28,7 +28,6 @@ import com.metamatrix.api.exception.MetaMatrixComponentException; import com.metamatrix.api.exception.MetaMatrixProcessingException; import com.metamatrix.api.exception.query.CriteriaEvaluationException; -import com.metamatrix.common.buffer.TupleSourceNotFoundException; public abstract class JoinStrategy { @@ -36,7 +35,7 @@ protected SourceState leftSource; protected SourceState rightSource; - public void close() throws TupleSourceNotFoundException, MetaMatrixComponentException { + public void close() { try { if (leftSource != null) { leftSource.close(); @@ -53,8 +52,7 @@ } } - public void initialize(JoinNode joinNode) - throws MetaMatrixComponentException { + public void initialize(JoinNode joinNode) { this.joinNode = joinNode; this.leftSource = new SourceState(joinNode.getChildren()[0], joinNode.getLeftExpressions()); this.leftSource.markDistinct(this.joinNode.isLeftDistinct()); @@ -63,19 +61,11 @@ } protected void loadLeft() throws MetaMatrixComponentException, MetaMatrixProcessingException { - this.leftSource.collectTuples(); } - protected void postLoadLeft() throws MetaMatrixComponentException, MetaMatrixProcessingException { - } - protected void loadRight() throws MetaMatrixComponentException, MetaMatrixProcessingException { - this.rightSource.collectTuples(); } - protected void postLoadRight() throws MetaMatrixComponentException, MetaMatrixProcessingException { - } - /** * Output a combined, projected tuple based on tuple parts from the left and right. * @param leftTuple Left tuple part Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/MergeJoinStrategy.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/MergeJoinStrategy.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/MergeJoinStrategy.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -28,8 +28,6 @@ import com.metamatrix.api.exception.MetaMatrixComponentException; import com.metamatrix.api.exception.MetaMatrixProcessingException; import com.metamatrix.api.exception.query.CriteriaEvaluationException; -import com.metamatrix.common.buffer.BlockedOnMemoryException; -import com.metamatrix.common.buffer.TupleSourceNotFoundException; import com.metamatrix.query.processor.relational.SortUtility.Mode; import com.metamatrix.query.sql.lang.JoinType; import com.metamatrix.query.sql.lang.OrderBy; @@ -116,7 +114,7 @@ * @see com.metamatrix.query.processor.relational.JoinStrategy#initialize(com.metamatrix.query.processor.relational.JoinNode) */ @Override - public void initialize(JoinNode joinNode) throws MetaMatrixComponentException { + public void initialize(JoinNode joinNode) { super.initialize(joinNode); this.outerState = this.leftSource; this.innerState = this.rightSource; @@ -134,8 +132,7 @@ * @see com.metamatrix.query.processor.relational.JoinStrategy#close() */ @Override - public void close() throws TupleSourceNotFoundException, - MetaMatrixComponentException { + public void close() { super.close(); this.outerState = null; this.innerState = null; @@ -331,27 +328,15 @@ return 0; } - /** - * @see com.metamatrix.query.processor.relational.JoinStrategy#loadLeft() - */ @Override protected void loadLeft() throws MetaMatrixComponentException, - MetaMatrixProcessingException { - if (sortLeft == SortOption.ALREADY_SORTED && !this.joinNode.isDependent() && !JoinType.JOIN_FULL_OUTER.equals(joinNode.getJoinType())) { - return; // don't buffer - } - super.loadLeft(); - } - - @Override - protected void postLoadLeft() throws MetaMatrixComponentException, MetaMatrixProcessingException { if (this.processingSortLeft == SortOption.SORT || this.processingSortLeft == SortOption.SORT_DISTINCT) { if (this.leftSort == null) { List expressions = this.joinNode.getLeftExpressions(); - this.leftSort = new SortUtility(this.leftSource.getTupleSourceID(), + this.leftSort = new SortUtility(this.leftSource.getIterator(), expressions, Collections.nCopies(expressions.size(), OrderBy.ASC), processingSortLeft == SortOption.SORT_DISTINCT?Mode.DUP_REMOVE_SORT:Mode.SORT, - this.joinNode.getBufferManager(), this.joinNode.getConnectionID(), true); + this.joinNode.getBufferManager(), this.joinNode.getConnectionID()); this.leftSource.markDistinct(processingSortLeft == SortOption.SORT_DISTINCT && expressions.size() == this.leftSource.getOuterVals().size()); } this.leftSource.setTupleSource(leftSort.sort()); @@ -360,19 +345,14 @@ } @Override - protected void postLoadRight() throws MetaMatrixComponentException, + protected void loadRight() throws MetaMatrixComponentException, MetaMatrixProcessingException { - sortRight(); - } - - protected void sortRight() throws MetaMatrixComponentException, - TupleSourceNotFoundException, BlockedOnMemoryException { if (this.processingSortRight == SortOption.SORT || this.processingSortRight == SortOption.SORT_DISTINCT) { if (this.rightSort == null) { List expressions = this.joinNode.getRightExpressions(); - this.rightSort = new SortUtility(this.rightSource.getTupleSourceID(), + this.rightSort = new SortUtility(this.rightSource.getIterator(), expressions, Collections.nCopies(expressions.size(), OrderBy.ASC), processingSortRight == SortOption.SORT_DISTINCT?Mode.DUP_REMOVE_SORT:Mode.SORT, - this.joinNode.getBufferManager(), this.joinNode.getConnectionID(), true); + this.joinNode.getBufferManager(), this.joinNode.getConnectionID()); this.rightSource.markDistinct(processingSortRight == SortOption.SORT_DISTINCT && expressions.size() == this.rightSource.getOuterVals().size()); } this.rightSource.setTupleSource(rightSort.sort()); Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/NestedLoopJoinStrategy.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/NestedLoopJoinStrategy.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/NestedLoopJoinStrategy.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -24,6 +24,9 @@ import java.util.List; +import com.metamatrix.api.exception.MetaMatrixComponentException; +import com.metamatrix.api.exception.MetaMatrixProcessingException; + /** * Nested loop is currently implemented as a degenerate case of merge join. * Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/PartitionedSortJoin.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/PartitionedSortJoin.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/PartitionedSortJoin.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -29,12 +29,9 @@ import com.metamatrix.api.exception.MetaMatrixComponentException; import com.metamatrix.api.exception.MetaMatrixProcessingException; import com.metamatrix.api.exception.query.CriteriaEvaluationException; -import com.metamatrix.common.buffer.BlockedOnMemoryException; import com.metamatrix.common.buffer.IndexedTupleSource; -import com.metamatrix.common.buffer.MemoryNotAvailableException; import com.metamatrix.common.buffer.TupleBatch; -import com.metamatrix.common.buffer.TupleSourceID; -import com.metamatrix.common.buffer.TupleSourceNotFoundException; +import com.metamatrix.common.buffer.TupleBuffer; /** * Extends the basic fully sorted merge join to check for conditions necessary @@ -59,8 +56,7 @@ private List[] endTuples; private List<Boolean> overlap = new ArrayList<Boolean>(); private List<Integer> endRows = new ArrayList<Integer>(); - private List<TupleSourceID> partitionIds = new ArrayList<TupleSourceID>(); - private List<TupleCollector> tupleCollectors = new ArrayList<TupleCollector>(); + private List<TupleBuffer> partitionIds = new ArrayList<TupleBuffer>(); private int currentPartition; private IndexedTupleSource currentSource; private SourceState sortedSource; @@ -75,21 +71,15 @@ } @Override - public void close() throws TupleSourceNotFoundException, - MetaMatrixComponentException { + public void close() { super.close(); - for (TupleSourceID tupleSourceID : this.partitionIds) { - try { - this.joinNode.getBufferManager().removeTupleSource(tupleSourceID); - } catch (TupleSourceNotFoundException e) { - - } + for (TupleBuffer tupleSourceID : this.partitionIds) { + tupleSourceID.remove(); } this.endTuples = null; this.overlap.clear(); this.endRows.clear(); this.partitionIds.clear(); - this.tupleCollectors.clear(); this.currentSource = null; this.sortedSource = null; this.partitionedSource = null; @@ -97,8 +87,7 @@ } @Override - public void initialize(JoinNode joinNode) - throws MetaMatrixComponentException { + public void initialize(JoinNode joinNode) { super.initialize(joinNode); this.currentPartition = 0; this.partitioned = false; @@ -106,8 +95,7 @@ this.matchEnd = -1; } - //TODO: save partial work or combine with the sort operation - public void computeBatchBounds(SourceState state) throws TupleSourceNotFoundException, MetaMatrixComponentException { + public void computeBatchBounds(SourceState state) throws MetaMatrixComponentException, MetaMatrixProcessingException { if (endTuples != null) { return; } @@ -115,22 +103,16 @@ ArrayList<List<?>> bounds = new ArrayList<List<?>>(); int beginRow = 1; while (beginRow <= state.getRowCount()) { - TupleBatch batch = null; - try { - batch = this.joinNode.getBufferManager().pinTupleBatch(state.getTupleSourceID(), beginRow); - if (batch.getRowCount() == 0) { - break; - } - beginRow = batch.getEndRow() + 1; - this.joinNode.getBufferManager().unpinTupleBatch(state.getTupleSourceID(), batch.getBeginRow()); - if (!bounds.isEmpty()) { - overlap.add(comp.compare(bounds.get(bounds.size() - 1), batch.getTuple(batch.getBeginRow())) == 0); - } - bounds.add(batch.getTuple(batch.getEndRow())); - endRows.add(batch.getEndRow()); - } catch (MemoryNotAvailableException e) { - throw BlockedOnMemoryException.INSTANCE; - } + TupleBatch batch = state.getTupleBuffer().getBatch(beginRow); + if (batch.getRowCount() == 0) { + break; + } + beginRow = batch.getEndRow() + 1; + if (!bounds.isEmpty()) { + overlap.add(comp.compare(bounds.get(bounds.size() - 1), batch.getTuple(batch.getBeginRow())) == 0); + } + bounds.add(batch.getTuple(batch.getEndRow())); + endRows.add(batch.getEndRow()); } this.endTuples = bounds.toArray(new List[bounds.size()]); } @@ -139,13 +121,13 @@ protected void loadLeft() throws MetaMatrixComponentException, MetaMatrixProcessingException { //always buffer to determine row counts - this.leftSource.collectTuples(); + this.leftSource.getTupleBuffer(); } @Override protected void loadRight() throws MetaMatrixComponentException, MetaMatrixProcessingException { - super.loadRight(); + this.rightSource.getTupleBuffer(); int maxRows = this.joinNode.getBatchSize() * MAX_PARTITIONS; if (processingSortRight == SortOption.SORT && this.leftSource.getRowCount() < maxRows @@ -156,42 +138,30 @@ && this.rightSource.getRowCount() * 4 < this.leftSource.getRowCount()) { this.processingSortLeft = SortOption.PARTITION; } - sortRight(); + super.loadRight(); //sort right if needed if (this.processingSortLeft == SortOption.PARTITION) { computeBatchBounds(this.rightSource); this.sortedSource = this.rightSource; this.partitionedSource = this.leftSource; } - } - - @Override - protected void postLoadLeft() throws MetaMatrixComponentException, - MetaMatrixProcessingException { - super.postLoadLeft(); - + super.loadLeft(); //sort left if needed if (this.processingSortRight == SortOption.PARTITION) { computeBatchBounds(this.leftSource); this.sortedSource = this.leftSource; this.partitionedSource = this.rightSource; } - if (this.processingSortLeft == SortOption.PARTITION) { partitionSource(true); - } - } - - @Override - protected void postLoadRight() throws MetaMatrixComponentException, - MetaMatrixProcessingException { - if (this.processingSortRight == SortOption.PARTITION) { + } + if (this.processingSortRight == SortOption.PARTITION) { partitionSource(false); if (this.processingSortRight == SortOption.SORT) { //degrade to a merge join - sortRight(); + super.loadRight(); } - } + } } - + private void partitionSource(boolean left) throws MetaMatrixComponentException, MetaMatrixProcessingException { if (partitioned) { @@ -206,14 +176,13 @@ return; } if (endTuples.length < 2) { - partitionIds.add(this.partitionedSource.getTupleSourceID()); + partitionIds.add(this.partitionedSource.getTupleBuffer()); } else { if (partitionIds.isEmpty()) { for (int i = 0; i < endTuples.length; i++) { - TupleCollector tc = new TupleCollector(this.partitionedSource.createSourceTupleSource(), this.joinNode.getBufferManager()); + TupleBuffer tc = this.partitionedSource.createSourceTupleBuffer(); tc.setBatchSize(Math.max(1, this.joinNode.getBatchSize()/4)); - this.tupleCollectors.add(tc); - this.partitionIds.add(tc.getTupleSourceID()); + this.partitionIds.add(tc); } } while (this.partitionedSource.getIterator().hasNext()) { @@ -222,14 +191,14 @@ if (index < 0) { index = -index - 1; } - if (index > this.tupleCollectors.size() -1) { + if (index > this.partitionIds.size() -1) { continue; } while (index > 0 && this.overlap.get(index - 1) && compare(tuple, this.endTuples[index - 1], this.partitionedSource.getExpressionIndexes(), this.sortedSource.getExpressionIndexes()) == 0) { index--; } - this.tupleCollectors.get(index).addTuple(tuple); + this.partitionIds.get(index).addTuple(tuple); } this.partitionedSource.getIterator().setPosition(1); } @@ -247,16 +216,15 @@ } while (currentPartition < partitionIds.size()) { if (currentSource == null) { - if (!this.tupleCollectors.isEmpty()) { - this.tupleCollectors.get(currentPartition).close(); + if (!this.partitionIds.isEmpty()) { + this.partitionIds.get(currentPartition).close(); } - currentSource = this.joinNode.getBufferManager().getTupleSource(partitionIds.get(currentPartition)); + currentSource = partitionIds.get(currentPartition).createIndexedTupleSource(); } int beginIndex = currentPartition>0?endRows.get(currentPartition - 1)+1:1; - this.sortedSource.getIterator().setPosition(beginIndex); - List[] batch = this.sortedSource.getIterator().getBatch().getAllTuples(); + List[] batch = this.sortedSource.getTupleBuffer().getBatch(beginIndex).getAllTuples(); while (partitionedTuple != null || currentSource.hasNext()) { if (partitionedTuple == null) { @@ -283,7 +251,7 @@ } } if (matchEnd == batch.length - 1 && currentPartition < overlap.size() && overlap.get(currentPartition)) { - this.tupleCollectors.get(currentPartition + 1).addTuple(partitionedTuple); + this.partitionIds.get(currentPartition + 1).addTuple(partitionedTuple); } } while (matchBegin <= matchEnd) { Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/RelationalNode.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/RelationalNode.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/RelationalNode.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -44,6 +44,7 @@ import com.metamatrix.query.processor.DescribableUtil; import com.metamatrix.query.processor.ProcessorDataManager; import com.metamatrix.query.processor.ProcessorPlan; +import com.metamatrix.query.processor.BatchCollector.BatchProducer; import com.metamatrix.query.sql.LanguageObject; import com.metamatrix.query.sql.lang.SubqueryContainer; import com.metamatrix.query.sql.symbol.AliasSymbol; @@ -52,7 +53,7 @@ import com.metamatrix.query.util.CommandContext; import com.metamatrix.query.util.ErrorMessageKeys; -public abstract class RelationalNode implements Cloneable, Describable{ +public abstract class RelationalNode implements Cloneable, Describable, BatchProducer { // External context and state private CommandContext context; @@ -155,6 +156,11 @@ public void setElements(List elements) { this.elements = elements; } + + @Override + public List getOutputElements() { + return getElements(); + } public List getElements() { return this.elements; Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SortNode.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SortNode.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SortNode.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -30,11 +30,9 @@ import com.metamatrix.api.exception.MetaMatrixComponentException; import com.metamatrix.api.exception.MetaMatrixProcessingException; import com.metamatrix.common.buffer.BlockedException; -import com.metamatrix.common.buffer.BlockedOnMemoryException; -import com.metamatrix.common.buffer.MemoryNotAvailableException; import com.metamatrix.common.buffer.TupleBatch; -import com.metamatrix.common.buffer.TupleSourceID; -import com.metamatrix.common.buffer.TupleSourceNotFoundException; +import com.metamatrix.common.buffer.TupleBuffer; +import com.metamatrix.query.processor.BatchIterator; import com.metamatrix.query.processor.relational.SortUtility.Mode; import com.metamatrix.query.sql.lang.OrderBy; @@ -45,13 +43,11 @@ private Mode mode = Mode.SORT; private SortUtility sortUtility; - private int phase = COLLECTION; - private TupleSourceID outputID; + private int phase = SORT; + private TupleBuffer outputID; private int rowCount = -1; private int outputBeginRow = 1; - private BatchCollector collector; - private static final int COLLECTION = 1; private static final int SORT = 2; private static final int OUTPUT = 3; @@ -62,11 +58,10 @@ public void reset() { super.reset(); sortUtility = null; - phase = COLLECTION; + phase = SORT; outputID = null; rowCount = -1; outputBeginRow = 1; - this.collector = null; } public void setSortElements(List sortElements, List<Boolean> sortTypes) { @@ -90,56 +85,33 @@ throws MetaMatrixComponentException, MetaMatrixProcessingException { super.open(); - this.collector = new BatchCollector(this.getChildren()[0]); } public TupleBatch nextBatchDirect() throws BlockedException, MetaMatrixComponentException, MetaMatrixProcessingException { - - try { - - if(this.phase == COLLECTION) { - collectionPhase(); - } - - if(this.phase == SORT) { - sortPhase(); - } - - return outputPhase(); - } catch(TupleSourceNotFoundException e) { - throw new MetaMatrixComponentException(e, e.getMessage()); + if(this.phase == SORT) { + sortPhase(); } + + return outputPhase(); } - private void collectionPhase() throws BlockedException, TupleSourceNotFoundException, MetaMatrixComponentException, MetaMatrixProcessingException { - try { - collector.collectTuples(); - } catch (BlockedOnMemoryException e) { - throw e; - } catch (BlockedException e) { - if (mode != Mode.DUP_REMOVE || !collector.collectedAny()) { - throw e; - } - } - if (this.sortUtility == null) { - this.sortUtility = new SortUtility(collector.getTupleSourceID(), sortElements, + private void sortPhase() throws BlockedException, MetaMatrixComponentException, MetaMatrixProcessingException { + if (this.sortUtility == null) { + this.sortUtility = new SortUtility(new BatchIterator(getChildren()[0]), sortElements, sortTypes, this.mode, getBufferManager(), - getConnectionID(), true); + getConnectionID()); } - this.phase = SORT; - } - - private void sortPhase() throws BlockedException, MetaMatrixComponentException { this.outputID = this.sortUtility.sort(); this.phase = OUTPUT; } - private TupleBatch outputPhase() throws BlockedException, TupleSourceNotFoundException, MetaMatrixComponentException { + private TupleBatch outputPhase() throws BlockedException, MetaMatrixComponentException { if (this.rowCount == -1) { - this.rowCount = getBufferManager().getFinalRowCount(outputID); - if (this.rowCount == -1) { - this.phase = this.collector.isDone()?SORT:COLLECTION; + if (this.outputID.isFinal()) { + this.rowCount = this.outputID.getRowCount(); + } else { + this.phase = SORT; } } if(this.rowCount == 0 || (this.rowCount != -1 && this.outputBeginRow > this.rowCount)) { @@ -148,23 +120,17 @@ return terminationBatch; } int beginPinned = this.outputBeginRow; - try { - TupleBatch outputBatch = getBufferManager().pinTupleBatch(outputID, beginPinned); - - this.outputBeginRow += outputBatch.getRowCount(); + TupleBatch outputBatch = this.outputID.getBatch(beginPinned); + + this.outputBeginRow += outputBatch.getRowCount(); - outputBatch = removeUnrelatedColumns(outputBatch); + outputBatch = removeUnrelatedColumns(outputBatch); - if(rowCount != -1 && outputBeginRow > rowCount) { - outputBatch.setTerminationFlag(true); - } - - return outputBatch; - } catch(MemoryNotAvailableException e) { - throw BlockedOnMemoryException.INSTANCE; - } finally { - getBufferManager().unpinTupleBatch(outputID, beginPinned); + if(rowCount != -1 && outputBeginRow > rowCount) { + outputBatch.setTerminationFlag(true); } + + return outputBatch; } private TupleBatch removeUnrelatedColumns(TupleBatch outputBatch) { @@ -182,17 +148,9 @@ public void close() throws MetaMatrixComponentException { if (!isClosed()) { super.close(); - try { - if(this.collector != null) { - this.collector.close(); - this.collector = null; - } - if(this.outputID != null) { - getBufferManager().removeTupleSource(outputID); - this.outputID = null; - } - } catch(TupleSourceNotFoundException e) { - throw new MetaMatrixComponentException(e, e.getMessage()); + if(this.outputID != null) { + this.outputID.remove(); + this.outputID = null; } } } Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SortUtility.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SortUtility.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SortUtility.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -23,417 +23,268 @@ package com.metamatrix.query.processor.relational; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.Iterator; import java.util.List; import com.metamatrix.api.exception.MetaMatrixComponentException; import com.metamatrix.api.exception.MetaMatrixProcessingException; -import com.metamatrix.common.buffer.BlockedOnMemoryException; +import com.metamatrix.common.buffer.BlockedException; import com.metamatrix.common.buffer.BufferManager; import com.metamatrix.common.buffer.IndexedTupleSource; -import com.metamatrix.common.buffer.MemoryNotAvailableException; -import com.metamatrix.common.buffer.TupleBatch; -import com.metamatrix.common.buffer.TupleSourceID; -import com.metamatrix.common.buffer.TupleSourceNotFoundException; -import com.metamatrix.common.buffer.BufferManager.TupleSourceStatus; +import com.metamatrix.common.buffer.TupleBuffer; +import com.metamatrix.common.buffer.TupleSource; import com.metamatrix.common.buffer.BufferManager.TupleSourceType; import com.metamatrix.core.util.Assertion; import com.metamatrix.query.sql.lang.OrderBy; import com.metamatrix.query.sql.symbol.SingleElementSymbol; /** + * Implements several modes of a multi-pass sort. */ public class SortUtility { public enum Mode { SORT, - DUP_REMOVE, + /** Removes duplicates, no sort elements need to be specified. + * may perform additional passes as new batches become available + */ + DUP_REMOVE, + /** Removes duplicates, but guarantees order based upon the sort elements. + */ DUP_REMOVE_SORT } + + /** + * state holder for the merge algorithm + */ + private class SortedSublist implements Comparable<SortedSublist> { + List<?> tuple; + int index; + boolean duplicate; + IndexedTupleSource its; + + @Override + public int compareTo(SortedSublist o) { + return comparator.compare(this.tuple, o.tuple); + } + } //constructor state - private TupleSourceID sourceID; - protected List sortElements; - protected List<Boolean> sortTypes; + private TupleSource sourceID; private Mode mode; protected BufferManager bufferManager; private String groupName; - private boolean useAllColumns; - - //init state - private int batchSize; protected List schema; - protected int[] sortCols; private ListNestedSortComparator comparator; - private TupleSourceID outputID; - private IndexedTupleSource outTs; + private TupleBuffer output; private boolean doneReading; - private int sortPhaseRow = 1; private int phase = INITIAL_SORT; - protected List<TupleSourceID> activeTupleIDs = new ArrayList<TupleSourceID>(); - private List<TupleBatch> workingBatches; + protected List<TupleBuffer> activeTupleBuffers = new ArrayList<TupleBuffer>(); private int masterSortIndex; - private TupleSourceID mergedID; - private TupleSourceID tempOutId; // Phase constants for readability private static final int INITIAL_SORT = 1; private static final int MERGE = 2; private static final int DONE = 3; - public SortUtility(TupleSourceID sourceID, List sortElements, List<Boolean> sortTypes, boolean removeDups, BufferManager bufferMgr, - String groupName) { - this(sourceID, sortElements, sortTypes, removeDups?Mode.DUP_REMOVE_SORT:Mode.SORT, bufferMgr, groupName, false); - } - - public SortUtility(TupleSourceID sourceID, List sortElements, List<Boolean> sortTypes, Mode mode, BufferManager bufferMgr, - String groupName, boolean useAllColumns) { + public SortUtility(TupleSource sourceID, List sortElements, List<Boolean> sortTypes, Mode mode, BufferManager bufferMgr, + String groupName) { this.sourceID = sourceID; - this.sortElements = sortElements; - this.sortTypes = sortTypes; this.mode = mode; this.bufferManager = bufferMgr; this.groupName = groupName; - this.useAllColumns = useAllColumns; + this.schema = this.sourceID.getSchema(); + int distinctIndex = sortElements != null? sortElements.size() - 1:0; + if (mode != Mode.SORT) { + if (sortElements != null && sortElements.size() < schema.size()) { + sortElements = new ArrayList(sortElements); + List toAdd = new ArrayList(schema); + toAdd.removeAll(sortElements); + sortElements.addAll(toAdd); + sortTypes = new ArrayList<Boolean>(sortTypes); + sortTypes.addAll(Collections.nCopies(sortElements.size() - sortTypes.size(), OrderBy.ASC)); + } else { + sortElements = this.schema; + sortTypes = Collections.nCopies(sortElements.size(), OrderBy.ASC); + } + } + + int[] cols = new int[sortElements.size()]; + + Iterator iter = sortElements.iterator(); + + for (int i = 0; i < cols.length; i++) { + SingleElementSymbol elem = (SingleElementSymbol)iter.next(); + + cols[i] = schema.indexOf(elem); + Assertion.assertTrue(cols[i] != -1); + } + this.comparator = new ListNestedSortComparator(cols, sortTypes); + this.comparator.setDistinctIndex(distinctIndex); } public boolean isDone() { return this.doneReading && this.phase == DONE; } - public TupleSourceID sort() - throws BlockedOnMemoryException, MetaMatrixComponentException { + public TupleBuffer sort() + throws MetaMatrixComponentException, MetaMatrixProcessingException { - try { - // One time setup - if(this.schema == null) { - initialize(); - } - - if(this.phase == INITIAL_SORT) { - initialSort(); - } - - if(this.phase == MERGE) { - try { - mergePhase(); - } finally { - if (this.mergedID != null) { - this.bufferManager.removeTupleSource(mergedID); - this.mergedID = null; - } - if (this.tempOutId != null) { - this.bufferManager.removeTupleSource(tempOutId); - this.tempOutId = null; - } - if (workingBatches != null) { - for (int i = 0; i < workingBatches.size(); i++) { - TupleBatch tupleBatch = workingBatches.get(i); - if (tupleBatch != null) { - unpinWorkingBatch(i, tupleBatch); - } - } - } - workingBatches = null; - } - } - if (this.outputID != null) { - return this.outputID; - } - return this.activeTupleIDs.get(0); - } catch(TupleSourceNotFoundException e) { - throw new MetaMatrixComponentException(e, e.getMessage()); + if(this.phase == INITIAL_SORT) { + initialSort(); } - + + if(this.phase == MERGE) { + mergePhase(); + } + if (this.output != null) { + return this.output; + } + return this.activeTupleBuffers.get(0); } - private TupleSourceID createTupleSource() throws MetaMatrixComponentException { - return bufferManager.createTupleSource(this.schema, this.groupName, TupleSourceType.PROCESSOR); + private TupleBuffer createTupleBuffer() throws MetaMatrixComponentException { + return bufferManager.createTupleBuffer(this.schema, this.groupName, TupleSourceType.PROCESSOR); } - - protected void initialSort() throws BlockedOnMemoryException, TupleSourceNotFoundException, MetaMatrixComponentException { + + /** + * creates sort sublists stored in tuplebuffers + */ + protected void initialSort() throws MetaMatrixComponentException, MetaMatrixProcessingException { while(!doneReading) { - ArrayList<int[]> pinned = new ArrayList<int[]>(); // of int[2] representing begin, end List<List<Object>> workingTuples = new ArrayList<List<Object>>(); - // Load data until out of memory - while(!doneReading) { + while(!doneReading) { //TODO: limit rows try { - // Load and pin batch - TupleBatch batch = bufferManager.pinTupleBatch(sourceID, sortPhaseRow); - - if (batch.getRowCount() == 0) { - if (bufferManager.getStatus(sourceID) == TupleSourceStatus.FULL) { - doneReading = true; - } - break; - } - // Remember pinned rows - pinned.add(new int[] { sortPhaseRow, batch.getEndRow() }); - - addTuples(workingTuples, batch); - - sortPhaseRow += batch.getRowCount(); - } catch(MemoryNotAvailableException e) { - break; - } + List<?> tuple = sourceID.nextTuple(); + + if (tuple == null) { + doneReading = true; + break; + } + + addTuple(workingTuples, tuple); + } catch(BlockedException e) { + if (workingTuples.isEmpty() && (mode != Mode.DUP_REMOVE || activeTupleBuffers.isEmpty())) { + throw e; //block if no work can be performed + } + break; + } } if(workingTuples.isEmpty()) { - break; + break; } - TupleSourceID activeID = createTupleSource(); - activeTupleIDs.add(activeID); - int sortedThisPass = workingTuples.size(); + TupleBuffer activeID = createTupleBuffer(); + activeTupleBuffers.add(activeID); if (this.mode == Mode.SORT) { //perform a stable sort Collections.sort(workingTuples, comparator); } - int writeBegin = 1; - while(writeBegin <= sortedThisPass) { - int writeEnd = Math.min(sortedThisPass, writeBegin + batchSize - 1); - - TupleBatch writeBatch = new TupleBatch(writeBegin, workingTuples.subList(writeBegin-1, writeEnd)); - bufferManager.addTupleBatch(activeID, writeBatch); - writeBegin += writeBatch.getRowCount(); - } - - // Clean up - unpin rows - for (int[] bounds : pinned) { - bufferManager.unpinTupleBatch(sourceID, bounds[0]); - } + for (List<Object> list : workingTuples) { + activeID.addTuple(list); + } } - - if (!doneReading && (mode != Mode.DUP_REMOVE || this.activeTupleIDs.isEmpty())) { - throw BlockedOnMemoryException.INSTANCE; - } - if (this.activeTupleIDs.isEmpty()) { - activeTupleIDs.add(createTupleSource()); + if (this.activeTupleBuffers.isEmpty()) { + activeTupleBuffers.add(createTupleBuffer()); } - // Clean up this.phase = MERGE; } - protected void addTuples(List workingTuples, TupleBatch batch) { + protected void addTuple(List workingTuples, List tuple) { if (this.mode == Mode.SORT) { - workingTuples.addAll(Arrays.asList(batch.getAllTuples())); + workingTuples.add(tuple); return; } - for (List<Object> list : batch.getAllTuples()) { - int index = Collections.binarySearch(workingTuples, list, comparator); - if (index >= 0) { - continue; - } - workingTuples.add(-index - 1, list); + int index = Collections.binarySearch(workingTuples, tuple, comparator); + if (index >= 0) { + return; //it's already there } + workingTuples.add(-index - 1, tuple); } - - protected void mergePhase() throws BlockedOnMemoryException, MetaMatrixComponentException, TupleSourceNotFoundException { - appendOutput(); - TupleCollector tempCollector = null; - while(this.activeTupleIDs.size() > 1) { - // Load and pin batch from sorted sublists while memory available - this.workingBatches = new ArrayList<TupleBatch>(activeTupleIDs.size()); + + protected void mergePhase() throws MetaMatrixComponentException, MetaMatrixProcessingException { + while(this.activeTupleBuffers.size() > 1) { + ArrayList<SortedSublist> workingTuples = new ArrayList<SortedSublist>(activeTupleBuffers.size()); + + TupleBuffer merged = createTupleBuffer(); + int sortedIndex = 0; - for(; sortedIndex<activeTupleIDs.size(); sortedIndex++) { - TupleSourceID activeID = activeTupleIDs.get(sortedIndex); - try { - TupleBatch sortedBatch = bufferManager.pinTupleBatch(activeID, 1); - workingBatches.add(sortedBatch); - } catch(MemoryNotAvailableException e) { - break; - } + for(; sortedIndex<activeTupleBuffers.size(); sortedIndex++) { //TODO: limit activeTupleIDs + TupleBuffer activeID = activeTupleBuffers.get(sortedIndex); + SortedSublist sortedSublist = new SortedSublist(); + sortedSublist.its = activeID.createIndexedTupleSource(); + sortedSublist.index = sortedIndex; + addWorkingTuple(workingTuples, sortedSublist); } - //if we cannot make progress, just block for now - if (workingBatches.size() < 2) { - throw BlockedOnMemoryException.INSTANCE; - } - - // Initialize pointers into working batches - int[] workingPointers = new int[workingBatches.size()]; - Arrays.fill(workingPointers, 1); - - mergedID = createTupleSource(); - - // Merge from working sorted batches - TupleCollector collector = new TupleCollector(mergedID, this.bufferManager); - while(true) { - // Find least valued row among working batches - List<?> currentRow = null; - int chosenBatchIndex = -1; - TupleBatch chosenBatch = null; - for(int i=0; i<workingBatches.size(); i++) { - TupleBatch batch = workingBatches.get(i); - if(batch == null) { - continue; + // iteratively process the lowest tuple + while (workingTuples.size() > 0) { + SortedSublist sortedSublist = workingTuples.remove(0); + if (!sortedSublist.duplicate) { + merged.addTuple(sortedSublist.tuple); + if (this.output != null && sortedSublist.index != masterSortIndex && sortedIndex > masterSortIndex) { + this.output.addTuple(sortedSublist.tuple); } - List<?> testRow = batch.getTuple(workingPointers[i]); - int compare = -1; - if (currentRow != null) { - compare = comparator.compare(testRow, currentRow); - } - if(compare < 0) { - // Found lower row - currentRow = testRow; - chosenBatchIndex = i; - chosenBatch = batch; - } else if (compare == 0 && this.mode != Mode.SORT) { - incrementWorkingBatch(i, workingPointers, batch); - } - } + } + addWorkingTuple(workingTuples, sortedSublist); + } - // Check for termination condition - all batches must have been null - if(currentRow == null) { - break; - } - // Output the row and update pointers - collector.addTuple(currentRow); - if (this.outputID != null && chosenBatchIndex != masterSortIndex && sortedIndex > masterSortIndex) { - if (tempCollector == null) { - tempOutId = createTupleSource(); - tempCollector = new TupleCollector(tempOutId, this.bufferManager); - } - tempCollector.addTuple(currentRow); - } - incrementWorkingBatch(chosenBatchIndex, workingPointers, chosenBatch); - } - - // Save without closing - collector.saveBatch(); - // Remove merged sublists for(int i=0; i<sortedIndex; i++) { - TupleSourceID id = activeTupleIDs.remove(0); - if (!id.equals(this.outputID)) { - bufferManager.removeTupleSource(id); + TupleBuffer id = activeTupleBuffers.remove(0); + if (id != this.output) { + id.remove(); } } - this.activeTupleIDs.add(mergedID); - this.mergedID = null; + this.activeTupleBuffers.add(merged); masterSortIndex = masterSortIndex - sortedIndex + 1; if (masterSortIndex < 0) { - masterSortIndex = this.activeTupleIDs.size() - 1; + masterSortIndex = this.activeTupleBuffers.size() - 1; } - } + } - if (tempCollector != null) { - tempCollector.close(); - this.outTs = this.bufferManager.getTupleSource(tempOutId); - appendOutput(); - } - // Close sorted source (all others have been removed) if (doneReading) { - bufferManager.setStatus(activeTupleIDs.get(0), TupleSourceStatus.FULL); - if (this.outputID != null) { - bufferManager.setStatus(outputID, TupleSourceStatus.FULL); + activeTupleBuffers.get(0).close(); + if (this.output != null) { + this.output.close(); } this.phase = DONE; return; } Assertion.assertTrue(mode == Mode.DUP_REMOVE); - if (this.outputID == null) { - this.outputID = activeTupleIDs.get(0); + if (this.output == null) { + this.output = activeTupleBuffers.get(0); } this.phase = INITIAL_SORT; } - private void appendOutput() throws TupleSourceNotFoundException, - MetaMatrixComponentException { - if (this.outTs != null) { - //transfer the new dup removed tuples to the output id - TupleCollector tc = new TupleCollector(outputID, this.bufferManager); - try { - try { - while (outTs.hasNext()) { - tc.addTuple(outTs.nextTuple()); - } - } catch (MetaMatrixProcessingException e) { - throw new MetaMatrixComponentException(e); - } - } finally { - tc.saveBatch(); - } - outTs.closeSource(); - outTs = null; + private void addWorkingTuple(ArrayList<SortedSublist> workingTuples, SortedSublist sortedSublist) throws MetaMatrixComponentException, MetaMatrixProcessingException { + sortedSublist.tuple = null; + try { + sortedSublist.tuple = sortedSublist.its.nextTuple(); + } catch (BlockedException e) { + //intermediate sources aren't closed + } + if (sortedSublist.tuple == null) { + return; // done with this sublist } - } + sortedSublist.duplicate = false; + int index = Collections.binarySearch(workingTuples, sortedSublist); + if (index >= 0) { + sortedSublist.duplicate = mode != Mode.SORT; + workingTuples.add(index, sortedSublist); + } else { + workingTuples.add(-index - 1, sortedSublist); + } + } - /** - * Increment the working batch at batchIndex. The currentBatch is the currentBatch - * for that batchIndex, which we already happen to have. Return whether the batch - * was changed or not. True = changed. - */ - private void incrementWorkingBatch(int batchIndex, int[] workingPointers, TupleBatch currentBatch) throws BlockedOnMemoryException, TupleSourceNotFoundException, MetaMatrixComponentException { - workingPointers[batchIndex] += 1; - if(workingPointers[batchIndex] > currentBatch.getEndRow()) { - TupleSourceID tsID = unpinWorkingBatch(batchIndex, currentBatch); - - int beginRow = workingPointers[batchIndex]; - - try { - TupleBatch newBatch = bufferManager.pinTupleBatch(tsID, beginRow); - if(newBatch.getRowCount() == 0) { - // Done with this working batch - workingBatches.set(batchIndex, null); - } else { - workingBatches.set(batchIndex, newBatch); - } - } catch(MemoryNotAvailableException e) { - throw BlockedOnMemoryException.INSTANCE; - } - } - } - - private TupleSourceID unpinWorkingBatch(int batchIndex, - TupleBatch currentBatch) throws TupleSourceNotFoundException, - MetaMatrixComponentException { - TupleSourceID tsID = activeTupleIDs.get(batchIndex); - int lastBeginRow = currentBatch.getBeginRow(); - bufferManager.unpinTupleBatch(tsID, lastBeginRow); - return tsID; - } - - private void initialize() throws TupleSourceNotFoundException, MetaMatrixComponentException { - this.schema = this.bufferManager.getTupleSchema(this.sourceID); - this.batchSize = bufferManager.getProcessorBatchSize(); - int distinctIndex = sortElements != null? sortElements.size() - 1:0; - if (useAllColumns && mode != Mode.SORT) { - if (this.sortElements != null) { - this.sortElements = new ArrayList(this.sortElements); - List toAdd = new ArrayList(schema); - toAdd.removeAll(this.sortElements); - this.sortElements.addAll(toAdd); - this.sortTypes = new ArrayList<Boolean>(this.sortTypes); - this.sortTypes.addAll(Collections.nCopies(this.sortElements.size() - this.sortTypes.size(), OrderBy.ASC)); - } else { - this.sortElements = this.schema; - this.sortTypes = Collections.nCopies(this.sortElements.size(), OrderBy.ASC); - } - } - - int[] cols = new int[sortElements.size()]; - - Iterator iter = sortElements.iterator(); - - for (int i = 0; i < cols.length; i++) { - SingleElementSymbol elem = (SingleElementSymbol)iter.next(); - - cols[i] = schema.indexOf(elem); - Assertion.assertTrue(cols[i] != -1); - } - this.sortCols = cols; - this.comparator = new ListNestedSortComparator(sortCols, sortTypes); - this.comparator.setDistinctIndex(distinctIndex); - } - public boolean isDistinct() { return this.comparator.isDistinct(); } Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SourceState.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SourceState.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SourceState.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -28,16 +28,16 @@ import com.metamatrix.api.exception.MetaMatrixComponentException; import com.metamatrix.api.exception.MetaMatrixProcessingException; import com.metamatrix.common.buffer.IndexedTupleSource; -import com.metamatrix.common.buffer.TupleSourceID; -import com.metamatrix.common.buffer.TupleSourceNotFoundException; +import com.metamatrix.common.buffer.TupleBuffer; import com.metamatrix.common.buffer.BufferManager.TupleSourceType; +import com.metamatrix.query.processor.BatchCollector; +import com.metamatrix.query.processor.BatchIterator; class SourceState { private RelationalNode source; private BatchCollector collector; - private IndexedTupleSource tupleSource; - private TupleSourceID tsID; + private TupleBuffer buffer; private List<Object> outerVals; private IndexedTupleSource iterator; private int[] expressionIndexes; @@ -45,6 +45,8 @@ private int maxProbeMatch = 1; private boolean distinct; + private boolean canBuffer = true; + public SourceState(RelationalNode source, List expressions) { this.source = source; List elements = source.getElements(); @@ -64,8 +66,8 @@ return indecies; } - TupleSourceID createSourceTupleSource() throws MetaMatrixComponentException { - return this.source.getBufferManager().createTupleSource(source.getElements(), source.getConnectionID(), TupleSourceType.PROCESSOR); + TupleBuffer createSourceTupleBuffer() throws MetaMatrixComponentException { + return this.source.getBufferManager().createTupleBuffer(source.getElements(), source.getConnectionID(), TupleSourceType.PROCESSOR); } public List saveNext() throws MetaMatrixComponentException, MetaMatrixProcessingException { @@ -79,52 +81,35 @@ this.currentTuple = null; } - public void close() throws TupleSourceNotFoundException, MetaMatrixComponentException { + public void close() { closeTupleSource(); } - private void closeTupleSource() throws TupleSourceNotFoundException, - MetaMatrixComponentException { - if (this.tsID != null) { - this.source.getBufferManager().removeTupleSource(tsID); - this.tsID = null; + private void closeTupleSource() { + if (this.buffer != null) { + this.buffer.remove(); + this.buffer = null; } + if (this.iterator != null) { + this.iterator = null; + } } - public int getRowCount() { - return this.collector.getRowCount(); + public int getRowCount() throws MetaMatrixComponentException, MetaMatrixProcessingException { + return this.getTupleBuffer().getRowCount(); } - /** - * Collect the underlying batches into a tuple source. Subsequent calls will return that tuple source - */ - public TupleSourceID collectTuples() throws MetaMatrixComponentException, - MetaMatrixProcessingException { - if (collector == null) { - collector = new BatchCollector(source); - } - TupleSourceID result = collector.collectTuples(); - if (this.tsID == null) { - setTupleSource(result); - } - return result; + void setTupleSource(TupleBuffer result) { + closeTupleSource(); + this.buffer = result; } - - void setTupleSource(TupleSourceID result) throws MetaMatrixComponentException, TupleSourceNotFoundException { - closeTupleSource(); - this.tsID = result; - try { - this.tupleSource = source.getBufferManager().getTupleSource(result); - } catch (TupleSourceNotFoundException e) { - throw new MetaMatrixComponentException(e, e.getMessage()); - } - } IndexedTupleSource getIterator() { if (this.iterator == null) { - if (this.tupleSource != null) { - iterator = this.tupleSource; + if (this.buffer != null) { + iterator = buffer.createIndexedTupleSource(); } else { + canBuffer = false; // return a TupleBatch tuplesource iterator iterator = new BatchIterator(this.source); } @@ -152,8 +137,18 @@ return maxProbeMatch; } - public TupleSourceID getTupleSourceID() { - return this.tsID; + public TupleBuffer getTupleBuffer() throws MetaMatrixComponentException, MetaMatrixProcessingException { + if (this.buffer == null) { + if (!canBuffer) { + throw new AssertionError("cannot buffer the source"); //$NON-NLS-1$ + } + if (collector == null) { + collector = new BatchCollector(source, createSourceTupleBuffer()); + } + TupleBuffer result = collector.collectTuples(); + setTupleSource(result); + } + return this.buffer; } public boolean isDistinct() { Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SubqueryAwareEvaluator.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SubqueryAwareEvaluator.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SubqueryAwareEvaluator.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -30,9 +30,8 @@ import com.metamatrix.api.exception.MetaMatrixProcessingException; import com.metamatrix.common.buffer.BlockedException; import com.metamatrix.common.buffer.BufferManager; -import com.metamatrix.common.buffer.TupleSourceID; -import com.metamatrix.common.buffer.TupleSourceNotFoundException; import com.metamatrix.query.eval.Evaluator; +import com.metamatrix.query.processor.BatchCollector; import com.metamatrix.query.processor.ProcessorDataManager; import com.metamatrix.query.processor.ProcessorPlan; import com.metamatrix.query.processor.QueryProcessor; @@ -52,6 +51,7 @@ public class SubqueryState { QueryProcessor processor; + BatchCollector collector; boolean done; List<?> tuple; ProcessorPlan plan; @@ -60,13 +60,8 @@ if (processor == null) { return; } - try { - processor.closeProcessing(); - TupleSourceID id = processor.getResultsID(); - manager.removeTupleSource(id); - } catch (TupleSourceNotFoundException e) { - - } + processor.closeProcessing(); + collector.getTupleBuffer().close(); processor = null; this.done = false; } @@ -125,12 +120,12 @@ } state.processor.getContext().pushVariableContext(currentContext); } + state.collector = state.processor.createBatchCollector(); } - state.processor.process(Integer.MAX_VALUE); state.done = true; state.processor.getProcessorPlan().reset(); } - return new DependentValueSource(state.processor.getResultsID(), this.manager).getValueIterator(ref.getValueExpression()); + return new DependentValueSource(state.collector.collectTuples(), this.manager.getProcessorBatchSize() / 2).getValueIterator(ref.getValueExpression()); } } Deleted: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/TupleCollector.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/TupleCollector.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/TupleCollector.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -1,112 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.query.processor.relational; - -import java.util.ArrayList; -import java.util.List; - -import com.metamatrix.api.exception.MetaMatrixComponentException; -import com.metamatrix.common.buffer.BufferManager; -import com.metamatrix.common.buffer.TupleBatch; -import com.metamatrix.common.buffer.TupleSourceID; -import com.metamatrix.common.buffer.TupleSourceNotFoundException; -import com.metamatrix.common.buffer.BufferManager.TupleSourceStatus; - -public class TupleCollector { - - private TupleSourceID tsid; - private BufferManager bm; - - private int batchSize; - private ArrayList<List<?>> batch; - private int index; - - public TupleCollector(TupleSourceID tsid, BufferManager bm) throws TupleSourceNotFoundException, MetaMatrixComponentException { - this.tsid = tsid; - this.batchSize = bm.getProcessorBatchSize(); - this.bm = bm; - this.index = bm.getRowCount(tsid) + 1; - } - - public TupleCollector(BufferManager bm) { - this.batchSize = bm.getProcessorBatchSize(); - this.bm = bm; - this.index = 1; - } - - public void setBatchSize(int batchSize) { - this.batchSize = batchSize; - } - - public void setTupleSourceID(TupleSourceID tsid) { - this.tsid = tsid; - } - - public boolean isEmpty() { - return this.batch == null || this.batch.isEmpty(); - } - - public TupleSourceID getTupleSourceID() { - return tsid; - } - - public ArrayList<List<?>> getBatch() { - if (batch == null) { - batch = new ArrayList<List<?>>(batchSize/4); - } - return batch; - } - - public void addTuple(List<?> tuple) throws TupleSourceNotFoundException, MetaMatrixComponentException { - if (batch == null) { - batch = new ArrayList<List<?>>(batchSize/4); - } - batch.add(tuple); - if (batch.size() == batchSize) { - saveBatch(); - } - } - - public void saveBatch() throws TupleSourceNotFoundException, - MetaMatrixComponentException { - if (batch == null || batch.isEmpty()) { - return; - } - int batchIndex = 0; - while(batchIndex < batch.size()) { - int writeEnd = Math.min(batch.size(), batchIndex + batchSize); - - TupleBatch writeBatch = new TupleBatch(index, batch.subList(batchIndex, writeEnd)); - bm.addTupleBatch(tsid, writeBatch); - index += writeBatch.getRowCount(); - batchIndex += writeBatch.getRowCount(); - } - batch = null; - } - - public void close() throws TupleSourceNotFoundException, MetaMatrixComponentException { - saveBatch(); - this.bm.setStatus(this.tsid, TupleSourceStatus.FULL); - } - -} Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/UnionAllNode.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/UnionAllNode.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/UnionAllNode.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -28,7 +28,6 @@ import com.metamatrix.api.exception.MetaMatrixComponentException; import com.metamatrix.api.exception.MetaMatrixProcessingException; import com.metamatrix.common.buffer.BlockedException; -import com.metamatrix.common.buffer.BlockedOnMemoryException; import com.metamatrix.common.buffer.TupleBatch; public class UnionAllNode extends RelationalNode { @@ -81,9 +80,6 @@ sourceDone[i] = true; activeSources--; } - - } catch (BlockedOnMemoryException e) { - throw e; } catch(BlockedException e) { if(i<children.length-1 && hasDependentProcedureExecutionNode(children[0])){ throw e; Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/xml/RelationalPlanExecutor.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/xml/RelationalPlanExecutor.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/xml/RelationalPlanExecutor.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -29,11 +29,11 @@ import com.metamatrix.api.exception.MetaMatrixProcessingException; import com.metamatrix.common.buffer.BlockedException; import com.metamatrix.common.buffer.BufferManager; -import com.metamatrix.common.buffer.TupleSource; -import com.metamatrix.common.buffer.TupleSourceNotFoundException; +import com.metamatrix.common.buffer.IndexedTupleSource; import com.metamatrix.common.log.LogManager; import com.metamatrix.query.execution.QueryExecPlugin; import com.metamatrix.query.mapping.xml.ResultSetInfo; +import com.metamatrix.query.processor.BatchIterator; import com.metamatrix.query.processor.ProcessorDataManager; import com.metamatrix.query.processor.ProcessorPlan; import com.metamatrix.query.processor.QueryProcessor; @@ -60,7 +60,7 @@ // flag to denote the end of rows boolean endOfRows = false; // results after the execution bucket. - TupleSource tupleSource; + IndexedTupleSource tupleSource; // cached current row of results. List currentRow; int currentRowNumber = 0; @@ -90,20 +90,15 @@ * @see com.metamatrix.query.processor.xml.PlanExecutor#execute(java.util.Map) */ public void execute(Map referenceValues) throws MetaMatrixComponentException, BlockedException, MetaMatrixProcessingException { - try { - // set the dynamic reference values - setReferenceValues(referenceValues); - - // execute - this.internalProcessor.process(Integer.MAX_VALUE); //TODO: but an actual value here - } catch (MetaMatrixComponentException e) { - throw e; - } catch (MetaMatrixProcessingException e) { - throw e; + if (this.tupleSource == null) { + setReferenceValues(referenceValues); + this.tupleSource = new BatchIterator(internalProcessor); } + //force execution + this.tupleSource.hasNext(); } - void setReferenceValues(Map<ElementSymbol, Object> referencesValues) throws MetaMatrixComponentException { + void setReferenceValues(Map<ElementSymbol, Object> referencesValues) { if (referencesValues == null || referencesValues.isEmpty()) { return; } @@ -118,14 +113,6 @@ * @throws MetaMatrixComponentException */ public List nextRow() throws MetaMatrixComponentException, MetaMatrixProcessingException { - if (this.tupleSource == null) { - try { - this.tupleSource = this.bufferMgr.getTupleSource(this.internalProcessor.getResultsID()); - } catch (TupleSourceNotFoundException e) { - throw new MetaMatrixComponentException(e, QueryExecPlugin.Util.getString("tuple_not_found", this.resultInfo.getResultSetName())); //$NON-NLS-1$ - } - } - // if we not already closed the tuple source; look for more results. if (!endOfRows) { @@ -166,12 +153,8 @@ * Close the executor and release all the resources. */ public void close() throws MetaMatrixComponentException { - try { - this.bufferMgr.removeTupleSource(this.internalProcessor.getResultsID()); - } catch (TupleSourceNotFoundException e) { - - } - LogManager.logTrace(LogConstants.CTX_XML_PLAN, new Object[]{"removed tuple source", this.internalProcessor.getResultsID(), "for result set", this.internalProcessor.getResultsID()}); //$NON-NLS-1$ //$NON-NLS-2$ + this.internalProcessor.closeProcessing(); + LogManager.logTrace(LogConstants.CTX_XML_PLAN, new Object[]{"closed executor", resultInfo.getResultSetName()}); //$NON-NLS-1$ } } Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/xml/XMLPlan.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/xml/XMLPlan.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/xml/XMLPlan.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -62,17 +62,16 @@ import com.metamatrix.api.exception.MetaMatrixException; import com.metamatrix.api.exception.MetaMatrixProcessingException; import com.metamatrix.common.buffer.BlockedException; -import com.metamatrix.common.buffer.BlockedOnMemoryException; import com.metamatrix.common.buffer.BufferManager; import com.metamatrix.common.buffer.TupleBatch; -import com.metamatrix.common.buffer.TupleSourceID; -import com.metamatrix.common.buffer.BufferManager.TupleSourceStatus; +import com.metamatrix.common.buffer.TupleBuffer; import com.metamatrix.common.lob.LobChunk; import com.metamatrix.common.log.LogManager; import com.metamatrix.common.types.DataTypeManager; import com.metamatrix.common.types.SQLXMLImpl; import com.metamatrix.common.types.Streamable; import com.metamatrix.common.types.XMLType; +import com.metamatrix.core.util.Assertion; import com.metamatrix.query.execution.QueryExecPlugin; import com.metamatrix.query.processor.BaseProcessorPlan; import com.metamatrix.query.processor.DescribableUtil; @@ -108,8 +107,7 @@ // is document in progress currently? boolean docInProgress = false; - TupleSourceID docInProgressTupleSourceId = null; - int chunkPosition = 0; + private TupleBuffer docInProgressTupleSourceId; // Post-processing private String styleSheet; @@ -177,25 +175,22 @@ public TupleBatch nextBatch() throws MetaMatrixComponentException, MetaMatrixProcessingException, BlockedException { - // If finished, processXML(); while(true){ - List rows = processXML(); - - // we have no more to send - if (rows == null){ - TupleBatch batch = new TupleBatch(nextBatchCount++, Collections.EMPTY_LIST); + LobChunk chunk = getNextXMLChunk(this.chunkSize); + + if (chunk == null) { + Assertion.assertTrue(!this.docInProgress); + TupleBatch batch = new TupleBatch(nextBatchCount++, Collections.EMPTY_LIST); batch.setTerminationFlag(true); return batch; - } + } + + XMLType doc = processXML(chunk); - // we got one more to send. - List listOfRows = new ArrayList(); - listOfRows.add(rows); - - TupleBatch batch = new TupleBatch(nextBatchCount++, listOfRows); - // when true; multiple doc return fails. - batch.setTerminationFlag(false); - return batch; + if (doc != null) { + TupleBatch batch = new TupleBatch(nextBatchCount++, Arrays.asList(Arrays.asList(doc))); + return batch; + } } } @@ -217,16 +212,10 @@ * many XML documents, if the root of the document model has * a mapping class (i.e. result set) associated with it.</p> * - * @return List a single tuple that contains the ValueID - * of the XML document for the first document chunk, or empty row - * for the rest of the document chunk; or will return null when - * there are no more documents to return */ - private List processXML() - throws MetaMatrixComponentException, MetaMatrixProcessingException, BlockedException { + private XMLType processXML(LobChunk chunk) + throws MetaMatrixComponentException, BlockedException { - - boolean postProcess = (this.styleSheet != null || this.shouldValidate); // Note that we need to stream the document right away, as multiple documents // results are generated from the same "env" object. So the trick here is either @@ -234,13 +223,8 @@ // right away so that we give way to next batch call. (this due to bad design of xml model) // also note that there could be "inst" object alive but no more chunks; that is reason // for "if" null block - LobChunk chunk = getNextXMLChunk(this.chunkSize); XMLType xml = null; - if (chunk == null) { - return null; - } - // if the chunk size is less than one chunk unit then send this as string based xml. if (!this.docInProgress && chunk.isLast()) { xml = new XMLType(new SQLXMLImpl(new String(chunk.getBytes(), Charset.forName("UTF-16")), getProperties())); //$NON-NLS-1$ @@ -251,29 +235,26 @@ if (!this.docInProgress) { this.docInProgress = true; this.docInProgressTupleSourceId = XMLUtil.createXMLTupleSource(this.bufferMgr, this.getContext().getConnectionID()); - this.chunkPosition = 1; } // now save the chunk of data to the buffer manager and move on. - this.bufferMgr.addStreamablePart(this.docInProgressTupleSourceId, chunk, chunkPosition++); + this.docInProgressTupleSourceId.addTuple(Arrays.asList(chunk)); // now document is finished, so create a xml object and return to the client. if (chunk.isLast()) { - this.bufferMgr.setStatus(this.docInProgressTupleSourceId, TupleSourceStatus.FULL); + this.docInProgressTupleSourceId.close(); // we want this to be naturally feed by chunks whether inside // or out side the processor - xml = new XMLType(XMLUtil.getFromBufferManager(bufferMgr, this.docInProgressTupleSourceId, getProperties())); - this.bufferMgr.setPersistentTupleSource(this.docInProgressTupleSourceId, xml); + xml = new XMLType(XMLUtil.getFromBufferManager(this.docInProgressTupleSourceId, getProperties())); + this.docInProgressTupleSourceId.setContainingLobReference(xml); //reset current document state. this.docInProgress = false; this.docInProgressTupleSourceId = null; - this.chunkPosition = 0; + } else { + return null; //need to continue processing } - else { - throw BlockedOnMemoryException.INSTANCE; - } } // check to see if we need to do any post validation on the document. @@ -281,9 +262,7 @@ xml = postProcessDocument(xml, getProperties()); } - List row = new ArrayList(1); - row.add(xml); - return row; + return xml; } Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/xml/XMLUtil.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/xml/XMLUtil.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/xml/XMLUtil.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -27,22 +27,26 @@ import java.sql.SQLException; import java.sql.SQLXML; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Properties; import com.metamatrix.api.exception.MetaMatrixComponentException; +import com.metamatrix.api.exception.MetaMatrixProcessingException; import com.metamatrix.common.buffer.BufferManager; -import com.metamatrix.common.buffer.BufferManagerLobChunkStream; -import com.metamatrix.common.buffer.TupleSourceID; -import com.metamatrix.common.buffer.TupleSourceNotFoundException; -import com.metamatrix.common.buffer.BufferManager.TupleSourceStatus; +import com.metamatrix.common.buffer.IndexedTupleSource; +import com.metamatrix.common.buffer.TupleBuffer; import com.metamatrix.common.buffer.BufferManager.TupleSourceType; import com.metamatrix.common.lob.ByteLobChunkStream; import com.metamatrix.common.lob.LobChunk; import com.metamatrix.common.lob.LobChunkInputStream; +import com.metamatrix.common.lob.LobChunkProducer; +import com.metamatrix.common.log.LogManager; import com.metamatrix.common.types.DataTypeManager; import com.metamatrix.common.types.SQLXMLImpl; import com.metamatrix.common.types.XMLReaderFactory; +import com.metamatrix.dqp.DQPPlugin; +import com.metamatrix.dqp.util.LogConstants; import com.metamatrix.query.sql.symbol.ElementSymbol; @@ -55,18 +59,17 @@ * This method saves the given XML object to the buffer manager's disk process * and returns the id which is saved under. */ - public static TupleSourceID saveToBufferManager(BufferManager bufferMgr, String tupleGroupName, SQLXML srcXML, int chunkSize) + public static TupleBuffer saveToBufferManager(BufferManager bufferMgr, String tupleGroupName, SQLXML srcXML, int chunkSize) throws MetaMatrixComponentException { try{ // first persist the XML to the Buffer Manager - TupleSourceID sourceId = createXMLTupleSource(bufferMgr, tupleGroupName); + TupleBuffer sourceId = createXMLTupleSource(bufferMgr, tupleGroupName); // since this object possibly will be streamed we need another streamable // wrapper layer. ByteLobChunkStream lobStream = new ByteLobChunkStream(srcXML.getBinaryStream(), chunkSize); // read all the globs and Store into storage manager - int batchCount = 1; LobChunk lobChunk = null; do { try { @@ -74,15 +77,13 @@ } catch (IOException e) { throw new MetaMatrixComponentException(e); } - bufferMgr.addStreamablePart(sourceId, lobChunk, batchCount++); + sourceId.addTuple(Arrays.asList(lobChunk)); } while (!lobChunk.isLast()); lobStream.close(); - bufferMgr.setStatus(sourceId, TupleSourceStatus.FULL); + sourceId.close(); return sourceId; - } catch (TupleSourceNotFoundException e) { - throw new MetaMatrixComponentException(e); } catch(SQLException e) { throw new MetaMatrixComponentException(e); } catch(IOException e) { @@ -94,15 +95,16 @@ * This will reconstruct the XML object from the buffer manager from given * buffer manager id. */ - public static SQLXML getFromBufferManager(final BufferManager bufferMgr, final TupleSourceID sourceId, Properties props) { - return new SQLXMLImpl(new BufferMangerXMLReaderFactory(bufferMgr, sourceId), props); + public static SQLXML getFromBufferManager(final TupleBuffer sourceId, Properties props) { + return new SQLXMLImpl(new BufferMangerXMLReaderFactory(sourceId), props); } /** * Creates a buffer manager's id for XML based data. */ - public static TupleSourceID createXMLTupleSource(BufferManager bufferMgr, String tupleGroupName) throws MetaMatrixComponentException { - TupleSourceID sourceID = bufferMgr.createTupleSource(getOutputElements(), tupleGroupName, TupleSourceType.PROCESSOR); + public static TupleBuffer createXMLTupleSource(BufferManager bufferMgr, String tupleGroupName) throws MetaMatrixComponentException { + TupleBuffer sourceID = bufferMgr.createTupleBuffer(getOutputElements(), tupleGroupName, TupleSourceType.PROCESSOR); + sourceID.setBatchSize(1); return sourceID; } @@ -119,16 +121,42 @@ * so that XML can be streamed by the clients more than once. */ static class BufferMangerXMLReaderFactory implements XMLReaderFactory { - BufferManager bufferMgr; - TupleSourceID sourceId; + private TupleBuffer tb; - BufferMangerXMLReaderFactory(BufferManager mgr, TupleSourceID id){ - this.bufferMgr = mgr; - this.sourceId = id; + BufferMangerXMLReaderFactory(TupleBuffer tb){ + this.tb = tb; } - public Reader getReader() { - return new LobChunkInputStream(new BufferManagerLobChunkStream(this.sourceId.getStringID(), this.bufferMgr)).getUTF16Reader(); + public Reader getReader() { + final IndexedTupleSource its = tb.createIndexedTupleSource(); + LobChunkProducer producer = new LobChunkProducer() { + public LobChunk getNextChunk() throws IOException { + try { + List<?> tuple = its.nextTuple(); + if (tuple == null) { + return null; + } + return (LobChunk)tuple.get(0); + } catch (MetaMatrixProcessingException e) { + String msg = DQPPlugin.Util.getString("BufferManagerLobChunkStream.no_tuple_source", new Object[] {tb}); //$NON-NLS-1$ + LogManager.logWarning(LogConstants.CTX_BUFFER_MGR, e, msg); + throw new IOException(msg); + } catch (MetaMatrixComponentException e) { + String msg = DQPPlugin.Util.getString("BufferManagerLobChunkStream.error_processing", new Object[] {tb}); //$NON-NLS-1$ + LogManager.logWarning(LogConstants.CTX_BUFFER_MGR, e, msg); + throw new IOException(msg); + } + } + + /** + * @see com.metamatrix.common.lob.LobChunkProducer#close() + */ + public void close() throws IOException { + // we could remove the buffer tuple here but, this is just a stream, so we need to delete + // that when we close th eplan. + } + }; + return new LobChunkInputStream(producer).getUTF16Reader(); } } } \ No newline at end of file Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/xquery/XQueryPlan.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/processor/xquery/XQueryPlan.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/processor/xquery/XQueryPlan.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -36,8 +36,7 @@ import com.metamatrix.common.buffer.BlockedException; import com.metamatrix.common.buffer.BufferManager; import com.metamatrix.common.buffer.TupleBatch; -import com.metamatrix.common.buffer.TupleSourceID; -import com.metamatrix.common.buffer.TupleSourceNotFoundException; +import com.metamatrix.common.buffer.TupleBuffer; import com.metamatrix.common.types.DataTypeManager; import com.metamatrix.common.types.Streamable; import com.metamatrix.common.types.XMLType; @@ -120,7 +119,7 @@ XQueryExpression expr = this.xQuery.getCompiledXQuery(); expr.setXMLFormat(xmlFormat); - SqlEval sqlEval = new SqlEval(bufferMgr, this.dataManager, getContext(), this.xQuery.getProcedureGroup(), this.xQuery.getVariables()); + SqlEval sqlEval = new SqlEval(this.dataManager, getContext(), this.xQuery.getProcedureGroup(), this.xQuery.getVariables()); try { SQLXML xml = expr.evaluateXQuery(sqlEval); TupleBatch batch = packResultsIntoBatch(xml); @@ -148,29 +147,25 @@ * @return */ private TupleBatch packResultsIntoBatch(SQLXML srcXML) throws MetaMatrixComponentException{ - try { - List rows = new ArrayList(1); - List row = new ArrayList(1); + List rows = new ArrayList(1); + List row = new ArrayList(1); - TupleSourceID savedId = XMLUtil.saveToBufferManager(this.bufferMgr, this.getContext().getConnectionID(), srcXML, this.chunkSize); + TupleBuffer savedId = XMLUtil.saveToBufferManager(this.bufferMgr, this.getContext().getConnectionID(), srcXML, this.chunkSize); - //for large documents use the buffermanager version instead - if (this.bufferMgr.getFinalRowCount(savedId) > 1) { - srcXML = XMLUtil.getFromBufferManager(this.bufferMgr, savedId, getFormatProperties()); - } - - XMLType xml = new XMLType(srcXML); - this.bufferMgr.setPersistentTupleSource(savedId, xml); - - // now build the top batch with information from the saved one. - row.add(xml); - rows.add(row); - TupleBatch batch = new TupleBatch(1, rows); - batch.setTerminationFlag(true); - return batch; - } catch (TupleSourceNotFoundException e) { - throw new MetaMatrixComponentException(e); + //for large documents use the buffermanager version instead + if (savedId.getRowCount() > 1) { + srcXML = XMLUtil.getFromBufferManager(savedId, getFormatProperties()); } + + XMLType xml = new XMLType(srcXML); + savedId.setContainingLobReference(xml); + + // now build the top batch with information from the saved one. + row.add(xml); + rows.add(row); + TupleBatch batch = new TupleBatch(1, rows); + batch.setTerminationFlag(true); + return batch; } /** Modified: trunk/engine/src/main/java/com/metamatrix/query/sql/lang/Command.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/sql/lang/Command.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/sql/lang/Command.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -319,18 +319,9 @@ */ public abstract int updatingModelCount(QueryMetadataInterface metadata) throws MetaMatrixComponentException; - protected int getSubCommandsUpdatingModelCount(QueryMetadataInterface metadata) throws MetaMatrixComponentException{ - return getSubCommandsUpdatingModelCount(this, metadata); - } - - public static int getSubCommandsUpdatingModelCount(Command object, QueryMetadataInterface metadata) throws MetaMatrixComponentException { - List<Command> subCommands = CommandCollectorVisitor.getCommands(object); - int numCommands = subCommands.size(); + List<Command> subCommands = CommandCollectorVisitor.getCommands(this); for (Command command : subCommands) { - if (numCommands == 1) { - return command.updatingModelCount(metadata); - } if (command.updatingModelCount(metadata) > 0) { return 2; //require a transaction if there is more than 1 statement } Modified: trunk/engine/src/main/java/com/metamatrix/query/sql/lang/Query.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/sql/lang/Query.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/sql/lang/Query.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -429,7 +429,14 @@ if(this.getInto() != null){ return false; } + if (isXML) { + return true; + } List projectedSymbols = getProjectedSymbols(); + return areResultsCachable(projectedSymbols); + } + + static boolean areResultsCachable(List projectedSymbols) { for(int i=0; i<projectedSymbols.size(); i++){ SingleElementSymbol projectedSymbol = (SingleElementSymbol)projectedSymbols.get(i); if(DataTypeManager.isLOB(projectedSymbol.getType())) { @@ -440,7 +447,7 @@ } public int updatingModelCount(QueryMetadataInterface metadata) throws MetaMatrixComponentException{ - if(isXML) { + if(isXML) { //TODO: this is incorrect return 0; } @@ -448,13 +455,7 @@ return 2; } - //any subcommand performing an update requires a transaction - //since we may not be able to roll it back other wise - int count = getSubCommandsUpdatingModelCount(metadata); - if (count > 0) { - return 2; - } - return 0; + return getSubCommandsUpdatingModelCount(metadata); } /** Modified: trunk/engine/src/main/java/com/metamatrix/query/sql/lang/StoredProcedure.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/sql/lang/StoredProcedure.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/sql/lang/StoredProcedure.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -335,7 +335,7 @@ * @see com.metamatrix.query.sql.lang.Command#areResultsCachable() */ public boolean areResultsCachable() { - return true; + return Query.areResultsCachable(getProjectedSymbols()); } /** Modified: trunk/engine/src/main/java/com/metamatrix/query/sql/lang/XQuery.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/sql/lang/XQuery.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/sql/lang/XQuery.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -147,7 +147,7 @@ * @see com.metamatrix.query.sql.lang.Command#areResultsCachable() */ public boolean areResultsCachable() { - return false; + return true; } public int updatingModelCount(QueryMetadataInterface metadata) throws MetaMatrixComponentException { Modified: trunk/engine/src/main/java/com/metamatrix/query/sql/proc/CreateUpdateProcedureCommand.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/sql/proc/CreateUpdateProcedureCommand.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/sql/proc/CreateUpdateProcedureCommand.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -294,7 +294,7 @@ return 0; } Command lastCommand = null; - Statement statement = (Statement)block.getStatements().get(block.getStatements().size() - 1); + Statement statement = block.getStatements().get(block.getStatements().size() - 1); if (statement instanceof CommandStatement) { CommandStatement cmdStatement = (CommandStatement)statement; lastCommand = cmdStatement.getCommand(); Modified: trunk/engine/src/main/java/com/metamatrix/query/sql/util/ValueIteratorSource.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/sql/util/ValueIteratorSource.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/sql/util/ValueIteratorSource.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -22,11 +22,10 @@ package com.metamatrix.query.sql.util; -import java.util.HashSet; +import java.util.Set; import com.metamatrix.api.exception.MetaMatrixComponentException; import com.metamatrix.api.exception.MetaMatrixProcessingException; -import com.metamatrix.common.buffer.TupleSourceNotFoundException; import com.metamatrix.query.sql.symbol.Expression; @@ -45,11 +44,10 @@ * @param valueExpression The expression we are retrieving an iterator for * @return ValueIterator if ready, null otherwise * @throws MetaMatrixComponentException - * @throws TupleSourceNotFoundException * @since 5.0.1 */ ValueIterator getValueIterator(Expression valueExpression) throws MetaMatrixComponentException; - HashSet<Object> getCachedSet(Expression valueExpression) throws MetaMatrixComponentException, MetaMatrixProcessingException; + Set<Object> getCachedSet(Expression valueExpression) throws MetaMatrixComponentException, MetaMatrixProcessingException; } Modified: trunk/engine/src/main/java/com/metamatrix/query/tempdata/TempTableStore.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/tempdata/TempTableStore.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/tempdata/TempTableStore.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -26,8 +26,8 @@ import com.metamatrix.api.exception.MetaMatrixComponentException; import com.metamatrix.api.exception.MetaMatrixProcessingException; +import com.metamatrix.common.buffer.TupleBuffer; import com.metamatrix.common.buffer.TupleSource; -import com.metamatrix.common.buffer.TupleSourceID; import com.metamatrix.query.metadata.TempMetadataStore; import com.metamatrix.query.sql.lang.Command; @@ -48,5 +48,5 @@ public void removeTempTableByName(String tempTableName) throws MetaMatrixComponentException; - public TupleSourceID getTupleSourceID(String tempTableName); + public TupleBuffer getTupleSourceID(String tempTableName); } Modified: trunk/engine/src/main/java/com/metamatrix/query/tempdata/TempTableStoreImpl.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/tempdata/TempTableStoreImpl.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/tempdata/TempTableStoreImpl.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -37,20 +37,15 @@ import com.metamatrix.api.exception.query.QueryProcessingException; import com.metamatrix.common.buffer.BlockedException; import com.metamatrix.common.buffer.BufferManager; -import com.metamatrix.common.buffer.TupleBatch; +import com.metamatrix.common.buffer.TupleBuffer; import com.metamatrix.common.buffer.TupleSource; -import com.metamatrix.common.buffer.TupleSourceID; -import com.metamatrix.common.buffer.TupleSourceNotFoundException; -import com.metamatrix.common.buffer.BufferManager.TupleSourceStatus; import com.metamatrix.common.buffer.BufferManager.TupleSourceType; -import com.metamatrix.core.util.Assertion; import com.metamatrix.query.eval.Evaluator; import com.metamatrix.query.execution.QueryExecPlugin; import com.metamatrix.query.metadata.TempMetadataAdapter; import com.metamatrix.query.metadata.TempMetadataStore; -import com.metamatrix.query.processor.proc.UpdateCountTupleSource; +import com.metamatrix.query.processor.CollectionTupleSource; import com.metamatrix.query.processor.relational.RelationalNode; -import com.metamatrix.query.processor.relational.TupleCollector; import com.metamatrix.query.resolver.util.ResolverUtil; import com.metamatrix.query.sql.lang.Command; import com.metamatrix.query.sql.lang.Create; @@ -74,24 +69,22 @@ private abstract class UpdateTupleSource implements TupleSource { private final String groupKey; - private final TupleSourceID tsId; + private final TupleBuffer oldBuffer; private final TupleSource ts; protected final Map lookup; - private final TupleCollector tc; - private final TupleSourceID newTs; + private final TupleBuffer newBuffer; protected final Evaluator eval; private final Criteria crit; protected int updateCount = 0; private boolean done; - private UpdateTupleSource(String groupKey, TupleSourceID tsId, Criteria crit) throws TupleSourceNotFoundException, MetaMatrixComponentException { + private UpdateTupleSource(String groupKey, TupleBuffer tsId, Criteria crit) throws MetaMatrixComponentException { this.groupKey = groupKey; - this.tsId = tsId; - this.ts = buffer.getTupleSource(tsId); - List columns = buffer.getTupleSchema(tsId); + this.oldBuffer = tsId; + this.ts = tsId.createIndexedTupleSource(); + List columns = tsId.getSchema(); this.lookup = RelationalNode.createLookupMap(columns); - this.newTs = buffer.createTupleSource(columns, sessionID, TupleSourceType.PROCESSOR); - this.tc = new TupleCollector(newTs, buffer); + this.newBuffer = buffer.createTupleBuffer(columns, sessionID, TupleSourceType.PROCESSOR); this.eval = new Evaluator(lookup, null, null); this.crit = crit; } @@ -112,23 +105,15 @@ tupleFailed(tuple); } } - tc.close(); - groupToTupleSourceID.put(groupKey, newTs); - try { - buffer.removeTupleSource(tsId); - }catch(TupleSourceNotFoundException e) { - - } + newBuffer.close(); + groupToTupleSourceID.put(groupKey, newBuffer); + oldBuffer.remove(); done = true; return Arrays.asList(updateCount); } protected void addTuple(List<?> tuple) throws MetaMatrixComponentException { - try { - tc.addTuple(tuple); - } catch (TupleSourceNotFoundException e) { - throw new MetaMatrixComponentException(e); - } + newBuffer.addTuple(tuple); } protected abstract void tuplePassed(List<?> tuple) throws ExpressionEvaluationException, BlockedException, MetaMatrixComponentException; @@ -148,7 +133,7 @@ private BufferManager buffer; private TempMetadataStore tempMetadataStore = new TempMetadataStore(); - private Map<String, TupleSourceID> groupToTupleSourceID = new HashMap<String, TupleSourceID>(); + private Map<String, TupleBuffer> groupToTupleSourceID = new HashMap<String, TupleBuffer>(); private String sessionID; private TempTableStore parentTempTableStore; @@ -169,23 +154,16 @@ //add metadata tempMetadataStore.addTempGroup(tempTableName, columns, false, true); //create tuple source - TupleSourceID tsId = buffer.createTupleSource(columns, sessionID, TupleSourceType.PROCESSOR); - try { - buffer.setStatus(tsId, TupleSourceStatus.FULL); - }catch(TupleSourceNotFoundException e) { - //should never happen because the TupleSourceID is just created - Assertion.failed("Could not find local tuple source for inserting into temp table."); //$NON-NLS-1$ - } - groupToTupleSourceID.put(tempTableName, tsId); + TupleBuffer tupleBuffer = buffer.createTupleBuffer(columns, sessionID, TupleSourceType.PROCESSOR); + tupleBuffer.setFinal(true); //final, but not closed so that we can append on insert + groupToTupleSourceID.put(tempTableName, tupleBuffer); } public void removeTempTableByName(String tempTableName) throws MetaMatrixComponentException { tempMetadataStore.removeTempGroup(tempTableName); - TupleSourceID tsId = this.groupToTupleSourceID.remove(tempTableName); + TupleBuffer tsId = this.groupToTupleSourceID.remove(tempTableName); if(tsId != null) { - try { - buffer.removeTupleSource(tsId); - }catch(TupleSourceNotFoundException e) {} + tsId.remove(); } } @@ -200,12 +178,8 @@ if (!group.isTempGroupSymbol()) { return null; } - TupleSourceID tsId = getTupleSourceID(group.getNonCorrelationName().toUpperCase(), command); - try { - return buffer.getTupleSource(tsId); - }catch(TupleSourceNotFoundException e) { - throw new MetaMatrixComponentException(e); - } + TupleBuffer tsId = getTupleSourceID(group.getNonCorrelationName().toUpperCase(), command); + return tsId.createIndexedTupleSource(); } if (command instanceof ProcedureContainer) { GroupSymbol group = ((ProcedureContainer)command).getGroup(); @@ -213,65 +187,61 @@ return null; } final String groupKey = group.getNonCorrelationName().toUpperCase(); - final TupleSourceID tsId = getTupleSourceID(groupKey, command); + final TupleBuffer tsId = getTupleSourceID(groupKey, command); if (command instanceof Insert) { return addTuple((Insert)command, tsId); } - try { - if (command instanceof Update) { - final Update update = (Update)command; - final Criteria crit = update.getCriteria(); - return new UpdateTupleSource(groupKey, tsId, crit) { - @Override - protected void tuplePassed(List<?> tuple) - throws ExpressionEvaluationException, - BlockedException, MetaMatrixComponentException { - List<Object> newTuple = new ArrayList<Object>(tuple); - for (Map.Entry<ElementSymbol, Expression> entry : update.getChangeList().getClauseMap().entrySet()) { - newTuple.set((Integer)lookup.get(entry.getKey()), eval.evaluate(entry.getValue(), tuple)); - } - updateCount++; - addTuple(newTuple); + if (command instanceof Update) { + final Update update = (Update)command; + final Criteria crit = update.getCriteria(); + return new UpdateTupleSource(groupKey, tsId, crit) { + @Override + protected void tuplePassed(List<?> tuple) + throws ExpressionEvaluationException, + BlockedException, MetaMatrixComponentException { + List<Object> newTuple = new ArrayList<Object>(tuple); + for (Map.Entry<ElementSymbol, Expression> entry : update.getChangeList().getClauseMap().entrySet()) { + newTuple.set((Integer)lookup.get(entry.getKey()), eval.evaluate(entry.getValue(), tuple)); } - - protected void tupleFailed(java.util.List<?> tuple) throws MetaMatrixComponentException { - addTuple(tuple); - } - }; - } - if (command instanceof Delete) { - final Delete delete = (Delete)command; - final Criteria crit = delete.getCriteria(); - if (crit == null) { - int rows = buffer.getRowCount(tsId); - addTempTable(groupKey, buffer.getTupleSchema(tsId), true); - return new UpdateCountTupleSource(rows); - } - return new UpdateTupleSource(groupKey, tsId, crit) { - @Override - protected void tuplePassed(List<?> tuple) - throws ExpressionEvaluationException, - BlockedException, MetaMatrixComponentException { - updateCount++; - } - - protected void tupleFailed(java.util.List<?> tuple) throws MetaMatrixComponentException { - addTuple(tuple); - } - }; - } - } catch (TupleSourceNotFoundException e) { - throw new MetaMatrixComponentException(e); + updateCount++; + addTuple(newTuple); + } + + protected void tupleFailed(java.util.List<?> tuple) throws MetaMatrixComponentException { + addTuple(tuple); + } + }; } + if (command instanceof Delete) { + final Delete delete = (Delete)command; + final Criteria crit = delete.getCriteria(); + if (crit == null) { + int rows = tsId.getRowCount(); + addTempTable(groupKey, tsId.getSchema(), true); + return CollectionTupleSource.createUpdateCountTupleSource(rows); + } + return new UpdateTupleSource(groupKey, tsId, crit) { + @Override + protected void tuplePassed(List<?> tuple) + throws ExpressionEvaluationException, + BlockedException, MetaMatrixComponentException { + updateCount++; + } + + protected void tupleFailed(java.util.List<?> tuple) throws MetaMatrixComponentException { + addTuple(tuple); + } + }; + } } if (command instanceof Create) { addTempTable(((Create)command).getTable().getName().toUpperCase(), ((Create)command).getColumns(), false); - return new UpdateCountTupleSource(0); + return CollectionTupleSource.createUpdateCountTupleSource(0); } if (command instanceof Drop) { String tempTableName = ((Drop)command).getTable().getName().toUpperCase(); removeTempTableByName(tempTableName); - return new UpdateCountTupleSource(0); + return CollectionTupleSource.createUpdateCountTupleSource(0); } return null; } @@ -282,8 +252,8 @@ } } - private TupleSourceID getTupleSourceID(String tempTableID, Command command) throws MetaMatrixComponentException, QueryProcessingException{ - TupleSourceID tsID = groupToTupleSourceID.get(tempTableID); + private TupleBuffer getTupleSourceID(String tempTableID, Command command) throws MetaMatrixComponentException, QueryProcessingException{ + TupleBuffer tsID = groupToTupleSourceID.get(tempTableID); if(tsID != null) { return tsID; } @@ -317,31 +287,24 @@ return groupToTupleSourceID.get(tempTableID); } - private TupleSource addTuple(Insert insert, TupleSourceID tsId) throws MetaMatrixComponentException, ExpressionEvaluationException { + private TupleSource addTuple(Insert insert, TupleBuffer tsId) throws MetaMatrixComponentException, ExpressionEvaluationException { GroupSymbol group = insert.getGroup(); int tuplesAdded = 0; try { - int rowCount = buffer.getRowCount(tsId); - TupleBatch tupleBatch; List<ElementSymbol> elements = ResolverUtil.resolveElementsInGroup(group, new TempMetadataAdapter(null, tempMetadataStore)); List<List<Object>> tuples = getBulkRows(insert, elements); - tuplesAdded = tuples.size(); - - // Buffer manager has 1 based index for the tuple sources - tupleBatch = new TupleBatch((++rowCount), tuples); + for (List<Object> list : tuples) { + tsId.addTuple(list); + } - buffer.addTupleBatch(tsId, tupleBatch); - } catch (TupleSourceNotFoundException err) { - throw new MetaMatrixComponentException(err); - } catch (BlockedException err) { - throw new MetaMatrixComponentException(err); + tuplesAdded = tuples.size(); } catch (QueryMetadataException err) { throw new MetaMatrixComponentException(err); } - return new UpdateCountTupleSource(tuplesAdded); + return CollectionTupleSource.createUpdateCountTupleSource(tuplesAdded); } public static List<List<Object>> getBulkRows(Insert insert, List<ElementSymbol> elements) throws ExpressionEvaluationException, BlockedException, MetaMatrixComponentException { @@ -403,7 +366,7 @@ return new HashSet<String>(this.groupToTupleSourceID.keySet()); } - public TupleSourceID getTupleSourceID(String tempTableName) { + public TupleBuffer getTupleSourceID(String tempTableName) { return groupToTupleSourceID.get(tempTableName.toUpperCase()); } } Modified: trunk/engine/src/main/java/com/metamatrix/query/util/CommandContext.java =================================================================== --- trunk/engine/src/main/java/com/metamatrix/query/util/CommandContext.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/com/metamatrix/query/util/CommandContext.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -49,57 +49,61 @@ * framework. */ public class CommandContext implements Cloneable { + + private static class GlobalState { + /** Uniquely identify the command being processed */ + private Object processorID; + + /** Identify a group of related commands, which typically get cleaned up together */ + private String connectionID; - /** Uniquely identify the command being processed */ - private Object processorID; - - /** Identify a group of related commands, which typically get cleaned up together */ - private String connectionID; + private int processorBatchSize = BufferConfig.DEFAULT_PROCESSOR_BATCH_SIZE; + + private int connectorBatchSize = BufferConfig.DEFAULT_CONNECTOR_BATCH_SIZE; - private int processorBatchSize = BufferConfig.DEFAULT_PROCESSOR_BATCH_SIZE; - - private int connectorBatchSize = BufferConfig.DEFAULT_CONNECTOR_BATCH_SIZE; + private String userName; + + private Serializable commandPayload; + + private String vdbName; + + private String vdbVersion; + + private Properties environmentProperties; + + /** Indicate whether data should be dumped for debugging purposes while processing the query */ + private boolean processDebug; + + /** Indicate whether statistics should be collected for relational node processing*/ + private boolean collectNodeStatistics; + + private int streamingBatchSize; + + private Random random = null; + + private SecurityFunctionEvaluator securityFunctionEvaluator; + + private TimeZone timezone = TimeZone.getDefault(); + + private PlanToProcessConverter planToProcessConverter; + + private QueryProcessor.ProcessorFactory queryProcessorFactory; + + private boolean sessionFunctionEvaluated; + + private Set<String> groups; + + private long timeSliceEnd = Long.MAX_VALUE; + + private long timeoutEnd = Long.MAX_VALUE; + } + + private GlobalState globalState = new GlobalState(); - private String userName; - - private Serializable commandPayload; - - private String vdbName; - - private String vdbVersion; - - private Properties environmentProperties; - - /** Indicate whether data should be dumped for debugging purposes while processing the query */ - private boolean processDebug; - - /** Indicate whether statistics should be collected for relational node processing*/ - private boolean collectNodeStatistics; - - private int streamingBatchSize; - - private Random random = null; - - private Stack<String> recursionStack = null; - - private SecurityFunctionEvaluator securityFunctionEvaluator; - - private Object tempTableStore; - - private TimeZone timezone = TimeZone.getDefault(); - - private PlanToProcessConverter planToProcessConverter; - - private QueryProcessor.ProcessorFactory queryProcessorFactory; - private VariableContext variableContext = new VariableContext(); - - private CommandContext parent; - - private boolean sessionFunctionEvaluated; - - private Set<String> groups; - + private Object tempTableStore; + private Stack<String> recursionStack; + /** * Construct a new context. * @param collectNodeStatistics TODO @@ -128,66 +132,30 @@ } - protected CommandContext(CommandContext context) { - setConnectionID(context.connectionID); - - // Reuse existing processor ID - may be overridden - setProcessorID(context.processorID); - - setUserName(context.userName); - setCommandPayload(context.commandPayload); - setVdbName(context.vdbName); - setVdbVersion(context.vdbVersion); - setEnvironmentProperties(context.environmentProperties); - setProcessDebug(context.processDebug); - setProcessorBatchSize(context.processorBatchSize); - setConnectorBatchSize(context.connectorBatchSize); - setRandom(context.random); - if (context.recursionStack != null) { - this.recursionStack = (Stack)context.recursionStack.clone(); - } - this.setSecurityFunctionEvaluator(context.getSecurityFunctionEvaluator()); - this.planToProcessConverter = context.planToProcessConverter; - this.queryProcessorFactory = context.queryProcessorFactory; - this.variableContext = context.variableContext; - this.parent = context; - } - public CommandContext() { } - public CommandContext getParent() { - return parent; - } - public boolean isSessionFunctionEvaluated() { - if (parent != null) { - return parent.isSessionFunctionEvaluated(); - } - return sessionFunctionEvaluated; + return globalState.sessionFunctionEvaluated; } public void setSessionFunctionEvaluated(boolean sessionFunctionEvaluated) { - if (parent != null) { - parent.setSessionFunctionEvaluated(sessionFunctionEvaluated); - } else { - this.sessionFunctionEvaluated = sessionFunctionEvaluated; - } + globalState.sessionFunctionEvaluated = sessionFunctionEvaluated; } /** * @return */ public Object getProcessorID() { - return processorID; + return globalState.processorID; } public boolean getProcessDebug() { - return this.processDebug; + return globalState.processDebug; } public void setProcessDebug(boolean processDebug) { - this.processDebug = processDebug; + globalState.processDebug = processDebug; } /** @@ -195,43 +163,49 @@ */ public void setProcessorID(Object object) { ArgCheck.isNotNull(object); - processorID = object; + globalState.processorID = object; } public Object clone() { - return new CommandContext(this); + CommandContext clone = new CommandContext(); + clone.globalState = this.globalState; + clone.variableContext = this.variableContext; + if (this.recursionStack != null) { + clone.recursionStack = (Stack<String>)this.recursionStack.clone(); + } + return clone; } public String toString() { - return "CommandContext: " + processorID; //$NON-NLS-1$ + return "CommandContext: " + globalState.processorID; //$NON-NLS-1$ } /** * @return String */ public String getConnectionID() { - return connectionID; + return globalState.connectionID; } /** * @return String */ public String getUserName() { - return userName; + return globalState.userName; } /** * @return String */ public String getVdbName() { - return vdbName; + return globalState.vdbName; } /** * @return String */ public String getVdbVersion() { - return vdbVersion; + return globalState.vdbVersion; } /** @@ -239,7 +213,7 @@ * @param connectionID The connectionID to set */ public void setConnectionID(String connectionID) { - this.connectionID = connectionID; + this.globalState.connectionID = connectionID; } /** @@ -247,7 +221,7 @@ * @param userName The userName to set */ public void setUserName(String userName) { - this.userName = userName; + this.globalState.userName = userName; } /** @@ -255,7 +229,7 @@ * @param vdbName The vdbName to set */ public void setVdbName(String vdbName) { - this.vdbName = vdbName; + this.globalState.vdbName = vdbName; } /** @@ -263,22 +237,22 @@ * @param vdbVersion The vdbVersion to set */ public void setVdbVersion(String vdbVersion) { - this.vdbVersion = vdbVersion; + this.globalState.vdbVersion = vdbVersion; } public Properties getEnvironmentProperties() { - return environmentProperties; + return globalState.environmentProperties; } public void setEnvironmentProperties(Properties properties) { - environmentProperties = properties; + globalState.environmentProperties = properties; } public Serializable getCommandPayload() { - return this.commandPayload; + return this.globalState.commandPayload; } public void setCommandPayload(Serializable commandPayload) { - this.commandPayload = commandPayload; + this.globalState.commandPayload = commandPayload; } /** @@ -286,64 +260,58 @@ * @since 4.2 */ public void setCollectNodeStatistics(boolean collectNodeStatistics) { - this.collectNodeStatistics = collectNodeStatistics; + this.globalState.collectNodeStatistics = collectNodeStatistics; } public boolean getCollectNodeStatistics() { - return this.collectNodeStatistics; + return this.globalState.collectNodeStatistics; } public int getStreamingBatchSize() { - return streamingBatchSize; + return globalState.streamingBatchSize; } public void setStreamingBatchSize(int streamingBatchSize) { - this.streamingBatchSize = streamingBatchSize; + this.globalState.streamingBatchSize = streamingBatchSize; } public int getConnectorBatchSize() { - return this.connectorBatchSize; + return this.globalState.connectorBatchSize; } public void setConnectorBatchSize(int connectorBatchSize) { - this.connectorBatchSize = connectorBatchSize; + this.globalState.connectorBatchSize = connectorBatchSize; } public int getProcessorBatchSize() { - return this.processorBatchSize; + return this.globalState.processorBatchSize; } public void setProcessorBatchSize(int processorBatchSize) { - this.processorBatchSize = processorBatchSize; + this.globalState.processorBatchSize = processorBatchSize; } public double getNextRand() { - if (parent != null) { - return parent.getNextRand(); - } - if (random == null) { - random = new Random(); + if (globalState.random == null) { + globalState.random = new Random(); } - return random.nextDouble(); + return globalState.random.nextDouble(); } public double getNextRand(long seed) { - if (parent != null) { - return parent.getNextRand(seed); - } - if (random == null) { - random = new Random(); + if (globalState.random == null) { + globalState.random = new Random(); } - random.setSeed(seed); - return random.nextDouble(); + globalState.random.setSeed(seed); + return globalState.random.nextDouble(); } void setRandom(Random random) { - this.random = random; + this.globalState.random = random; } public void pushCall(String value) throws QueryProcessingException { @@ -373,14 +341,14 @@ * @return Returns the securityFunctionEvaluator. */ public SecurityFunctionEvaluator getSecurityFunctionEvaluator() { - return this.securityFunctionEvaluator; + return this.globalState.securityFunctionEvaluator; } /** * @param securityFunctionEvaluator The securityFunctionEvaluator to set. */ public void setSecurityFunctionEvaluator(SecurityFunctionEvaluator securityFunctionEvaluator) { - this.securityFunctionEvaluator = securityFunctionEvaluator; + this.globalState.securityFunctionEvaluator = securityFunctionEvaluator; } public Object getTempTableStore() { @@ -392,23 +360,23 @@ } public TimeZone getServerTimeZone() { - return timezone; + return globalState.timezone; } public void setPlanToProcessConverter(PlanToProcessConverter planToProcessConverter) { - this.planToProcessConverter = planToProcessConverter; + this.globalState.planToProcessConverter = planToProcessConverter; } public PlanToProcessConverter getPlanToProcessConverter() { - return planToProcessConverter; + return globalState.planToProcessConverter; } public QueryProcessor.ProcessorFactory getQueryProcessorFactory() { - return this.queryProcessorFactory; + return this.globalState.queryProcessorFactory; } public void setQueryProcessorFactory(QueryProcessor.ProcessorFactory queryProcessorFactory) { - this.queryProcessorFactory = queryProcessorFactory; + this.globalState.queryProcessorFactory = queryProcessorFactory; } public VariableContext getVariableContext() { @@ -436,11 +404,27 @@ } public Set<String> getGroups() { - return groups; + return globalState.groups; } public void setGroups(Set<String> groups) { - this.groups = groups; + this.globalState.groups = groups; } + public long getTimeSliceEnd() { + return globalState.timeSliceEnd; + } + + public long getTimeoutEnd() { + return globalState.timeoutEnd; + } + + public void setTimeSliceEnd(long timeSliceEnd) { + globalState.timeSliceEnd = timeSliceEnd; + } + + public void setTimeoutEnd(long timeoutEnd) { + globalState.timeoutEnd = timeoutEnd; + } + } Deleted: trunk/engine/src/main/java/org/teiid/dqp/internal/cache/CacheID.java =================================================================== --- trunk/engine/src/main/java/org/teiid/dqp/internal/cache/CacheID.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/org/teiid/dqp/internal/cache/CacheID.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -1,78 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package org.teiid.dqp.internal.cache; - -import java.io.Serializable; -import java.util.List; - -import com.metamatrix.core.util.HashCodeUtil; - -public class CacheID implements Serializable { - private String scopeID; - private String command; - private int hashCode; - private List preparedStatementValues; - - public CacheID(String scopeID, String command){ - this(scopeID, command, null); - } - - public CacheID(String scopeID, String command, List preparedStatementValue){ - this.scopeID = scopeID; - this.command = command; - this.preparedStatementValues = preparedStatementValue; - hashCode = HashCodeUtil.expHashCode(HashCodeUtil.hashCode(HashCodeUtil.hashCode(0, scopeID), command), preparedStatementValues); - } - - public String getCommand() { - return command; - } - - public boolean equals(Object obj){ - if(obj == this) { - return true; - } else if(! (obj instanceof CacheID)) { - return false; - } else { - CacheID that = (CacheID)obj; - return this.scopeID.equals(that.scopeID) - && this.command.equals(that.command) - && compareParamValues(preparedStatementValues, that.preparedStatementValues); - } - } - - private boolean compareParamValues(List thisPreparedStatementValues, List thatPreparedStatementValues) { - if(thisPreparedStatementValues == null && thatPreparedStatementValues == null){ - return true; - } - if(thisPreparedStatementValues == null || thatPreparedStatementValues == null){ - return false; - } - return thisPreparedStatementValues.equals(thatPreparedStatementValues); - } - - public int hashCode() { - return hashCode; - } - -} Deleted: trunk/engine/src/main/java/org/teiid/dqp/internal/cache/CacheResults.java =================================================================== --- trunk/engine/src/main/java/org/teiid/dqp/internal/cache/CacheResults.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/org/teiid/dqp/internal/cache/CacheResults.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -1,125 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package org.teiid.dqp.internal.cache; - -import java.io.Serializable; -import java.util.Arrays; -import java.util.List; - -import com.metamatrix.core.MetaMatrixRuntimeException; -import com.metamatrix.dqp.DQPPlugin; -import com.metamatrix.query.analysis.AnalysisRecord; -import com.metamatrix.query.sql.lang.Command; -import com.metamatrix.query.sql.symbol.SingleElementSymbol; - -public class CacheResults implements Serializable { - private List[] results; - private AnalysisRecord analysisRecord; - private Command command; - - private boolean isFinal; - private int firstRow = 1; - //size of this results in memory - private long size= -1; - - public CacheResults(List[] results, int firstRow, boolean isFinal){ - this.results = results; - this.firstRow = firstRow; - this.isFinal = isFinal; - } - - public int getFirstRow() { - return firstRow; - } - - public boolean isFinal() { - return isFinal; - } - - public List<?>[] getResults() { - return results; - } - - public List<SingleElementSymbol> getElements() { - if (command == null) { - return null; - } - return command.getProjectedSymbols(); - } - - public Command getCommand() { - return command; - } - - public void setCommand(Command command) { - this.command = command; - } - - public long getSize() { - return size; - } - - public void setSize(long size) { - this.size = size; - } - - public AnalysisRecord getAnalysisRecord() { - return analysisRecord; - } - - public void setAnalysisRecord(AnalysisRecord analysisRecord) { - this.analysisRecord = analysisRecord; - } - - public int getFinalRow() { - if (this.isFinal) { - return results.length; - } - return -1; - } - - //add the results to the existing one, this is used - //when building the batched results - boolean addResults(CacheResults cacheResults){ - if (this.firstRow + results.length != cacheResults.getFirstRow()) { - throw new MetaMatrixRuntimeException(DQPPlugin.Util.getString("ResultSetCache.1"));//$NON-NLS-1$ - } - this.size += cacheResults.size; - List[] batchResults = cacheResults.getResults(); - if (results == null) { - this.results = batchResults; - } else if (batchResults.length > 0){ - this.results = Arrays.copyOf(this.results, this.results.length + batchResults.length); - System.arraycopy(batchResults, 0, this.results, this.results.length - batchResults.length, batchResults.length); - } - - if(cacheResults.isFinal()){ - this.command = cacheResults.getCommand(); - this.analysisRecord = cacheResults.getAnalysisRecord(); - this.firstRow = 1; - this.isFinal = true; - } - return true; - } - -} Deleted: trunk/engine/src/main/java/org/teiid/dqp/internal/cache/ResultSetCache.java =================================================================== --- trunk/engine/src/main/java/org/teiid/dqp/internal/cache/ResultSetCache.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/org/teiid/dqp/internal/cache/ResultSetCache.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -1,232 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package org.teiid.dqp.internal.cache; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.teiid.dqp.internal.process.DQPWorkContext; - -import com.metamatrix.cache.Cache; -import com.metamatrix.cache.CacheConfiguration; -import com.metamatrix.cache.CacheFactory; -import com.metamatrix.cache.Cache.Type; -import com.metamatrix.cache.CacheConfiguration.Policy; -import com.metamatrix.common.buffer.TupleBatch; -import com.metamatrix.common.buffer.impl.BufferManagerImpl; -import com.metamatrix.common.buffer.impl.SizeUtility; -import com.metamatrix.core.util.HashCodeUtil; - -/** - * Used to cache ResultSet based on the exact match of sql string. - */ -public class ResultSetCache { - - static class TempKey { - CacheID cacheID; - Object requestID; - - public TempKey(CacheID cacheID, Object requestID) { - this.cacheID = cacheID; - this.requestID = requestID; - } - @Override - public int hashCode() { - return HashCodeUtil.hashCode(cacheID.hashCode(), requestID.hashCode()); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (!(obj instanceof TempKey)) { - return false; - } - TempKey other = (TempKey) obj; - return (cacheID.equals(other.cacheID) && requestID.equals(other.requestID)); - } - } - - //configurable parameters - public static final String RS_CACHE_MAX_ENTRIES = "maxEntries"; //$NON-NLS-1$ - public static final String RS_CACHE_MAX_SIZE = "maxEntrySize"; //$NON-NLS-1$ - public static final String RS_CACHE_MAX_AGE = "maxAge"; //$NON-NLS-1$ - public static final String RS_CACHE_SCOPE = "scope"; //$NON-NLS-1$ - - //constants - public static final String RS_CACHE_SCOPE_VDB = "vdb"; //$NON-NLS-1$ - public static final String RS_CACHE_SCOPE_CONN = "connection"; //$NON-NLS-1$ - private static final char DELIMITOR = '.'; - - private Cache<CacheID, CacheResults> cache; - private String scope = RS_CACHE_SCOPE_VDB; - private Map<TempKey, CacheResults> tempBatchResults = new HashMap<TempKey, CacheResults>(); - private int maxEntrySize = 100 * 1024; //bytes - private int maxAge = 60 * 60; // seconds - private int maxEntries = 1024; - - public void start(CacheFactory cacheFactory) { - this.cache = cacheFactory.get(Type.RESULTSET, new CacheConfiguration(Policy.MRU, maxAge, maxEntries)); - } - - public void setMaxEntries(int maxEntries) { - this.maxEntries = maxEntries; - } - - public void setMaxEntrySize(int maxEntrySize) { - this.maxEntrySize = maxEntrySize; - } - - public void setMaxAge(int maxAge) { - if (maxAge <= 0) { - this.maxAge = Integer.MAX_VALUE; - } - this.maxAge = Math.max(1, maxAge / 1000); - } - - public void setScope(String scope) { - this.scope = scope; - } - - //interval is 1 based. - public final CacheResults getResults(CacheID cacheID, int[] interval){ - CacheResults cacheResults = null; - cacheResults = cache.get(cacheID); - if(cacheResults == null){ - return null; - } - if (interval == null) { - return cacheResults; - } - int firstRow = interval[0] - 1; - int resultSize = cacheResults.getResults().length; - int finalRow = resultSize - 1; - int lastRow = Math.min(finalRow, interval[1] - 1); - if(resultSize == 0 || (firstRow ==0 && lastRow == finalRow)){ - //the whole results - return cacheResults; - } - int batchSize = lastRow - firstRow + 1; - List<?>[] resultsPart = new List[batchSize]; - System.arraycopy(cacheResults.getResults(), firstRow, resultsPart, 0, batchSize); - CacheResults newCacheResults = new CacheResults(resultsPart, firstRow + 1, lastRow == finalRow); - newCacheResults.setCommand(cacheResults.getCommand()); - newCacheResults.setAnalysisRecord(cacheResults.getAnalysisRecord()); - return newCacheResults; - } - - public boolean hasResults(CacheID cacheID){ - boolean hasResults = cache.get(cacheID) != null; - return hasResults; - } - - /** - * if valueID is not null, it is lob (XML document) chunk - * @return true if the result was cachable - */ - public boolean setResults(CacheID cacheID, CacheResults cacheResults, Object requestID){ - List<?>[] results = cacheResults.getResults(); - if(cacheResults.getSize() == TupleBatch.UNKNOWN_SIZE){ - cacheResults.setSize(SizeUtility.getBatchSize(BufferManagerImpl.getTypeNames(cacheResults.getElements()), results)); - } - - TempKey key = new TempKey(cacheID, requestID); - //do not cache if it is over cache limit - if(isOverCacheLimit(cacheResults.getSize())){ - removeTempResults(key); - return false; - } - - synchronized(tempBatchResults){ - CacheResults savedResults = tempBatchResults.get(key); - if(savedResults == null){ - if (tempBatchResults.size() >= maxEntries) { - return false; - } - savedResults = cacheResults; - tempBatchResults.put(key, cacheResults); - } else if(!savedResults.addResults(cacheResults)){ - removeTempResults(key); - return false; - } - - //do not cache if it is over cache limit - if(isOverCacheLimit(savedResults.getSize())){ - removeTempResults(key); - return false; - } - - if(savedResults.isFinal()){ - tempBatchResults.remove(cacheID); - cache.put(cacheID, savedResults); - } - } - - return true; - } - - private boolean isOverCacheLimit(long sizeToAdd) { - if(maxEntrySize == 0 || sizeToAdd == 0){ - return false; - } - return sizeToAdd > maxEntrySize; - } - - public void removeTempResults(CacheID cacheID, Object requestID){ - removeTempResults(new TempKey(cacheID, requestID)); - } - - public void removeTempResults(TempKey cacheID){ - synchronized(tempBatchResults){ - tempBatchResults.remove(cacheID); - } - } - - public void clear(){ - cache.clear(); - synchronized(tempBatchResults){ - tempBatchResults.clear(); - } - } - - public void shutDown(){ - clear(); - } - - public String getCacheScope(){ - return scope; - } - - public CacheID createCacheID(DQPWorkContext workContext, String command, List<?> parameterValues){ - String scopeID = null; - if(RS_CACHE_SCOPE_VDB.equalsIgnoreCase(getCacheScope())){ - scopeID = workContext.getVdbName() + DELIMITOR + workContext.getVdbVersion(); - }else{ - scopeID = workContext.getConnectionID(); - } - return new CacheID(scopeID, command, parameterValues); - } - -} Copied: trunk/engine/src/main/java/org/teiid/dqp/internal/process/CacheResults.java (from rev 1666, trunk/engine/src/main/java/org/teiid/dqp/internal/cache/CacheResults.java) =================================================================== --- trunk/engine/src/main/java/org/teiid/dqp/internal/process/CacheResults.java (rev 0) +++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/CacheResults.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -0,0 +1,68 @@ +/* + * JBoss, Home of Professional Open Source. + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA. + */ + +package org.teiid.dqp.internal.process; + +import java.io.Serializable; +import java.util.List; + +import com.metamatrix.common.buffer.TupleSourceID; +import com.metamatrix.query.analysis.AnalysisRecord; +import com.metamatrix.query.sql.lang.Command; +import com.metamatrix.query.sql.symbol.SingleElementSymbol; + +public class CacheResults implements Serializable { + private TupleSourceID results; + private AnalysisRecord analysisRecord; + private Command command; + + public CacheResults(TupleSourceID results){ + this.results = results; + } + + public TupleSourceID getResults() { + return results; + } + + public List<SingleElementSymbol> getElements() { + if (command == null) { + return null; + } + return command.getProjectedSymbols(); + } + + public Command getCommand() { + return command; + } + + public void setCommand(Command command) { + this.command = command; + } + + public AnalysisRecord getAnalysisRecord() { + return analysisRecord; + } + + public void setAnalysisRecord(AnalysisRecord analysisRecord) { + this.analysisRecord = analysisRecord; + } +} Deleted: trunk/engine/src/main/java/org/teiid/dqp/internal/process/CachedRequestWorkItem.java =================================================================== --- trunk/engine/src/main/java/org/teiid/dqp/internal/process/CachedRequestWorkItem.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/CachedRequestWorkItem.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -1,102 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package org.teiid.dqp.internal.process; - -import java.util.List; - -import javax.transaction.SystemException; - -import org.teiid.dqp.internal.cache.CacheResults; - -import com.metamatrix.api.exception.MetaMatrixComponentException; -import com.metamatrix.api.exception.MetaMatrixProcessingException; -import com.metamatrix.common.buffer.BlockedException; -import com.metamatrix.common.buffer.BlockedOnMemoryException; -import com.metamatrix.common.buffer.TupleBatch; -import com.metamatrix.common.buffer.TupleSourceNotFoundException; -import com.metamatrix.common.comm.api.ResultsReceiver; -import com.metamatrix.core.MetaMatrixCoreException; -import com.metamatrix.dqp.DQPPlugin; -import com.metamatrix.dqp.message.RequestID; -import com.metamatrix.dqp.message.RequestMessage; -import com.metamatrix.dqp.message.ResultsMessage; -import com.metamatrix.query.sql.lang.Command; - -public class CachedRequestWorkItem extends RequestWorkItem { - - public CachedRequestWorkItem(DQPCore dqpCore, RequestMessage requestMsg, Request request, ResultsReceiver<ResultsMessage> receiver, RequestID requestID, DQPWorkContext workContext, Command originalCommand) { - super(dqpCore, requestMsg, request, receiver, requestID, workContext); - this.originalCommand = originalCommand; - } - - @Override - protected void processNew() throws MetaMatrixComponentException, MetaMatrixProcessingException { - request.initMetadata(); - request.validateAccess(originalCommand); - this.request = null; - } - - @Override - protected void processMore() throws SystemException, BlockedException, - MetaMatrixCoreException { - //do nothing - } - - - @Override - protected void sendResultsIfNeeded(TupleBatch batch) - throws BlockedOnMemoryException, MetaMatrixComponentException, - TupleSourceNotFoundException { - synchronized (this.resultsCursor) { - if (!this.resultsCursor.resultsRequested) { - return; - } - } - CacheResults cResult = rsCache.getResults(cid, new int[] {this.resultsCursor.begin, this.resultsCursor.end}); - if (cResult == null) { - throw new MetaMatrixComponentException(DQPPlugin.Util.getString("CachedRequestWorkItem.not_available")); //$NON-NLS-1$ - } - List results[] = cResult.getResults(); - int firstRow = cResult.getFirstRow(); - - ResultsMessage response = createResultsMessage(requestMsg, results, cResult.getElements(), cResult.getAnalysisRecord()); - response.setFirstRow(firstRow); - response.setLastRow(firstRow + results.length - 1); - - boolean isFinal = cResult.isFinal(); - if(isFinal){ - response.setFinalRow(cResult.getFinalRow()); - } - - this.resultsCursor.resultsSent(); - this.resultsReceiver.receiveResults(response); - } - - @Override - protected void attemptClose() { - this.isClosed = true; - dqpCore.logMMCommand(this, false, false, -1); - this.dqpCore.removeRequest(this); - } - -} \ No newline at end of file Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/process/DQPCore.java =================================================================== --- trunk/engine/src/main/java/org/teiid/dqp/internal/process/DQPCore.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/DQPCore.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -39,10 +39,7 @@ import javax.transaction.xa.Xid; import org.teiid.connector.xa.api.TransactionContext; -import org.teiid.dqp.internal.cache.CacheID; -import org.teiid.dqp.internal.cache.CacheResults; import org.teiid.dqp.internal.cache.DQPContextCache; -import org.teiid.dqp.internal.cache.ResultSetCache; import com.google.inject.Inject; import com.google.inject.Singleton; @@ -149,7 +146,7 @@ private PreparedPlanCache prepPlanCache; private TransactionService transactionService; private MetadataService metadataService; - private ResultSetCache rsCache; + //private ResultSetCache rsCache; // Query worker pool for processing plans private WorkerPool processWorkerPool; @@ -266,21 +263,10 @@ state.tempTableStoreImpl, workContext, chunkSize); - RequestWorkItem workItem = null; - ResultsFuture<ResultsMessage> resultsFuture = new ResultsFuture<ResultsMessage>(); - - if(this.rsCache != null && requestMsg.useResultSetCache()) { - CacheID cID = this.rsCache.createCacheID(workContext, requestMsg.getCommandString(), requestMsg.getParameterValues()); - CacheResults cr = this.rsCache.getResults(cID, new int[]{1, 1}); - if (cr != null) { - workItem = new CachedRequestWorkItem(this, requestMsg, request, resultsFuture.getResultsReceiver(), requestID, workContext, cr.getCommand()); - } - } - if (workItem == null) { - workItem = new RequestWorkItem(this, requestMsg, request, resultsFuture.getResultsReceiver(), requestID, workContext); - } - + + RequestWorkItem workItem = new RequestWorkItem(this, requestMsg, request, resultsFuture.getResultsReceiver(), requestID, workContext); + logMMCommand(workItem, true, false, 0); //TODO: there is no transaction at this point addRequest(requestID, workItem, state); @@ -423,7 +409,7 @@ // cleanup the buffer manager try { - bufferManager.removeTupleSources(sessionId); + bufferManager.removeTupleBuffers(sessionId); } catch (Exception e) { LogManager.logWarning(LogConstants.CTX_DQP, e, "Failed to remove buffered tuples for connection " + sessionId); //$NON-NLS-1$ } @@ -461,7 +447,7 @@ return markCancelled; } - public ResultsFuture<?> closeRequest(long requestId) throws MetaMatrixProcessingException { + public ResultsFuture<?> closeRequest(long requestId) throws MetaMatrixProcessingException, MetaMatrixComponentException { DQPWorkContext workContext = DQPWorkContext.getWorkContext(); closeRequest(workContext.getRequestID(requestId)); return null; @@ -470,8 +456,9 @@ /** * Close the request with given ID * @param requestID + * @throws MetaMatrixComponentException */ - void closeRequest(RequestID requestID) { + void closeRequest(RequestID requestID) throws MetaMatrixComponentException { if (LogManager.isMessageToBeRecorded(LogConstants.CTX_DQP, MessageLevel.DETAIL)) { LogManager.logDetail(LogConstants.CTX_DQP, "closeQuery for requestID=" + requestID); //$NON-NLS-1$ } @@ -496,9 +483,9 @@ public void clearResultSetCache() { //clear cache in server - if(rsCache != null){ + /*if(rsCache != null){ rsCache.clear(); - } + }*/ } void logMMCommand(RequestWorkItem workItem, boolean isBegin, boolean isCancel, int rowCount) { @@ -569,9 +556,9 @@ return transactionService; } - ResultSetCache getRsCache() { + /*ResultSetCache getRsCache() { return rsCache; - } + }*/ int getProcessorTimeSlice() { return this.processorTimeslice; @@ -632,11 +619,11 @@ this.chunkSize = PropertiesUtils.getIntProperty(props, DQPEmbeddedProperties.STREAMING_BATCH_SIZE, 10) * 1024; //result set cache - if(PropertiesUtils.getBooleanProperty(props, DQPEmbeddedProperties.USE_RESULTSET_CACHE, false)){ + /*if(PropertiesUtils.getBooleanProperty(props, DQPEmbeddedProperties.USE_RESULTSET_CACHE, false)){ this.rsCache = new ResultSetCache(); PropertiesUtils.setBeanProperties(this.rsCache, props, "ResultSetCache"); //$NON-NLS-1$ this.rsCache.start(cacheFactory); - } + }*/ //prepared plan cache int maxSizeTotal = PropertiesUtils.getIntProperty(props, DQPEmbeddedProperties.MAX_PLAN_CACHE_SIZE, PreparedPlanCache.DEFAULT_MAX_SIZE_TOTAL); Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/process/DataTierManagerImpl.java =================================================================== --- trunk/engine/src/main/java/org/teiid/dqp/internal/process/DataTierManagerImpl.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/DataTierManagerImpl.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -54,7 +54,6 @@ import com.metamatrix.common.buffer.TupleBatch; import com.metamatrix.common.buffer.TupleSource; import com.metamatrix.common.comm.api.ResultsReceiver; -import com.metamatrix.common.log.LogManager; import com.metamatrix.common.vdb.api.ModelInfo; import com.metamatrix.core.CoreConstants; import com.metamatrix.core.util.Assertion; @@ -70,8 +69,8 @@ import com.metamatrix.dqp.service.DataService; import com.metamatrix.dqp.service.MetadataService; import com.metamatrix.dqp.service.VDBService; -import com.metamatrix.dqp.util.LogConstants; import com.metamatrix.metadata.runtime.api.MetadataSourceUtil; +import com.metamatrix.query.processor.CollectionTupleSource; import com.metamatrix.query.processor.ProcessorDataManager; import com.metamatrix.query.processor.QueryProcessor; import com.metamatrix.query.sql.ReservedWords; @@ -80,7 +79,6 @@ import com.metamatrix.query.sql.lang.StoredProcedure; import com.metamatrix.query.sql.lang.UnaryFromClause; import com.metamatrix.query.sql.symbol.GroupSymbol; -import com.metamatrix.query.sql.symbol.SingleElementSymbol; import com.metamatrix.query.util.CommandContext; public class DataTierManagerImpl implements ProcessorDataManager { @@ -105,37 +103,6 @@ GETVDBRESOURCEPATHS } - private class CollectionTupleSource implements TupleSource { - - private Iterator<List<Object>> tuples; - private List<SingleElementSymbol> schema; - - public CollectionTupleSource(Iterator<List<Object>> tuples, - List<SingleElementSymbol> schema) { - this.tuples = tuples; - this.schema = schema; - } - - @Override - public List<?> nextTuple() throws MetaMatrixComponentException, - MetaMatrixProcessingException { - if (tuples.hasNext()) { - return tuples.next(); - } - return null; - } - - @Override - public List<SingleElementSymbol> getSchema() { - return schema; - } - - @Override - public void closeSource() throws MetaMatrixComponentException { - - } - } - // Resources private DQPCore requestMgr; private DataService dataService; @@ -294,11 +261,11 @@ break; case REFERENCEKEYCOLUMNS: for (ForeignKey key : table.getForeignKeys()) { - int postition = 0; + short postition = 0; for (Column column : key.getColumns()) { Table pkTable = key.getPrimaryKey().getTable(); rows.add(Arrays.asList(vdbName, pkTable.getSchema().getName(), pkTable.getName(), key.getPrimaryKey().getColumns().get(postition).getName(), vdbName, schema.getName(), table.getName(), column.getName(), - ++postition, (short)DatabaseMetaData.importedKeyNoAction, (short)DatabaseMetaData.importedKeyNoAction, key.getName(), key.getPrimaryKey().getName(), (short)DatabaseMetaData.importedKeyInitiallyDeferred)); + ++postition, DatabaseMetaData.importedKeyNoAction, DatabaseMetaData.importedKeyNoAction, key.getName(), key.getPrimaryKey().getName(), DatabaseMetaData.importedKeyInitiallyDeferred)); } } break; @@ -467,19 +434,16 @@ final CacheKey codeRequestId = this.codeTableCache.createCacheRequest(codeTableName, returnElementName, keyElementName, context); boolean success = false; - QueryProcessor processor = null; try { - processor = context.getQueryProcessorFactory().createQueryProcessor(query, codeTableName.toUpperCase(), context); - - processor.setBatchHandler(new QueryProcessor.BatchHandler() { - @Override - public void batchProduced(TupleBatch batch) throws MetaMatrixProcessingException { - codeTableCache.loadTable(codeRequestId, batch.getAllTuples()); + QueryProcessor processor = context.getQueryProcessorFactory().createQueryProcessor(query, codeTableName.toUpperCase(), context); + processor.setNonBlocking(true); //process lookup as fully blocking + while (true) { + TupleBatch batch = processor.nextBatch(); + codeTableCache.loadTable(codeRequestId, batch.getAllTuples()); + if (batch.getTerminationFlag()) { + break; } - }); - - //process lookup as fully blocking - processor.process(); + } success = true; } finally { Collection requests = null; @@ -489,13 +453,6 @@ requests = codeTableCache.errorLoadingCache(codeRequestId); } notifyWaitingCodeTableRequests(requests); - if (processor != null) { - try { - this.bufferService.getBufferManager().removeTupleSource(processor.getResultsID()); - } catch (MetaMatrixComponentException e1) { - LogManager.logDetail(LogConstants.CTX_DQP, "Exception closing code table request"); //$NON-NLS-1$ - } - } } } Deleted: trunk/engine/src/main/java/org/teiid/dqp/internal/process/LobChunkStream.java =================================================================== --- trunk/engine/src/main/java/org/teiid/dqp/internal/process/LobChunkStream.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/LobChunkStream.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -1,83 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package org.teiid.dqp.internal.process; - -import java.io.IOException; -import java.nio.charset.Charset; -import java.sql.SQLException; - -import com.metamatrix.common.lob.ByteLobChunkStream; -import com.metamatrix.common.lob.LobChunk; -import com.metamatrix.common.lob.LobChunkProducer; -import com.metamatrix.common.lob.ReaderInputStream; -import com.metamatrix.common.types.BlobType; -import com.metamatrix.common.types.ClobType; -import com.metamatrix.common.types.Streamable; -import com.metamatrix.common.types.XMLType; - -/** - * A Lob Stream builder class. Given the Lob object this object can build - * an object which can produce a stream lob chunks, which can be sent to the - * client one by one. - */ -class LobChunkStream implements LobChunkProducer { - - LobChunkProducer internalStream = null; - - public LobChunkStream(Streamable<?> streamable, int chunkSize) - throws IOException { - - try { - if (streamable instanceof XMLType) { - XMLType xml = (XMLType)streamable; - this.internalStream = new ByteLobChunkStream(new ReaderInputStream(xml.getCharacterStream(), Charset.forName("UTF-16")), chunkSize); //$NON-NLS-1$ - } - else if (streamable instanceof ClobType) { - ClobType clob = (ClobType)streamable; - this.internalStream = new ByteLobChunkStream(new ReaderInputStream(clob.getCharacterStream(), Charset.forName("UTF-16")), chunkSize); //$NON-NLS-1$ - } - else if (streamable instanceof BlobType) { - BlobType blob = (BlobType)streamable; - this.internalStream = new ByteLobChunkStream(blob.getBinaryStream(), chunkSize); - } - } catch(SQLException e) { - IOException ex = new IOException(); - ex.initCause(e); - throw ex; - } - } - - /** - * @see com.metamatrix.common.lob.LobChunkProducer#getNextChunk(int) - */ - public LobChunk getNextChunk() throws IOException { - return internalStream.getNextChunk(); - } - - /** - * @see com.metamatrix.common.lob.LobChunkProducer#close() - */ - public void close() throws IOException { - internalStream.close(); - } -} Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/process/LobWorkItem.java =================================================================== --- trunk/engine/src/main/java/org/teiid/dqp/internal/process/LobWorkItem.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/LobWorkItem.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -23,14 +23,19 @@ package org.teiid.dqp.internal.process; import java.io.IOException; +import java.nio.charset.Charset; +import java.sql.SQLException; import com.metamatrix.api.exception.MetaMatrixComponentException; -import com.metamatrix.common.buffer.BlockedOnMemoryException; -import com.metamatrix.common.buffer.TupleSourceNotFoundException; import com.metamatrix.common.comm.api.ResultsReceiver; +import com.metamatrix.common.lob.ByteLobChunkStream; import com.metamatrix.common.lob.LobChunk; +import com.metamatrix.common.lob.ReaderInputStream; import com.metamatrix.common.log.LogManager; +import com.metamatrix.common.types.BlobType; +import com.metamatrix.common.types.ClobType; import com.metamatrix.common.types.Streamable; +import com.metamatrix.common.types.XMLType; import com.metamatrix.core.util.Assertion; import com.metamatrix.dqp.DQPPlugin; import com.metamatrix.dqp.message.RequestID; @@ -46,7 +51,7 @@ /* private work item state */ private String streamId; - private LobChunkStream stream; + private ByteLobChunkStream stream; private int streamRequestId; private ResultsReceiver<LobChunk> resultsReceiver; @@ -74,13 +79,6 @@ // now get the chunk from stream chunk = stream.getNextChunk(); shouldClose = chunk.isLast(); - } catch (BlockedOnMemoryException e) { - LogManager.logDetail(LogConstants.CTX_DQP, new Object[] {"Reenqueueing LOB chunk request due to lack of available memory ###########", requestID}); //$NON-NLS-1$ - this.dqpCore.addWork(this); - return; - } catch (TupleSourceNotFoundException e) { - LogManager.logWarning(LogConstants.CTX_DQP, e, DQPPlugin.Util.getString("BufferManagerLobChunkStream.no_tuple_source", streamId)); //$NON-NLS-1$ - ex = e; } catch (MetaMatrixComponentException e) { LogManager.logWarning(LogConstants.CTX_DQP, e, DQPPlugin.Util.getString("ProcessWorker.LobError")); //$NON-NLS-1$ ex = e; @@ -114,13 +112,29 @@ * Create a object which can create a sequence of LobChunk objects on a given * LOB object */ - private LobChunkStream createLobStream(String referenceStreamId) - throws BlockedOnMemoryException, MetaMatrixComponentException, IOException, TupleSourceNotFoundException { + private ByteLobChunkStream createLobStream(String referenceStreamId) + throws MetaMatrixComponentException, IOException { // get the reference object in the buffer manager, and try to stream off // the original sources. - Streamable<?> streamable = dqpCore.getBufferManager().getStreamable(parent.resultsID, referenceStreamId); - return new LobChunkStream(streamable, chunkSize); + Streamable<?> streamable = parent.resultsBuffer.getLobReference(referenceStreamId); + + try { + if (streamable instanceof XMLType) { + XMLType xml = (XMLType)streamable; + return new ByteLobChunkStream(new ReaderInputStream(xml.getCharacterStream(), Charset.forName("UTF-16")), chunkSize); //$NON-NLS-1$ + } + else if (streamable instanceof ClobType) { + ClobType clob = (ClobType)streamable; + return new ByteLobChunkStream(new ReaderInputStream(clob.getCharacterStream(), Charset.forName("UTF-16")), chunkSize); //$NON-NLS-1$ + } + BlobType blob = (BlobType)streamable; + return new ByteLobChunkStream(blob.getBinaryStream(), chunkSize); + } catch(SQLException e) { + IOException ex = new IOException(); + ex.initCause(e); + throw ex; + } } synchronized void setResultsReceiver(ResultsReceiver<LobChunk> resultsReceiver) { Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/process/PreparedPlanCache.java =================================================================== --- trunk/engine/src/main/java/org/teiid/dqp/internal/process/PreparedPlanCache.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/PreparedPlanCache.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -131,7 +131,6 @@ static class PreparedPlan{ private ProcessorPlan plan; private Command command; - private Command rewritenCommand; private List<Reference> refs; private AnalysisRecord analysisRecord; @@ -191,14 +190,6 @@ refs = refsValue; } - public void setRewritenCommand(Command rewritenCommand) { - this.rewritenCommand = rewritenCommand; - } - - public Command getRewritenCommand() { - return rewritenCommand; - } - } //for testing purpose Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/process/PreparedStatementRequest.java =================================================================== --- trunk/engine/src/main/java/org/teiid/dqp/internal/process/PreparedStatementRequest.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/PreparedStatementRequest.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -28,6 +28,7 @@ import java.util.List; import org.teiid.dqp.internal.process.PreparedPlanCache.CacheID; +import org.teiid.dqp.internal.process.PreparedPlanCache.PreparedPlan; import com.metamatrix.api.exception.MetaMatrixComponentException; import com.metamatrix.api.exception.query.ExpressionEvaluationException; @@ -62,7 +63,7 @@ */ public class PreparedStatementRequest extends Request { private PreparedPlanCache prepPlanCache; - private PreparedPlanCache.PreparedPlan prepPlan; + private PreparedPlan prepPlan; public PreparedStatementRequest(PreparedPlanCache prepPlanCache) { this.prepPlanCache = prepPlanCache; @@ -129,20 +130,20 @@ * @throws QueryPlannerException * @see org.teiid.dqp.internal.process.Request#generatePlan() */ - protected Command generatePlan() throws QueryPlannerException, QueryParserException, QueryResolverException, QueryValidatorException, MetaMatrixComponentException { + protected void generatePlan() throws QueryPlannerException, QueryParserException, QueryResolverException, QueryValidatorException, MetaMatrixComponentException { String sqlQuery = requestMsg.getCommands()[0]; CacheID id = new PreparedPlanCache.CacheID(this.workContext, Request.createParseInfo(this.requestMsg), sqlQuery); prepPlan = prepPlanCache.getPreparedPlan(id); if (prepPlan == null) { //if prepared plan does not exist, create one - prepPlan = new PreparedPlanCache.PreparedPlan(); + prepPlan = new PreparedPlan(); LogManager.logTrace(LogConstants.CTX_DQP, new Object[] { "Query does not exist in cache: ", sqlQuery}); //$NON-NLS-1$ } ProcessorPlan cachedPlan = prepPlan.getPlan(); if (cachedPlan == null) { - prepPlan.setRewritenCommand(super.generatePlan()); + super.generatePlan(); if (!this.addedLimit) { //TODO: this is a little problematic prepPlan.setCommand(this.userCommand); // Defect 13751: Clone the plan in its current state (i.e. before processing) so that it can be used for later queries @@ -168,7 +169,6 @@ PreparedStatementRequest.resolveParameterValues(params, values, this.context); } - return prepPlan.getRewritenCommand(); } /** @@ -191,7 +191,7 @@ throw new QueryValidatorException("No batch values sent for prepared batch update"); //$NON-NLS-1$ } boolean supportPreparedBatchUpdate = false; - if (this.processPlan instanceof RelationalPlan && this.prepPlan.getRewritenCommand().getSubCommands().isEmpty()) { + if (this.processPlan instanceof RelationalPlan) { RelationalPlan rPlan = (RelationalPlan)this.processPlan; if (rPlan.getRootNode() instanceof AccessNode) { AccessNode aNode = (AccessNode)rPlan.getRootNode(); @@ -218,7 +218,7 @@ } continue; } - Command c = (Command)this.prepPlan.getRewritenCommand().clone(); + Command c = (Command)this.prepPlan.getCommand().clone(); commands.add(c); c.setProcessorPlan((ProcessorPlan)this.processPlan.clone()); } Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/process/Request.java =================================================================== --- trunk/engine/src/main/java/org/teiid/dqp/internal/process/Request.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/Request.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -303,7 +303,7 @@ // Create generic sql validation visitor AbstractValidationVisitor visitor = new ValidationVisitor(); - validateWithVisitor(visitor, metadata, command, false); + validateWithVisitor(visitor, metadata, command); } private Command parseCommand() throws QueryParserException { @@ -330,8 +330,7 @@ public static void validateWithVisitor( AbstractValidationVisitor visitor, QueryMetadataInterface metadata, - Command command, - boolean validateOnlyEmbedded) + Command command) throws QueryValidatorException, MetaMatrixComponentException { // Validate with visitor @@ -422,7 +421,7 @@ * @throws QueryResolverException * @throws QueryValidatorException */ - protected Command generatePlan() throws MetaMatrixComponentException, QueryPlannerException, QueryParserException, QueryResolverException, QueryValidatorException { + protected void generatePlan() throws MetaMatrixComponentException, QueryPlannerException, QueryParserException, QueryResolverException, QueryValidatorException { Command command = parseCommand(); List<Reference> references = ReferenceCollectorVisitor.getReferences(command); @@ -486,7 +485,6 @@ String msg = DQPPlugin.Util.getString("DQPCore.Unknown_query_metadata_exception_while_registering_query__{0}.", params); //$NON-NLS-1$ throw new QueryPlannerException(e, msg); } - return command; } private void setSchemasForXMLPlan(Command command, QueryMetadataInterface metadata) @@ -555,13 +553,13 @@ initMetadata(); - Command processingCommand = generatePlan(); + generatePlan(); validateAccess(userCommand); setSchemasForXMLPlan(userCommand, metadata); - createProcessor(processingCommand); + createProcessor(userCommand); } public QueryProcessor createQueryProcessor(String query, String recursionGroup, CommandContext commandContext) throws MetaMatrixProcessingException, MetaMatrixComponentException { @@ -601,6 +599,6 @@ // See if entitlement checking is turned on AuthorizationValidationVisitor visitor = new AuthorizationValidationVisitor(this.workContext.getConnectionID(), authSvc, this.vdbService, this.vdbName, this.vdbVersion); - validateWithVisitor(visitor, this.metadata, command, true); + validateWithVisitor(visitor, this.metadata, command); } } Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java =================================================================== --- trunk/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -36,21 +36,14 @@ import javax.transaction.SystemException; import org.teiid.connector.xa.api.TransactionContext; -import org.teiid.dqp.internal.cache.CacheID; -import org.teiid.dqp.internal.cache.CacheResults; -import org.teiid.dqp.internal.cache.ResultSetCache; import com.metamatrix.api.exception.MetaMatrixComponentException; import com.metamatrix.api.exception.MetaMatrixException; import com.metamatrix.api.exception.MetaMatrixProcessingException; import com.metamatrix.common.buffer.BlockedException; -import com.metamatrix.common.buffer.BlockedOnMemoryException; import com.metamatrix.common.buffer.BufferManager; -import com.metamatrix.common.buffer.MemoryNotAvailableException; import com.metamatrix.common.buffer.TupleBatch; -import com.metamatrix.common.buffer.TupleSourceID; -import com.metamatrix.common.buffer.TupleSourceNotFoundException; -import com.metamatrix.common.buffer.BufferManager.TupleSourceStatus; +import com.metamatrix.common.buffer.TupleBuffer; import com.metamatrix.common.comm.api.ResultsReceiver; import com.metamatrix.common.lob.LobChunk; import com.metamatrix.common.log.LogManager; @@ -71,8 +64,9 @@ import com.metamatrix.query.analysis.AnalysisRecord; import com.metamatrix.query.analysis.QueryAnnotation; import com.metamatrix.query.execution.QueryExecPlugin; +import com.metamatrix.query.processor.BatchCollector; import com.metamatrix.query.processor.QueryProcessor; -import com.metamatrix.query.processor.QueryProcessor.BatchHandler; +import com.metamatrix.query.processor.BatchCollector.BatchHandler; import com.metamatrix.query.sql.lang.Command; import com.metamatrix.query.sql.lang.Option; import com.metamatrix.query.sql.lang.SPParameter; @@ -125,8 +119,8 @@ protected Request request; //provides the processing plan, held on a temporary basis final private BufferManager bufferMgr; final private int processorTimeslice; - protected ResultSetCache rsCache; - protected CacheID cid; + //protected ResultSetCache rsCache; + //protected CacheID cid; final private TransactionService transactionService; final DQPWorkContext dqpWorkContext; ResultsReceiver<ResultsMessage> resultsReceiver; @@ -135,10 +129,11 @@ * obtained during new */ private volatile QueryProcessor processor; + private BatchCollector collector; protected Command originalCommand; private AnalysisRecord analysisRecord; private TransactionContext transactionContext; - protected TupleSourceID resultsID; + protected TupleBuffer resultsBuffer; private Collection schemas; // These are schemas associated with XML results private boolean returnsUpdateCount; @@ -166,10 +161,10 @@ this.resultsCursor.requestResults(1, requestMsg.getFetchSize(), false); this.bufferMgr = dqpCore.getBufferManager(); this.processorTimeslice = dqpCore.getProcessorTimeSlice(); - this.rsCache = dqpCore.getRsCache(); + /*this.rsCache = dqpCore.getRsCache(); if (this.rsCache != null) { this.cid = this.rsCache.createCacheID(workContext, requestMsg.getCommandString(), requestMsg.getParameterValues()); - } + }*/ this.transactionService = dqpCore.getTransactionServiceDirect(); this.dqpCore = dqpCore; this.request = request; @@ -208,17 +203,10 @@ this.state = ProcessingState.CLOSE; } } - } catch (BlockedOnMemoryException e) { - moreWork(false); - LogManager.logDetail(LogConstants.CTX_DQP, "############# PW EXITING on", requestID, "- reenqueueing for more processing due to lack of available memory ###########"); //$NON-NLS-1$ //$NON-NLS-2$ } catch (BlockedException e) { LogManager.logDetail(LogConstants.CTX_DQP, "############# PW EXITING on", requestID, "- processor blocked ###########"); //$NON-NLS-1$ //$NON-NLS-2$ } catch (Throwable e) { LogManager.logDetail(LogConstants.CTX_DQP, e, "############# PW EXITING on", requestID, "- error occurred ###########"); //$NON-NLS-1$ //$NON-NLS-2$ - //if there is a cache, remove temp results if there is any - if(this.rsCache != null){ - rsCache.removeTempResults(cid, requestID); - } if (!isCanceled()) { logCommandError(); @@ -255,11 +243,19 @@ } protected void processMore() throws SystemException, BlockedException, MetaMatrixCoreException { + this.processor.getContext().setTimeSliceEnd(System.currentTimeMillis() + this.processorTimeslice); if (!doneProducingBatches) { sendResultsIfNeeded(null); - doneProducingBatches = processor.process(this.processorTimeslice); + collector.collectTuples(); + doneProducingBatches = collector.isDone(); } if (doneProducingBatches) { + /*if(rsCache != null && requestMsg.useResultSetCache() && originalCommand.areResultsCachable()){ + boolean sessionScope = this.processor.getContext().isSessionFunctionEvaluated(); + CacheResults cr = new CacheResults(); + cr.setCommand(originalCommand); + cr.setAnalysisRecord(analysisRecord); + }*/ if (this.transactionState == TransactionState.ACTIVE) { boolean end = true; /* @@ -296,28 +292,20 @@ * Any errors that occur will not make it to the client, instead we just log them here. */ protected void attemptClose() { - if (this.resultsID != null) { + if (this.resultsBuffer != null) { try { - if (LogManager.isMessageToBeRecorded(LogConstants.CTX_DQP, MessageLevel.DETAIL)) { - LogManager.logDetail(LogConstants.CTX_DQP, "Removing tuplesource for the request " + requestID); //$NON-NLS-1$ - } - this.dqpCore.getBufferManager().removeTupleSource(resultsID); - } catch(TupleSourceNotFoundException e) { - // ignore - } catch (MetaMatrixComponentException e) { - LogManager.logDetail(LogConstants.CTX_DQP, e, e.getMessage()); - } - - try { this.processor.closeProcessing(); - } catch (TupleSourceNotFoundException e) { - // ignore } catch (MetaMatrixComponentException e) { if (LogManager.isMessageToBeRecorded(LogConstants.CTX_DQP, MessageLevel.DETAIL)) { LogManager.logDetail(LogConstants.CTX_DQP, e, e.getMessage()); } } + if (LogManager.isMessageToBeRecorded(LogConstants.CTX_DQP, MessageLevel.DETAIL)) { + LogManager.logDetail(LogConstants.CTX_DQP, "Removing tuplesource for the request " + requestID); //$NON-NLS-1$ + } + resultsBuffer.remove(); + for (DataTierTupleSource connectorRequest : this.connectorInfo.values()) { try { connectorRequest.fullyCloseSource(); @@ -328,7 +316,7 @@ } } - this.resultsID = null; + this.resultsBuffer = null; } if (this.transactionState == TransactionState.ACTIVE) { @@ -364,22 +352,13 @@ request.processRequest(); originalCommand = request.userCommand; processor = request.processor; - processor.setBatchHandler(new BatchHandler() { - public void batchProduced(TupleBatch batch) throws BlockedOnMemoryException, TupleSourceNotFoundException, MetaMatrixComponentException { - //if there is a cache, and it is a query, save it - if(rsCache != null && requestMsg.useResultSetCache() && originalCommand.areResultsCachable() && transactionState == TransactionState.NONE && !rsCache.hasResults(cid)){ - CacheResults cr = new CacheResults(batch.getAllTuples(), batch.getBeginRow(), !doneProducingBatches); - cr.setCommand(originalCommand); - cr.setSize(batch.getSize()); - cr.setAnalysisRecord(analysisRecord); - if (!rsCache.setResults(cid, cr, requestID)) { - rsCache = null; //disable caching if we are over size - } - } + collector = processor.createBatchCollector(); + collector.setBatchHandler(new BatchHandler() { + public void batchProduced(TupleBatch batch) throws MetaMatrixComponentException { sendResultsIfNeeded(batch); } }); - resultsID = processor.getResultsID(); + resultsBuffer = collector.getTupleBuffer(); analysisRecord = request.analysisRecord; schemas = request.schemas; transactionContext = request.transactionContext; @@ -389,7 +368,7 @@ Option option = originalCommand.getOption(); if (option != null && option.getPlanOnly()) { doneProducingBatches = true; - this.bufferMgr.setStatus(resultsID, TupleSourceStatus.FULL); + resultsBuffer.close(); } if (analysisRecord.recordQueryPlan()) { @@ -402,8 +381,7 @@ /** * Send results if they have been requested. This should only be called from the processing thread. */ - protected void sendResultsIfNeeded(TupleBatch batch) throws BlockedOnMemoryException, - MetaMatrixComponentException, TupleSourceNotFoundException { + protected void sendResultsIfNeeded(TupleBatch batch) throws MetaMatrixComponentException { synchronized (resultsCursor) { if (!this.resultsCursor.resultsRequested @@ -414,76 +392,61 @@ } if (LogManager.isMessageToBeRecorded(LogConstants.CTX_DQP, MessageLevel.DETAIL)) { - LogManager.logDetail(LogConstants.CTX_DQP, "[RequestWorkItem.sendResultsIfNeeded] requestID: " + requestID + " resultsID: " + this.resultsID + " done: " + doneProducingBatches ); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + LogManager.logDetail(LogConstants.CTX_DQP, "[RequestWorkItem.sendResultsIfNeeded] requestID: " + requestID + " resultsID: " + this.resultsBuffer + " done: " + doneProducingBatches ); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ } - boolean pinned = false; - - try { - if (batch == null || batch.getBeginRow() > this.resultsCursor.begin) { - batch = this.bufferMgr.pinTupleBatch(resultsID, resultsCursor.begin); - pinned = true; - //TODO: support fetching more than 1 batch - int count = this.resultsCursor.end - this.resultsCursor.begin + 1; - if (batch.getRowCount() > count) { - int beginRow = Math.min(this.resultsCursor.begin, batch.getEndRow() - count + 1); - int endRow = Math.min(beginRow + count - 1, batch.getEndRow()); - int firstOffset = beginRow - batch.getBeginRow(); - List[] memoryRows = batch.getAllTuples(); - List[] rows = new List[count]; - System.arraycopy(memoryRows, firstOffset, rows, 0, endRow - beginRow + 1); - batch = new TupleBatch(beginRow, rows); - } - } - int finalRowCount = doneProducingBatches?this.processor.getHighestRow():-1; - - ResultsMessage response = createResultsMessage(requestMsg, batch.getAllTuples(), this.processor.getProcessorPlan().getOutputElements(), analysisRecord); - response.setFirstRow(batch.getBeginRow()); - response.setLastRow(batch.getEndRow()); - response.setUpdateResult(this.returnsUpdateCount); - // set final row - response.setFinalRow(finalRowCount); + if (batch == null || batch.getBeginRow() > this.resultsCursor.begin) { + batch = resultsBuffer.getBatch(resultsCursor.begin); + //TODO: support fetching more than 1 batch + int count = this.resultsCursor.end - this.resultsCursor.begin + 1; + if (batch.getRowCount() > count) { + int beginRow = Math.min(this.resultsCursor.begin, batch.getEndRow() - count + 1); + int endRow = Math.min(beginRow + count - 1, batch.getEndRow()); + int firstOffset = beginRow - batch.getBeginRow(); + List[] memoryRows = batch.getAllTuples(); + List[] rows = new List[count]; + System.arraycopy(memoryRows, firstOffset, rows, 0, endRow - beginRow + 1); + batch = new TupleBatch(beginRow, rows); + } + } + int finalRowCount = doneProducingBatches?this.processor.getHighestRow():-1; + + ResultsMessage response = createResultsMessage(requestMsg, batch.getAllTuples(), this.processor.getProcessorPlan().getOutputElements(), analysisRecord); + response.setFirstRow(batch.getBeginRow()); + response.setLastRow(batch.getEndRow()); + response.setUpdateResult(this.returnsUpdateCount); + // set final row + response.setFinalRow(finalRowCount); - // send any schemas associated with the results - response.setSchemas(this.schemas); - - // send any warnings with the response object - List<Throwable> responseWarnings = new ArrayList<Throwable>(); - List<Exception> currentWarnings = processor.getAndClearWarnings(); - if (currentWarnings != null) { - responseWarnings.addAll(currentWarnings); - } - synchronized (warnings) { - responseWarnings.addAll(this.warnings); - this.warnings.clear(); - } - response.setWarnings(responseWarnings); - - // If it is stored procedure, set parameters - if (originalCommand instanceof StoredProcedure) { - StoredProcedure proc = (StoredProcedure)originalCommand; - if (proc.returnParameters()) { - response.setParameters(getParameterInfo(proc)); - } - } - - /* - * mark the results sent at this point. - * communication exceptions will be treated as non-recoverable - */ - this.resultsCursor.resultsSent(); - this.resultsReceiver.receiveResults(response); - } catch (MemoryNotAvailableException e) { - throw BlockedOnMemoryException.INSTANCE; - } finally { - try { - if (pinned) { - this.bufferMgr.unpinTupleBatch(this.resultsID, batch.getBeginRow()); - } - } catch (Exception e) { - // ignore - nothing more we can do - } + // send any schemas associated with the results + response.setSchemas(this.schemas); + + // send any warnings with the response object + List<Throwable> responseWarnings = new ArrayList<Throwable>(); + List<Exception> currentWarnings = processor.getAndClearWarnings(); + if (currentWarnings != null) { + responseWarnings.addAll(currentWarnings); + } + synchronized (warnings) { + responseWarnings.addAll(this.warnings); + this.warnings.clear(); + } + response.setWarnings(responseWarnings); + + // If it is stored procedure, set parameters + if (originalCommand instanceof StoredProcedure) { + StoredProcedure proc = (StoredProcedure)originalCommand; + if (proc.returnParameters()) { + response.setParameters(getParameterInfo(proc)); + } } + + /* + * mark the results sent at this point. + * communication exceptions will be treated as non-recoverable + */ + this.resultsCursor.resultsSent(); + this.resultsReceiver.receiveResults(response); } public static ResultsMessage createResultsMessage(RequestMessage message, List[] batch, List columnSymbols, AnalysisRecord analysisRecord) { @@ -616,17 +579,17 @@ return false; } - public synchronized void requestClose() { - if (this.state == ProcessingState.CLOSE || this.closeRequested) { - if (LogManager.isMessageToBeRecorded(LogConstants.CTX_DQP, MessageLevel.DETAIL)) { - LogManager.logDetail(LogConstants.CTX_DQP, "Request already closing" + requestID); //$NON-NLS-1$ - } - return; - } + public void requestClose() throws MetaMatrixComponentException { + synchronized (this) { + if (this.state == ProcessingState.CLOSE || this.closeRequested) { + if (LogManager.isMessageToBeRecorded(LogConstants.CTX_DQP, MessageLevel.DETAIL)) { + LogManager.logDetail(LogConstants.CTX_DQP, "Request already closing" + requestID); //$NON-NLS-1$ + } + return; + } + } this.closeRequested = true; - if (this.processor != null) { - this.processor.requestClosed(); - } + this.requestCancel(); //pending work should be canceled for fastest clean up this.moreWork(); } Modified: trunk/engine/src/test/java/com/metamatrix/common/buffer/BufferManagerFactory.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/common/buffer/BufferManagerFactory.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/test/java/com/metamatrix/common/buffer/BufferManagerFactory.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -22,14 +22,11 @@ package com.metamatrix.common.buffer; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; import java.util.Properties; import com.metamatrix.api.exception.MetaMatrixComponentException; import com.metamatrix.common.buffer.impl.BufferManagerImpl; -import com.metamatrix.query.execution.QueryExecPlugin; +import com.metamatrix.common.buffer.impl.MemoryStorageManager; /** * <p>Factory for BufferManager instances. One method will get @@ -44,75 +41,6 @@ */ public class BufferManagerFactory { - public static class MemoryStorageManager implements StorageManager { - - // TupleSourceID -> List<TupleBatch> (ordered by startRow) - private Map<TupleSourceID, Map<Integer, TupleBatch>> storage = Collections.synchronizedMap(new HashMap<TupleSourceID, Map<Integer, TupleBatch>>()); - - /** - * @see StorageManager#initialize(Properties) - */ - public void initialize(Properties props) throws MetaMatrixComponentException { - } - - /** - * @see StorageManager#addBatch(TupleSourceID, TupleBatch) - */ - public void addBatch(TupleSourceID storageID, TupleBatch batch, String[] types) - throws MetaMatrixComponentException { - - Map<Integer, TupleBatch> batches = null; - synchronized(this.storage) { - batches = storage.get(storageID); - if(batches == null) { - batches = new HashMap<Integer, TupleBatch>(); - this.storage.put(storageID, batches); - } - } - - synchronized(batches) { - batches.put(batch.getBeginRow(), batch); - } - } - - /** - * @see StorageManager#getBatch(TupleSourceID, int, int) - */ - public TupleBatch getBatch(TupleSourceID storageID, int beginRow, String[] types) - throws TupleSourceNotFoundException, MetaMatrixComponentException { - - Map<Integer, TupleBatch> batches = storage.get(storageID); - - if(batches == null) { - throw new TupleSourceNotFoundException(QueryExecPlugin.Util.getString("BufferManagerImpl.tuple_source_not_found", storageID)); //$NON-NLS-1$ - } - - synchronized(batches) { - TupleBatch batch = batches.get(beginRow); - if(batch == null) { - throw new MetaMatrixComponentException("unknown batch"); //$NON-NLS-1$ - } - return batch; - } - } - - /** - * @see StorageManager#removeStorageArea(TupleSourceID) - */ - public void removeBatches(TupleSourceID storageID) throws MetaMatrixComponentException { - storage.remove(storageID); - } - - /** - * @see StorageManager#shutdown() - */ - public void shutdown() { - this.storage.clear(); - this.storage = null; - } - - } - private static BufferManager INSTANCE; /** @@ -128,7 +56,7 @@ props.setProperty(BufferManagerPropertyNames.SESSION_USE_PERCENTAGE, "100"); //$NON-NLS-1$ props.setProperty(BufferManagerPropertyNames.LOG_STATS_INTERVAL, "0"); //$NON-NLS-1$ props.setProperty(BufferManagerPropertyNames.MANAGEMENT_INTERVAL, "0"); //$NON-NLS-1$ - bufferMgr.initialize("local", props); //$NON-NLS-1$ + bufferMgr.initialize(props); // Add unmanaged memory storage manager bufferMgr.setStorageManager(new MemoryStorageManager()); Modified: trunk/engine/src/test/java/com/metamatrix/common/buffer/storage/file/TestFileStorageManager.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/common/buffer/storage/file/TestFileStorageManager.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/test/java/com/metamatrix/common/buffer/storage/file/TestFileStorageManager.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -36,11 +36,11 @@ import junit.framework.TestCase; import com.metamatrix.api.exception.MetaMatrixComponentException; -import com.metamatrix.api.exception.MetaMatrixException; import com.metamatrix.common.buffer.BufferManagerPropertyNames; import com.metamatrix.common.buffer.StorageManager; import com.metamatrix.common.buffer.TupleBatch; import com.metamatrix.common.buffer.TupleSourceID; +import com.metamatrix.common.buffer.impl.FileStorageManager; import com.metamatrix.core.util.UnitTestUtil; /** @@ -250,43 +250,37 @@ } - public void testAddGetBatch1() { + public void testAddGetBatch1() throws Exception { StorageManager sm = getStorageManager(null); TupleSourceID tsID = new TupleSourceID("local,1:0"); //$NON-NLS-1$ TupleBatch batch = exampleBatch(1, 10); - try { - // Add one batch - sm.addBatch(tsID, batch, null); - - // Get that batch - TupleBatch actual = sm.getBatch(tsID, batch.getBeginRow(), null); - helpCompareBatches(batch, actual); - - // Remove the batches - sm.removeBatches(tsID); - - } catch(MetaMatrixException e) { - fail("Unexpected exception of type " + e.getClass().getName() + ": " + e.getMessage()); //$NON-NLS-1$ //$NON-NLS-2$ - } + // Add one batch + sm.addBatch(tsID, batch, null); + + // Get that batch + TupleBatch actual = sm.getBatch(tsID, batch.getBeginRow(), null); + helpCompareBatches(batch, actual); + + // Remove the batches + sm.removeBatches(tsID); } - public void testAddGetBatch2() { + public void testAddGetBatch2() throws Exception { StorageManager sm = getStorageManager(null); TupleSourceID tsID = new TupleSourceID("local,1:0"); //$NON-NLS-1$ int numBatches = 20; int batchSize = 100; - try { - // Add batches - for(int i=0; i<numBatches; i++) { - int begin = (i*batchSize)+1; - int end = begin + batchSize - 1; - TupleBatch batch = exampleBatch(begin, end); - sm.addBatch(tsID, batch, null); - } + // Add batches + for(int i=0; i<numBatches; i++) { + int begin = (i*batchSize)+1; + int end = begin + batchSize - 1; + TupleBatch batch = exampleBatch(begin, end); + sm.addBatch(tsID, batch, null); + } - // Get batches + // Get batches for(int i=0; i<numBatches; i++) { int begin = (i*batchSize)+1; int end = begin + batchSize - 1; @@ -295,31 +289,21 @@ } // Remove the batches - sm.removeBatches(tsID); - - } catch(MetaMatrixException e) { - fail("Unexpected exception of type " + e.getClass().getName() + ": " + e.getMessage()); //$NON-NLS-1$ //$NON-NLS-2$ - } + sm.removeBatches(tsID); } - public void testCreateNewDirectory() { + public void testCreateNewDirectory() throws Exception { Properties resourceProps = new Properties(); String nonExistentDirectory = UnitTestUtil.getTestScratchPath() + File.separator + "GONZO"; //$NON-NLS-1$ resourceProps.put(BufferManagerPropertyNames.BUFFER_STORAGE_DIRECTORY, nonExistentDirectory); StorageManager sm = new FileStorageManager(); - try { - sm.initialize(resourceProps); - - File file = new File(nonExistentDirectory); - assertTrue("Directory doesn't exist", file.exists()); //$NON-NLS-1$ - assertTrue("Directory was created as a file", file.isDirectory()); //$NON-NLS-1$ - - } catch(Exception e) { - e.printStackTrace(); - fail("Unexpected exception: " + e.getMessage()); //$NON-NLS-1$ - } + sm.initialize(resourceProps); + + File file = new File(nonExistentDirectory); + assertTrue("Directory doesn't exist", file.exists()); //$NON-NLS-1$ + assertTrue("Directory was created as a file", file.isDirectory()); //$NON-NLS-1$ } public void testAddTwice() throws Exception { @@ -400,9 +384,9 @@ assertTrue(storageFile1.exists()); assertTrue(storageFile2.exists()); assertTrue(storageFile3.exists()); - assertEquals(1048435, storageFile1.length()); - assertEquals(211806, storageFile2.length()); - assertEquals(2108106, storageFile3.length()); + assertEquals(1048427, storageFile1.length()); + assertEquals(211798, storageFile2.length()); + assertEquals(2108098, storageFile3.length()); sm.removeBatches(tsID); assertFalse(storageFile1.exists()); @@ -454,14 +438,8 @@ } catch(Throwable e) { this.error = e; } finally { - // Remove all the batches - //System.out.println(tsID.toString() + ": removing batches"); - try { - sm.removeBatches(tsID); - } catch(MetaMatrixComponentException e) { - } + sm.removeBatches(tsID); } - //System.out.println(tsID.toString() + ": ending"); } } @@ -521,7 +499,6 @@ } else { // Remove if(added[batch]) { - //System.out.println(tsID.toString() + ": removing batch " + batch); added[batch] = false; } } @@ -531,14 +508,8 @@ } catch(Throwable e) { this.error = e; } finally { - // Remove all the batches - //System.out.println(tsID.toString() + ": removing batches"); - try { - sm.removeBatches(tsID); - } catch(MetaMatrixComponentException e) { - } + sm.removeBatches(tsID); } - //System.out.println(tsID.toString() + ": ending"); } } } Modified: trunk/engine/src/test/java/com/metamatrix/query/optimizer/relational/rules/TestRuleAccessPatternValidation.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/query/optimizer/relational/rules/TestRuleAccessPatternValidation.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/test/java/com/metamatrix/query/optimizer/relational/rules/TestRuleAccessPatternValidation.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -28,7 +28,6 @@ import java.util.Collection; import org.junit.Test; -import org.teiid.dqp.internal.process.TestPreparedStatement; import com.metamatrix.api.exception.MetaMatrixComponentException; import com.metamatrix.api.exception.query.QueryMetadataException; Copied: trunk/engine/src/test/java/com/metamatrix/query/processor/TestCollectionTupleSource.java (from rev 1666, trunk/engine/src/test/java/com/metamatrix/query/processor/TestNullTupleSource.java) =================================================================== --- trunk/engine/src/test/java/com/metamatrix/query/processor/TestCollectionTupleSource.java (rev 0) +++ trunk/engine/src/test/java/com/metamatrix/query/processor/TestCollectionTupleSource.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -0,0 +1,70 @@ +/* + * JBoss, Home of Professional Open Source. + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA. + */ + +package com.metamatrix.query.processor; + +import static org.junit.Assert.*; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.junit.Test; + +import com.metamatrix.query.sql.lang.Command; +import com.metamatrix.query.sql.symbol.ElementSymbol; +import com.metamatrix.query.sql.symbol.SingleElementSymbol; +/** + */ +public class TestCollectionTupleSource { + + @Test public void testNullSource() { + List<SingleElementSymbol> elements = new ArrayList<SingleElementSymbol>(); + elements.add(new ElementSymbol("x")); //$NON-NLS-1$ + elements.add(new ElementSymbol("y")); //$NON-NLS-1$ + CollectionTupleSource nts = CollectionTupleSource.createNullTupleSource(elements); + + // Check schema + assertEquals("Didn't get expected schema", elements, nts.getSchema()); //$NON-NLS-1$ + + // Walk it and get no data + List tuple = nts.nextTuple(); + nts.closeSource(); + + assertEquals("Didn't get termination tuple for first tuple", null, tuple); //$NON-NLS-1$ + } + + @Test public void testUpdateCountSource() { + CollectionTupleSource nts = CollectionTupleSource.createUpdateCountTupleSource(5); + + // Check schema + assertEquals("Didn't get expected schema", Command.getUpdateCommandSymbol(), nts.getSchema()); //$NON-NLS-1$ + + // Walk it and get no data + List tuple = nts.nextTuple(); + nts.closeSource(); + + assertEquals("Didn't get termination tuple for first tuple", Arrays.asList(5), tuple); //$NON-NLS-1$ + } + + +} Deleted: trunk/engine/src/test/java/com/metamatrix/query/processor/TestNullTupleSource.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/query/processor/TestNullTupleSource.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/test/java/com/metamatrix/query/processor/TestNullTupleSource.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -1,61 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package com.metamatrix.query.processor; - -import java.util.ArrayList; -import java.util.List; - -import junit.framework.TestCase; - -import com.metamatrix.query.sql.symbol.ElementSymbol; - -/** - */ -public class TestNullTupleSource extends TestCase { - - /** - * Constructor for TestNullTupleSource. - * @param name - */ - public TestNullTupleSource(String name) { - super(name); - } - - public void testSource() { - List elements = new ArrayList(); - elements.add(new ElementSymbol("x")); //$NON-NLS-1$ - elements.add(new ElementSymbol("y")); //$NON-NLS-1$ - NullTupleSource nts = new NullTupleSource(elements); - - // Check schema - assertEquals("Didn't get expected schema", elements, nts.getSchema()); //$NON-NLS-1$ - - // Walk it and get no data - nts.openSource(); - List tuple = nts.nextTuple(); - nts.closeSource(); - - assertEquals("Didn't get termination tuple for first tuple", null, tuple); //$NON-NLS-1$ - } - -} Modified: trunk/engine/src/test/java/com/metamatrix/query/processor/TestQueryProcessor.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/query/processor/TestQueryProcessor.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/test/java/com/metamatrix/query/processor/TestQueryProcessor.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -30,13 +30,16 @@ import junit.framework.TestCase; import com.metamatrix.api.exception.MetaMatrixComponentException; +import com.metamatrix.api.exception.MetaMatrixProcessingException; import com.metamatrix.common.buffer.BlockedException; import com.metamatrix.common.buffer.BufferManager; import com.metamatrix.common.buffer.BufferManagerFactory; import com.metamatrix.common.buffer.TupleBatch; +import com.metamatrix.common.buffer.TupleBuffer; import com.metamatrix.common.buffer.TupleSource; import com.metamatrix.common.buffer.TupleSourceNotFoundException; import com.metamatrix.core.MetaMatrixCoreException; +import com.metamatrix.query.processor.BatchCollector.BatchHandler; import com.metamatrix.query.sql.symbol.ElementSymbol; import com.metamatrix.query.util.CommandContext; @@ -58,22 +61,21 @@ CommandContext context = new CommandContext("pid", "group", null, null, null); //$NON-NLS-1$ //$NON-NLS-2$ QueryProcessor processor = new QueryProcessor(plan, context, bufferMgr, dataManager); - + BatchCollector collector = processor.createBatchCollector(); + TupleBuffer tsID = null; while(true) { try { - boolean done = processor.process(timeslice); - if(done) { - break; - } + tsID = collector.collectTuples(); + break; } catch(BlockedException e) { } } // Compare # of rows in actual and expected - assertEquals("Did not get expected # of rows", expectedResults.length, bufferMgr.getFinalRowCount(processor.getResultsID())); //$NON-NLS-1$ + assertEquals("Did not get expected # of rows", expectedResults.length, tsID.getRowCount()); //$NON-NLS-1$ // Compare actual with expected results - TupleSource actual = bufferMgr.getTupleSource(processor.getResultsID()); + TupleSource actual = tsID.createIndexedTupleSource(); if(expectedResults.length > 0) { for(int i=0; i<expectedResults.length; i++) { List actRecord = actual.nextTuple(); @@ -81,6 +83,7 @@ assertEquals("Did not match row at row index " + i, expRecord, actRecord); //$NON-NLS-1$ } } + tsID.remove(); } public void testNoResults() throws Exception { @@ -138,58 +141,4 @@ helpTestProcessor(plan, 1000, expectedResults); } - public void testProcessWhenClientNeedsBatch() throws Exception { - List elements = new ArrayList(); - elements.add(new ElementSymbol("a")); //$NON-NLS-1$ - - HashSet blocked = new HashSet(Arrays.asList(new Integer[] { new Integer(2)})); - int numBatches = 5; - int batchRow = 1; - int rowsPerBatch = 10; - List[] expectedResults = new List[rowsPerBatch*(numBatches-blocked.size())]; - List batches = new ArrayList(); - for(int b=0; b<numBatches; b++) { - if(blocked.contains(new Integer(b))) { - batches.add(BlockedException.INSTANCE); - } else { - List[] rows = new List[rowsPerBatch]; - for(int i=0; i<rowsPerBatch; i++) { - rows[i] = new ArrayList(); - rows[i].add(new Integer(batchRow)); - expectedResults[batchRow-1] = rows[i]; - batchRow++; - } - - TupleBatch batch = new TupleBatch(batchRow-rows.length, rows); - if(b == numBatches-1) { - batch.setTerminationFlag(true); - } - batches.add(batch); - } - } - - final FakeProcessorPlan plan = new FakeProcessorPlan(elements, batches); - - BufferManager bufferMgr = BufferManagerFactory.getStandaloneBufferManager(); - FakeDataManager dataManager = new FakeDataManager(); - - CommandContext context = new CommandContext("pid", "group", null, null, null); //$NON-NLS-1$ //$NON-NLS-2$ - final QueryProcessor processor = new QueryProcessor(plan, context, bufferMgr, dataManager); - - processor.setBatchHandler(new QueryProcessor.BatchHandler() { - - int count = 0; - - public void batchProduced(TupleBatch batch) throws TupleSourceNotFoundException, MetaMatrixComponentException { - - assertEquals(++count, plan.batchIndex); - if (count == 2) { - processor.closeProcessing(); - } - } - }); - - // Give the processor plenty of time to process - processor.process(30000); - } } Modified: trunk/engine/src/test/java/com/metamatrix/query/processor/TestVirtualDepJoin.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/query/processor/TestVirtualDepJoin.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/test/java/com/metamatrix/query/processor/TestVirtualDepJoin.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -39,9 +39,10 @@ import com.metamatrix.api.exception.MetaMatrixException; import com.metamatrix.common.buffer.BufferManager; import com.metamatrix.common.buffer.BufferManagerPropertyNames; +import com.metamatrix.common.buffer.TupleBuffer; import com.metamatrix.common.buffer.TupleSourceNotFoundException; -import com.metamatrix.common.buffer.BufferManagerFactory.MemoryStorageManager; import com.metamatrix.common.buffer.impl.BufferManagerImpl; +import com.metamatrix.common.buffer.impl.MemoryStorageManager; import com.metamatrix.common.types.DataTypeManager; import com.metamatrix.query.mapping.relational.QueryNode; import com.metamatrix.query.optimizer.TestOptimizer; @@ -484,9 +485,11 @@ // Run query BufferManager bufferMgr = createCustomBufferMgr(2); QueryProcessor processor = new QueryProcessor(plan, context, bufferMgr, dataManager); - processor.process(); + processor.setNonBlocking(true); + BatchCollector collector = processor.createBatchCollector(); + TupleBuffer id = collector.collectTuples(); - TestProcessor.examineResults((List[])expected.toArray(new List[expected.size()]), bufferMgr, processor.getResultsID()); + TestProcessor.examineResults((List[])expected.toArray(new List[expected.size()]), bufferMgr, id); } private BufferManager createCustomBufferMgr(int batchSize) throws MetaMatrixComponentException { @@ -498,7 +501,7 @@ props.setProperty(BufferManagerPropertyNames.MANAGEMENT_INTERVAL, "0"); //$NON-NLS-1$ props.setProperty(BufferManagerPropertyNames.PROCESSOR_BATCH_SIZE, String.valueOf(batchSize)); props.setProperty(BufferManagerPropertyNames.CONNECTOR_BATCH_SIZE, String.valueOf(batchSize)); - bufferMgr.initialize("local", props); //$NON-NLS-1$ + bufferMgr.initialize(props); // Add unmanaged memory storage manager bufferMgr.setStorageManager(new MemoryStorageManager()); Modified: trunk/engine/src/test/java/com/metamatrix/query/processor/dynamic/TestSqlEval.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/query/processor/dynamic/TestSqlEval.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/test/java/com/metamatrix/query/processor/dynamic/TestSqlEval.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -69,7 +69,7 @@ CommandContext cc = TestProcessor.createCommandContext(); cc.setQueryProcessorFactory(factory); - SqlEval sqlEval = new SqlEval(bufferMgr, null, cc, null, Collections.EMPTY_MAP); + SqlEval sqlEval = new SqlEval(null, cc, null, Collections.EMPTY_MAP); Source results = sqlEval.executeSQL(sql); String result = toXMLString(results); sqlEval.close(); Modified: trunk/engine/src/test/java/com/metamatrix/query/processor/proc/TestProcedureProcessor.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/query/processor/proc/TestProcedureProcessor.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/test/java/com/metamatrix/query/processor/proc/TestProcedureProcessor.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -40,6 +40,7 @@ import com.metamatrix.api.exception.query.QueryValidatorException; import com.metamatrix.common.buffer.BufferManager; import com.metamatrix.common.buffer.BufferManagerFactory; +import com.metamatrix.common.buffer.TupleBuffer; import com.metamatrix.common.buffer.TupleSource; import com.metamatrix.common.buffer.TupleSourceID; import com.metamatrix.common.types.DataTypeManager; @@ -54,6 +55,7 @@ import com.metamatrix.query.optimizer.capabilities.CapabilitiesFinder; import com.metamatrix.query.optimizer.capabilities.DefaultCapabilitiesFinder; import com.metamatrix.query.parser.QueryParser; +import com.metamatrix.query.processor.BatchCollector; import com.metamatrix.query.processor.FakeDataManager; import com.metamatrix.query.processor.ProcessorDataManager; import com.metamatrix.query.processor.ProcessorPlan; @@ -117,16 +119,17 @@ context.getNextRand(0); context.setProcessDebug(DEBUG); QueryProcessor processor = new QueryProcessor(procPlan, context, bufferMgr, dataMgr); - TupleSourceID tsID = processor.getResultsID(); - processor.process(); + processor.setNonBlocking(true); + BatchCollector collector = processor.createBatchCollector(); + TupleBuffer tsID = collector.collectTuples(); // Create QueryResults from TupleSource - TupleSource ts = bufferMgr.getTupleSource(tsID); - + TupleSource ts = tsID.createIndexedTupleSource(); + int count = tsID.getRowCount(); + if(DEBUG) { - int count = bufferMgr.getFinalRowCount(tsID); - System.out.println("\nResults:\n" + bufferMgr.getTupleSchema(tsID)); //$NON-NLS-1$ - TupleSource ts2 = bufferMgr.getTupleSource(tsID); + System.out.println("\nResults:\n" + tsID.getSchema()); //$NON-NLS-1$ + TupleSource ts2 = tsID.createIndexedTupleSource(); for(int j=0; j<count; j++) { System.out.println("" + j + ": " + ts2.nextTuple()); //$NON-NLS-1$ //$NON-NLS-2$ } @@ -137,8 +140,6 @@ fail("Expected processing to fail"); //$NON-NLS-1$ } - int count = bufferMgr.getFinalRowCount(tsID); - if (expectedResults == null) { assertEquals("Incorrect number of rows: ", 1, count); //$NON-NLS-1$ List tuple = ts.nextTuple(); @@ -150,7 +151,7 @@ if (expectedResults != null) { //Walk results and compare - ts = bufferMgr.getTupleSource(tsID); + ts = tsID.createIndexedTupleSource(); assertEquals("Incorrect number of rows: ", expectedResults.length, count); //$NON-NLS-1$ for(int j=0; j<count; j++) { List record = ts.nextTuple(); Modified: trunk/engine/src/test/java/com/metamatrix/query/processor/relational/NodeTestUtil.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/query/processor/relational/NodeTestUtil.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/test/java/com/metamatrix/query/processor/relational/NodeTestUtil.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -23,19 +23,13 @@ package com.metamatrix.query.processor.relational; import java.util.Properties; -import java.util.Set; import com.metamatrix.api.exception.MetaMatrixComponentException; import com.metamatrix.common.buffer.BufferManager; import com.metamatrix.common.buffer.BufferManagerPropertyNames; -import com.metamatrix.common.buffer.MemoryNotAvailableException; import com.metamatrix.common.buffer.StorageManager; -import com.metamatrix.common.buffer.TupleBatch; -import com.metamatrix.common.buffer.TupleSourceID; -import com.metamatrix.common.buffer.TupleSourceNotFoundException; -import com.metamatrix.common.buffer.BufferManagerFactory.MemoryStorageManager; import com.metamatrix.common.buffer.impl.BufferManagerImpl; -import com.metamatrix.core.MetaMatrixRuntimeException; +import com.metamatrix.common.buffer.impl.MemoryStorageManager; /** @@ -63,13 +57,13 @@ } static BufferManager createBufferManager(Properties bmProps) { - BufferManagerImpl bufferManager = new TestableBufferManagerImpl(); + BufferManagerImpl bufferManager = new BufferManagerImpl(); bmProps.setProperty(BufferManagerPropertyNames.MANAGEMENT_INTERVAL, "0"); //$NON-NLS-1$ try { - bufferManager.initialize("local", bmProps); //$NON-NLS-1$ - } catch (MetaMatrixComponentException err) { - throw new MetaMatrixRuntimeException(err); - } + bufferManager.initialize(bmProps); + } catch (MetaMatrixComponentException e) { + throw new RuntimeException(e); + } // Add storage managers @@ -82,41 +76,4 @@ return new MemoryStorageManager(); } - public static class TestableBufferManagerImpl extends BufferManagerImpl { - - private Set blockOn; - private int pinCount = 0; - private boolean wasBlocked; - - /** - * @see com.metamatrix.common.buffer.impl.BufferManagerImpl#pinTupleBatch(com.metamatrix.common.buffer.TupleSourceID, int) - */ - public TupleBatch pinTupleBatch(TupleSourceID tupleSourceID, - int beginRow) throws TupleSourceNotFoundException, - MemoryNotAvailableException, - MetaMatrixComponentException { - if (blockOn != null && blockOn.contains(new Integer(++pinCount))) { - wasBlocked = true; - throw new MemoryNotAvailableException(); - } - - wasBlocked = false; - - return super.pinTupleBatch(tupleSourceID, beginRow); - } - - - /** - * @param blockOn The blockOn to set. - */ - public void setBlockOn(Set blockOn) { - this.blockOn = blockOn; - } - - public boolean wasBlocked() { - return wasBlocked; - } - - } - } Modified: trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestDuplicateFilter.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestDuplicateFilter.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestDuplicateFilter.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -46,7 +46,7 @@ public void helpTestDuplicateFilter(Object[] input, Class dataType, int expected) throws MetaMatrixComponentException, MetaMatrixProcessingException { BufferManager mgr = BufferManagerFactory.getStandaloneBufferManager(); - DuplicateFilter filter = new DuplicateFilter(new Count(), mgr, "test", mgr.getProcessorBatchSize()); //$NON-NLS-1$ + DuplicateFilter filter = new DuplicateFilter(new Count(), mgr, "test"); //$NON-NLS-1$ filter.initialize(dataType, dataType); filter.reset(); Modified: trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestSelectNode.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestSelectNode.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestSelectNode.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -39,6 +39,7 @@ import com.metamatrix.common.types.DataTypeManager; import com.metamatrix.query.function.FunctionDescriptor; import com.metamatrix.query.function.FunctionLibraryManager; +import com.metamatrix.query.processor.BatchIterator; import com.metamatrix.query.processor.FakeDataManager; import com.metamatrix.query.processor.ProcessorDataManager; import com.metamatrix.query.sql.lang.CompareCriteria; Modified: trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestSortNode.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestSortNode.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestSortNode.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -36,16 +36,13 @@ import com.metamatrix.api.exception.MetaMatrixComponentException; import com.metamatrix.api.exception.MetaMatrixProcessingException; import com.metamatrix.common.buffer.BlockedException; -import com.metamatrix.common.buffer.BlockedOnMemoryException; import com.metamatrix.common.buffer.BufferManager; import com.metamatrix.common.buffer.BufferManagerFactory; import com.metamatrix.common.buffer.TupleBatch; +import com.metamatrix.common.buffer.TupleBuffer; import com.metamatrix.common.buffer.TupleSource; -import com.metamatrix.common.buffer.TupleSourceID; import com.metamatrix.common.buffer.BufferManager.TupleSourceType; -import com.metamatrix.common.buffer.impl.SizeUtility; import com.metamatrix.common.types.DataTypeManager; -import com.metamatrix.query.processor.relational.NodeTestUtil.TestableBufferManagerImpl; import com.metamatrix.query.processor.relational.SortUtility.Mode; import com.metamatrix.query.sql.lang.OrderBy; import com.metamatrix.query.sql.symbol.ElementSymbol; @@ -54,15 +51,9 @@ public class TestSortNode { public static final int BATCH_SIZE = 100; - public static final int INT_BATCH_SIZE = TestSortNode.getIntBatchSize(); //the size of 100 integers - private void helpTestSort(long bytesInMemory, List elements, List[] data, List sortElements, List sortTypes, List[] expected, Set blockOn, Mode mode) throws MetaMatrixComponentException, MetaMatrixProcessingException { - BufferManager mgr = NodeTestUtil.getTestBufferManager(bytesInMemory, BATCH_SIZE, BATCH_SIZE); - TestableBufferManagerImpl impl = (TestableBufferManagerImpl) mgr; - impl.setBlockOn(blockOn); - impl.getConfig().setTotalAvailableMemory(bytesInMemory); - impl.getConfig().setGroupUsePercentage(100); - impl.getConfig().setManagementInterval(0); + private void helpTestSort(List elements, List[] data, List sortElements, List sortTypes, List[] expected, Set blockOn, Mode mode) throws MetaMatrixComponentException, MetaMatrixProcessingException { + BufferManager mgr = NodeTestUtil.getTestBufferManager(100, BATCH_SIZE, BATCH_SIZE); CommandContext context = new CommandContext ("pid", "test", null, null, null); //$NON-NLS-1$ //$NON-NLS-2$ FakeRelationalNode dataNode = new FakeRelationalNode(2, data); @@ -86,36 +77,20 @@ int currentRow = 1; while(true) { - try { - TupleBatch batch = sortNode.nextBatch(); - if (mode != Mode.DUP_REMOVE) { - for(int row = currentRow; row <= batch.getEndRow(); row++) { - assertEquals("Rows don't match at " + row, expected[row-1], batch.getTuple(row)); //$NON-NLS-1$ - } + TupleBatch batch = sortNode.nextBatch(); + if (mode != Mode.DUP_REMOVE) { + for(int row = currentRow; row <= batch.getEndRow(); row++) { + assertEquals("Rows don't match at " + row, expected[row-1], batch.getTuple(row)); //$NON-NLS-1$ } - currentRow += batch.getRowCount(); - if(batch.getTerminationFlag()) { - break; - } - } catch (BlockedOnMemoryException e) { - if (!impl.wasBlocked()) { - throw new BlockedOnMemoryException(); - } } + currentRow += batch.getRowCount(); + if(batch.getTerminationFlag()) { + break; + } } assertEquals(expected.length, currentRow - 1); } - public static int getIntBatchSize() { - List[] expected = new List[BATCH_SIZE]; - Arrays.fill(expected, Arrays.asList(1)); - - String[] types = { "integer" }; //$NON-NLS-1$ - - int size = (int)SizeUtility.getBatchSize( types, expected ); - return size; - } - /* * 1 batch all in memory */ @@ -158,7 +133,7 @@ if (i > 0) { blockedOn.add(new Integer(i)); } - helpTestSort(INT_BATCH_SIZE*2, elements, data, sortElements, sortTypes, expected, blockedOn, mode); + helpTestSort(elements, data, sortElements, sortTypes, expected, blockedOn, mode); } } @@ -209,7 +184,7 @@ blockedOn.add(new Integer(batches*(i+1))); } //5 batches in memory out of 10 total - helpTestSort(INT_BATCH_SIZE * inMemoryBatches, elements, data, sortElements, sortTypes, expected, blockedOn, mode); + helpTestSort(elements, data, sortElements, sortTypes, expected, blockedOn, mode); } } } @@ -305,7 +280,7 @@ helpTestBiggerSort(10, 5); } - @Test(expected=BlockedOnMemoryException.class) public void testBiggerSortLowMemory() throws Exception { + @Test public void testBiggerSortLowMemory() throws Exception { helpTestBiggerSort(5, 1); } @@ -322,11 +297,11 @@ ElementSymbol es1 = new ElementSymbol("e1"); //$NON-NLS-1$ es1.setType(DataTypeManager.DefaultDataClasses.INTEGER); BufferManager bm = BufferManagerFactory.getStandaloneBufferManager(); - TupleSourceID tsid = bm.createTupleSource(Arrays.asList(es1), "test", TupleSourceType.PROCESSOR); //$NON-NLS-1$ - bm.addTupleBatch(tsid, new TupleBatch(1, new List[] {Arrays.asList(1)})); - SortUtility su = new SortUtility(tsid, Arrays.asList(es1), Arrays.asList(Boolean.TRUE), Mode.DUP_REMOVE, bm, "test", true); //$NON-NLS-1$ - TupleSourceID out = su.sort(); - TupleSource ts = bm.getTupleSource(out); + TupleBuffer tsid = bm.createTupleBuffer(Arrays.asList(es1), "test", TupleSourceType.PROCESSOR); //$NON-NLS-1$ + tsid.addTuple(Arrays.asList(1)); + SortUtility su = new SortUtility(tsid.createIndexedTupleSource(), Arrays.asList(es1), Arrays.asList(Boolean.TRUE), Mode.DUP_REMOVE, bm, "test"); //$NON-NLS-1$ + TupleBuffer out = su.sort(); + TupleSource ts = out.createIndexedTupleSource(); assertEquals(Arrays.asList(1), ts.nextTuple()); try { ts.nextTuple(); @@ -334,7 +309,7 @@ } catch (BlockedException e) { } - bm.addTupleBatch(tsid, new TupleBatch(2, new List[] {Arrays.asList(2)})); + tsid.addTuple(Arrays.asList(2)); su.sort(); assertEquals(Arrays.asList(2), ts.nextTuple()); } Modified: trunk/engine/src/test/java/com/metamatrix/query/processor/xml/TestXMLProcessor.java =================================================================== --- trunk/engine/src/test/java/com/metamatrix/query/processor/xml/TestXMLProcessor.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/test/java/com/metamatrix/query/processor/xml/TestXMLProcessor.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -42,9 +42,9 @@ import com.metamatrix.api.exception.query.QueryPlannerException; import com.metamatrix.api.exception.query.QueryResolverException; import com.metamatrix.api.exception.query.QueryValidatorException; -import com.metamatrix.common.buffer.BlockedException; import com.metamatrix.common.buffer.BufferManager; import com.metamatrix.common.buffer.BufferManagerFactory; +import com.metamatrix.common.buffer.TupleBuffer; import com.metamatrix.common.buffer.TupleSource; import com.metamatrix.common.types.DataTypeManager; import com.metamatrix.common.types.XMLType; @@ -71,6 +71,7 @@ import com.metamatrix.query.optimizer.capabilities.SourceCapabilities.Capability; import com.metamatrix.query.optimizer.xml.TestXMLPlanner; import com.metamatrix.query.parser.QueryParser; +import com.metamatrix.query.processor.BatchCollector; import com.metamatrix.query.processor.FakeDataManager; import com.metamatrix.query.processor.QueryProcessor; import com.metamatrix.query.resolver.QueryResolver; @@ -2964,20 +2965,11 @@ CommandContext context = new CommandContext("pID", "TestConn", "testUser", null, null); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ context.setProcessDebug(DEBUG); QueryProcessor processor = new QueryProcessor(plan, context, bufferMgr, dataMgr); - - while(true) { - try { - processor.process(); - break; - } catch(BlockedException e) { - // retry - } - } + processor.setNonBlocking(true); + BatchCollector collector = processor.createBatchCollector(); + TupleBuffer id = collector.collectTuples(); - //int count = bufferMgr.getFinalRowCount(tsID); - //assertEquals("Incorrect number of records: ", 1, count); //$NON-NLS-1$ - - TupleSource ts = bufferMgr.getTupleSource(processor.getResultsID()); + TupleSource ts = id.createIndexedTupleSource(); List row = ts.nextTuple(); assertEquals("Incorrect number of columns: ", 1, row.size()); //$NON-NLS-1$ @@ -2985,7 +2977,7 @@ XMLType result = (XMLType)row.get(0); String actualDoc = result.getString(); - bufferMgr.removeTupleSource(processor.getResultsID()); + id.remove(); if(DEBUG) { System.out.println("expectedDoc = \n" + expectedDoc); //$NON-NLS-1$ @@ -3009,7 +3001,9 @@ BufferManager bufferMgr = BufferManagerFactory.getStandaloneBufferManager(); CommandContext context = new CommandContext("pID", null, null, null, null); //$NON-NLS-1$ QueryProcessor processor = new QueryProcessor(plan, context, bufferMgr, dataMgr); - processor.process(); + processor.setNonBlocking(true); + BatchCollector collector = processor.createBatchCollector(); + collector.collectTuples(); } catch (Exception e){ if (expectedException.isInstance(e)){ expected = e; @@ -3053,12 +3047,13 @@ BufferManager bufferMgr = BufferManagerFactory.getStandaloneBufferManager(); CommandContext context = new CommandContext("pID", null, null, null, null); //$NON-NLS-1$ QueryProcessor processor = new QueryProcessor(plan, context, bufferMgr, dataMgr); - processor.process(); + processor.setNonBlocking(true); + BatchCollector collector = processor.createBatchCollector(); + TupleBuffer tupleBuffer = collector.collectTuples(); + int count = tupleBuffer.getRowCount(); + assertEquals("Incorrect number of records: ", expectedDocs.length, count); //$NON-NLS-1$ - int count = bufferMgr.getFinalRowCount(processor.getResultsID()); - assertEquals("Incorrect number of records: ", expectedDocs.length, count); //$NON-NLS-1$ - - TupleSource ts = bufferMgr.getTupleSource(processor.getResultsID()); + TupleSource ts = tupleBuffer.createIndexedTupleSource(); for (int i=0; i<expectedDocs.length; i++){ List row = ts.nextTuple(); if(row.isEmpty()){ @@ -3071,7 +3066,7 @@ //assertEquals("XML doc result # " + i +" mismatch: ", expectedDocs[i], actualDoc); //$NON-NLS-1$ //$NON-NLS-2$ compareDocuments(expectedDocs[i], actualDoc); } - bufferMgr.removeTupleSource(processor.getResultsID()); + tupleBuffer.remove(); } // ============================================================================================= @@ -6060,10 +6055,11 @@ BufferManager bufferMgr = BufferManagerFactory.getStandaloneBufferManager(); CommandContext context = new CommandContext("pID", null, null, null, null); //$NON-NLS-1$ QueryProcessor processor = new QueryProcessor(plan, context, bufferMgr, dataMgr); - + processor.setNonBlocking(true); + BatchCollector collector = processor.createBatchCollector(); MetaMatrixComponentException failOnDefaultException = null; try{ - processor.process(); + collector.collectTuples(); } catch (MetaMatrixComponentException e){ failOnDefaultException = e; } Deleted: trunk/engine/src/test/java/org/teiid/dqp/internal/cache/TestCacheID.java =================================================================== --- trunk/engine/src/test/java/org/teiid/dqp/internal/cache/TestCacheID.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/test/java/org/teiid/dqp/internal/cache/TestCacheID.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -1,107 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package org.teiid.dqp.internal.cache; - -import java.util.ArrayList; -import java.util.List; - -import org.teiid.dqp.internal.cache.CacheID; - -import junit.framework.TestCase; - -public class TestCacheID extends TestCase{ - public TestCacheID(final String name) { - super(name); - } - - public void testIDEquals1(){ - CacheID id1 = new CacheID("12345", "select * from table1"); //$NON-NLS-1$//$NON-NLS-2$ - CacheID id2 = new CacheID("12345", "select * from table1"); //$NON-NLS-1$//$NON-NLS-2$ - assertEquals(id1, id2); - } - - public void testIDEquals2(){ - List variables1 = new ArrayList(); - variables1.add(new Integer(6)); - variables1.add("aaa"); //$NON-NLS-1$ - CacheID id1 = new CacheID("12345", "select * from table1",variables1); //$NON-NLS-1$//$NON-NLS-2$ - List variables2 = new ArrayList(); - variables2.add(new Integer(6)); - variables2.add("aaa"); //$NON-NLS-1$ - CacheID id2 = new CacheID("12345", "select * from table1",variables2); //$NON-NLS-1$//$NON-NLS-2$ - assertEquals(id1, id2); - } - - public void testIDEquals3(){ - CacheID id1 = new CacheID("12345", "select * from table1", null); //$NON-NLS-1$//$NON-NLS-2$ - CacheID id2 = new CacheID("12345", "select * from table1"); //$NON-NLS-1$//$NON-NLS-2$ - assertEquals(id1, id2); - } - - public void testIDNotEquals1(){ - CacheID id1 = new CacheID("12345", "select * from table1"); //$NON-NLS-1$//$NON-NLS-2$ - CacheID id2 = new CacheID("2345", "select * from table1"); //$NON-NLS-1$//$NON-NLS-2$ - assertTrue(!id1.equals(id2)); - } - - public void testIDNotEquals2(){ - CacheID id1 = new CacheID("12345", "select * from table1"); //$NON-NLS-1$//$NON-NLS-2$ - CacheID id2 = new CacheID("12345", "select * from table2"); //$NON-NLS-1$//$NON-NLS-2$ - assertTrue(!id1.equals(id2)); - } - - public void testIDNotEquals3(){ - List variables1 = new ArrayList(); - variables1.add(new Integer(6)); - variables1.add("aba"); //$NON-NLS-1$ - CacheID id1 = new CacheID("12345", "select * from table1",variables1); //$NON-NLS-1$//$NON-NLS-2$ - List variables2 = new ArrayList(); - variables2.add(new Integer(6)); - variables2.add("aaa"); //$NON-NLS-1$ - CacheID id2 = new CacheID("12345", "select * from table1",variables2); //$NON-NLS-1$//$NON-NLS-2$ - assertTrue(!id1.equals(id2)); - } - - public void testIDNotEquals4(){ - List variables1 = null; - CacheID id1 = new CacheID("12345", "select * from table1",variables1); //$NON-NLS-1$//$NON-NLS-2$ - List variables2 = new ArrayList(); - variables2.add(new Integer(6)); - variables2.add("aaa"); //$NON-NLS-1$ - CacheID id2 = new CacheID("12345", "select * from table1",variables2); //$NON-NLS-1$//$NON-NLS-2$ - assertTrue(!id1.equals(id2)); - } - - public void testIDNotEquals5(){ - List variables1 = new ArrayList(); - variables1.add(new Integer(6)); - variables1.add("aaa"); //$NON-NLS-1$ - variables1.add("aaa"); //$NON-NLS-1$ - CacheID id1 = new CacheID("12345", "select * from table1",variables1); //$NON-NLS-1$//$NON-NLS-2$ - List variables2 = new ArrayList(); - variables2.add(new Integer(6)); - variables2.add("aaa"); //$NON-NLS-1$ - CacheID id2 = new CacheID("12345", "select * from table1",variables2); //$NON-NLS-1$//$NON-NLS-2$ - assertTrue(!id1.equals(id2)); - } -} Deleted: trunk/engine/src/test/java/org/teiid/dqp/internal/cache/TestResultSetCache.java =================================================================== --- trunk/engine/src/test/java/org/teiid/dqp/internal/cache/TestResultSetCache.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/test/java/org/teiid/dqp/internal/cache/TestResultSetCache.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -1,190 +0,0 @@ -/* - * JBoss, Home of Professional Open Source. - * See the COPYRIGHT.txt file distributed with this work for information - * regarding copyright ownership. Some portions may be licensed - * to Red Hat, Inc. under one or more contributor license agreements. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301 USA. - */ - -package org.teiid.dqp.internal.cache; - -import static org.junit.Assert.*; - -import java.util.ArrayList; -import java.util.List; -import java.util.Properties; - -import org.junit.Test; - -import com.metamatrix.cache.FakeCache; -import com.metamatrix.common.buffer.impl.SizeUtility; -import com.metamatrix.core.MetaMatrixRuntimeException; - -public class TestResultSetCache { - - @Test public void testSetAndGetResultsForSession() throws Exception{ - ResultSetCache cache = new ResultSetCache(); - cache.setScope(ResultSetCache.RS_CACHE_SCOPE_CONN); - cache.start(new FakeCache.FakeCacheFactory()); - CacheID id1 = new CacheID("12345", "select * from table1"); //$NON-NLS-1$//$NON-NLS-2$ - List[] result1 = new List[]{new ArrayList()}; - cache.setResults(id1, new CacheResults(result1, 1, true ), "req1"); //$NON-NLS-1$ - CacheID id2 = new CacheID("12346", "select * from table2"); //$NON-NLS-1$//$NON-NLS-2$ - List[] result2 = new List[]{new ArrayList()}; - cache.setResults(id2, new CacheResults(result2, 1, true), "req2" ); //$NON-NLS-1$ - assertEquals(result1, cache.getResults(id1, new int[]{1, 500}).getResults()); - assertEquals(result2, cache.getResults(id2, new int[]{1, 500}).getResults()); - } - - @Test public void testSetAndGetResultsForVDB() throws Exception{ - ResultSetCache cache = new ResultSetCache(); - cache.start(new FakeCache.FakeCacheFactory()); - CacheID id1 = new CacheID("vdb1", "select * from table1"); //$NON-NLS-1$//$NON-NLS-2$ - List[] result1 = new List[]{new ArrayList()}; - cache.setResults(id1, new CacheResults(result1, 1, true) , "req1"); //$NON-NLS-1$ - CacheID id2 = new CacheID("vdb2", "select * from table2"); //$NON-NLS-1$//$NON-NLS-2$ - List[] result2 = new List[]{new ArrayList()}; - cache.setResults(id2, new CacheResults(result2, 1, true ), "req2"); //$NON-NLS-1$ - assertEquals(cache.getResults(id1, new int[]{1, 500}).getResults(), result1); - assertEquals(cache.getResults(id2, new int[]{1, 500}).getResults(), result2); - } - - @Test public void testClearAllCache() throws Exception{ - ResultSetCache cache = new ResultSetCache(); - cache.start(new FakeCache.FakeCacheFactory()); - CacheID id1 = new CacheID("vdb1", "select * from table1"); //$NON-NLS-1$//$NON-NLS-2$ - List[] result1 = new List[]{new ArrayList()}; - cache.setResults(id1, new CacheResults(result1, 1, true), "req1" ); //$NON-NLS-1$ - CacheID id2 = new CacheID("vdb2", "select * from table2"); //$NON-NLS-1$//$NON-NLS-2$ - List[] result2 = new List[]{new ArrayList()}; - cache.setResults(id2, new CacheResults(result2, 1, true ), "req2"); //$NON-NLS-1$ - cache.clear(); - assertNull(cache.getResults(id1, new int[]{1, 500})); - } - - @Test public void testSetAndGetResultsForSession1() throws Exception{ - ResultSetCache cache = new ResultSetCache(); - cache.setScope(ResultSetCache.RS_CACHE_SCOPE_CONN); - cache.start(new FakeCache.FakeCacheFactory()); - CacheID id1 = new CacheID("12345", "select * from table1"); //$NON-NLS-1$//$NON-NLS-2$ - List row1 = new ArrayList(); - row1.add("1"); //$NON-NLS-1$ - List row2 = new ArrayList(); - row2.add("2"); //$NON-NLS-1$ - List row3 = new ArrayList(); - row3.add("3"); //$NON-NLS-1$ - List row4 = new ArrayList(); - row4.add("4"); //$NON-NLS-1$ - List[] result1 = new List[]{row1, row2, row3, row4}; - cache.setResults(id1, new CacheResults(result1, 1, true ), "req1"); //$NON-NLS-1$ - assertEquals(cache.getResults(id1, new int[]{1, 500}).getResults(), result1); - assertEquals(cache.getResults(id1, new int[]{1, 2}).getResults()[0], row1); - } - - @Test public void testSetAndGetResultsForSession2() throws Exception{ - ResultSetCache cache = new ResultSetCache(); - cache.setScope(ResultSetCache.RS_CACHE_SCOPE_CONN); - cache.start(new FakeCache.FakeCacheFactory()); - CacheID id1 = new CacheID("12345", "select * from table1"); //$NON-NLS-1$//$NON-NLS-2$ - List row1 = new ArrayList(); - row1.add("1"); //$NON-NLS-1$ - List row2 = new ArrayList(); - row2.add("2"); //$NON-NLS-1$ - List row3 = new ArrayList(); - row3.add("3"); //$NON-NLS-1$ - List row4 = new ArrayList(); - row4.add("4"); //$NON-NLS-1$ - List[] result1 = new List[]{row1, row2, row3, row4}; - cache.setResults(id1, new CacheResults(result1, 1, false) , "req1"); //$NON-NLS-1$ - List row5 = new ArrayList(); - row5.add("5"); //$NON-NLS-1$ - List[] result2 = new List[]{row5}; - cache.setResults(id1, new CacheResults(result2, 5, true), "req1" ); //$NON-NLS-1$ - assertEquals(cache.getResults(id1, new int[]{1, 500}).getResults()[4], row5); - assertEquals(cache.getResults(id1, new int[]{1, 2}).getResults()[0], row1); - } - - @Test public void testMaxSize() throws Exception{ - ResultSetCache cache = new ResultSetCache(); - cache.setMaxEntrySize(600000); - cache.start(new FakeCache.FakeCacheFactory()); - CacheID id1 = new CacheID("vdb1", "select * from table1"); //$NON-NLS-1$//$NON-NLS-2$ - CacheResults result1 = createResults(500000, 1, 1, true); - CacheID id2 = new CacheID("vdb2", "select * from table2"); //$NON-NLS-1$//$NON-NLS-2$ - CacheResults result2 = createResults(500000, 1, 1, true); - CacheID id3 = new CacheID("vdb1", "select * from table3"); //$NON-NLS-1$//$NON-NLS-2$ - CacheResults result3 = createResults(700000, 1, 1, true); - //add two results - cache.setResults(id1, result1, "req1" ); //$NON-NLS-1$ - cache.setResults(id2, result2, "req2" ); //$NON-NLS-1$ - //use the first one - assertNotNull(cache.getResults(id1, new int[]{1, 500})); - //add one more result. Because the max size is 1mb, the - //third one can not be added - cache.setResults(id3, result3, "req3" ); //$NON-NLS-1$ - assertNull(cache.getResults(id3, new int[]{1, 500})); - //reduced the size and try again. The second one should - //be removed - result3 = createResults(1000, 1, 1, true); - cache.setResults(id3, result3, "req4" ); //$NON-NLS-1$ - assertNotNull(cache.getResults(id3, new int[]{1, 500})); - } - - private CacheResults createResults(int size, int firstRow, int rowCnt, boolean isFinal){ - List[] results = new List[rowCnt]; - for(int i=0; i<rowCnt; i++){ - results[i] = new ArrayList(); - } - CacheResults cr = new CacheResults(results, firstRow, isFinal); - cr.setSize(size); - return cr; - } - - @Test public void testComputeSize() throws Exception{ - int cnt = 1000000; - List[] results = new List[cnt]; - List row = new ArrayList(); - row.add("ajwuiotbn0w49yunq9 tjfvwioprkpo23bltplql;galkg"); //$NON-NLS-1$ - row.add(new Long(1242534776)); - row.add(new Double(24235.5476884)); - for(int i=0; i<cnt; i++){ - results[i] = row; - } - Properties props = new Properties(); - props.setProperty(ResultSetCache.RS_CACHE_MAX_AGE, "0"); //$NON-NLS-1$ - props.setProperty(ResultSetCache.RS_CACHE_MAX_SIZE, "0"); //$NON-NLS-1$ - props.setProperty(ResultSetCache.RS_CACHE_SCOPE, ResultSetCache.RS_CACHE_SCOPE_CONN); - ResultSetCache cache = new ResultSetCache(); - CacheID id1 = new CacheID("vdb1", "select * from table1"); //$NON-NLS-1$//$NON-NLS-2$ - CacheResults result = new CacheResults(results, 1, true); - cache.setResults(id1, result, "req1" ); //$NON-NLS-1$ - - int size = (SizeUtility.IS_64BIT ? 304000016 : 260000016); - assertEquals(size, result.getSize()); - } - - @Test(expected=MetaMatrixRuntimeException.class) public void testBatchNotContiguous() throws Exception{ - ResultSetCache cache = new ResultSetCache(); - cache.start(new FakeCache.FakeCacheFactory()); - CacheID id1 = new CacheID("vdb1", "select * from table1"); //$NON-NLS-1$//$NON-NLS-2$ - CacheResults result1 = createResults(1000, 1, 100, false); - CacheResults result3 = createResults(2000, 102, 200, true); - cache.setResults(id1, result1 , "req1"); //$NON-NLS-1$ - cache.setResults(id1, result3, "req1" ); //$NON-NLS-1$ - } - -} Modified: trunk/engine/src/test/java/org/teiid/dqp/internal/process/TestPreparedPlanCache.java =================================================================== --- trunk/engine/src/test/java/org/teiid/dqp/internal/process/TestPreparedPlanCache.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/test/java/org/teiid/dqp/internal/process/TestPreparedPlanCache.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -22,16 +22,16 @@ package org.teiid.dqp.internal.process; +import static org.junit.Assert.*; + import java.util.ArrayList; import java.util.HashMap; import java.util.Map; -import static org.junit.Assert.*; - import org.junit.BeforeClass; import org.junit.Test; -import org.teiid.dqp.internal.process.PreparedPlanCache; import org.teiid.dqp.internal.process.PreparedPlanCache.CacheID; +import org.teiid.dqp.internal.process.PreparedPlanCache.PreparedPlan; import com.metamatrix.api.exception.query.QueryParserException; import com.metamatrix.query.analysis.AnalysisRecord; @@ -59,12 +59,12 @@ @Test public void testPutPreparedPlan(){ PreparedPlanCache cache = new PreparedPlanCache(); - CacheID id = new PreparedPlanCache.CacheID(token, pi, EXAMPLE_QUERY + 1); + CacheID id = new CacheID(token, pi, EXAMPLE_QUERY + 1); //No PreparedPlan at the begining assertNull(cache.getPreparedPlan(id)); //create one - cache.putPreparedPlan(id, true, new PreparedPlanCache.PreparedPlan()); + cache.putPreparedPlan(id, true, new PreparedPlan()); //should have one now assertNotNull("Unable to get prepared plan from cache", cache.getPreparedPlan(id)); //$NON-NLS-1$ } @@ -75,7 +75,7 @@ helpPutPreparedPlans(cache, token2, 0, 15); //read an entry for session2 (token2) - PreparedPlanCache.PreparedPlan pPlan = cache.getPreparedPlan(new PreparedPlanCache.CacheID(token2, pi, EXAMPLE_QUERY + 12)); + PreparedPlan pPlan = cache.getPreparedPlan(new CacheID(token2, pi, EXAMPLE_QUERY + 12)); assertNotNull("Unable to get prepared plan from cache", pPlan); //$NON-NLS-1$ assertEquals("Error getting plan from cache", new RelationalPlan(new ProjectNode(12)).toString(), pPlan.getPlan().toString()); //$NON-NLS-1$ assertEquals("Error getting command from cache", EXAMPLE_QUERY + 12, pPlan.getCommand().toString()); //$NON-NLS-1$ @@ -90,24 +90,24 @@ helpPutPreparedPlans(cache, token, 1, 1); helpPutPreparedPlans(cache, token2, 1, 1); //should have one - assertNotNull("Unable to get prepared plan from cache for token", cache.getPreparedPlan(new PreparedPlanCache.CacheID(token, pi, EXAMPLE_QUERY + 1))); //$NON-NLS-1$ + assertNotNull("Unable to get prepared plan from cache for token", cache.getPreparedPlan(new CacheID(token, pi, EXAMPLE_QUERY + 1))); //$NON-NLS-1$ cache.clearAll(); //should not exist for token - assertNull("Failed remove from cache", cache.getPreparedPlan(new PreparedPlanCache.CacheID(token, pi, EXAMPLE_QUERY + 1))); //$NON-NLS-1$ + assertNull("Failed remove from cache", cache.getPreparedPlan(new CacheID(token, pi, EXAMPLE_QUERY + 1))); //$NON-NLS-1$ //should not exist for token2 - assertNull("Unable to get prepared plan from cache for token2", cache.getPreparedPlan(new PreparedPlanCache.CacheID(token2, pi, EXAMPLE_QUERY + 1))); //$NON-NLS-1$ + assertNull("Unable to get prepared plan from cache for token2", cache.getPreparedPlan(new CacheID(token2, pi, EXAMPLE_QUERY + 1))); //$NON-NLS-1$ } @Test public void testMaxSize(){ PreparedPlanCache cache = new PreparedPlanCache(100); helpPutPreparedPlans(cache, token, 0, 101); //the first one should be gone because the max size is 100 - assertNull(cache.getPreparedPlan(new PreparedPlanCache.CacheID(token, pi, EXAMPLE_QUERY + 0))); + assertNull(cache.getPreparedPlan(new CacheID(token, pi, EXAMPLE_QUERY + 0))); - assertNotNull(cache.getPreparedPlan(new PreparedPlanCache.CacheID(token, pi, EXAMPLE_QUERY + 12))); + assertNotNull(cache.getPreparedPlan(new CacheID(token, pi, EXAMPLE_QUERY + 12))); helpPutPreparedPlans(cache, token, 102, 50); //"sql12" should still be there based on lru policy - assertNotNull(cache.getPreparedPlan(new PreparedPlanCache.CacheID(token, pi, EXAMPLE_QUERY + 12))); + assertNotNull(cache.getPreparedPlan(new CacheID(token, pi, EXAMPLE_QUERY + 12))); helpPutPreparedPlans(cache, token2, 0, 121); helpPutPreparedPlans(cache, token, 0, 50); @@ -121,12 +121,12 @@ // Add 1 plan and verify it is not in the cache helpPutPreparedPlans(cache, token, 0, 1); - assertNull(cache.getPreparedPlan(new PreparedPlanCache.CacheID(token, pi, EXAMPLE_QUERY + 0))); + assertNull(cache.getPreparedPlan(new CacheID(token, pi, EXAMPLE_QUERY + 0))); assertEquals(0, cache.getSpaceUsed()); // Add another plan and verify it is not in the cache helpPutPreparedPlans(cache, token, 1, 1); - assertNull(cache.getPreparedPlan(new PreparedPlanCache.CacheID(token, pi, EXAMPLE_QUERY + 1))); + assertNull(cache.getPreparedPlan(new CacheID(token, pi, EXAMPLE_QUERY + 1))); assertEquals(0, cache.getSpaceUsed()); } @@ -148,9 +148,9 @@ } catch (QueryParserException e) { throw new RuntimeException(e); } - CacheID id = new PreparedPlanCache.CacheID(session, pi, dummy.toString()); + CacheID id = new CacheID(session, pi, dummy.toString()); - PreparedPlanCache.PreparedPlan pPlan = new PreparedPlanCache.PreparedPlan(); + PreparedPlan pPlan = new PreparedPlan(); cache.putPreparedPlan(id, true, pPlan); pPlan.setCommand(dummy); pPlan.setPlan(new RelationalPlan(new ProjectNode(i))); Modified: trunk/engine/src/test/java/org/teiid/dqp/internal/process/TestPreparedStatement.java =================================================================== --- trunk/engine/src/test/java/org/teiid/dqp/internal/process/TestPreparedStatement.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/test/java/org/teiid/dqp/internal/process/TestPreparedStatement.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -84,21 +84,19 @@ } static public void helpTestProcessing(String preparedSql, List values, List[] expected, ProcessorDataManager dataManager, CapabilitiesFinder capFinder, QueryMetadataInterface metadata, TestablePreparedPlanCache prepPlanCache, boolean callableStatement, boolean isSessionSpecific, boolean isAlreadyCached) throws Exception { - TestablePreparedPlanCache pPlanCache = null; - CapabilitiesFinder cFinder = null; - ProcessorDataManager dManager = null; - if ( dataManager == null ) { // Construct data manager with data - dManager = new FakeDataManager(); - TestProcessor.sampleData1((FakeDataManager)dManager); - } else dManager = dataManager; + dataManager = new FakeDataManager(); + TestProcessor.sampleData1((FakeDataManager)dataManager); + } - if ( capFinder == null ) cFinder = new DefaultCapabilitiesFinder(); - else cFinder = capFinder; + if ( capFinder == null ) { + capFinder = new DefaultCapabilitiesFinder(); + } - if ( prepPlanCache == null ) pPlanCache = new TestablePreparedPlanCache(); - else pPlanCache = prepPlanCache; + if ( prepPlanCache == null ) { + prepPlanCache = new TestablePreparedPlanCache(); + } // expected cache hit count int exHitCount = -1; @@ -109,28 +107,28 @@ * get the plan twice. Otherwise, we want it to be 1. */ if ( isAlreadyCached ) { - exHitCount = pPlanCache.hitCount + 2; + exHitCount = prepPlanCache.hitCount + 2; } else { - exHitCount = pPlanCache.hitCount + 1; + exHitCount = prepPlanCache.hitCount + 1; } //Create plan or used cache plan if isPlanCached - PreparedStatementRequest plan = TestPreparedStatement.helpGetProcessorPlan(preparedSql, values, cFinder, metadata, pPlanCache, SESSION_ID, callableStatement, false); + PreparedStatementRequest plan = TestPreparedStatement.helpGetProcessorPlan(preparedSql, values, capFinder, metadata, prepPlanCache, SESSION_ID, callableStatement, false); // Run query TestProcessor.doProcess(plan.processPlan, dataManager, expected, plan.context); //test cached plan - plan = TestPreparedStatement.helpGetProcessorPlan(preparedSql, values, cFinder, metadata, pPlanCache, SESSION_ID, callableStatement, false); + plan = TestPreparedStatement.helpGetProcessorPlan(preparedSql, values, capFinder, metadata, prepPlanCache, SESSION_ID, callableStatement, false); //make sure the plan is only created once - assertEquals("should reuse the plan", exHitCount, pPlanCache.hitCount); //$NON-NLS-1$ + assertEquals("should reuse the plan", exHitCount, prepPlanCache.hitCount); //$NON-NLS-1$ // Run query again TestProcessor.doProcess(plan.processPlan, dataManager, expected, plan.context); //get the plan again with a new connection - assertNotNull(TestPreparedStatement.helpGetProcessorPlan(preparedSql, values, cFinder, metadata, pPlanCache, 7, callableStatement, false)); + assertNotNull(TestPreparedStatement.helpGetProcessorPlan(preparedSql, values, capFinder, metadata, prepPlanCache, 7, callableStatement, false)); /* * If the command is not specific to a session we expect @@ -139,7 +137,7 @@ * created and the hit count will be unchanged. */ if ( !isSessionSpecific ) exHitCount++; - assertEquals(exHitCount, pPlanCache.hitCount); + assertEquals(exHitCount, prepPlanCache.hitCount); } @Test public void testWhere() throws Exception { Modified: trunk/engine/src/test/java/org/teiid/dqp/internal/process/validator/TestAuthorizationValidationVisitor.java =================================================================== --- trunk/engine/src/test/java/org/teiid/dqp/internal/process/validator/TestAuthorizationValidationVisitor.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/engine/src/test/java/org/teiid/dqp/internal/process/validator/TestAuthorizationValidationVisitor.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -251,7 +251,7 @@ String sql = "select lookup('pm1.g1', 'e1', 'e2', 1)"; //$NON-NLS-1$ Command command = QueryParser.getQueryParser().parseCommand(sql); - Request.validateWithVisitor(mvvv, FakeMetadataFactory.example1Cached(), command, true); + Request.validateWithVisitor(mvvv, FakeMetadataFactory.example1Cached(), command); } public void testLookupVisibility() throws Exception { Modified: trunk/runtime/src/main/java/com/metamatrix/dqp/embedded/services/EmbeddedBufferService.java =================================================================== --- trunk/runtime/src/main/java/com/metamatrix/dqp/embedded/services/EmbeddedBufferService.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/runtime/src/main/java/com/metamatrix/dqp/embedded/services/EmbeddedBufferService.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -37,7 +37,8 @@ import com.metamatrix.common.buffer.BufferManagerPropertyNames; import com.metamatrix.common.buffer.StorageManager; import com.metamatrix.common.buffer.impl.BufferManagerImpl; -import com.metamatrix.common.buffer.storage.file.FileStorageManager; +import com.metamatrix.common.buffer.impl.FileStorageManager; +import com.metamatrix.common.buffer.impl.MemoryStorageManager; import com.metamatrix.dqp.embedded.DQPEmbeddedPlugin; import com.metamatrix.dqp.service.BufferService; import com.metamatrix.dqp.service.ConfigurationService; @@ -113,7 +114,7 @@ // Construct and initialize the buffer manager this.bufferMgr = new BufferManagerImpl(); - this.bufferMgr.initialize("local", bufferProps); //$NON-NLS-1$ + this.bufferMgr.initialize(bufferProps); // If necessary, add disk storage manager if(useDisk) { @@ -125,9 +126,11 @@ fsm.initialize(fsmProps); this.bufferMgr.setStorageManager(fsm); - // start the file storrage manager in clean state + // start the file storage manager in clean state // wise FileStorageManager is smart enough to clen up after itself cleanDirectory(bufferDir); + } else { + this.bufferMgr.setStorageManager(new MemoryStorageManager()); } } catch(MetaMatrixComponentException e) { @@ -141,7 +144,7 @@ * @see com.metamatrix.common.application.ApplicationService#stop() */ public void stopService() throws ApplicationLifecycleException { - bufferMgr.stop(); + bufferMgr.shutdown(); // Delete the buffer directory if (bufferDir != null) { Modified: trunk/test-integration/common/src/test/java/com/metamatrix/server/integration/BaseQueryTest.java =================================================================== --- trunk/test-integration/common/src/test/java/com/metamatrix/server/integration/BaseQueryTest.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/test-integration/common/src/test/java/com/metamatrix/server/integration/BaseQueryTest.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -21,18 +21,13 @@ */ package com.metamatrix.server.integration; -import java.sql.SQLXML; import java.util.List; import java.util.Properties; -import org.teiid.metadata.index.VDBMetadataFactory; - import junit.framework.TestCase; -import com.metamatrix.common.buffer.BufferManager; -import com.metamatrix.common.buffer.BufferManagerFactory; -import com.metamatrix.common.buffer.TupleSource; -import com.metamatrix.common.buffer.TupleSourceID; +import org.teiid.metadata.index.VDBMetadataFactory; + import com.metamatrix.query.analysis.AnalysisRecord; import com.metamatrix.query.metadata.QueryMetadataInterface; import com.metamatrix.query.optimizer.QueryOptimizer; @@ -40,7 +35,7 @@ import com.metamatrix.query.optimizer.capabilities.CapabilitiesFinder; import com.metamatrix.query.processor.ProcessorDataManager; import com.metamatrix.query.processor.ProcessorPlan; -import com.metamatrix.query.processor.QueryProcessor; +import com.metamatrix.query.processor.TestProcessor; import com.metamatrix.query.sql.lang.Command; import com.metamatrix.query.util.CommandContext; @@ -61,9 +56,10 @@ public static QueryMetadataInterface createMetadata(String vdbFile, String systemVDBFile) { return VDBMetadataFactory.getVDBMetadata(new String[] {vdbFile, systemVDBFile}); } - - public ProcessorPlan createPlan(QueryMetadataInterface metadata, String sql, CapabilitiesFinder capFinder, boolean debug) throws Exception { - + + protected void doProcess(QueryMetadataInterface metadata, String sql, CapabilitiesFinder capFinder, ProcessorDataManager dataManager, List[] expectedResults, boolean debug) throws Exception { + CommandContext context = createCommandContext(); + context.setProcessDebug(debug); Command command = TestOptimizer.helpGetCommand(sql, metadata, null); // plan @@ -76,80 +72,10 @@ System.out.println(analysisRecord.getDebugLog()); } } - - return plan; - } - - protected void doProcess(ProcessorPlan plan, ProcessorDataManager dataManager, List[] expectedResults, boolean debug) throws Exception { - BufferManager bufferMgr = BufferManagerFactory.getStandaloneBufferManager(); - CommandContext context = createCommandContext(); - context.setProcessDebug(debug); - QueryProcessor processor = new QueryProcessor(plan, context, bufferMgr, dataManager); - TupleSourceID tsID = processor.getResultsID(); - processor.process(); - // Create QueryResults from TupleSource - TupleSource ts = bufferMgr.getTupleSource(tsID); - int count = bufferMgr.getFinalRowCount(tsID); - - if(debug) { - System.out.println("\nResults:\n" + bufferMgr.getTupleSchema(tsID)); //$NON-NLS-1$ - TupleSource ts2 = bufferMgr.getTupleSource(tsID); - for(int j=0; j<count; j++) { - System.out.println("" + j + ": " + ts2.nextTuple()); //$NON-NLS-1$ //$NON-NLS-2$ - } - ts2.closeSource(); - } - - // Compare actual to expected row count - assertEquals("Did not get expected row count: ", expectedResults.length, count); //$NON-NLS-1$ - - // Walk results and compare - for(int i=0; i<count; i++) { - List record = ts.nextTuple(); - - Object value = record.get(0); - if(value instanceof SQLXML) { - record.set(0, ((SQLXML)value).getString()); - } - assertEquals(expectedResults[i].get(0), record.get(0)); - } - ts.closeSource(); - - bufferMgr.removeTupleSource(tsID); + TestProcessor.doProcess(plan, dataManager, expectedResults, context); } - protected void doProcessNoResultsCheck(ProcessorPlan plan, ProcessorDataManager dataManager, int expectedRowCount, boolean debug) throws Exception { - BufferManager bufferMgr = BufferManagerFactory.getStandaloneBufferManager(); - CommandContext context = createCommandContext(); - QueryProcessor processor = new QueryProcessor(plan, context, bufferMgr, dataManager); - TupleSourceID tsID = processor.getResultsID(); - processor.process(); - - // Create QueryResults from TupleSource - TupleSource ts = bufferMgr.getTupleSource(tsID); - int count = bufferMgr.getFinalRowCount(tsID); - - if(debug) { - System.out.println("\nResults:\n" + bufferMgr.getTupleSchema(tsID)); //$NON-NLS-1$ - TupleSource ts2 = bufferMgr.getTupleSource(tsID); - for(int j=0; j<count; j++) { - System.out.println("" + j + ": " + ts2.nextTuple()); //$NON-NLS-1$ //$NON-NLS-2$ - } - ts2.closeSource(); - } - - // Compare actual to expected row count - assertEquals("Did not get expected row count: ", expectedRowCount, count); //$NON-NLS-1$ - - // Walk results - for(int i=0; i<count; i++) { - ts.nextTuple(); - } - ts.closeSource(); - bufferMgr.removeTupleSource(tsID); - } - protected CommandContext createCommandContext() { Properties props = new Properties(); //props.setProperty(ContextProperties.SOAP_HOST, "my.host.com"); //$NON-NLS-1$ @@ -158,8 +84,4 @@ return context; } - public void verifyQueryPlan(String[] expectedAtomic, ProcessorPlan plan, QueryMetadataInterface md, CapabilitiesFinder capFinder) throws Exception { - TestOptimizer.checkAtomicQueries(expectedAtomic, plan, md, capFinder); - } - } Modified: trunk/test-integration/common/src/test/java/com/metamatrix/server/integration/TestTPCR.java =================================================================== --- trunk/test-integration/common/src/test/java/com/metamatrix/server/integration/TestTPCR.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/test-integration/common/src/test/java/com/metamatrix/server/integration/TestTPCR.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -59,16 +59,6 @@ FakeCapabilitiesFinder finder = new FakeCapabilitiesFinder(); finder.addCapabilities("TPCR_Oracle_9i", CapabilitiesConverter.convertCapabilities(new OracleCapabilities())); //$NON-NLS-1$ - ProcessorPlan plan = createPlan(METADATA, - "select l_orderkey, sum(l_extendedprice*(1-l_discount)) as revenue, o_orderdate, o_shippriority " + //$NON-NLS-1$ - "from customer, orders, lineitem " + //$NON-NLS-1$ - "where c_mktsegment = 'BUILDING' and c_custkey = o_custkey and l_orderkey = o_orderkey " + //$NON-NLS-1$ - "and o_orderdate < {ts'1995-03-15 00:00:00'} " + //$NON-NLS-1$ - "and l_shipdate > {ts'1995-03-15 00:00:00'} " + //$NON-NLS-1$ - "group by l_orderkey, o_orderdate, o_shippriority " + //$NON-NLS-1$ - "order by revenue desc, o_orderdate", //$NON-NLS-1$ - finder, DEBUG); - HardcodedDataManager dataMgr = new HardcodedDataManager(); List[] expected = @@ -79,7 +69,15 @@ dataMgr.addData("SELECT g_2.l_orderkey AS c_0, SUM((g_2.l_extendedprice * (1 - g_2.l_discount))) AS c_1, g_1.o_orderdate AS c_2, g_1.o_shippriority AS c_3 FROM TPCR_Oracle_9i.CUSTOMER AS g_0, TPCR_Oracle_9i.ORDERS AS g_1, TPCR_Oracle_9i.LINEITEM AS g_2 WHERE (g_0.c_custkey = g_1.o_custkey) AND (g_2.l_orderkey = g_1.o_orderkey) AND (g_0.c_mktsegment = 'BUILDING') AND (g_1.o_orderdate < {d'1995-03-15'}) AND (g_2.l_shipdate > {ts'1995-03-15 00:00:00.0'}) GROUP BY g_2.l_orderkey, g_1.o_orderdate, g_1.o_shippriority ORDER BY c_1 DESC, c_2", //$NON-NLS-1$ expected); - doProcess(plan, dataMgr, expected, DEBUG); + doProcess(METADATA, + "select l_orderkey, sum(l_extendedprice*(1-l_discount)) as revenue, o_orderdate, o_shippriority " + //$NON-NLS-1$ + "from customer, orders, lineitem " + //$NON-NLS-1$ + "where c_mktsegment = 'BUILDING' and c_custkey = o_custkey and l_orderkey = o_orderkey " + //$NON-NLS-1$ + "and o_orderdate < {ts'1995-03-15 00:00:00'} " + //$NON-NLS-1$ + "and l_shipdate > {ts'1995-03-15 00:00:00'} " + //$NON-NLS-1$ + "group by l_orderkey, o_orderdate, o_shippriority " + //$NON-NLS-1$ + "order by revenue desc, o_orderdate", //$NON-NLS-1$ + finder, dataMgr, expected, DEBUG); } @@ -87,13 +85,6 @@ FakeCapabilitiesFinder finder = new FakeCapabilitiesFinder(); finder.addCapabilities("TPCR_Ora", CapabilitiesConverter.convertCapabilities(new OracleCapabilities())); //$NON-NLS-1$ - ProcessorPlan plan = createPlan(BaseQueryTest.createMetadata(UnitTestUtil.getTestDataPath()+"/tpcr/TPCR_3.vdb"), //$NON-NLS-1$ - "SELECT count (*) " + //$NON-NLS-1$ - "FROM TPCR_Ora.CUSTOMER LEFT OUTER JOIN TPCR_Ora.ORDERS ON C_CUSTKEY = O_CUSTKEY " + //$NON-NLS-1$ - "WHERE (O_ORDERKEY IS NULL) OR O_ORDERDATE < '1992-01-02 00:00:00' " + //$NON-NLS-1$ - "AND C_ACCTBAL > 0", //$NON-NLS-1$ - finder, DEBUG); - HardcodedDataManager dataMgr = new HardcodedDataManager(); List[] expected = new List[] { Arrays.asList(new Object[] { new Integer(5) } ) }; @@ -101,7 +92,12 @@ dataMgr.addData("SELECT COUNT(*) FROM TPCR_Ora.CUSTOMER AS g_0 LEFT OUTER JOIN TPCR_Ora.ORDERS AS g_1 ON g_0.C_CUSTKEY = g_1.O_CUSTKEY WHERE (g_1.O_ORDERKEY IS NULL) OR ((g_1.O_ORDERDATE < {ts'1992-01-02 00:00:00.0'}) AND (g_0.C_ACCTBAL > 0))", //$NON-NLS-1$ expected); - doProcess(plan, dataMgr, expected, DEBUG); + doProcess(BaseQueryTest.createMetadata(UnitTestUtil.getTestDataPath()+"/tpcr/TPCR_3.vdb"), //$NON-NLS-1$ + "SELECT count (*) " + //$NON-NLS-1$ + "FROM TPCR_Ora.CUSTOMER LEFT OUTER JOIN TPCR_Ora.ORDERS ON C_CUSTKEY = O_CUSTKEY " + //$NON-NLS-1$ + "WHERE (O_ORDERKEY IS NULL) OR O_ORDERDATE < '1992-01-02 00:00:00' " + //$NON-NLS-1$ + "AND C_ACCTBAL > 0", //$NON-NLS-1$ + finder, dataMgr, expected, DEBUG); } @@ -122,13 +118,6 @@ finder.addCapabilities("TPCR_Ora", CapabilitiesConverter.convertCapabilities(new OracleCapabilities())); //$NON-NLS-1$ finder.addCapabilities("TPCR_SQLS", CapabilitiesConverter.convertCapabilities(new SqlServerCapabilities())); //$NON-NLS-1$ - ProcessorPlan plan = createPlan(BaseQueryTest.createMetadata(UnitTestUtil.getTestDataPath()+"/tpcr/TPCR_3.vdb"), //$NON-NLS-1$ - "SELECT C_CUSTKEY, C_NAME, C_ADDRESS, C_PHONE, C_ACCTBAL, O_ORDERKEY FROM TPCR_Ora.CUSTOMER " + //$NON-NLS-1$ - "LEFT OUTER JOIN TPCR_SQLS.ORDERS ON C_CUSTKEY = O_CUSTKEY " + //$NON-NLS-1$ - "AND O_ORDERDATE < {ts'1992-01-02 00:00:00.0'} " + //$NON-NLS-1$ - "WHERE (C_ACCTBAL > 50)", //$NON-NLS-1$ - finder, DEBUG); - HardcodedDataManager dataMgr = new HardcodedDataManager(); List[] oracleExpected = @@ -149,7 +138,12 @@ Arrays.asList(new Object[] { new Long(5), "Bill", "101 Fake St.", "392839283", "21.12", new Integer(13) } ), //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ Arrays.asList(new Object[] { new Long(6), "Stu", "102 Fake St.", "385729385", "51.50", null } )}; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ - doProcess(plan, dataMgr, expected, DEBUG); + doProcess(BaseQueryTest.createMetadata(UnitTestUtil.getTestDataPath()+"/tpcr/TPCR_3.vdb"), //$NON-NLS-1$ + "SELECT C_CUSTKEY, C_NAME, C_ADDRESS, C_PHONE, C_ACCTBAL, O_ORDERKEY FROM TPCR_Ora.CUSTOMER " + //$NON-NLS-1$ + "LEFT OUTER JOIN TPCR_SQLS.ORDERS ON C_CUSTKEY = O_CUSTKEY " + //$NON-NLS-1$ + "AND O_ORDERDATE < {ts'1992-01-02 00:00:00.0'} " + //$NON-NLS-1$ + "WHERE (C_ACCTBAL > 50)", //$NON-NLS-1$ + finder, dataMgr, expected, DEBUG); } @@ -164,14 +158,6 @@ finder.addCapabilities("TPCR_Ora", CapabilitiesConverter.convertCapabilities(new OracleCapabilities())); //$NON-NLS-1$ finder.addCapabilities("TPCR_SQLS", CapabilitiesConverter.convertCapabilities(new SqlServerCapabilities())); //$NON-NLS-1$ - ProcessorPlan plan = createPlan(BaseQueryTest.createMetadata(UnitTestUtil.getTestDataPath()+"/tpcr/TPCR_3.vdb"), //$NON-NLS-1$ - "SELECT C_CUSTKEY, C_NAME, C_ADDRESS, C_PHONE, C_ACCTBAL, O_ORDERKEY FROM TPCR_Ora.CUSTOMER " + //$NON-NLS-1$ - "LEFT OUTER JOIN " + //$NON-NLS-1$ - "(SELECT O_CUSTKEY, O_ORDERKEY FROM TPCR_SQLS.ORDERS WHERE O_ORDERDATE < {ts'1992-01-02 00:00:00.0'}) AS X " + //$NON-NLS-1$ - "ON C_CUSTKEY = O_CUSTKEY " + //$NON-NLS-1$ - "WHERE (C_ACCTBAL > 50)", //$NON-NLS-1$ - finder, DEBUG); - HardcodedDataManager dataMgr = new HardcodedDataManager(); List[] oracleExpected = @@ -192,7 +178,13 @@ Arrays.asList(new Object[] { new Long(5), "Bill", "101 Fake St.", "392839283", "51.12", new Integer(13) } ), //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ Arrays.asList(new Object[] { new Long(6), "Stu", "102 Fake St.", "385729385", "51.50", null } )}; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ - doProcess(plan, dataMgr, expected, DEBUG); + doProcess(BaseQueryTest.createMetadata(UnitTestUtil.getTestDataPath()+"/tpcr/TPCR_3.vdb"), //$NON-NLS-1$ + "SELECT C_CUSTKEY, C_NAME, C_ADDRESS, C_PHONE, C_ACCTBAL, O_ORDERKEY FROM TPCR_Ora.CUSTOMER " + //$NON-NLS-1$ + "LEFT OUTER JOIN " + //$NON-NLS-1$ + "(SELECT O_CUSTKEY, O_ORDERKEY FROM TPCR_SQLS.ORDERS WHERE O_ORDERDATE < {ts'1992-01-02 00:00:00.0'}) AS X " + //$NON-NLS-1$ + "ON C_CUSTKEY = O_CUSTKEY " + //$NON-NLS-1$ + "WHERE (C_ACCTBAL > 50)", //$NON-NLS-1$ + finder, dataMgr, expected, DEBUG); } Modified: trunk/test-integration/common/src/test/java/com/metamatrix/server/integration/TestXMLTypeTranslations.java =================================================================== --- trunk/test-integration/common/src/test/java/com/metamatrix/server/integration/TestXMLTypeTranslations.java 2009-12-22 17:45:11 UTC (rev 1697) +++ trunk/test-integration/common/src/test/java/com/metamatrix/server/integration/TestXMLTypeTranslations.java 2009-12-22 19:36:54 UTC (rev 1698) @@ -35,7 +35,6 @@ import com.metamatrix.query.metadata.QueryMetadataInterface; import com.metamatrix.query.optimizer.capabilities.FakeCapabilitiesFinder; import com.metamatrix.query.processor.HardcodedDataManager; -import com.metamatrix.query.processor.ProcessorPlan; import com.metamatrix.query.unittest.TimestampUtil; public class TestXMLTypeTranslations extends BaseQueryTest { @@ -55,11 +54,6 @@ String sql = "select * from testdocument.testDocument"; //$NON-NLS-1$ - - ProcessorPlan plan = createPlan(metadata, - sql, - finder, DEBUG); - HardcodedDataManager dataMgr = new HardcodedDataManager(); Set models = new HashSet(); @@ -82,7 +76,9 @@ List[] expected = new List[] { Arrays.asList(new Object[] {"<?xml version=\"1.0\" encoding=\"UTF-8\"?><XSDTypesNS:test xmlns:xsd=\"http://www.w3.org/2001/XMLSchema\" xmlns:XSDTypesNS=\"http://www.metamatrix.com/XMLSchema/DataSets/XSDT... })}; //$NON-NLS-1$ - doProcess(plan, dataMgr , expected, DEBUG); + doProcess(metadata, + sql, + finder, dataMgr , expected, DEBUG); }