[teiid-commits] teiid SVN: r1698 - in trunk: client/src/main/java/com/metamatrix/common/lob and 37 other directories.
teiid-commits at lists.jboss.org
teiid-commits at lists.jboss.org
Tue Dec 22 14:36:57 EST 2009
Author: shawkins
Date: 2009-12-22 14:36:54 -0500 (Tue, 22 Dec 2009)
New Revision: 1698
Added:
trunk/engine/src/main/java/com/metamatrix/common/buffer/ManagedBatch.java
trunk/engine/src/main/java/com/metamatrix/common/buffer/TupleBuffer.java
trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/FileStorageManager.java
trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/MemoryStorageManager.java
trunk/engine/src/main/java/com/metamatrix/dqp/internal/
trunk/engine/src/main/java/com/metamatrix/dqp/internal/datamgr/
trunk/engine/src/main/java/com/metamatrix/query/processor/BatchCollector.java
trunk/engine/src/main/java/com/metamatrix/query/processor/BatchIterator.java
trunk/engine/src/main/java/com/metamatrix/query/processor/CollectionTupleSource.java
trunk/engine/src/main/java/org/teiid/dqp/internal/process/CacheResults.java
trunk/engine/src/test/java/com/metamatrix/query/processor/TestCollectionTupleSource.java
Removed:
trunk/client/src/main/java/com/metamatrix/dqp/internal/datamgr/
trunk/engine/src/main/java/com/metamatrix/common/buffer/BlockedOnMemoryException.java
trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManagerLobChunkStream.java
trunk/engine/src/main/java/com/metamatrix/common/buffer/MemoryNotAvailableException.java
trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferStats.java
trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/ManagedBatch.java
trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/SizeUtility.java
trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleGroupInfo.java
trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleSourceImpl.java
trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleSourceInfo.java
trunk/engine/src/main/java/com/metamatrix/common/buffer/storage/file/FileStorageManager.java
trunk/engine/src/main/java/com/metamatrix/connector/metadata/internal/
trunk/engine/src/main/java/com/metamatrix/query/processor/NullTupleSource.java
trunk/engine/src/main/java/com/metamatrix/query/processor/proc/ExecSqlInstruction.java
trunk/engine/src/main/java/com/metamatrix/query/processor/proc/UpdateCountTupleSource.java
trunk/engine/src/main/java/com/metamatrix/query/processor/relational/BatchCollector.java
trunk/engine/src/main/java/com/metamatrix/query/processor/relational/BatchIterator.java
trunk/engine/src/main/java/com/metamatrix/query/processor/relational/TupleCollector.java
trunk/engine/src/main/java/org/teiid/dqp/internal/cache/CacheID.java
trunk/engine/src/main/java/org/teiid/dqp/internal/cache/CacheResults.java
trunk/engine/src/main/java/org/teiid/dqp/internal/cache/ResultSetCache.java
trunk/engine/src/main/java/org/teiid/dqp/internal/process/CachedRequestWorkItem.java
trunk/engine/src/main/java/org/teiid/dqp/internal/process/LobChunkStream.java
trunk/engine/src/test/java/com/metamatrix/common/buffer/impl/
trunk/engine/src/test/java/com/metamatrix/query/processor/TestNullTupleSource.java
trunk/engine/src/test/java/org/teiid/dqp/internal/cache/TestCacheID.java
trunk/engine/src/test/java/org/teiid/dqp/internal/cache/TestResultSetCache.java
Modified:
trunk/cache-jbosscache/src/main/java/com/metamatrix/cache/jboss/JBossCacheFactory.java
trunk/client/src/main/java/com/metamatrix/common/lob/ByteLobChunkStream.java
trunk/engine/src/main/java/com/metamatrix/cache/CacheConfiguration.java
trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManager.java
trunk/engine/src/main/java/com/metamatrix/common/buffer/IndexedTupleSource.java
trunk/engine/src/main/java/com/metamatrix/common/buffer/StorageManager.java
trunk/engine/src/main/java/com/metamatrix/common/buffer/TupleBatch.java
trunk/engine/src/main/java/com/metamatrix/common/buffer/TupleSourceID.java
trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferManagerImpl.java
trunk/engine/src/main/java/com/metamatrix/query/eval/Evaluator.java
trunk/engine/src/main/java/com/metamatrix/query/optimizer/proc/ProcedurePlanner.java
trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/RelationalPlanner.java
trunk/engine/src/main/java/com/metamatrix/query/processor/ProcessorPlan.java
trunk/engine/src/main/java/com/metamatrix/query/processor/QueryProcessor.java
trunk/engine/src/main/java/com/metamatrix/query/processor/dynamic/SqlEval.java
trunk/engine/src/main/java/com/metamatrix/query/processor/proc/CreateCursorResultSetInstruction.java
trunk/engine/src/main/java/com/metamatrix/query/processor/proc/ExecDynamicSqlInstruction.java
trunk/engine/src/main/java/com/metamatrix/query/processor/proc/ProcedurePlan.java
trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentCriteriaProcessor.java
trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentProcedureExecutionNode.java
trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentValueSource.java
trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DuplicateFilter.java
trunk/engine/src/main/java/com/metamatrix/query/processor/relational/GroupingNode.java
trunk/engine/src/main/java/com/metamatrix/query/processor/relational/JoinNode.java
trunk/engine/src/main/java/com/metamatrix/query/processor/relational/JoinStrategy.java
trunk/engine/src/main/java/com/metamatrix/query/processor/relational/MergeJoinStrategy.java
trunk/engine/src/main/java/com/metamatrix/query/processor/relational/NestedLoopJoinStrategy.java
trunk/engine/src/main/java/com/metamatrix/query/processor/relational/PartitionedSortJoin.java
trunk/engine/src/main/java/com/metamatrix/query/processor/relational/RelationalNode.java
trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SortNode.java
trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SortUtility.java
trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SourceState.java
trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SubqueryAwareEvaluator.java
trunk/engine/src/main/java/com/metamatrix/query/processor/relational/UnionAllNode.java
trunk/engine/src/main/java/com/metamatrix/query/processor/xml/RelationalPlanExecutor.java
trunk/engine/src/main/java/com/metamatrix/query/processor/xml/XMLPlan.java
trunk/engine/src/main/java/com/metamatrix/query/processor/xml/XMLUtil.java
trunk/engine/src/main/java/com/metamatrix/query/processor/xquery/XQueryPlan.java
trunk/engine/src/main/java/com/metamatrix/query/sql/lang/Command.java
trunk/engine/src/main/java/com/metamatrix/query/sql/lang/Query.java
trunk/engine/src/main/java/com/metamatrix/query/sql/lang/StoredProcedure.java
trunk/engine/src/main/java/com/metamatrix/query/sql/lang/XQuery.java
trunk/engine/src/main/java/com/metamatrix/query/sql/proc/CreateUpdateProcedureCommand.java
trunk/engine/src/main/java/com/metamatrix/query/sql/util/ValueIteratorSource.java
trunk/engine/src/main/java/com/metamatrix/query/tempdata/TempTableStore.java
trunk/engine/src/main/java/com/metamatrix/query/tempdata/TempTableStoreImpl.java
trunk/engine/src/main/java/com/metamatrix/query/util/CommandContext.java
trunk/engine/src/main/java/org/teiid/dqp/internal/process/DQPCore.java
trunk/engine/src/main/java/org/teiid/dqp/internal/process/DataTierManagerImpl.java
trunk/engine/src/main/java/org/teiid/dqp/internal/process/LobWorkItem.java
trunk/engine/src/main/java/org/teiid/dqp/internal/process/PreparedPlanCache.java
trunk/engine/src/main/java/org/teiid/dqp/internal/process/PreparedStatementRequest.java
trunk/engine/src/main/java/org/teiid/dqp/internal/process/Request.java
trunk/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java
trunk/engine/src/test/java/com/metamatrix/common/buffer/BufferManagerFactory.java
trunk/engine/src/test/java/com/metamatrix/common/buffer/storage/file/TestFileStorageManager.java
trunk/engine/src/test/java/com/metamatrix/query/optimizer/relational/rules/TestRuleAccessPatternValidation.java
trunk/engine/src/test/java/com/metamatrix/query/processor/TestQueryProcessor.java
trunk/engine/src/test/java/com/metamatrix/query/processor/TestVirtualDepJoin.java
trunk/engine/src/test/java/com/metamatrix/query/processor/dynamic/TestSqlEval.java
trunk/engine/src/test/java/com/metamatrix/query/processor/proc/TestProcedureProcessor.java
trunk/engine/src/test/java/com/metamatrix/query/processor/relational/NodeTestUtil.java
trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestDuplicateFilter.java
trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestSelectNode.java
trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestSortNode.java
trunk/engine/src/test/java/com/metamatrix/query/processor/xml/TestXMLProcessor.java
trunk/engine/src/test/java/org/teiid/dqp/internal/process/TestPreparedPlanCache.java
trunk/engine/src/test/java/org/teiid/dqp/internal/process/TestPreparedStatement.java
trunk/engine/src/test/java/org/teiid/dqp/internal/process/validator/TestAuthorizationValidationVisitor.java
trunk/runtime/src/main/java/com/metamatrix/dqp/embedded/services/EmbeddedBufferService.java
trunk/test-integration/common/src/test/java/com/metamatrix/server/integration/BaseQueryTest.java
trunk/test-integration/common/src/test/java/com/metamatrix/server/integration/TestTPCR.java
trunk/test-integration/common/src/test/java/com/metamatrix/server/integration/TestXMLTypeTranslations.java
Log:
TEIID-168 TEIID-913 intermediate commit in removing old implementation of result set caching and addressing buffermanager shortcomings.
Modified: trunk/cache-jbosscache/src/main/java/com/metamatrix/cache/jboss/JBossCacheFactory.java
===================================================================
--- trunk/cache-jbosscache/src/main/java/com/metamatrix/cache/jboss/JBossCacheFactory.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/cache-jbosscache/src/main/java/com/metamatrix/cache/jboss/JBossCacheFactory.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -30,7 +30,6 @@
import org.jboss.cache.eviction.FIFOAlgorithmConfig;
import org.jboss.cache.eviction.LFUAlgorithmConfig;
import org.jboss.cache.eviction.LRUAlgorithmConfig;
-import org.jboss.cache.eviction.MRUAlgorithmConfig;
import org.jboss.cache.jmx.JmxRegistrationManager;
import com.google.inject.Inject;
@@ -91,11 +90,6 @@
lru.setTimeToLive(-1); // -1 no limit
evictionConfig = lru;
}
- else if (config.getPolicy() == Policy.MRU) {
- MRUAlgorithmConfig mru = new MRUAlgorithmConfig();
- mru.setMaxNodes(config.getMaxNodes());
- evictionConfig = mru;
- }
else if (config.getPolicy() == Policy.FIFO) {
FIFOAlgorithmConfig fifo = new FIFOAlgorithmConfig();
fifo.setMaxNodes(config.getMaxNodes());
Modified: trunk/client/src/main/java/com/metamatrix/common/lob/ByteLobChunkStream.java
===================================================================
--- trunk/client/src/main/java/com/metamatrix/common/lob/ByteLobChunkStream.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/client/src/main/java/com/metamatrix/common/lob/ByteLobChunkStream.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -31,9 +31,6 @@
/**
* A wrapper class, given a InputStream object can convert a underlying
* stream into sequence of ByteLobChunk objects of given chunk size.
- * Opposite of the ByteLobChunkInputStream.
- *
- * @see ByteLobChunkInputStream
*/
public class ByteLobChunkStream implements LobChunkProducer {
private PushbackInputStream stream;
Modified: trunk/engine/src/main/java/com/metamatrix/cache/CacheConfiguration.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/cache/CacheConfiguration.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/cache/CacheConfiguration.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -24,11 +24,10 @@
public class CacheConfiguration {
- public static CacheConfiguration DEFAULT = new CacheConfiguration(Policy.MRU, 60*60, 100); // 1 hours with 100 nodes.
+ public static CacheConfiguration DEFAULT = new CacheConfiguration(Policy.LRU, 60*60, 100); // 1 hours with 100 nodes.
public enum Policy {
LRU, // Least Recently Used
- MRU, // Most Recently Used
FIFO, // First in First Out
LFU; // Least frequently Used
}
Deleted: trunk/engine/src/main/java/com/metamatrix/common/buffer/BlockedOnMemoryException.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/common/buffer/BlockedOnMemoryException.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/common/buffer/BlockedOnMemoryException.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -1,80 +0,0 @@
-/*
- * JBoss, Home of Professional Open Source.
- * See the COPYRIGHT.txt file distributed with this work for information
- * regarding copyright ownership. Some portions may be licensed
- * to Red Hat, Inc. under one or more contributor license agreements.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301 USA.
- */
-
-package com.metamatrix.common.buffer;
-
-/**
- * This exception is thrown by the plan if it receives a MemoryNotAvailableException while
- * attempting to pin a batch in the BufferManager.
- */
-public class BlockedOnMemoryException extends BlockedException {
-
- public static final BlockedOnMemoryException INSTANCE = new BlockedOnMemoryException(""); //$NON-NLS-1$
-
- /**
- * No-arg costructor required by Externalizable semantics
- */
- public BlockedOnMemoryException() {
- super();
- }
-
- /**
- * Construct an instance with the message specified.
- *
- * @param message A message describing the exception
- */
- public BlockedOnMemoryException( String message ) {
- super( message );
- }
-
- /**
- * Construct an instance with the message and error code specified.
- *
- * @param message A message describing the exception
- * @param code The error code
- */
- public BlockedOnMemoryException( String code, String message ) {
- super( code, message );
- }
-
- /**
- * Construct an instance from a message and an exception to chain to this one.
- *
- * @param message A message describing the exception
- * @param e An exception to nest within this one
- */
- public BlockedOnMemoryException( Throwable e, String message ) {
- super( e, message );
- }
-
- /**
- * Construct an instance from a message and a code and an exception to
- * chain to this one.
- *
- * @param e An exception to nest within this one
- * @param message A message describing the exception
- * @param code A code denoting the exception
- */
- public BlockedOnMemoryException( Throwable e, String code, String message ) {
- super( e, code, message );
- }
-}
Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManager.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManager.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManager.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -25,8 +25,6 @@
import java.util.List;
import com.metamatrix.api.exception.MetaMatrixComponentException;
-import com.metamatrix.common.lob.LobChunk;
-import com.metamatrix.common.types.Streamable;
/**
* The buffer manager controls how memory is used and how data flows through
@@ -38,23 +36,6 @@
*/
public interface BufferManager {
- public enum TupleSourceStatus {
- /**
- * Indicates the status of a {@link TupleSource} is active; the
- * TupleSource is itself currently still receiving data.
- * @see #getStatus
- * @see #setStatus
- */
- ACTIVE,
- /**
- * Indicates the status of a {@link TupleSource} is full; the
- * TupleSource has loaded all of its tuples.
- * @see #getStatus
- * @see #setStatus
- */
- FULL
- }
-
public enum TupleSourceType {
/**
* Indicates that a tuple source is use during query processing as a
@@ -79,29 +60,9 @@
*/
int getConnectorBatchSize();
- /**
- * Creates a tuple source based on a schema and properties describing
- * hints about the source
- * @param elements Elements of the tuple source
- * @param groupName Tuple source group name
- * @param tupleSourceType Type of tuple source
- * @return Identifier for tuple source
- * @throws MetaMatrixComponentException indicating a non-business-related
- * exception (such as a communication exception)
- */
- TupleSourceID createTupleSource(List elements, String groupName, TupleSourceType tupleSourceType)
+ TupleBuffer createTupleBuffer(List elements, String groupName, TupleSourceType tupleSourceType)
throws MetaMatrixComponentException;
- /**
- * Removes a tuple source by ID
- * @param tupleSourceID Tuple source identifier
- * @throws TupleSourceNotFoundException if tuple source could not be found
- * @throws MetaMatrixComponentException indicating a non-business-related
- * exception (such as a communication exception)
- */
- void removeTupleSource(TupleSourceID tupleSourceID)
- throws TupleSourceNotFoundException, MetaMatrixComponentException;
-
/**
* Removes all tuple sources by group name
* @param groupName Tuple source group name
@@ -109,164 +70,7 @@
* @throws MetaMatrixComponentException indicating a non-business-related
* exception (such as a communication exception)
*/
- void removeTupleSources(String groupName)
+ void removeTupleBuffers(String groupName)
throws MetaMatrixComponentException;
-
- /**
- * Gets a tuple source by ID
- * @param tupleSourceID Tuple source identifier
- * @return Tuple source to get tuples from the specified source
- * @throws TupleSourceNotFoundException if tuple source could not be found
- * @throws MetaMatrixComponentException indicating a non-business-related
- * exception (such as a communication exception)
- */
- IndexedTupleSource getTupleSource(TupleSourceID tupleSourceID)
- throws TupleSourceNotFoundException, MetaMatrixComponentException;
-
- /**
- * Gets a tuple batch by ID and indexes. Pins this tuple batch in memory until
- * it is unpinned or tuple source is removed. If memory does not exist to pin the
- * batch, a MemoryNotAvailableException is thrown.
- *
- * @param tupleSourceID Tuple source identifier
- * @param beginRow First row index to return
- * @return Batch of rows starting from beginRow and not past maxEndRow
- * @throws TupleSourceNotFoundException if tuple source could not be found
- * @throws MetaMatrixComponentException indicating a non-business-related
- * exception (such as a communication exception)
- * @throws MemoryNotAvailableException If memory was not available for the pin
- */
- TupleBatch pinTupleBatch(TupleSourceID tupleSourceID, int beginRow)
- throws TupleSourceNotFoundException, MemoryNotAvailableException, MetaMatrixComponentException;
-
- /**
- * Unpins a range of rows from the given tuple source
- * @param tupleSourceID Tuple source identifier
- * @param firstRow First row to unpin
- * @throws TupleSourceNotFoundException if tuple source could not be found
- * @throws MetaMatrixComponentException indicating a non-business-related
- * exception (such as a communication exception)
- */
- void unpinTupleBatch(TupleSourceID tupleSourceID, int firstRow)
- throws TupleSourceNotFoundException, MetaMatrixComponentException;
-
- /**
- * Gets a tuple source schema by ID
- * @param tupleSourceID Tuple source identifier
- * @return List of ElementSymbol describing elements of tuple source
- * @throws TupleSourceNotFoundException if tuple source could not be found
- * @throws MetaMatrixComponentException indicating a non-business-related
- * exception (such as a communication exception)
- */
- List getTupleSchema(TupleSourceID tupleSourceID)
- throws TupleSourceNotFoundException, MetaMatrixComponentException;
-
-
- /**
- * Adds a batch of tuples for the specified tuple source
- * @param tupleSourceID Tuple source identifier
- * @param tupleBatch Batch of rows to add
- * @throws TupleSourceNotFoundException if tuple source could not be found
- * @throws MetaMatrixComponentException indicating a non-business-related
- * exception (such as a communication exception)
- */
- void addTupleBatch(TupleSourceID tupleSourceID, TupleBatch tupleBatch)
- throws TupleSourceNotFoundException, MetaMatrixComponentException;
-
- /**
- * Gets the current row count
- * @param tupleSourceID Tuple source identifier
- * @return Current known number of rows in tuple source
- * @throws TupleSourceNotFoundException if tuple source could not be found
- * @throws MetaMatrixComponentException indicating a non-business-related
- * exception (such as a communication exception)
- */
- int getRowCount(TupleSourceID tupleSourceID)
- throws TupleSourceNotFoundException, MetaMatrixComponentException;
-
- /**
- * Sets the status of the tuple source
- * @param tupleSourceID Tuple source identifier
- * @param status New status
- * @throws TupleSourceNotFoundException if tuple source could not be found
- * @throws MetaMatrixComponentException indicating a non-business-related
- * exception (such as a communication exception)
- * @see TupleSourceStatus#ACTIVE
- * @see TupleSourceStatus#FULL
- */
- void setStatus(TupleSourceID tupleSourceID, TupleSourceStatus status)
- throws TupleSourceNotFoundException, MetaMatrixComponentException;
-
- /**
- * Gets the status of the tuple source
- * @param tupleSourceID Tuple source identifier
- * @return Status of tuple source
- * @throws TupleSourceNotFoundException if tuple source could not be found
- * @throws MetaMatrixComponentException indicating a non-business-related
- * exception (such as a communication exception)
- * @see TupleSourceStatus#ACTIVE
- * @see TupleSourceStatus#FULL
- */
- TupleSourceStatus getStatus(TupleSourceID tupleSourceID)
- throws TupleSourceNotFoundException, MetaMatrixComponentException;
-
- /**
- * Gets the final row count if tuple source is FULL, otherwise returns -1.
- * @param tupleSourceID Tuple source identifier
- * @return Final row count if status == FULL, -1 otherwise
- */
- int getFinalRowCount(TupleSourceID tupleSourceID)
- throws TupleSourceNotFoundException, MetaMatrixComponentException;
- /**
- * Add a streamable object to the persistent store. The distinction is made between
- * regular tuple sources and streamable object beacuse, streamable objects are fairly
- * large in size can not be loaded totally into memory, as other tuple sources are done
- * this mechanism allows to stream these objects chunk by chunk.
- * @param tupleSourceID
- * @param streamGlob part of the stream
- * @param stream
- * @throws TupleSourceNotFoundException
- * @throws MetaMatrixComponentException
- */
- void addStreamablePart(TupleSourceID tupleSourceID, LobChunk streamGlob, int beginRow)
- throws TupleSourceNotFoundException, MetaMatrixComponentException;
-
- /**
- * Returns the streamable batch object's part stored with specified identifier
- * @param tupleSourceID - identifier
- * @return LobChunk a part of the Streamable object stored.
- * @throws TupleSourceNotFoundException
- * @throws MetaMatrixComponentException
- */
- LobChunk getStreamablePart(TupleSourceID tupleSourceID, int beginRow)
- throws TupleSourceNotFoundException, MetaMatrixComponentException;
-
- /**
- * Release batches that have been pinned by this thread.
- * This method should be called when processing is terminated
- * to ensure that the memory can be freed.
- */
- void releasePinnedBatches() throws MetaMatrixComponentException;
-
- /**
- * Return the LOB associated with the referenceId
- * @param id
- * @param referenceId
- * @return
- * @throws TupleSourceNotFoundException
- * @throws MetaMatrixComponentException
- */
- Streamable<?> getStreamable(TupleSourceID id, String referenceId)
- throws TupleSourceNotFoundException, MetaMatrixComponentException;
-
- /**
- * Assign the {@link TupleSource} as the persistent stream for the {@link Streamable}
- * @param id
- * @param s
- * @throws TupleSourceNotFoundException
- */
- void setPersistentTupleSource(TupleSourceID id, Streamable<? extends Object> s)
- throws TupleSourceNotFoundException;
-
}
Deleted: trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManagerLobChunkStream.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManagerLobChunkStream.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManagerLobChunkStream.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -1,67 +0,0 @@
-/*
- * JBoss, Home of Professional Open Source.
- * See the COPYRIGHT.txt file distributed with this work for information
- * regarding copyright ownership. Some portions may be licensed
- * to Red Hat, Inc. under one or more contributor license agreements.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301 USA.
- */
-
-package com.metamatrix.common.buffer;
-
-
-import java.io.IOException;
-
-import com.metamatrix.api.exception.MetaMatrixComponentException;
-import com.metamatrix.common.lob.LobChunk;
-import com.metamatrix.common.lob.LobChunkProducer;
-import com.metamatrix.common.log.LogManager;
-import com.metamatrix.dqp.DQPPlugin;
-import com.metamatrix.dqp.util.LogConstants;
-
-public class BufferManagerLobChunkStream implements LobChunkProducer {
- private TupleSourceID sourceId;
- private BufferManager bufferMgr;
- private int position = 0;
-
- public BufferManagerLobChunkStream(String persitentId, BufferManager bufferMgr) {
- this.sourceId = new TupleSourceID(persitentId);
- this.bufferMgr = bufferMgr;
- }
-
- public LobChunk getNextChunk() throws IOException {
- try {
- this.position++;
- return bufferMgr.getStreamablePart(sourceId, position);
- } catch (TupleSourceNotFoundException e) {
- String msg = DQPPlugin.Util.getString("BufferManagerLobChunkStream.no_tuple_source", new Object[] {sourceId}); //$NON-NLS-1$
- LogManager.logWarning(LogConstants.CTX_BUFFER_MGR, e, msg);
- throw new IOException(msg);
- } catch (MetaMatrixComponentException e) {
- String msg = DQPPlugin.Util.getString("BufferManagerLobChunkStream.error_processing", new Object[] {sourceId}); //$NON-NLS-1$
- LogManager.logWarning(LogConstants.CTX_BUFFER_MGR, e, msg);
- throw new IOException(msg);
- }
- }
-
- /**
- * @see com.metamatrix.common.lob.LobChunkProducer#close()
- */
- public void close() throws IOException {
- // we could remove the buffer tuple here but, this is just a stream, so we need to delete
- // that when we close th eplan.
- }
-}
\ No newline at end of file
Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/IndexedTupleSource.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/common/buffer/IndexedTupleSource.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/common/buffer/IndexedTupleSource.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -57,6 +57,4 @@
*/
int getCurrentIndex();
- TupleBatch getBatch()
- throws MetaMatrixComponentException;
}
Added: trunk/engine/src/main/java/com/metamatrix/common/buffer/ManagedBatch.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/common/buffer/ManagedBatch.java (rev 0)
+++ trunk/engine/src/main/java/com/metamatrix/common/buffer/ManagedBatch.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -0,0 +1,70 @@
+/*
+ * JBoss, Home of Professional Open Source.
+ * See the COPYRIGHT.txt file distributed with this work for information
+ * regarding copyright ownership. Some portions may be licensed
+ * to Red Hat, Inc. under one or more contributor license agreements.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301 USA.
+ */
+
+package com.metamatrix.common.buffer;
+
+import java.lang.ref.SoftReference;
+
+class ManagedBatch {
+
+ private int beginRow;
+ private int endRow;
+ private SoftReference<TupleBatch> batchReference;
+
+ /**
+ * Constructor for ManagedBatch.
+ */
+ public ManagedBatch(TupleBatch batch) {
+ this.beginRow = batch.getBeginRow();
+ this.endRow = batch.getEndRow();
+ this.batchReference = new SoftReference<TupleBatch>(batch);
+ }
+
+ /**
+ * Get the begin row, must be >= 1
+ * @return Begin row
+ */
+ public int getBeginRow() {
+ return this.beginRow;
+ }
+
+ /**
+ * Get the end row, inclusive
+ * @return End row
+ */
+ public int getEndRow() {
+ return this.endRow;
+ }
+
+ public String toString() {
+ return "ManagedBatch[" + beginRow + ", " + endRow + "]"; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
+ }
+
+ public TupleBatch getBatch() {
+ return this.batchReference.get();
+ }
+
+ public void setBatchReference(TupleBatch batch) {
+ this.batchReference = new SoftReference<TupleBatch>(batch);
+ }
+
+}
Property changes on: trunk/engine/src/main/java/com/metamatrix/common/buffer/ManagedBatch.java
___________________________________________________________________
Name: svn:mime-type
+ text/plain
Deleted: trunk/engine/src/main/java/com/metamatrix/common/buffer/MemoryNotAvailableException.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/common/buffer/MemoryNotAvailableException.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/common/buffer/MemoryNotAvailableException.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -1,75 +0,0 @@
-/*
- * JBoss, Home of Professional Open Source.
- * See the COPYRIGHT.txt file distributed with this work for information
- * regarding copyright ownership. Some portions may be licensed
- * to Red Hat, Inc. under one or more contributor license agreements.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301 USA.
- */
-
-package com.metamatrix.common.buffer;
-
-import com.metamatrix.api.exception.MetaMatrixException;
-
-/**
- * Indicates memory was not available for the requested operation.
- */
-public class MemoryNotAvailableException extends MetaMatrixException {
-
- /**
- * No-arg costructor required by Externalizable semantics
- */
- public MemoryNotAvailableException() {
- super();
- }
-
- /**
- * Constructor for MemoryNotAvailableException.
- * @param message
- */
- public MemoryNotAvailableException(String message) {
- super(message);
- }
-
- /**
- * Constructor for MemoryNotAvailableException.
- * @param code
- * @param message
- */
- public MemoryNotAvailableException(String code, String message) {
- super(code, message);
- }
-
- /**
- * Constructor for MemoryNotAvailableException.
- * @param e
- * @param message
- */
- public MemoryNotAvailableException(Throwable e, String message) {
- super(e, message);
- }
-
- /**
- * Constructor for MemoryNotAvailableException.
- * @param e
- * @param code
- * @param message
- */
- public MemoryNotAvailableException(Throwable e, String code, String message) {
- super(e, code, message);
- }
-
-}
Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/StorageManager.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/common/buffer/StorageManager.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/common/buffer/StorageManager.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -89,8 +89,7 @@
* @throws MetaMatrixComponentException indicating a non-business-related
* exception (such as a communication exception)
*/
- void removeBatches(TupleSourceID sourceID)
- throws MetaMatrixComponentException;
+ void removeBatches(TupleSourceID sourceID);
/**
* Shut down the Storage Manager.
Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/TupleBatch.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/common/buffer/TupleBatch.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/common/buffer/TupleBatch.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -41,14 +41,11 @@
*/
public class TupleBatch implements Externalizable {
- public static final long UNKNOWN_SIZE = -1;
-
private int rowOffset;
private List[] tuples;
// Optional state
private boolean terminationFlag = false;
- private long size = UNKNOWN_SIZE;
/**
* Contains ordered data types of each of the columns in the batch. Although it is not serialized,
@@ -143,22 +140,6 @@
this.terminationFlag = terminationFlag;
}
- /**
- * Get the size of this batch - may be constant indicating unknown.
- * @return Size of batch in bytes or UNKNOWN_SIZE
- */
- public long getSize() {
- return this.size;
- }
-
- /**
- * Set the size of this batch.
- * @param size Size in bytes or UNKNOWN_SIZE
- */
- public void setSize(long size) {
- this.size = size;
- }
-
public void setDataTypes(String[] types) {
this.types = types;
}
@@ -179,13 +160,11 @@
}
public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- size = in.readLong();
rowOffset = in.readInt();
terminationFlag = in.readBoolean();
tuples = BatchSerializer.readBatch(in, types);
}
public void writeExternal(ObjectOutput out) throws IOException {
- out.writeLong(size);
out.writeInt(rowOffset);
out.writeBoolean(terminationFlag);
BatchSerializer.writeBatch(out, types, tuples);
Added: trunk/engine/src/main/java/com/metamatrix/common/buffer/TupleBuffer.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/common/buffer/TupleBuffer.java (rev 0)
+++ trunk/engine/src/main/java/com/metamatrix/common/buffer/TupleBuffer.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -0,0 +1,390 @@
+/*
+ * JBoss, Home of Professional Open Source.
+ * See the COPYRIGHT.txt file distributed with this work for information
+ * regarding copyright ownership. Some portions may be licensed
+ * to Red Hat, Inc. under one or more contributor license agreements.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301 USA.
+ */
+
+package com.metamatrix.common.buffer;
+
+import java.lang.ref.PhantomReference;
+import java.lang.ref.ReferenceQueue;
+import java.lang.ref.SoftReference;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.concurrent.atomic.AtomicLong;
+
+import com.metamatrix.api.exception.MetaMatrixComponentException;
+import com.metamatrix.common.types.DataTypeManager;
+import com.metamatrix.common.types.Streamable;
+import com.metamatrix.core.util.Assertion;
+import com.metamatrix.dqp.DQPPlugin;
+
+public class TupleBuffer {
+
+ class TupleSourceImpl implements IndexedTupleSource {
+ private SoftReference<TupleBatch> currentBatch;
+ private int currentRow = 1;
+ private int mark = 1;
+ private List<?> currentTuple;
+
+ @Override
+ public int getCurrentIndex() {
+ return this.currentRow;
+ }
+
+ @Override
+ public List getSchema(){
+ return schema;
+ }
+
+ @Override
+ public List<?> nextTuple()
+ throws MetaMatrixComponentException{
+ List<?> result = null;
+ if (currentTuple != null){
+ result = currentTuple;
+ currentTuple = null;
+ } else {
+ result = getCurrentTuple();
+ }
+ if (result != null) {
+ currentRow++;
+ }
+ return result;
+ }
+
+ private List<?> getCurrentTuple() throws MetaMatrixComponentException,
+ BlockedException {
+ TupleBatch batch = getBatch();
+ if(batch.getRowCount() == 0) {
+ // Check if last
+ if(isFinal) {
+ currentBatch = null;
+ return null;
+ }
+ throw BlockedException.INSTANCE;
+ }
+
+ return batch.getTuple(currentRow);
+ }
+
+ @Override
+ public void closeSource()
+ throws MetaMatrixComponentException{
+ currentBatch = null;
+ mark = 1;
+ reset();
+ }
+
+ // Retrieves the necessary batch based on the currentRow
+ TupleBatch getBatch()
+ throws MetaMatrixComponentException{
+ TupleBatch batch = null;
+ if (currentBatch != null) {
+ batch = currentBatch.get();
+ }
+ if (batch != null) {
+ if (currentRow <= batch.getEndRow() && currentRow >= batch.getBeginRow()) {
+ return batch;
+ }
+ currentBatch = null;
+ }
+
+ batch = TupleBuffer.this.getBatch(currentRow);
+ if (batch != null) {
+ currentBatch = new SoftReference<TupleBatch>(batch);
+ }
+ return batch;
+ }
+
+ @Override
+ public boolean hasNext() throws MetaMatrixComponentException {
+ if (this.currentTuple != null) {
+ return true;
+ }
+
+ this.currentTuple = getCurrentTuple();
+ return this.currentTuple != null;
+ }
+
+ @Override
+ public void reset() {
+ this.setPosition(mark);
+ this.mark = 1;
+ }
+
+ @Override
+ public void mark() {
+ this.mark = currentRow;
+ }
+
+ @Override
+ public void setPosition(int position) {
+ if (this.currentRow != position) {
+ this.currentRow = position;
+ this.currentTuple = null;
+ }
+ }
+ }
+
+ private static class LobReference extends PhantomReference<Streamable<?>> {
+
+ String persistentStreamId;
+
+ public LobReference(Streamable<?> lob) {
+ super(lob, LOB_QUEUE);
+ this.persistentStreamId = lob.getPersistenceStreamId();
+ }
+ }
+
+ private static final AtomicLong LOB_ID = new AtomicLong();
+ private static final ReferenceQueue<Streamable<?>> LOB_QUEUE = new ReferenceQueue<Streamable<?>>();
+
+ //construction state
+ private StorageManager manager;
+ private String groupName;
+ private TupleSourceID tupleSourceID;
+ private List<?> schema;
+ private String[] types;
+ private int batchSize;
+
+ private int rowCount;
+ private boolean isFinal;
+ private TreeMap<Integer, ManagedBatch> batches = new TreeMap<Integer, ManagedBatch>();
+ private ArrayList<List<?>> batchBuffer;
+
+ //lob management
+ private Map<String, Streamable<?>> lobReferences; //references to contained lobs
+ private boolean lobs = true;
+ private LobReference containingLobReference; //reference to containing lob
+
+ public TupleBuffer(StorageManager manager, String groupName, TupleSourceID id, List<?> schema, String[] types, int batchSize) {
+ this.manager = manager;
+ this.tupleSourceID = id;
+ this.schema = schema;
+ this.types = types;
+ this.batchSize = batchSize;
+ if (types != null) {
+ int i = 0;
+ for (i = 0; i < types.length; i++) {
+ if (DataTypeManager.isLOB(types[i]) || types[i] == DataTypeManager.DefaultDataTypes.OBJECT) {
+ break;
+ }
+ }
+ if (i == types.length) {
+ lobs = false;
+ }
+ }
+ this.groupName = groupName;
+ }
+
+ public String getGroupName() {
+ return groupName;
+ }
+
+ public boolean isBatchBufferEmpty() {
+ return this.batchBuffer == null || this.batchBuffer.isEmpty();
+ }
+
+ public void addTuple(List<?> tuple) throws MetaMatrixComponentException {
+ this.rowCount++;
+ if (batchBuffer == null) {
+ batchBuffer = new ArrayList<List<?>>(batchSize/4);
+ }
+ batchBuffer.add(tuple);
+ if (batchBuffer.size() == batchSize) {
+ saveBatch(false);
+ }
+ }
+
+ void saveBatch(boolean finalBatch) throws MetaMatrixComponentException {
+ if ((batchBuffer == null || batchBuffer.isEmpty()) && !finalBatch) {
+ return;
+ }
+ List rows = batchBuffer==null?Collections.emptyList():batchBuffer;
+ TupleBatch writeBatch = new TupleBatch(rowCount - rows.size() + 1, rows);
+ if (finalBatch) {
+ writeBatch.setTerminationFlag(true);
+ }
+ if (writeBatch.getTerminationFlag()) {
+ this.isFinal = true;
+ }
+ correctLobReferences(writeBatch);
+ ManagedBatch mbatch = new ManagedBatch(writeBatch);
+ this.batches.put(mbatch.getBeginRow(), mbatch);
+ manager.addBatch(tupleSourceID, writeBatch, types);
+ batchBuffer = null;
+ }
+
+ public void close() throws MetaMatrixComponentException {
+ saveBatch(true);
+ }
+
+ public TupleBatch getBatch(int row) throws MetaMatrixComponentException {
+ if (row > rowCount) {
+ TupleBatch batch = new TupleBatch(rowCount + 1, new List[] {});
+ if (isFinal) {
+ batch.setTerminationFlag(true);
+ }
+ return batch;
+ }
+ if (this.batchBuffer != null && row > rowCount - this.batchBuffer.size()) {
+ return new TupleBatch(rowCount - this.batchBuffer.size() + 1, batchBuffer);
+ }
+ Map.Entry<Integer, ManagedBatch> entry = batches.floorEntry(row);
+ ManagedBatch batch = null;
+ if (entry != null && entry.getValue().getEndRow() >= row) {
+ batch = entry.getValue();
+ TupleBatch result = batch.getBatch();
+ if (result != null) {
+ return result;
+ }
+ row = batch.getBeginRow();
+ }
+ try {
+ TupleBatch result = manager.getBatch(tupleSourceID, row, types);
+ correctLobReferences(result);
+ if (batch != null) {
+ batch.setBatchReference(result);
+ }
+ return result;
+ } catch (TupleSourceNotFoundException e) {
+ throw new MetaMatrixComponentException(e);
+ }
+ }
+
+ public void remove() {
+ manager.removeBatches(tupleSourceID);
+ }
+
+ public int getRowCount() {
+ return rowCount;
+ }
+
+ public boolean isFinal() {
+ return isFinal;
+ }
+
+ public void setFinal(boolean isFinal) {
+ this.isFinal = isFinal;
+ }
+
+ public List<?> getSchema() {
+ return schema;
+ }
+
+ public int getBatchSize() {
+ return batchSize;
+ }
+
+ public void setBatchSize(int batchSize) {
+ this.batchSize = batchSize;
+ }
+
+ public void setContainingLobReference(Streamable<?> s) {
+ cleanLobTupleSource();
+ s.setPersistenceStreamId(tupleSourceID.getStringID());
+ Assertion.isNull(this.containingLobReference);
+ this.containingLobReference = new LobReference(s);
+ }
+
+ /**
+ * Attempt to clean up lingering tuplebuffers to lobs that are no longer referenced
+ */
+ private void cleanLobTupleSource() {
+ for (int i = 0; i < 10; i++) {
+ String tupleSourceId = null;
+ LobReference ref = (LobReference)LOB_QUEUE.poll();
+ if (ref != null) {
+ tupleSourceId = ref.persistentStreamId;
+ }
+ if (tupleSourceId == null) {
+ break;
+ }
+ this.manager.removeBatches(new TupleSourceID(tupleSourceId));
+ }
+ }
+
+ public Streamable<?> getLobReference(String id) throws MetaMatrixComponentException {
+ Streamable<?> lob = null;
+ if (this.lobReferences != null) {
+ lob = this.lobReferences.get(id);
+ }
+ if (lob == null) {
+ throw new MetaMatrixComponentException(DQPPlugin.Util.getString("ProcessWorker.wrongdata")); //$NON-NLS-1$
+ }
+ return lob;
+ }
+
+ /**
+ * If a tuple batch is being added with Lobs, then references to
+ * the lobs will be held on the {@link TupleSourceInfo}
+ * @param batch
+ * @throws MetaMatrixComponentException
+ */
+ @SuppressWarnings("unchecked")
+ private void correctLobReferences(TupleBatch batch) throws MetaMatrixComponentException {
+ if (!lobs) {
+ return;
+ }
+ List[] rows = batch.getAllTuples();
+ int columns = schema.size();
+ // walk through the results and find all the lobs
+ for (int row = 0; row < rows.length; row++) {
+ for (int col = 0; col < columns; col++) {
+ Object anObj = rows[row].get(col);
+
+ if (!(anObj instanceof Streamable<?>)) {
+ continue;
+ }
+ Streamable lob = (Streamable)anObj;
+ String id = lob.getReferenceStreamId();
+ if (id == null) {
+ id = String.valueOf(LOB_ID.getAndIncrement());
+ lob.setReferenceStreamId(id);
+ }
+ if (this.lobReferences == null) {
+ this.lobReferences = Collections.synchronizedMap(new HashMap<String, Streamable<?>>());
+ }
+ this.lobReferences.put(id, lob);
+ if (lob.getReference() == null) {
+ lob.setReference(getLobReference(lob.getReferenceStreamId()).getReference());
+ }
+ }
+ }
+ }
+
+ /**
+ * Create a new iterator for this buffer
+ * @return
+ */
+ public IndexedTupleSource createIndexedTupleSource() {
+ return new TupleSourceImpl();
+ }
+
+ @Override
+ public String toString() {
+ return this.tupleSourceID.toString();
+ }
+
+}
Property changes on: trunk/engine/src/main/java/com/metamatrix/common/buffer/TupleBuffer.java
___________________________________________________________________
Name: svn:mime-type
+ text/plain
Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/TupleSourceID.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/common/buffer/TupleSourceID.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/common/buffer/TupleSourceID.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -28,9 +28,7 @@
* Identifier for a tuple source. The tuple source ID contains two
* pieces of information: a unique ID (unique across locations) and
* a location. Both must be represented by a string but different
- * systems may use different location descriptions. The
- * {@link com.metamatrix.common.buffer.BufferManagerLookup}
- * is the only party responsible for providing and decoding the location string.
+ * systems may use different location descriptions.
*/
public class TupleSourceID implements Serializable {
Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferManagerImpl.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferManagerImpl.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferManagerImpl.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -26,130 +26,47 @@
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashSet;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
-import java.util.Timer;
-import java.util.TimerTask;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import com.metamatrix.api.exception.MetaMatrixComponentException;
import com.metamatrix.common.buffer.BufferManager;
-import com.metamatrix.common.buffer.IndexedTupleSource;
-import com.metamatrix.common.buffer.MemoryNotAvailableException;
import com.metamatrix.common.buffer.StorageManager;
import com.metamatrix.common.buffer.TupleBatch;
+import com.metamatrix.common.buffer.TupleBuffer;
import com.metamatrix.common.buffer.TupleSourceID;
import com.metamatrix.common.buffer.TupleSourceNotFoundException;
-import com.metamatrix.common.lob.LobChunk;
import com.metamatrix.common.log.LogManager;
import com.metamatrix.common.types.DataTypeManager;
-import com.metamatrix.common.types.Streamable;
import com.metamatrix.core.log.MessageLevel;
import com.metamatrix.core.util.Assertion;
-import com.metamatrix.dqp.DQPPlugin;
import com.metamatrix.dqp.util.LogConstants;
import com.metamatrix.query.execution.QueryExecPlugin;
import com.metamatrix.query.sql.symbol.Expression;
/**
- * <p>Default implementation of BufferManager. This buffer manager implementation
- * assumes the usage of a StorageManager of type memory and optionally (preferred)
- * an additional StorageManager of type FILE.</p>
+ * <p>Default implementation of BufferManager.</p>
+ * Responsible for creating/tracking TupleBuffers and providing access to the StorageManager
*/
-public class BufferManagerImpl implements BufferManager {
+public class BufferManagerImpl implements BufferManager, StorageManager {
- //memory availability when reserveMemory() is called
- static final int MEMORY_AVAILABLE = 1;
- static final int MEMORY_EXCEED_MAX = 2; //exceed buffer manager max memory
- static final int MEMORY_EXCEED_SESSION_MAX = 3; //exceed session max memory
-
- private static ThreadLocal<Set<ManagedBatch>> PINNED_BY_THREAD = new ThreadLocal<Set<ManagedBatch>>() {
- protected Set<ManagedBatch> initialValue() {
- return new HashSet<ManagedBatch>();
- };
- };
-
// Initialized stuff
private String lookup;
private BufferConfig config;
- // Cache tuple source info in memory
- private Map<TupleSourceID, TupleSourceInfo> tupleSourceMap = new ConcurrentHashMap<TupleSourceID, TupleSourceInfo>();
- // groupName (String) -> TupleGroupInfo map
- private Map<String, TupleGroupInfo> groupInfos = new HashMap<String, TupleGroupInfo>();
+ private Map<TupleSourceID, TupleBuffer> tupleSourceMap = new ConcurrentHashMap<TupleSourceID, TupleBuffer>();
+ private Map<String, Set<TupleSourceID>> groupInfos = new HashMap<String, Set<TupleSourceID>>();
- // Storage manager
private StorageManager diskMgr;
- // ID creator
private AtomicLong currentTuple = new AtomicLong(0);
- // Keep track of how memory usage
- private volatile long memoryUsed = 0;
-
- // Track the currently unpinned stuff in a sorted set
- private Set<ManagedBatch> unpinned = Collections.synchronizedSet(new LinkedHashSet<ManagedBatch>());
-
- // Trigger to handle management and stats logging
- private Timer timer;
-
- // Collected stats
- private AtomicInteger pinRequests = new AtomicInteger(0);
- private AtomicInteger pinFailures = new AtomicInteger(0);
- private AtomicInteger pinnedFromDisk = new AtomicInteger(0);
- private AtomicInteger cleanings = new AtomicInteger(0);
- private AtomicLong totalCleaned = new AtomicLong(0);
- private AtomicInteger pinned = new AtomicInteger();
-
/**
- * See {@link com.metamatrix.common.buffer.BufferManagerPropertyNames} for a
- * description of all the properties.
- * @param lookup An object telling the buffer manager what his location is and
- * how to find other buffer managers of different locations
- * @param properties Properties to configure the buffer manager
- */
- public synchronized void initialize(String lookup, Properties properties) throws MetaMatrixComponentException {
- this.lookup = lookup;
-
- // Set up config based on properties
- this.config = new BufferConfig(properties);
-
- // Set up alarms based on config
- if(this.config.getManagementInterval() > 0) {
- TimerTask mgmtTask = new TimerTask() {
- public void run() {
- clean(0, null);
- }
-
- };
- getTimer().schedule(mgmtTask, 0, this.config.getManagementInterval());
- }
- if(this.config.getLogStatInterval() > 0) {
- TimerTask statTask = new TimerTask() {
- public void run() {
- BufferStats stats = getStats();
- stats.log();
- }
- };
- getTimer().schedule(statTask, 0, this.config.getLogStatInterval());
- }
- }
-
- private Timer getTimer() {
- if (timer == null) {
- timer = new Timer("BufferManagementThread", true); //$NON-NLS-1$
- }
- return timer;
- }
-
- /**
* Get the configuration of the buffer manager
* @return Configuration
*/
@@ -158,66 +75,6 @@
}
/**
- * Construct a BufferStats object by looking at all the state. This should
- * be multi-thread safe but calling this method clears several counters and
- * resets that state.
- * @return Buffer statistics
- */
- public BufferStats getStats() {
- BufferStats stats = new BufferStats();
-
- // Get memory info
- stats.memoryUsed = this.memoryUsed;
- stats.memoryFree = config.getTotalAvailableMemory() - memoryUsed;
-
- // Get picture of what's happening
- Set<TupleSourceID> copyKeys = tupleSourceMap.keySet();
- Iterator<TupleSourceID> infoIter = copyKeys.iterator();
- stats.numTupleSources = copyKeys.size();
-
- // Walk through each info and count where all the batches are -
- // lock only a single info at a time to minimize locking
- while(infoIter.hasNext()) {
- TupleSourceID key = infoIter.next();
- TupleSourceInfo info = tupleSourceMap.get(key);
- if(info == null) {
- continue;
- }
-
- synchronized(info) {
- Iterator<ManagedBatch> batchIter = info.getBatchIterator();
- while(batchIter.hasNext()) {
- ManagedBatch batch = batchIter.next();
- switch(batch.getLocation()) {
- case ManagedBatch.PERSISTENT:
- stats.numPersistentBatches++;
- break;
- case ManagedBatch.PINNED:
- stats.numPinnedBatches++;
-
- if ( LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.TRACE )) {
- stats.pinnedManagedBatches.add(batch);
- }
-
- break;
- case ManagedBatch.UNPINNED:
- stats.numUnpinnedBatches++;
- break;
- }
- }
- }
- }
-
- stats.pinRequests = this.pinRequests.get();
- stats.pinSuccesses = this.pinRequests.get() - this.pinFailures.get();
- stats.pinnedFromMemory = this.pinRequests.get() - this.pinnedFromDisk.get();
- stats.numCleanings = this.cleanings.get();
- stats.totalCleaned = this.totalCleaned.get();
-
- return stats;
- }
-
- /**
* Get processor batch size
* @return Number of rows in a processor batch
*/
@@ -242,28 +99,20 @@
Assertion.isNull(diskMgr);
this.diskMgr = storageManager;
}
-
- /**
- * Register a new tuple source and return a unique ID for it.
- * @param groupName Group name
- * @param tupleSourceType Type of tuple source as defined in BufferManager constants
- * @param schema List of ElementSymbol
- * @return New unique ID for this tuple source
- * @throws MetaMatrixComponentException If internal server error occurs
- */
- public TupleSourceID createTupleSource(List schema, String groupName, TupleSourceType tupleSourceType)
- throws MetaMatrixComponentException {
-
+
+ @Override
+ public TupleBuffer createTupleBuffer(List elements, String groupName,
+ TupleSourceType tupleSourceType)
+ throws MetaMatrixComponentException {
TupleSourceID newID = new TupleSourceID(String.valueOf(this.currentTuple.getAndIncrement()), this.lookup);
- TupleGroupInfo tupleGroupInfo = getGroupInfo(groupName);
- TupleSourceInfo info = new TupleSourceInfo(newID, schema, getTypeNames(schema), tupleGroupInfo, tupleSourceType);
- tupleGroupInfo.getTupleSourceIDs().add(newID);
- tupleSourceMap.put(newID, info);
+ Set<TupleSourceID> tupleGroupInfo = getGroupInfo(groupName, true);
+ TupleBuffer tupleBuffer = new TupleBuffer(this, groupName, newID, elements, getTypeNames(elements), getProcessorBatchSize());
+ tupleGroupInfo.add(newID);
+ tupleSourceMap.put(newID, tupleBuffer);
if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.DETAIL)) {
- LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, new Object[]{"Creating TupleSource:", newID, "of type "+tupleSourceType}); //$NON-NLS-1$ //$NON-NLS-2$
+ LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, new Object[]{"Creating TupleBuffer:", newID, "of type "+tupleSourceType}); //$NON-NLS-1$ //$NON-NLS-2$
}
-
- return newID;
+ return tupleBuffer;
}
/**
@@ -272,7 +121,7 @@
* @return
* @since 4.2
*/
- public static String[] getTypeNames(List expressions) {
+ private static String[] getTypeNames(List expressions) {
if (expressions == null) {
return null;
}
@@ -285,73 +134,20 @@
}
/**
- * Remove a tuple source based on ID
- * @throws TupleSourceNotFoundException If tuple source not found
- * @throws MetaMatrixComponentException If internal server error occurs
- */
- public void removeTupleSource(TupleSourceID tupleSourceID)
- throws TupleSourceNotFoundException, MetaMatrixComponentException {
-
- if(tupleSourceID == null || tupleSourceID.getLocation() == null) {
- // this is a bogus tuple source ID
- return;
- }
- if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.DETAIL)) {
- LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, new Object[]{"Removing TupleSource:", tupleSourceID}); //$NON-NLS-1$
- }
-
- // Remove from main map first
- TupleSourceInfo info = this.tupleSourceMap.remove(tupleSourceID);
- if(info == null) {
- throw new TupleSourceNotFoundException(QueryExecPlugin.Util.getString("BufferManagerImpl.tuple_source_not_found", tupleSourceID)); //$NON-NLS-1$
- }
-
- // Walk through batches and determine whether memory or disk cleanup needs to occur
- synchronized(info) {
- if(! info.isRemoved()) {
- info.setRemoved();
-
- Iterator<ManagedBatch> iter = info.getBatchIterator();
- while(iter.hasNext()) {
- ManagedBatch batch = iter.next();
- switch(batch.getLocation()) {
- case ManagedBatch.UNPINNED:
- this.unpinned.remove(batch);
- releaseMemory(batch.getSize(), info.getGroupInfo());
- break;
- case ManagedBatch.PINNED:
- PINNED_BY_THREAD.get().remove(batch);
- this.pinned.getAndDecrement();
- releaseMemory(batch.getSize(), info.getGroupInfo());
- break;
- }
- }
- }
- }
- TupleGroupInfo tupleGroupInfo = info.getGroupInfo();
- tupleGroupInfo.getTupleSourceIDs().remove(tupleSourceID);
-
- // Remove disk storage
- if (this.diskMgr != null){
- this.diskMgr.removeBatches(tupleSourceID);
- }
- }
-
- /**
* Remove all the tuple sources with the specified group name. Typically the group
* name is really a session identifier and this is called to remove all tuple sources
* used by a particular session.
* @param groupName Name of the group
* @throws MetaMatrixComponentException If internal server error occurs
*/
- public void removeTupleSources(String groupName)
+ public void removeTupleBuffers(String groupName)
throws MetaMatrixComponentException {
if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.DETAIL)) {
- LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, new Object[]{"Removing TupleSources for group", groupName}); //$NON-NLS-1$
+ LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, new Object[]{"Removing TupleBuffers for group", groupName}); //$NON-NLS-1$
}
- TupleGroupInfo tupleGroupInfo = null;
+ Set<TupleSourceID> tupleGroupInfo = null;
synchronized(groupInfos) {
tupleGroupInfo = groupInfos.remove(groupName);
}
@@ -359,613 +155,97 @@
return;
}
List<TupleSourceID> tupleSourceIDs = null;
- synchronized (tupleGroupInfo.getTupleSourceIDs()) {
- tupleSourceIDs = new ArrayList<TupleSourceID>(tupleGroupInfo.getTupleSourceIDs());
+ synchronized (tupleGroupInfo) {
+ tupleSourceIDs = new ArrayList<TupleSourceID>(tupleGroupInfo);
}
- MetaMatrixComponentException ex = null;
-
for (TupleSourceID tsID : tupleSourceIDs) {
try {
- this.removeTupleSource(tsID);
+ TupleBuffer tupleBuffer = getTupleBuffer(tsID);
+ tupleBuffer.remove();
} catch(TupleSourceNotFoundException e) {
// ignore and go on
- } catch(MetaMatrixComponentException e) {
- if(ex == null) {
- ex = e;
- }
}
}
-
- if(ex != null) {
- throw ex;
- }
}
/**
- * Get a tuple source to walk through the rows for a particular
- * tupleSourceID.
- * @param tupleSourceID Tuple source identifier
- * @return TupleSource
- * @throws TupleSourceNotFoundException If tuple source not found
- * @throws MetaMatrixComponentException If an internal server error occurred
- */
- public IndexedTupleSource getTupleSource(TupleSourceID tupleSourceID)
- throws TupleSourceNotFoundException, MetaMatrixComponentException {
-
- TupleSourceInfo info = getTupleSourceInfo(tupleSourceID, true);
- int batchSize = this.config.getProcessorBatchSize();
- return new TupleSourceImpl(this, tupleSourceID, info.getTupleSchema());
- }
-
- /**
- * Get the tuple schema for a particular tuple source
- * @param tupleSourceID Tuple source identifier
- * @return List of elements describing tuple source
- * @throws TupleSourceNotFoundException If tuple source not found
- * @throws MetaMatrixComponentException If an internal server error occurred
- */
- public List getTupleSchema(TupleSourceID tupleSourceID)
- throws TupleSourceNotFoundException, MetaMatrixComponentException {
-
- TupleSourceInfo info = getTupleSourceInfo(tupleSourceID, true);
- return info.getTupleSchema();
- }
-
- /**
- * Set the status for a particular tuple source
- * @param tupleSourceID Tuple source identifier
- * @param status New status
- * @throws TupleSourceNotFoundException If tuple source not found
- * @throws MetaMatrixComponentException If an internal server error occurred
- */
- public void setStatus(TupleSourceID tupleSourceID, TupleSourceStatus status)
- throws TupleSourceNotFoundException, MetaMatrixComponentException {
-
- if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.DETAIL)) {
- LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, new Object[]{"Setting status for", tupleSourceID, "to", status}); //$NON-NLS-1$ //$NON-NLS-2$
- }
-
- TupleSourceInfo info = getTupleSourceInfo(tupleSourceID, true);
- synchronized(info) {
- info.setStatus(status);
- }
- }
-
- /**
- * Get the status for a particular tuple source
- * @param tupleSourceID Tuple source identifier
- * @return Status
- * @throws TupleSourceNotFoundException If tuple source not found
- * @throws MetaMatrixComponentException If an internal server error occurred
- */
- public TupleSourceStatus getStatus(TupleSourceID tupleSourceID)
- throws TupleSourceNotFoundException, MetaMatrixComponentException {
-
- TupleSourceInfo info = getTupleSourceInfo(tupleSourceID, true);
- return info.getStatus();
- }
-
- /**
- * Get the row count for a particular tuple source
- * @param tupleSourceID Tuple source identifier
- * @return Row count
- * @throws TupleSourceNotFoundException If tuple source not found
- * @throws MetaMatrixComponentException If an internal server error occurred
- */
- public int getRowCount(TupleSourceID tupleSourceID)
- throws TupleSourceNotFoundException, MetaMatrixComponentException {
-
- TupleSourceInfo info = getTupleSourceInfo(tupleSourceID, true);
- return info.getRowCount();
- }
-
- /**
- * Add a batch to the given tuple source. It is assumed that batches are
- * added in order, so the row count is reset to the end row of this batch.
- * @param tupleSourceID Tuple source identifier
- * @param tupleBatch New batch
- * @throws TupleSourceNotFoundException If tuple source not found
- * @throws MetaMatrixComponentException If an internal server error occurred
- */
- public void addTupleBatch(TupleSourceID tupleSourceID, TupleBatch tupleBatch)
- throws TupleSourceNotFoundException, MetaMatrixComponentException {
-
- if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.TRACE)) {
- LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, new Object[]{"AddTupleBatch for", tupleSourceID, "with " + tupleBatch.getRowCount() + " rows"}); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
- }
-
- if (tupleBatch.getRowCount() == 0 && !tupleBatch.getTerminationFlag()) {
- return;
- }
-
- // Look up info
- TupleSourceInfo info = getTupleSourceInfo(tupleSourceID, true);
-
- if (info.lobsInSource()) {
- correctLobReferences(info, tupleBatch);
- }
-
- // Determine where to store
- long bytes = SizeUtility.getBatchSize(info.getTypes(), tupleBatch.getAllTuples());
- tupleBatch.setSize(bytes);
- short location = ManagedBatch.PERSISTENT;
-
- if(reserveMemory(bytes, info.getGroupInfo()) == BufferManagerImpl.MEMORY_AVAILABLE) {
- location = ManagedBatch.UNPINNED;
- }
-
- synchronized(info) {
- if(info.isRemoved()) {
- if(location != ManagedBatch.PERSISTENT) {
- releaseMemory(bytes, info.getGroupInfo());
- }
- throw new TupleSourceNotFoundException(QueryExecPlugin.Util.getString("BufferManagerImpl.tuple_source_not_found", tupleSourceID)); //$NON-NLS-1$
- }
-
- // Update tuple source state
- ManagedBatch managedBatch = new ManagedBatch(tupleSourceID, tupleBatch.getBeginRow(), tupleBatch.getEndRow(), bytes);
- managedBatch.setLocation(location);
-
- // Store into storage manager
- try {
- if(location == ManagedBatch.PERSISTENT) {
- this.diskMgr.addBatch(tupleSourceID, tupleBatch, info.getTypes());
- } else {
- managedBatch.setBatch(tupleBatch);
- }
- } catch(MetaMatrixComponentException e) {
- // If we were storing to memory, clean up memory we reserved
- if(location != ManagedBatch.PERSISTENT) {
- releaseMemory(bytes, info.getGroupInfo());
- }
- throw e;
- }
-
- // Add to memory state if in memory
- if(location == ManagedBatch.UNPINNED) {
- this.unpinned.add(managedBatch);
- }
-
- // Update info with new rows
- info.addBatch(managedBatch);
- info.setRowCount(tupleBatch.getEndRow());
- }
- }
-
- /**
- * Pin a tuple batch in memory and return it. This batch must be unpinned by
- * passed the identical tuple source ID and beginning row.
- * NOTE: the returned {@link TupleBatch} will have the exact same bounds as the {@link ManagedBatch}
- * @param tupleSourceID Tuple source identifier
- * @param beginRow Beginning row
- * @throws TupleSourceNotFoundException If tuple source not found
- * @throws MetaMatrixComponentException If an internal server error occurred
- * @throws MemoryNotAvailableException If memory was not available for the pin
- */
- public TupleBatch pinTupleBatch(TupleSourceID tupleSourceID, int beginRow)
- throws TupleSourceNotFoundException, MemoryNotAvailableException, MetaMatrixComponentException {
-
- if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.TRACE)) {
- LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, new Object[]{"Pinning tupleBatch for", tupleSourceID, "beginRow:", beginRow}); //$NON-NLS-1$ //$NON-NLS-2$
- }
-
- this.pinRequests.incrementAndGet();
-
- TupleSourceInfo info = getTupleSourceInfo(tupleSourceID, true);
- int memoryAvailability = BufferManagerImpl.MEMORY_AVAILABLE;
-
- ManagedBatch mbatch = null;
- synchronized (info) {
- mbatch = info.getBatch(beginRow);
- }
- if (mbatch == null || beginRow == mbatch.getEndRow() + 1) {
- return new TupleBatch(beginRow, Collections.EMPTY_LIST);
- }
- long memoryRequiredByBatch = mbatch.getSize();
-
- TupleBatch memoryBatch = null;
-
- for (int pass = 0; pass < 2; pass++) {
- if (memoryAvailability == BufferManagerImpl.MEMORY_EXCEED_MAX) {
- clean(memoryRequiredByBatch, null);
- } else if (memoryAvailability == BufferManagerImpl.MEMORY_EXCEED_SESSION_MAX) {
- clean(memoryRequiredByBatch, info.getGroupInfo());
- }
-
- synchronized(info) {
- if(mbatch.getLocation() == ManagedBatch.PINNED) {
- // Load batch from memory - already pinned
- memoryBatch = mbatch.getBatch();
- break;
- }
- if(mbatch.getLocation() == ManagedBatch.UNPINNED) {
- // Already in memory - just move from unpinned to pinned
- this.unpinned.remove(mbatch);
- pin(mbatch);
-
- // Load batch from memory
- memoryBatch = mbatch.getBatch();
- break;
- }
- memoryRequiredByBatch = mbatch.getSize();
-
- // Try to reserve some memory
- if((memoryAvailability = reserveMemory(memoryRequiredByBatch, info.getGroupInfo())) != BufferManagerImpl.MEMORY_AVAILABLE) {
- continue;
- }
-
- this.pinnedFromDisk.incrementAndGet();
-
- // Memory was reserved, so move from persistent to memory and pin
- int internalBeginRow = mbatch.getBeginRow();
- memoryBatch = diskMgr.getBatch(tupleSourceID, internalBeginRow, info.getTypes());
-
- mbatch.setBatch(memoryBatch);
- if (info.lobsInSource()) {
- correctLobReferences(info, memoryBatch);
- }
- pin(mbatch);
- }
- }
-
- if (memoryBatch == null) {
- // Failed to reserve the memory even after a clean, so record the failure and fail the pin
- this.pinFailures.incrementAndGet();
- throw new MemoryNotAvailableException(QueryExecPlugin.Util.getString("BufferManagerImpl.no_memory_available")); //$NON-NLS-1$
- }
-
- return memoryBatch;
- }
-
- private void pin(ManagedBatch mbatch) {
- mbatch.setLocation(ManagedBatch.PINNED);
- PINNED_BY_THREAD.get().add(mbatch);
- if (!mbatch.hasPinnedRows()) {
- pinned.getAndIncrement();
- }
- mbatch.pin();
- }
-
- /**
- * Unpin a tuple source batch.
- * @param tupleSourceID Tuple source identifier
- * @param beginRow Beginning row
- * @throws TupleSourceNotFoundException If tuple source not found
- * @throws MetaMatrixComponentException If an internal server error occurred
- */
- public void unpinTupleBatch(TupleSourceID tupleSourceID, int beginRow)
- throws TupleSourceNotFoundException, MetaMatrixComponentException {
-
- if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.TRACE)) {
- LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, new Object[]{"Unpinning tupleBatch for", tupleSourceID, "beginRow: " + beginRow}); //$NON-NLS-1$ //$NON-NLS-2$
- }
-
- TupleSourceInfo info = getTupleSourceInfo(tupleSourceID, true);
- synchronized(info) {
- ManagedBatch mbatch = info.getBatch(beginRow);
- if(mbatch == null || mbatch.getLocation() != ManagedBatch.PINNED) {
- return;
- }
-
- mbatch.unpin();
-
- // Determine whether batch itself should be unpinned
- if(! mbatch.hasPinnedRows()) {
- mbatch.setLocation(ManagedBatch.UNPINNED);
- PINNED_BY_THREAD.get().remove(mbatch);
- this.unpinned.add(mbatch);
- pinned.getAndDecrement();
- }
- }
- }
-
- /**
- * Gets the final row count if tuple source is FULL, otherwise returns -1.
- * @param tupleSourceID Tuple source identifier
- * @return Final row count if status == FULL, -1 otherwise
- * @throws TupleSourceNotFoundException If tuple source not found
- * @throws MetaMatrixComponentException If an internal server error occurred
- */
- public int getFinalRowCount(TupleSourceID tupleSourceID)
- throws TupleSourceNotFoundException, MetaMatrixComponentException {
-
- TupleSourceInfo info = getTupleSourceInfo(tupleSourceID, true);
- synchronized(info) {
- if(info.getStatus() == TupleSourceStatus.FULL) {
- return info.getRowCount();
- }
- return -1;
- }
- }
-
- /**
* Get the tuple source info for the specified tuple source ID
* @param tupleSourceID Tuple source to get
- * @param throwException Whether to throw exception if the TupleSourceInfo is not found
* @return Tuple source info
* @throws TupleSourceNotFoundException If not found
*/
- TupleSourceInfo getTupleSourceInfo(TupleSourceID tupleSourceID, boolean throwException)
- throws TupleSourceNotFoundException {
+ public TupleBuffer getTupleBuffer(TupleSourceID tupleSourceID) throws TupleSourceNotFoundException {
+ TupleBuffer info = this.tupleSourceMap.get(tupleSourceID);
- // Look up info
- TupleSourceInfo info = this.tupleSourceMap.get(tupleSourceID);
-
- if(info == null && throwException) {
+ if(info == null) {
throw new TupleSourceNotFoundException(QueryExecPlugin.Util.getString("BufferManagerImpl.tuple_source_not_found", tupleSourceID)); //$NON-NLS-1$
}
return info;
}
/**
- * This can be done actively if someone wants memory and none is free.
- */
- protected void clean(long memoryRequired, TupleGroupInfo targetGroupInfo) {
- cleanLobTupleSource();
-
- long released = 0;
-
- long targetLevel = config.getActiveMemoryLevel();
- long totalMemory = config.getTotalAvailableMemory();
-
- boolean generalCleaningDone = false;
- List<ManagedBatch> toClean = null;
- synchronized (unpinned) {
- //TODO: re-implement without having to scan and compete
- toClean = new ArrayList<ManagedBatch>(unpinned);
- }
- for (ManagedBatch batch : toClean) {
- TupleSourceID tsID = batch.getTupleSourceID();
- TupleSourceInfo tsInfo = this.tupleSourceMap.get(tsID);
- if(tsInfo == null) {
- //may be removed by another thread
- continue;
- }
-
- long currentMemoryUsed = memoryUsed;
- if (!generalCleaningDone && (memoryRequired <= totalMemory - currentMemoryUsed && // if the memory needed is more than what's available, or
- currentMemoryUsed <= targetLevel)) { // if we've crossed the active memory threshold, then cleanup
- generalCleaningDone = true;
- }
- if (generalCleaningDone) {
- if (targetGroupInfo == null) {
- break;
- }
- if (targetGroupInfo == tsInfo.getGroupInfo()) {
- //if the memory needed is more than what is available for the session, then cleanup. Otherwise, break the loop.
- if(memoryRequired <= config.getMaxAvailableSession() - targetGroupInfo.getGroupMemoryUsed()) {
- break;
- }
- }
- }
-
- released += releaseMemory(batch, tsInfo);
- }
-
- if(released > 0) {
- this.cleanings.incrementAndGet();
- this.totalCleaned.addAndGet(released);
- }
- }
-
- private void cleanLobTupleSource() {
- String tupleSourceId = TupleSourceInfo.getStaleLobTupleSource();
- if (tupleSourceId != null) {
- try {
- removeTupleSource(new TupleSourceID(tupleSourceId));
- } catch (TupleSourceNotFoundException e) {
- } catch (MetaMatrixComponentException e) {
- LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, e, "Exception removing stale lob tuple source"); //$NON-NLS-1$
- }
- }
- }
-
- /**
- * Release the memory for the given unpinned batch.
- * @param batch Batch to be released from memory
- * @param tsID ID of the tuple source to be released from memory
- * @return The size of memory released in bytes
- * @since 4.3
- */
- private long releaseMemory(ManagedBatch batch, TupleSourceInfo info) {
- // Find info and lock on it
- synchronized(info) {
- if(info.isRemoved() || batch.getLocation() != ManagedBatch.UNPINNED) {
- return 0;
- }
-
- // This batch is still unpinned - move to persistent storage
- TupleBatch dataBatch = batch.getBatch();
-
- try {
- diskMgr.addBatch(info.getTupleSourceID(), dataBatch, info.getTypes());
- } catch(MetaMatrixComponentException e) {
- // Can't move
- return 0;
- }
-
- batch.setBatch(null);
-
- // Update memory
- batch.setLocation(ManagedBatch.PERSISTENT);
- this.unpinned.remove(batch);
- releaseMemory(batch.getSize(), info.getGroupInfo());
-
- return batch.getSize();
- }
- }
-
- /**
* Gets a TupleGroupInfo representing the group
* @param groupName Logical grouping name for tuple sources. Can be null.
* @return TupleGroupInfo for the group. Never null.
* @since 4.3
*/
- private TupleGroupInfo getGroupInfo(String groupName) {
- TupleGroupInfo info = null;
+ private Set<TupleSourceID> getGroupInfo(String groupName, boolean create) {
+ Set<TupleSourceID> info = null;
synchronized(groupInfos) {
info = groupInfos.get(groupName);
- if (info == null) {
+ if (info == null && create) {
// If it doesn't exist, create a new one
- info = new TupleGroupInfo(groupName);
+ info = Collections.synchronizedSet(new HashSet<TupleSourceID>());
groupInfos.put(groupName, info);
}
}
return info;
}
- /**
- * @see com.metamatrix.common.buffer.BufferManager#stop()
- */
- public synchronized void stop() {
- if (timer != null) {
- timer.cancel();
- }
- }
-
- /**
- * If a tuple batch is being added with Lobs, then references to
- * the lobs will be held on the {@link TupleSourceInfo}
- * @param batch
- */
- @SuppressWarnings("unchecked")
- private void correctLobReferences(TupleSourceInfo info, TupleBatch batch) {
- List parentSchema = info.getTupleSchema();
- List[] rows = batch.getAllTuples();
- int columns = parentSchema.size();
- // walk through the results and find all the lobs
- for (int row = 0; row < rows.length; row++) {
- for (int col = 0; col < columns; col++) {
- Object anObj = rows[row].get(col);
-
- if (!(anObj instanceof Streamable<?>)) {
- continue;
- }
- Streamable lob = (Streamable)anObj;
- info.addLobReference(lob);
- if (lob.getReference() == null) {
- lob.setReference(info.getLobReference(lob.getReferenceStreamId()).getReference());
- }
- }
+ @Override
+ public void addBatch(TupleSourceID sourceID, TupleBatch batch,
+ String[] types) throws MetaMatrixComponentException {
+ if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.TRACE)) {
+ LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, new Object[]{"AddTupleBatch for", sourceID, "with " + batch.getRowCount() + " rows"}); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
}
- }
-
- @Override
- public Streamable<?> getStreamable(TupleSourceID id, String referenceId) throws TupleSourceNotFoundException, MetaMatrixComponentException {
- TupleSourceInfo tsInfo = getTupleSourceInfo(id, true);
- Streamable<?> s = tsInfo.getLobReference(referenceId);
- if (s == null) {
- throw new MetaMatrixComponentException(DQPPlugin.Util.getString("ProcessWorker.wrongdata")); //$NON-NLS-1$
- }
- return s;
- }
-
- @Override
- public void setPersistentTupleSource(TupleSourceID id, Streamable<?> s) throws TupleSourceNotFoundException {
- cleanLobTupleSource();
- TupleSourceInfo tsInfo = getTupleSourceInfo(id, true);
- s.setPersistenceStreamId(id.getStringID());
- tsInfo.setContainingLobReference(s);
- }
+ this.diskMgr.addBatch(sourceID, batch, types);
+ }
- /**
- * @see com.metamatrix.common.buffer.BufferManager#addStreamablePart(com.metamatrix.common.buffer.TupleSourceID, com.metamatrix.common.lob.LobChunk, int)
- */
- public void addStreamablePart(TupleSourceID tupleSourceID, LobChunk streamChunk, int beginRow)
- throws TupleSourceNotFoundException, MetaMatrixComponentException {
-
- // Look up info
- TupleSourceInfo info = getTupleSourceInfo(tupleSourceID, true);
-
- synchronized(info) {
- List<LobChunk> data = new ArrayList<LobChunk>();
- data.add(streamChunk);
- TupleBatch batch = new TupleBatch(beginRow, new List[] {data});
- this.diskMgr.addBatch(tupleSourceID, batch, info.getTypes());
-
- // Update tuple source state (we could calculate the size of stream if need to)
- ManagedBatch managedBatch = new ManagedBatch(tupleSourceID, beginRow, batch.getEndRow(), 0);
- managedBatch.setLocation(ManagedBatch.PERSISTENT);
+ @Override
+ public TupleBatch getBatch(TupleSourceID sourceID, int beginRow,
+ String[] types) throws TupleSourceNotFoundException,
+ MetaMatrixComponentException {
+ if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.TRACE)) {
+ LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, new Object[]{"Getting tupleBatch for", sourceID, "beginRow:", beginRow}); //$NON-NLS-1$ //$NON-NLS-2$
+ }
+ return this.diskMgr.getBatch(sourceID, beginRow, types);
+ }
- // Update info with new rows
- info.addBatch(managedBatch);
- info.setRowCount(batch.getEndRow());
- }
- }
+ @Override
+ public void initialize(Properties props)
+ throws MetaMatrixComponentException {
+ this.config = new BufferConfig(props);
+ this.lookup = "local"; //$NON-NLS-1$
+ }
- /**
- * @see com.metamatrix.common.buffer.BufferManager#getStreamable(com.metamatrix.common.buffer.TupleSourceID)
- */
- public LobChunk getStreamablePart(final TupleSourceID tupleSourceID, int beginRow)
- throws TupleSourceNotFoundException, MetaMatrixComponentException {
-
- final TupleSourceInfo info = getTupleSourceInfo(tupleSourceID, true);
- TupleBatch batch = diskMgr.getBatch(tupleSourceID, beginRow, info.getTypes());
- LobChunk chunk = (LobChunk)batch.getAllTuples()[0].get(0);
-
- return chunk;
- }
-
- /**
- * @see com.metamatrix.common.buffer.BufferManager#releasePinnedBatches()
- */
- public void releasePinnedBatches() throws MetaMatrixComponentException {
- MetaMatrixComponentException e = null;
- List<ManagedBatch> pinnedByThread = new ArrayList<ManagedBatch>(PINNED_BY_THREAD.get());
- for (ManagedBatch managedBatch : pinnedByThread) {
- try {
- //TODO: add trace logging about the batch that is being unpinned
- unpinTupleBatch(managedBatch.getTupleSourceID(), managedBatch.getBeginRow());
- } catch (TupleSourceNotFoundException err) {
- } catch (MetaMatrixComponentException err) {
- e = err;
- }
- }
- if (e != null) {
- throw e;
- }
- }
-
- /**
- * Check for whether the specified amount of memory can be reserved,
- * and if so reserve it. This is done in the same method so that the
- * memory is not taken away by a different thread between checking and
- * reserving - standard "test and set" behavior.
- * @param bytes Bytes requested
- * @return One of MEMORY_AVAILABLE, MEMORY_EXCEED_MAX, or MEMORY_EXCEED_SESSION_MAX
- */
- private synchronized int reserveMemory(long bytes, TupleGroupInfo groupInfo) {
- //check session limit first
- long sessionMax = config.getMaxAvailableSession();
- if(sessionMax - groupInfo.getGroupMemoryUsed() < bytes) {
- return BufferManagerImpl.MEMORY_EXCEED_SESSION_MAX;
+ @Override
+ public void removeBatches(TupleSourceID sourceID) {
+ if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.DETAIL)) {
+ LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, new Object[]{"Removing TupleBuffer:", sourceID}); //$NON-NLS-1$
}
-
- //then check the total memory limit
- long max = config.getTotalAvailableMemory();
- if(max - memoryUsed < bytes) {
- return BufferManagerImpl.MEMORY_EXCEED_MAX;
- }
-
- groupInfo.reserveMemory(bytes);
- memoryUsed += bytes;
-
- return BufferManagerImpl.MEMORY_AVAILABLE;
- }
+ TupleBuffer tupleBuffer = this.tupleSourceMap.remove(sourceID);
+ if (tupleBuffer != null) {
+ Set<TupleSourceID> ids = getGroupInfo(tupleBuffer.getGroupName(), false);
+ if (ids != null) {
+ ids.remove(sourceID);
+ }
+ }
+ this.diskMgr.removeBatches(sourceID);
+ }
- /**
- * Release memory
- * @param bytes Bytes to release
- */
- private synchronized void releaseMemory(long bytes, TupleGroupInfo groupInfo) {
- groupInfo.releaseMemory(bytes);
- memoryUsed -= bytes;
- }
+ @Override
+ public void shutdown() {
+ this.diskMgr.shutdown();
+ }
- /**
- * for testing purposes
- */
- public int getPinnedCount() {
- return pinned.get();
- }
}
Deleted: trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferStats.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferStats.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferStats.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -1,128 +0,0 @@
-/*
- * JBoss, Home of Professional Open Source.
- * See the COPYRIGHT.txt file distributed with this work for information
- * regarding copyright ownership. Some portions may be licensed
- * to Red Hat, Inc. under one or more contributor license agreements.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301 USA.
- */
-
-package com.metamatrix.common.buffer.impl;
-
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-
-import com.metamatrix.common.log.LogManager;
-import com.metamatrix.core.log.MessageLevel;
-import com.metamatrix.dqp.util.LogConstants;
-
-/**
- * This encapsulates all the statistics for the BufferManagerImpl. The attribute
- * values are set by direct attribute manipulation (no getter/setters).
- */
-public class BufferStats {
-
- // Basic stats
- public long memoryUsed;
- public long memoryFree;
- public int numTupleSources;
- public int numPersistentBatches;
- public int numPinnedBatches;
- public int numUnpinnedBatches;
- public String sLocalDrive;
-
- // Accumulated stats
- public int pinRequests;
- public int pinSuccesses;
- public int pinnedFromMemory;
- public int numCleanings;
- public long totalCleaned;
-
- // Pinned batch details
- public List<ManagedBatch> pinnedManagedBatches = new LinkedList<ManagedBatch>();
-
- /**
- * Constructor for BufferStats.
- */
- BufferStats() {
- }
-
- /**
- * Log this set of stats.
- */
- void log() {
- LogManager.logInfo(LogConstants.CTX_BUFFER_MGR, "\n"); //$NON-NLS-1$
- LogManager.logInfo(LogConstants.CTX_BUFFER_MGR, "BUFFER MANAGER STATS"); //$NON-NLS-1$
- LogManager.logInfo(LogConstants.CTX_BUFFER_MGR, " memoryFree = " + memoryFree); //$NON-NLS-1$
- LogManager.logInfo(LogConstants.CTX_BUFFER_MGR, " memoryUsed = " + memoryUsed); //$NON-NLS-1$
- LogManager.logInfo(LogConstants.CTX_BUFFER_MGR, " numTupleSources = " + numTupleSources); //$NON-NLS-1$
- LogManager.logInfo(LogConstants.CTX_BUFFER_MGR, " numPersistentBatches = " + numPersistentBatches); //$NON-NLS-1$
- LogManager.logInfo(LogConstants.CTX_BUFFER_MGR, " numPinnedBatches = " + numPinnedBatches); //$NON-NLS-1$
- LogManager.logInfo(LogConstants.CTX_BUFFER_MGR, " numUnpinnedBatches = " + numUnpinnedBatches); //$NON-NLS-1$
- LogManager.logInfo(LogConstants.CTX_BUFFER_MGR, " pinRequests = " + pinRequests); //$NON-NLS-1$
-
- // pinSuccesses / pinRequests
- double pinSuccessRate = (pinRequests > 0) ? (((double)pinSuccesses / (double)pinRequests) * 100) : 100;
- LogManager.logInfo(LogConstants.CTX_BUFFER_MGR, " pinSuccessRate = " + pinSuccessRate); //$NON-NLS-1$
-
- // pinnedFromMemory / pinRequests
- double memoryHitRate = (pinRequests > 0) ? (((double)pinnedFromMemory / (double)pinRequests) * 100): 100;
- LogManager.logInfo(LogConstants.CTX_BUFFER_MGR, " memoryHitRate = " + memoryHitRate); //$NON-NLS-1$
- LogManager.logInfo(LogConstants.CTX_BUFFER_MGR, " numCleanings = " + numCleanings); //$NON-NLS-1$
-
- // totalCleaned / numCleanings
- long avgCleaned = (numCleanings > 0) ? (totalCleaned / numCleanings) : 0;
- LogManager.logInfo(LogConstants.CTX_BUFFER_MGR, " avgCleaned = " + avgCleaned); //$NON-NLS-1$
-
- if ( LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.TRACE) ) {
- HashMap<List<String>, Integer> stackTraces = new HashMap<List<String>, Integer>();
-
- if ( pinnedManagedBatches.isEmpty() ) {
- return;
- }
-
- LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, "\n"); //$NON-NLS-1$
- LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, " =========== Pinned Batch Details: ==========="); //$NON-NLS-1$
-
- int stackNumber = 1;
-
- // pinned batch details
- for (ManagedBatch batch : pinnedManagedBatches) {
- LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, " TupleSourceID: " + batch.getTupleSourceID() + " Begin: " + batch.getBeginRow() + " End: " + batch.getEndRow()); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
-
- Integer stackKey = stackTraces.get(batch.getCallStack());
-
- boolean isFirst = false;
-
- if (stackKey == null) {
- isFirst = true;
- stackKey = new Integer(stackNumber++);
- stackTraces.put(batch.getCallStack(), stackKey);
- }
-
- LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, " Pinned at: " + batch.getCallStackTimeStamp() + " by call# " + stackKey); //$NON-NLS-1$ //$NON-NLS-2$
- if (isFirst) {
- for (Iterator<String> j = batch.getCallStack().iterator(); j.hasNext();) {
- LogManager.logTrace( LogConstants.CTX_BUFFER_MGR, " " + j.next() ); //$NON-NLS-1$
- }
- }
- }
- }
- }
-
-}
Copied: trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/FileStorageManager.java (from rev 1678, trunk/engine/src/main/java/com/metamatrix/common/buffer/storage/file/FileStorageManager.java)
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/FileStorageManager.java (rev 0)
+++ trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/FileStorageManager.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -0,0 +1,425 @@
+/*
+ * JBoss, Home of Professional Open Source.
+ * See the COPYRIGHT.txt file distributed with this work for information
+ * regarding copyright ownership. Some portions may be licensed
+ * to Red Hat, Inc. under one or more contributor license agreements.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301 USA.
+ */
+
+package com.metamatrix.common.buffer.impl;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.RandomAccessFile;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import com.metamatrix.api.exception.MetaMatrixComponentException;
+import com.metamatrix.common.buffer.BufferManagerPropertyNames;
+import com.metamatrix.common.buffer.StorageManager;
+import com.metamatrix.common.buffer.TupleBatch;
+import com.metamatrix.common.buffer.TupleSourceID;
+import com.metamatrix.common.buffer.TupleSourceNotFoundException;
+import com.metamatrix.common.log.LogManager;
+import com.metamatrix.common.util.PropertiesUtils;
+import com.metamatrix.core.log.MessageLevel;
+import com.metamatrix.core.util.Assertion;
+import com.metamatrix.dqp.util.LogConstants;
+import com.metamatrix.query.execution.QueryExecPlugin;
+/**
+ * This class stores batches in files on disk in a specified directory. Every
+ * tuple source gets it's own set of files, named b_<i>id</i>_<i>num</i> where <i>id</i> is the
+ * unique id value of the TupleSourceID, and <i>num</i> is a file counter for the files
+ * associated with the TupleSourceID. Batches are stored random access into the
+ * file, typically but not necessarily in order. An in memory data structure stores
+ * info about all open files and where the batches are in the file (file pointers and
+ * lengths).
+ */
+public class FileStorageManager implements StorageManager {
+
+ private static final String FILE_PREFIX = "b_"; //$NON-NLS-1$
+
+ // Initialization
+ private int maxOpenFiles = 10;
+ private long maxFileSize = 2L * 1024L * 1024L * 1024L; // 2GB
+ private String directory;
+ private File dirFile;
+
+ // State
+ private Map<TupleSourceID, TupleSourceInfo> tupleSourceMap = new HashMap<TupleSourceID, TupleSourceInfo>();
+ private Map<File, RandomAccessFile> fileCache = Collections.synchronizedMap(new LinkedHashMap<File, RandomAccessFile>() {
+ @Override
+ protected boolean removeEldestEntry(
+ java.util.Map.Entry<File, RandomAccessFile> eldest) {
+ if (this.size() > maxOpenFiles) {
+ try {
+ eldest.getValue().close();
+ } catch (IOException e) {
+ }
+ return true;
+ }
+ return false;
+ }
+ });
+
+ /**
+ * Initialize with properties
+ * @param props Initialization properties
+ * @see com.metamatrix.common.buffer.BufferManagerPropertyNames#BUFFER_STORAGE_DIRECTORY
+ * @see com.metamatrix.common.buffer.BufferManagerPropertyNames#MAX_OPEN_FILES
+ * @see com.metamatrix.common.buffer.BufferManagerPropertyNames#MAX_FILE_SIZE
+ */
+ public void initialize(Properties props) throws MetaMatrixComponentException {
+ this.directory = props.getProperty(BufferManagerPropertyNames.BUFFER_STORAGE_DIRECTORY);
+ if(this.directory == null) {
+ throw new MetaMatrixComponentException(QueryExecPlugin.Util.getString("FileStoreageManager.no_directory")); //$NON-NLS-1$
+ }
+
+ dirFile = new File(this.directory);
+ if(dirFile.exists()) {
+ if(! dirFile.isDirectory()) {
+ throw new MetaMatrixComponentException(QueryExecPlugin.Util.getString("FileStoreageManager.not_a_directory", dirFile.getAbsoluteFile())); //$NON-NLS-1$
+
+ }
+ } else if(! dirFile.mkdirs()) {
+ throw new MetaMatrixComponentException(QueryExecPlugin.Util.getString("FileStoreageManager.error_creating", dirFile.getAbsoluteFile())); //$NON-NLS-1$
+ }
+
+ // Set up max number of open file descriptors
+ maxOpenFiles = PropertiesUtils.getIntProperty(props, BufferManagerPropertyNames.MAX_OPEN_FILES, 10);
+
+ // Set the max file size
+ maxFileSize = PropertiesUtils.getIntProperty(props, BufferManagerPropertyNames.MAX_FILE_SIZE, 2048) * 1024L * 1024L; // Multiply by 1MB
+ }
+
+ /**
+ * Look up tuple source info and possibly create. First the file map is used to find an
+ * existing file info. If the info is found it is returned. If not, then
+ * a TupleSourceInfo is created according to shouldCreate flag
+ * @param sourceID Source identifier
+ * @param shouldCreate true if this method should create info related to the tuple source ID if it does not exist.
+ * @return All the tuple source info or null if shouldCreate == false and no tuple source info was found
+ */
+ private TupleSourceInfo getTupleSourceInfo(TupleSourceID sourceID, boolean shouldCreate) {
+
+ // Try to find in cache
+ synchronized(tupleSourceMap) {
+ TupleSourceInfo info = tupleSourceMap.get(sourceID);
+ if(info == null && shouldCreate) {
+ info = new TupleSourceInfo();
+ tupleSourceMap.put(sourceID, info);
+ }
+ return info;
+ }
+ }
+
+ /**
+ * Creates a new file to the specified TupleSourceID
+ * @param sourceID The TupleSourceID
+ * @param fileNumber a number uniquely identifying this file within the set of tuple source files.
+ * @return the newly created file with the name b_<i>sourceID</i>_<i>fileNumber</i>
+ * @throws MetaMatrixComponentException if a file with this name already exists in the directory, or if it cannot be created
+ * @since 4.2
+ */
+ private File createFile(TupleSourceID sourceID, int fileNumber) throws MetaMatrixComponentException {
+ File storageFile = new File(this.directory, FILE_PREFIX + sourceID.getIDValue() + "_" + fileNumber); //$NON-NLS-1$
+ if (LogManager.isMessageToBeRecorded(LogConstants.CTX_STORAGE_MGR, MessageLevel.DETAIL)) {
+ LogManager.logDetail(LogConstants.CTX_STORAGE_MGR, "Creating temporary storage area file " + storageFile.getAbsoluteFile()); //$NON-NLS-1$
+ }
+ try {
+ boolean created = storageFile.createNewFile();
+ if(!created) {
+ throw new MetaMatrixComponentException(QueryExecPlugin.Util.getString("FileStoreageManager.file_exists", storageFile.getAbsoluteFile())); //$NON-NLS-1$
+ }
+ } catch(IOException e) {
+ throw new MetaMatrixComponentException(e, QueryExecPlugin.Util.getString("FileStoreageManager.error_creating", storageFile.getAbsoluteFile())); //$NON-NLS-1$
+ }
+ return storageFile;
+ }
+
+ /**
+ * Add a batch to the storage manager. Because we don't implement remove, it's
+ * possible that this batch already exists, in which case nothing is done.
+ * @param sourceID Source identifier
+ * @param batch Batch to add
+ */
+ public void addBatch(TupleSourceID sourceID, TupleBatch batch, String[] types)
+ throws MetaMatrixComponentException {
+
+ // Defect 13342 - addBatch method now creates spill files if the total bytes exceeds the max file size limit
+ TupleSourceInfo tsInfo = getTupleSourceInfo(sourceID, true);
+ synchronized (tsInfo) {
+ if (tsInfo.isRemoved) {
+ return;
+ }
+ Integer batchKey = new Integer(batch.getBeginRow());
+ if (tsInfo.tupleBatchPointers.containsKey(batchKey)) {
+ return;
+ }
+ byte[] bytes = convertToBytes(batch, types);
+ if (bytes.length > maxFileSize) {
+ LogManager.logWarning(LogConstants.CTX_STORAGE_MGR, "Detected an attempt to save a batch (" + sourceID + ", begin=" + batch.getBeginRow()+ ", size=" + bytes.length + ") larger than the buffer max file size setting of " + maxFileSize + " bytes. The buffer manager will ignore the max file size setting for this batch, and create a buffer file dedicated to this batch. It may be necessary to reduce the buffer batch setting or increase the buffer max file size setting."); //$NON-NLS-1$//$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$
+ }
+ FileInfo fileInfo = tsInfo.getMostRecentlyCreatedFile();
+ if (fileInfo == null ||
+ (/*fileInfo.file.length() != 0 && */fileInfo.file.length() + bytes.length > maxFileSize)) {
+ // Create and add
+ fileInfo = new FileInfo(createFile(sourceID, tsInfo.storageFiles.size()));
+ tsInfo.storageFiles.add(fileInfo);
+ }
+ long pointer = 0;
+
+ if (LogManager.isMessageToBeRecorded(LogConstants.CTX_STORAGE_MGR, MessageLevel.DETAIL)) {
+ LogManager.logDetail(LogConstants.CTX_STORAGE_MGR, "Adding batch to storage area file " + fileInfo.file.getAbsoluteFile() + " [ sourceID: " + sourceID + "batch: " + batch + " ]"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
+ }
+ try {
+ // Get access to the file and remember whether we had to open it or not
+ fileInfo.open();
+ RandomAccessFile fileAccess = fileInfo.getAccess();
+
+ // Store the batch in the file
+ pointer = fileAccess.length();
+ fileAccess.setLength(pointer + bytes.length);
+ fileAccess.seek(pointer);
+ fileAccess.write(bytes);
+ } catch(IOException e) {
+ throw new MetaMatrixComponentException(e, QueryExecPlugin.Util.getString("FileStoreageManager.error_reading", fileInfo.file.getAbsoluteFile())); //$NON-NLS-1$
+ } finally {
+ fileInfo.close();
+ }
+
+ // Update the pointers
+ tsInfo.tupleBatchPointers.put(batchKey, new PointerInfo(fileInfo, pointer, bytes.length));
+ }
+ }
+
+ /**
+ * Convert from an object to a byte array
+ * @param object Object to convert
+ * @return Byte array
+ */
+ private byte[] convertToBytes(TupleBatch batch, String[] types) throws MetaMatrixComponentException {
+ ObjectOutputStream oos = null;
+ try {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ oos = new ObjectOutputStream(baos);
+
+ batch.setDataTypes(types);
+ batch.writeExternal(oos);
+ oos.flush();
+ return baos.toByteArray();
+
+ } catch(IOException e) {
+ throw new MetaMatrixComponentException(e, QueryExecPlugin.Util.getString("FileStorageManager.batch_error")); //$NON-NLS-1$
+ } finally {
+ if(oos != null) {
+ try {
+ oos.close();
+ } catch(IOException e) {
+ }
+ }
+ }
+ }
+
+ /**
+ * Get a batch from the storage manager based on the beginRow.
+ * @param sourceID Source identifier
+ * @param beginRow Beginning row of batch to retrieve
+ * @return Batch retrieved
+ */
+ public TupleBatch getBatch(TupleSourceID sourceID, int beginRow, String[] types)
+ throws TupleSourceNotFoundException, MetaMatrixComponentException {
+
+ TupleSourceInfo info = getTupleSourceInfo(sourceID, false);
+ if(info == null) {
+ throw new TupleSourceNotFoundException(QueryExecPlugin.Util.getString("BufferManagerImpl.tuple_source_not_found", sourceID)); //$NON-NLS-1$
+ }
+
+ byte[] bytes = null;
+ synchronized(info) {
+ if(info.isRemoved) {
+ throw new TupleSourceNotFoundException(QueryExecPlugin.Util.getString("BufferManagerImpl.tuple_source_not_found", sourceID)); //$NON-NLS-1$
+ }
+ // Find pointer
+ PointerInfo pointerInfo = info.tupleBatchPointers.get(new Integer(beginRow));
+ Assertion.isNotNull(pointerInfo);
+
+ FileInfo fileInfo = pointerInfo.fileInfo;
+ // Get access to the file
+ RandomAccessFile fileAccess = null;
+ try {
+ fileInfo.open();
+ fileAccess = fileInfo.getAccess();
+ fileAccess.seek(pointerInfo.pointer);
+ bytes = new byte[pointerInfo.length];
+ fileAccess.readFully(bytes);
+ ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
+ ObjectInputStream ois = new ObjectInputStream(bais);
+
+ TupleBatch batch = new TupleBatch();
+ batch.setDataTypes(types);
+ batch.readExternal(ois);
+ return batch;
+ } catch(IOException e) {
+ throw new MetaMatrixComponentException(e, QueryExecPlugin.Util.getString("FileStoreageManager.error_reading", fileInfo.file.getAbsoluteFile())); //$NON-NLS-1$
+ } catch (ClassNotFoundException e) {
+ throw new MetaMatrixComponentException(e, QueryExecPlugin.Util.getString("FileStoreageManager.error_reading", fileInfo.file.getAbsoluteFile())); //$NON-NLS-1$
+ } finally {
+ fileInfo.close();
+ }
+ }
+ }
+
+ /**
+ * Remove all batches for a sourceID. Before removal, the file is closed.
+ * @param sourceID Tuple source ID
+ */
+ public void removeBatches(TupleSourceID sourceID) {
+ TupleSourceInfo info = null;
+ // Remove info from the file map
+ synchronized(tupleSourceMap) {
+ info = tupleSourceMap.remove(sourceID);
+ }
+
+ // Didn't find a file
+ if(info == null) {
+ return;
+ }
+
+ if (LogManager.isMessageToBeRecorded(LogConstants.CTX_STORAGE_MGR, MessageLevel.DETAIL)) {
+ LogManager.logDetail(LogConstants.CTX_STORAGE_MGR, "Removing storage for " + sourceID); //$NON-NLS-1$
+ }
+
+ synchronized(info) {
+ // If open, close the file and decrement the open file counter
+ for (int i = 0; i < info.storageFiles.size(); i++) {
+ FileInfo fileInfo = info.storageFiles.get(i);
+ fileInfo.delete();
+ }
+ // Delete the file and mark info as being removed
+ info.isRemoved = true;
+ }
+ }
+
+ /**
+ * This method removes all storage area files by walking through the file info
+ * map and closing and removing each file.
+ */
+ public synchronized void shutdown() {
+
+ LogManager.logDetail(LogConstants.CTX_STORAGE_MGR, "Removing all storage area files "); //$NON-NLS-1$
+
+ List<TupleSourceID> ids = null;
+ synchronized (tupleSourceMap) {
+ ids = new LinkedList<TupleSourceID>(tupleSourceMap.keySet());
+ }
+
+ for (TupleSourceID key : ids) {
+ removeBatches(key);
+ }
+
+ tupleSourceMap = null;
+ }
+
+ public int getOpenFiles() {
+ return this.fileCache.size();
+ }
+
+ private class FileInfo {
+ private File file;
+ private RandomAccessFile fileData; // may be null if not open
+
+ public FileInfo(File file) {
+ this.file = file;
+ }
+
+ public void open() throws FileNotFoundException {
+ if(this.fileData == null) {
+ this.fileData = fileCache.remove(this.file);
+ if (this.fileData == null) {
+ this.fileData = new RandomAccessFile(file, "rw"); //$NON-NLS-1$
+ }
+ }
+ }
+
+ public RandomAccessFile getAccess() {
+ return this.fileData;
+ }
+
+ public void close() {
+ if(this.fileData != null) {
+ fileCache.put(file, this.fileData);
+ this.fileData = null;
+ }
+ }
+
+ public void delete() {
+ if (this.fileData == null) {
+ this.fileData = fileCache.remove(this.file);
+ }
+ if (this.fileData != null) {
+ try {
+ this.fileData.close();
+ } catch(Exception e) {
+ }
+ this.fileData = null;
+ }
+ file.delete();
+ }
+
+ public String toString() {
+ return "FileInfo<" + file.getName() + ", has fileData = " + (fileData != null) + ">"; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
+ }
+ }
+
+ private static class PointerInfo {
+ FileInfo fileInfo;
+ public long pointer;
+ public int length;
+
+ public PointerInfo(FileInfo fileInfo, long pointer, int length) {
+ this.fileInfo = fileInfo;
+ this.pointer = pointer;
+ this.length = length;
+ }
+ }
+
+ private static class TupleSourceInfo {
+ Map<Integer, PointerInfo> tupleBatchPointers = new HashMap<Integer, PointerInfo>();
+ List<FileInfo> storageFiles = new ArrayList<FileInfo>(2); // Stores all the FileInfos for this tupleSource
+ private boolean isRemoved = false;
+
+ FileInfo getMostRecentlyCreatedFile() {
+ if (storageFiles.isEmpty()) {
+ return null;
+ }
+ return storageFiles.get(storageFiles.size() - 1);
+ }
+ }
+
+}
Deleted: trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/ManagedBatch.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/ManagedBatch.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/ManagedBatch.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -1,217 +0,0 @@
-/*
- * JBoss, Home of Professional Open Source.
- * See the COPYRIGHT.txt file distributed with this work for information
- * regarding copyright ownership. Some portions may be licensed
- * to Red Hat, Inc. under one or more contributor license agreements.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301 USA.
- */
-
-package com.metamatrix.common.buffer.impl;
-
-import java.sql.Timestamp;
-import java.util.LinkedList;
-import java.util.List;
-
-import com.metamatrix.common.buffer.TupleBatch;
-import com.metamatrix.common.buffer.TupleSourceID;
-import com.metamatrix.core.util.HashCodeUtil;
-
-/**
- * A ManagedBatch basically describes a TupleBatch that is being managed and
- * appends a bunch of additional state used within the BufferManagerImpl. The
- * batch is described by it's beginRow and endRow and it's size is in bytes.
- * The lastAccessed field records the last accessed time in memory. The location
- * tells where the batch is currently stored, as defined in the TupleSourceInfo
- * constants. The pinned intervals describe which rows within the batch are
- * currently pinned.
- */
-class ManagedBatch {
-
- /** Constant for the location of this batch - stored in persistent storage */
- public final static short PERSISTENT = 0;
-
- /** Constant for the location of this batch - stored in memory, but not currently in use */
- public final static short UNPINNED = 1;
-
- /** Constant for the location of this batch - stored in memory and currently in use */
- public final static short PINNED = 2;
-
- private TupleSourceID tupleSourceID;
- private int beginRow;
- private int endRow;
- private long size;
- private int location;
- private int pinnedCount;
- private int hashCode;
- private TupleBatch batch;
-
- // logging
- private static int STACK_LEVELS_TO_OMIT = 2;
- private static int STACK_LEVELS_TO_CAPTURE = 5;
-
- private List<String> whoCalledUs;
- private String sCallStackTimeStamp;
-
- /**
- * Constructor for ManagedBatch.
- */
- public ManagedBatch(TupleSourceID tupleSourceID, int begin, int end, long size) {
- this.tupleSourceID = tupleSourceID;
- this.beginRow = begin;
- this.endRow = end;
- this.size = size;
- this.hashCode = HashCodeUtil.hashCode(tupleSourceID.hashCode(), beginRow);
- }
-
- public TupleBatch getBatch() {
- return batch;
- }
-
- public void setBatch(TupleBatch batch) {
- this.batch = batch;
- }
-
- /**
- * Get the tuple source ID
- * @return Tuple sourceID
- */
- public TupleSourceID getTupleSourceID() {
- return this.tupleSourceID;
- }
-
- /**
- * Get the begin row, must be >= 1
- * @return Begin row
- */
- public int getBeginRow() {
- return this.beginRow;
- }
-
- /**
- * Get the end row, inclusive
- * @return End row
- */
- public int getEndRow() {
- return this.endRow;
- }
-
- /**
- * Get the size of the batch in bytes
- * @return Size, in bytes
- */
- public long getSize() {
- return this.size;
- }
-
- /**
- * Get the location of the batch, as defined in constants
- * @return Location
- */
- public int getLocation() {
- return this.location;
- }
-
- /**
- * Set location of the batch, as defined in constants
- */
- public void setLocation(int location) {
- this.location = location;
- }
-
- /**
- * Check whether this managed batch has any pinned rows
- * @return True if any are pinned
- */
- public boolean hasPinnedRows() {
- return pinnedCount > 0;
- }
-
- public void pin() {
- this.pinnedCount++;
- }
-
- public void unpin() {
- this.pinnedCount--;
- }
-
- /**
- * Capture partial stack trace to express who owns this batch
- */
- public void captureCallStack() {
- /*
- * If more detail is needed StackTraceElement has several useful methods.
- */
- StackTraceElement[] elements = new Exception().getStackTrace();
-
- whoCalledUs = new LinkedList<String>();
-
- for ( int i = STACK_LEVELS_TO_OMIT; i < elements.length && i < STACK_LEVELS_TO_OMIT + STACK_LEVELS_TO_CAPTURE; i++ ) {
- whoCalledUs.add(elements[ i ].toString());
- }
- sCallStackTimeStamp = new Timestamp(System.currentTimeMillis()).toString();
- }
-
- /**
- * Returns call stack
- */
- public List<String> getCallStack() {
-
- return whoCalledUs;
- }
-
- /**
- * Returns call stack timestamp
- */
- public String getCallStackTimeStamp() {
-
- return this.sCallStackTimeStamp;
- }
-
- /**
- * Compare two managed batches for equality - this check
- * is made based on TupleSourceID and beginRow.
- * @param obj Other object
- * @return True if equal
- */
- public boolean equals(Object obj) {
- if(obj == this) {
- return true;
- }
-
- if(! (obj instanceof ManagedBatch)) {
- return false;
- }
-
- ManagedBatch other = (ManagedBatch) obj;
- return ( this.getTupleSourceID().equals(other.getTupleSourceID()) &&
- this.getBeginRow() == other.getBeginRow() );
-
- }
-
- /**
- * Get hash code, based on tupleSourceID and beginRow
- * @return Hash code
- */
- public int hashCode() {
- return hashCode;
- }
-
- public String toString() {
- return "ManagedBatch[" + tupleSourceID + ", " + beginRow + ", " + endRow + "]"; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
- }
-
-}
Added: trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/MemoryStorageManager.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/MemoryStorageManager.java (rev 0)
+++ trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/MemoryStorageManager.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -0,0 +1,104 @@
+/*
+ * JBoss, Home of Professional Open Source.
+ * See the COPYRIGHT.txt file distributed with this work for information
+ * regarding copyright ownership. Some portions may be licensed
+ * to Red Hat, Inc. under one or more contributor license agreements.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301 USA.
+ */
+
+package com.metamatrix.common.buffer.impl;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+import com.metamatrix.api.exception.MetaMatrixComponentException;
+import com.metamatrix.common.buffer.StorageManager;
+import com.metamatrix.common.buffer.TupleBatch;
+import com.metamatrix.common.buffer.TupleSourceID;
+import com.metamatrix.common.buffer.TupleSourceNotFoundException;
+import com.metamatrix.query.execution.QueryExecPlugin;
+
+public class MemoryStorageManager implements StorageManager {
+
+ // TupleSourceID -> List<TupleBatch> (ordered by startRow)
+ private Map<TupleSourceID, Map<Integer, TupleBatch>> storage = Collections.synchronizedMap(new HashMap<TupleSourceID, Map<Integer, TupleBatch>>());
+
+ /**
+ * @see StorageManager#initialize(Properties)
+ */
+ public void initialize(Properties props) throws MetaMatrixComponentException {
+ }
+
+ /**
+ * @see StorageManager#addBatch(TupleSourceID, TupleBatch)
+ */
+ public void addBatch(TupleSourceID storageID, TupleBatch batch, String[] types)
+ throws MetaMatrixComponentException {
+
+ Map<Integer, TupleBatch> batches = null;
+ synchronized(this.storage) {
+ batches = storage.get(storageID);
+ if(batches == null) {
+ batches = new HashMap<Integer, TupleBatch>();
+ this.storage.put(storageID, batches);
+ }
+ }
+
+ synchronized(batches) {
+ batches.put(batch.getBeginRow(), batch);
+ }
+ }
+
+ /**
+ * @see StorageManager#getBatch(TupleSourceID, int, int)
+ */
+ public TupleBatch getBatch(TupleSourceID storageID, int beginRow, String[] types)
+ throws TupleSourceNotFoundException, MetaMatrixComponentException {
+
+ Map<Integer, TupleBatch> batches = storage.get(storageID);
+
+ if(batches == null) {
+ throw new TupleSourceNotFoundException(QueryExecPlugin.Util.getString("BufferManagerImpl.tuple_source_not_found", storageID)); //$NON-NLS-1$
+ }
+
+ synchronized(batches) {
+ TupleBatch batch = batches.get(beginRow);
+ if(batch == null) {
+ throw new MetaMatrixComponentException("unknown batch"); //$NON-NLS-1$
+ }
+ return batch;
+ }
+ }
+
+ /**
+ * @see StorageManager#removeStorageArea(TupleSourceID)
+ */
+ public void removeBatches(TupleSourceID storageID) {
+ storage.remove(storageID);
+ }
+
+ /**
+ * @see StorageManager#shutdown()
+ */
+ public void shutdown() {
+ this.storage.clear();
+ this.storage = null;
+ }
+
+}
\ No newline at end of file
Property changes on: trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/MemoryStorageManager.java
___________________________________________________________________
Name: svn:mime-type
+ text/plain
Deleted: trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/SizeUtility.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/SizeUtility.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/SizeUtility.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -1,243 +0,0 @@
-/*
- * JBoss, Home of Professional Open Source.
- * See the COPYRIGHT.txt file distributed with this work for information
- * regarding copyright ownership. Some portions may be licensed
- * to Red Hat, Inc. under one or more contributor license agreements.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301 USA.
- */
-
-package com.metamatrix.common.buffer.impl;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.ObjectOutputStream;
-import java.io.Serializable;
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Vector;
-
-import com.metamatrix.common.types.DataTypeManager;
-import com.metamatrix.core.MetaMatrixRuntimeException;
-import com.metamatrix.core.util.Assertion;
-
-
-/**
- * Utility methods to determine the size of Java objects, particularly with
- * respect to the MetaMatrix runtime types.
- */
-public final class SizeUtility {
- private static final int DEFAULT_BUFFER_SIZE = 4096;
- public static final boolean IS_64BIT = System.getProperty("sun.arch.data.model", "32").indexOf("64") != -1; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
-// public static final boolean IS_64BIT = true;
- public static final int REFERENCE_SIZE = IS_64BIT?8:4;
-
- private static final int SAMPLE_RATE = 10;
-
-
- /**
- * Constructor for SizeUtility - utility class, can't construct
- */
- private SizeUtility() {
- }
-
- /**
- * This method calculates the sizes of the tuples based on their list sizes
- * Since we know the type and length, it jsut calculates size from that
- * information, instead of walking whole object tree.
- * @param types - Data type for the each column
- * @param data - data
- * @return size in total
- */
- public static long getBatchSize(String[] types, List[] data) {
-
- // If the type information is not available then, use the long route
- // and calculate each and every size.
- if (types == null) {
- return getSize(data);
- }
-
- int colLength = types.length;
- int rowLength = data.length;
-
- // Array overhead for row array
- long size = 16 + alignMemory(rowLength * REFERENCE_SIZE);
- // array overhead for all the columns ( 8 object overhead + 4 ref + 4 int)
- size += (rowLength * (48 + alignMemory(colLength * REFERENCE_SIZE)));
- for (int col = 0; col < colLength; col++) {
- Class type = DataTypeManager.getDataTypeClass(types[col]);
-
- if (type == DataTypeManager.DefaultDataClasses.CHAR
- || type == DataTypeManager.DefaultDataClasses.BOOLEAN
- || type == DataTypeManager.DefaultDataClasses.BYTE
- || type == DataTypeManager.DefaultDataClasses.SHORT
- || type == DataTypeManager.DefaultDataClasses.INTEGER
- || type == DataTypeManager.DefaultDataClasses.LONG
- || type == DataTypeManager.DefaultDataClasses.FLOAT
- || type == DataTypeManager.DefaultDataClasses.DOUBLE) {
- size += (20*rowLength);
- } else if (type == DataTypeManager.DefaultDataClasses.DATE
- || type == DataTypeManager.DefaultDataClasses.TIME
- || type == DataTypeManager.DefaultDataClasses.TIMESTAMP) {
- // Even though Timestamp contains an extra int, these are
- // the same size because of rounding
- size += (32*rowLength);
- } else if (type == DataTypeManager.DefaultDataClasses.NULL) {
- //do nothing
- }
- else {
- for (int row = 0; row < rowLength; row+=SAMPLE_RATE) {
- size += getSize(data[row].get(col)) * Math.min(rowLength - row, SAMPLE_RATE);
- }
- }
- }
- return size;
- }
-
- /**
- * Get size of object
- * @return Size in bytes
- */
- public static long getSize(Object obj) {
- if(obj == null) {
- return 0;
- }
-
- Class type = obj.getClass();
- // Defect 14530 - this code represents the size of the object in the heap,
- // and not necessarily the number of bytes serialized.
- if (type == DataTypeManager.DefaultDataClasses.CHAR ||
- type == DataTypeManager.DefaultDataClasses.BOOLEAN ||
- type == DataTypeManager.DefaultDataClasses.BYTE ||
- type == DataTypeManager.DefaultDataClasses.SHORT ||
- type == DataTypeManager.DefaultDataClasses.INTEGER ||
- type == DataTypeManager.DefaultDataClasses.LONG ||
- type == DataTypeManager.DefaultDataClasses.FLOAT ||
- type == DataTypeManager.DefaultDataClasses.DOUBLE) {
- return 20;
- } else if (type == DataTypeManager.DefaultDataClasses.DATE ||
- type == DataTypeManager.DefaultDataClasses.TIME ||
- type == DataTypeManager.DefaultDataClasses.TIMESTAMP) { // Even though Timestamp contains an extra int, these are the same size because of rounding
- return 32;
- } else if(type == DataTypeManager.DefaultDataClasses.STRING) {
- int length = ((String)obj).length();
- if (length > 0) {
- return alignMemory(40 + (2 * length));
- }
- return 40;
- } else if(obj instanceof List) {
- int total = 16; // Object overhead + 4(int modcount), rounded
- if (obj instanceof LinkedList) {
- total += 8; // 4 (Entry ref) + 4 (int size)
- for (Iterator i = ((List)obj).iterator(); i.hasNext();) {
- total+= 16 + SizeUtility.getSize(i.next()); // rounded(4 (Object ref + 4 (next Entry ref) + 4 (previous Entry ref)) + actual object
- }
- } else if (obj instanceof ArrayList) {
- List arrayList = (ArrayList)obj;
- int arraySize = arrayList.size();
- int maxEnsuredSize = (arraySize < 10) ? 10 : arraySize * 3 / 2; // assuming default size
- total += 24 /*4(char[] ref) + 4(int) + 16 (array overhead)*/
- +alignMemory(maxEnsuredSize * REFERENCE_SIZE); /*number of references held in the array*/ if (arraySize > 0) {
- for (int i = 0; i < arraySize; i++) {
- total += SizeUtility.getSize(arrayList.get(i));
- }
- }
- } else if (obj instanceof Vector) {
- List arrayList = (List)obj;
- int arraySize = arrayList.size();
- int maxEnsuredSize = (arraySize < 10) ? 10 : arraySize * 2; // assuming default size, default capacity growth
- total += 16 // 4(Object[] ref) + 4 (int capacity) + 4 (int capIncrement), rounded
- + alignMemory(maxEnsuredSize * REFERENCE_SIZE); // Array overhead
- } else { // Assume java.util.Arrays.ArrayList
- List list = (List)obj;
- int arraySize = list.size();
- total += 16 + alignMemory(arraySize * REFERENCE_SIZE); // For the Object[]
- if (arraySize > 0) {
- for (int i = 0; i < arraySize; i++) {
- total += SizeUtility.getSize(list.get(i));
- }
- }
- }
- return total;
- } else if(type == DataTypeManager.DefaultDataClasses.BIG_DECIMAL) {
- int bitLength = ((BigDecimal)obj).unscaledValue().bitLength();
- return // 8 // Object overhead
- // + 4 // BigInteger reference
- // + 4 // int scale
- // Not aligning the above since they are already a multiple of 8
- // The remaining should be the same as BigInteger
- // + 8 // object overhead
- // + 24 // 6 int fields
- // + 4 // int[] ref
- // + 16 + alignMemory(4 * ((bitLength/32) + 1)) // int[] overhead
- 72 + alignMemory(4 + (bitLength >> 3)); // Same as above calculation
- } else if(type == DataTypeManager.DefaultDataClasses.BIG_INTEGER) {
- int bitLength = ((BigInteger)obj).bitLength();
- return // + 8 // object overhead
- // + 24 // 6 int fields
- // + 4 // int[] ref
- // + 16 + alignMemory(4 * ((bitLength/32) + 1)) // int[] overhead. BigInteger represents all values in all bases as a concatenation of chunks of 32-bits.
- 56 + alignMemory(4 + (bitLength >> 3)); // Same as above calculation
- } else if(type.isArray() && !type.getComponentType().isPrimitive()) {
- Object[] rows = (Object[]) obj;
- long total = 16 + alignMemory(rows.length * REFERENCE_SIZE); // Array overhead
- for(int i=0; i<rows.length; i++) {
- total += SizeUtility.getSize(rows[i]);
- }
- return total;
- } else {
- // Unknown object
- // 8 - for object overhead
- return 8+alignMemory(4+getSerializedSize(obj)); // Assume some max value for unknown objects.
- }
- }
-
- /**
- * Most current VMs have memory alignment that places objects into heap space that is a multiple of 8 Bytes.
- * This utility method helps with calculating the aligned size of an object.
- * @param numBytes
- * @return
- * @since 4.2
- */
- private static long alignMemory(long numBytes) {
- long remainder = numBytes % 8;
- if (remainder != 0) {
- numBytes += (8 - remainder);
- }
- return numBytes;
- }
-
- private static long getSerializedSize(Object anObj) {
- Assertion.assertTrue(anObj instanceof Serializable);
- try {
- ByteArrayOutputStream bout = new ByteArrayOutputStream(DEFAULT_BUFFER_SIZE);
- ObjectOutputStream out = new ObjectOutputStream(bout);
- out.writeObject(anObj);
- out.flush();
- long size = bout.size();
- out.close();
- bout.close();
- return size;
- } catch (IOException e) {
- // just return exception, this should not happen
- throw new MetaMatrixRuntimeException(e);
- }
- }
-}
\ No newline at end of file
Deleted: trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleGroupInfo.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleGroupInfo.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleGroupInfo.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -1,68 +0,0 @@
-/*
- * JBoss, Home of Professional Open Source.
- * See the COPYRIGHT.txt file distributed with this work for information
- * regarding copyright ownership. Some portions may be licensed
- * to Red Hat, Inc. under one or more contributor license agreements.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301 USA.
- */
-
-package com.metamatrix.common.buffer.impl;
-
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Set;
-
-import com.metamatrix.common.buffer.TupleSourceID;
-
-
-
-/**
- * Represents a logical grouping of tuple sources managed by the buffer manager.
- * Tuple sources are typically grouped by session, and the groupName is typically a sessionID/connectionID.
- * @since 4.3
- */
-class TupleGroupInfo {
-
- private String groupName;
- /** The bytes of memory used by this tuple group*/
- private volatile long memoryUsed;
- private Set<TupleSourceID> tupleSourceIDs = Collections.synchronizedSet(new HashSet<TupleSourceID>());
-
- TupleGroupInfo(String groupName) {
- this.groupName = groupName;
- }
-
- public Set<TupleSourceID> getTupleSourceIDs() {
- return tupleSourceIDs;
- }
-
- String getGroupName() {
- return groupName;
- }
-
- long getGroupMemoryUsed() {
- return memoryUsed;
- }
-
- long reserveMemory(long bytes) {
- return memoryUsed += bytes;
- }
-
- long releaseMemory(long bytes) {
- return memoryUsed -= bytes;
- }
-}
Deleted: trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleSourceImpl.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleSourceImpl.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleSourceImpl.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -1,188 +0,0 @@
-/*
- * JBoss, Home of Professional Open Source.
- * See the COPYRIGHT.txt file distributed with this work for information
- * regarding copyright ownership. Some portions may be licensed
- * to Red Hat, Inc. under one or more contributor license agreements.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301 USA.
- */
-
-package com.metamatrix.common.buffer.impl;
-
-import java.lang.ref.WeakReference;
-import java.util.List;
-
-import com.metamatrix.api.exception.ComponentNotFoundException;
-import com.metamatrix.api.exception.MetaMatrixComponentException;
-import com.metamatrix.common.buffer.BlockedException;
-import com.metamatrix.common.buffer.BlockedOnMemoryException;
-import com.metamatrix.common.buffer.IndexedTupleSource;
-import com.metamatrix.common.buffer.MemoryNotAvailableException;
-import com.metamatrix.common.buffer.TupleBatch;
-import com.metamatrix.common.buffer.TupleSourceID;
-import com.metamatrix.common.buffer.TupleSourceNotFoundException;
-import com.metamatrix.common.buffer.BufferManager.TupleSourceStatus;
-
-class TupleSourceImpl implements IndexedTupleSource {
- private final BufferManagerImpl bufferManagerImpl;
- private TupleSourceID tupleSourceID;
- private List<?> schema;
- private WeakReference<TupleBatch> currentBatch;
- private int currentRow = 1;
- private int mark = 1;
- private List<?> currentTuple;
-
- TupleSourceImpl(BufferManagerImpl bufferManagerImpl, TupleSourceID tupleSourceID, List schema){
- this.bufferManagerImpl = bufferManagerImpl;
- this.tupleSourceID = tupleSourceID;
- this.schema = schema;
- }
-
- @Override
- public int getCurrentIndex() {
- return this.currentRow;
- }
-
- @Override
- public List getSchema(){
- return this.schema;
- }
-
- @Override
- public List<?> nextTuple()
- throws MetaMatrixComponentException{
- List<?> result = null;
- if (currentTuple != null){
- result = currentTuple;
- currentTuple = null;
- } else {
- result = getCurrentTuple();
- }
- if (result != null) {
- currentRow++;
- }
- return result;
- }
-
- private List getCurrentTuple() throws MetaMatrixComponentException,
- BlockedException, ComponentNotFoundException {
- TupleBatch batch = getBatch();
- if(batch.getRowCount() == 0) {
- // Check if last
- try {
- TupleSourceStatus status = this.bufferManagerImpl.getStatus(this.tupleSourceID);
- if(status == TupleSourceStatus.FULL) {
- unpinCurrentBatch();
- return null;
- }
- throw BlockedException.INSTANCE;
- } catch(TupleSourceNotFoundException e) {
- throw new ComponentNotFoundException(e, e.getMessage());
- }
- }
-
- return batch.getTuple(currentRow);
- }
-
- @Override
- public void closeSource()
- throws MetaMatrixComponentException{
- // Reset to same state as newly-instantiated TupleSourceImpl
- unpinCurrentBatch();
- mark = 1;
- reset();
- //TODO: this is not quite correct wrt the javadoc, close does
- //not need to ensure that we are back at the beginning
- }
-
- private TupleBatch getCurrentBatch() {
- if (currentBatch != null) {
- return currentBatch.get();
- }
- return null;
- }
-
- private void unpinCurrentBatch()
- throws MetaMatrixComponentException {
- TupleBatch batch = getCurrentBatch();
- if(batch != null ) {
- try {
- this.bufferManagerImpl.unpinTupleBatch(this.tupleSourceID, batch.getBeginRow());
- } catch (TupleSourceNotFoundException e) {
- throw new MetaMatrixComponentException(e);
- } finally {
- currentBatch = null;
- }
- }
- }
-
- // Retrieves the necessary batch based on the currentRow
- public TupleBatch getBatch()
- throws MetaMatrixComponentException{
- TupleBatch batch = getCurrentBatch();
- if (batch != null) {
- if (currentRow <= batch.getEndRow() && currentRow >= batch.getBeginRow()) {
- return batch;
- }
- unpinCurrentBatch();
- }
-
- try{
- batch = this.bufferManagerImpl.pinTupleBatch(this.tupleSourceID, currentRow);
- currentBatch = new WeakReference<TupleBatch>(batch);
- } catch (MemoryNotAvailableException e) {
- /* Defect 18499 - ProcessWorker doesn't know how to handle MemoryNotAvailableException properly,
- * and this should always be converted to a BlockedOnMemoryException during processing so that
- * the work can be requeued.
- */
- throw BlockedOnMemoryException.INSTANCE;
- } catch(MetaMatrixComponentException e) {
- throw e;
- } catch(TupleSourceNotFoundException e){
- throw new MetaMatrixComponentException(e);
- }
- return batch;
- }
-
- @Override
- public boolean hasNext() throws MetaMatrixComponentException {
- if (this.currentTuple != null) {
- return true;
- }
-
- this.currentTuple = getCurrentTuple();
- return this.currentTuple != null;
- }
-
- @Override
- public void reset() {
- this.setPosition(mark);
- this.mark = 1;
- }
-
- @Override
- public void mark() {
- this.mark = currentRow;
- }
-
- @Override
- public void setPosition(int position) {
- if (this.currentRow != position) {
- this.currentRow = position;
- this.currentTuple = null;
- }
- }
-}
\ No newline at end of file
Deleted: trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleSourceInfo.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleSourceInfo.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/TupleSourceInfo.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -1,259 +0,0 @@
-/*
- * JBoss, Home of Professional Open Source.
- * See the COPYRIGHT.txt file distributed with this work for information
- * regarding copyright ownership. Some portions may be licensed
- * to Red Hat, Inc. under one or more contributor license agreements.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301 USA.
- */
-
-package com.metamatrix.common.buffer.impl;
-
-import java.lang.ref.PhantomReference;
-import java.lang.ref.ReferenceQueue;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-import java.util.concurrent.atomic.AtomicLong;
-
-import com.metamatrix.common.buffer.TupleSourceID;
-import com.metamatrix.common.buffer.BufferManager.TupleSourceStatus;
-import com.metamatrix.common.buffer.BufferManager.TupleSourceType;
-import com.metamatrix.common.types.DataTypeManager;
-import com.metamatrix.common.types.Streamable;
-
-/**
- * Describe a TupleSource and all important information about it.
- */
-public class TupleSourceInfo {
-
- private static final AtomicLong LOB_ID = new AtomicLong();
- private static final ReferenceQueue<Streamable<?>> LOB_QUEUE = new ReferenceQueue<Streamable<?>>();
-
- private static class LobReference extends PhantomReference<Streamable<?>> {
-
- String persistentStreamId;
-
- public LobReference(Streamable<?> lob) {
- super(lob, LOB_QUEUE);
- this.persistentStreamId = lob.getPersistenceStreamId();
- }
- }
-
- private TupleSourceType type; // Type of TupleSource, as defined in BufferManager constants
- private TupleSourceID tsID;
- private List schema;
- private String[] types;
- private int rowCount;
- private TupleSourceStatus status = TupleSourceStatus.ACTIVE;
- private TupleGroupInfo groupInfo;
- private boolean removed = false;
- private TreeMap<Integer, ManagedBatch> batches = new TreeMap<Integer, ManagedBatch>();
- private Map<String, Streamable<?>> lobReferences; //references to contained lobs
- private boolean lobs;
-
- @SuppressWarnings("unused")
- private LobReference containingLobReference; //reference to containing lob
-
- /**
- * Construct a TupleSourceInfo given information about it.
- * @param tsID Identifier
- * @param schema Schema describing tuple source
- * @param groupInfo The tuple gorup to which this tuple source belongs
- * @param type Type of tuple source, as defined in BufferManager constants
- */
- public TupleSourceInfo(TupleSourceID tsID, List schema, String[] types, TupleGroupInfo groupInfo, TupleSourceType type) {
- this.tsID = tsID;
- this.schema = schema;
- this.types = types;
- this.groupInfo = groupInfo;
- this.type = type;
- this.lobs = checkForLobs();
- }
-
- public void setContainingLobReference(Streamable<?> s) {
- this.containingLobReference = new LobReference(s);
- }
-
- public void addLobReference(Streamable<Object> lob) {
- String id = lob.getReferenceStreamId();
- if (id == null) {
- id = String.valueOf(LOB_ID.getAndIncrement());
- lob.setReferenceStreamId(id);
- }
- if (this.lobReferences == null) {
- this.lobReferences = Collections.synchronizedMap(new HashMap<String, Streamable<?>>());
- }
- this.lobReferences.put(id, lob);
- }
-
- public static String getStaleLobTupleSource() {
- LobReference ref = (LobReference)LOB_QUEUE.poll();
- if (ref == null) {
- return null;
- }
- return ref.persistentStreamId;
- }
-
- public Streamable<?> getLobReference(String id) {
- if (this.lobReferences == null) {
- return null;
- }
- return this.lobReferences.get(id);
- }
-
- public void addBatch(ManagedBatch batch) {
- batches.put(batch.getBeginRow(), batch);
- }
-
- /**
- * Returns the batch containing the begin row or null
- * if it doesn't exist
- * @param beginRow
- * @return
- */
- public ManagedBatch getBatch(int beginRow) {
- Map.Entry<Integer, ManagedBatch> entry = batches.floorEntry(beginRow);
- if (entry != null && entry.getValue().getEndRow() >= beginRow) {
- return entry.getValue();
- }
- return null;
- }
-
- public void removeBatch(int beginRow) {
- batches.remove(new Integer(beginRow));
- }
-
- public Iterator<ManagedBatch> getBatchIterator() {
- return batches.values().iterator();
- }
-
- /**
- * Get the tuple source identifier
- * @return Tuple source identifier
- */
- public TupleSourceID getTupleSourceID() {
- return this.tsID;
- }
-
- /**
- * Get the tuple schema describing a tuple source
- * @return Schema
- */
- public List getTupleSchema() {
- return this.schema;
- }
-
- /**
- * Get group this tuple source is in. Never null
- * @return TupleGroupInfo instance representing the tuple group to which this tuple source belongs.
- */
- public TupleGroupInfo getGroupInfo() {
- return this.groupInfo;
- }
-
- void setGroupInfo(TupleGroupInfo info) {
- this.groupInfo = info;
- }
-
- /**
- * Get current row count
- * @return Row count
- */
- public int getRowCount() {
- return this.rowCount;
- }
-
- /**
- * Set the current row count
- * @param rows New row count
- */
- public void setRowCount(int rows) {
- this.rowCount = rows;
- }
-
- /**
- * Get status of this tuple source, as defined in BufferManager constants
- * @return Status
- */
- public TupleSourceStatus getStatus() {
- return this.status;
- }
-
- /**
- * Set status of this tuple source
- * @param status New status
- */
- public void setStatus(TupleSourceStatus status) {
- this.status = status;
- }
-
- /**
- * Get type of this tuple source
- * @return Type of tuple source
- */
- public TupleSourceType getType() {
- return this.type;
- }
-
- /**
- * Check whether this tuple source has been removed. Due to
- * synchronization, this tuple source may have been removed
- * between the time it was retrieved and now.
- */
- public boolean isRemoved() {
- return this.removed;
- }
-
- /**
- * Set a flag that this tuple source has been removed
- */
- public void setRemoved() {
- this.removed = true;
- }
-
- public boolean lobsInSource() {
- return this.lobs;
- }
-
- public String[] getTypes() {
- return types;
- }
-
- /**
- * Get string representation
- * @return String representation
- */
- public String toString() {
- return "TupleSourceInfo[" + this.tsID + "]"; //$NON-NLS-1$ //$NON-NLS-2$
- }
-
- private boolean checkForLobs() {
- if (types == null) {
- // assume the worst
- return true;
- }
- for (int i = 0; i < types.length; i++) {
- if (DataTypeManager.isLOB(types[i]) || types[i] == DataTypeManager.DefaultDataTypes.OBJECT) {
- return true;
- }
- }
- return false;
- }
-}
Deleted: trunk/engine/src/main/java/com/metamatrix/common/buffer/storage/file/FileStorageManager.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/common/buffer/storage/file/FileStorageManager.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/common/buffer/storage/file/FileStorageManager.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -1,429 +0,0 @@
-/*
- * JBoss, Home of Professional Open Source.
- * See the COPYRIGHT.txt file distributed with this work for information
- * regarding copyright ownership. Some portions may be licensed
- * to Red Hat, Inc. under one or more contributor license agreements.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301 USA.
- */
-
-package com.metamatrix.common.buffer.storage.file;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import java.io.RandomAccessFile;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-import com.metamatrix.api.exception.MetaMatrixComponentException;
-import com.metamatrix.common.buffer.BufferManagerPropertyNames;
-import com.metamatrix.common.buffer.StorageManager;
-import com.metamatrix.common.buffer.TupleBatch;
-import com.metamatrix.common.buffer.TupleSourceID;
-import com.metamatrix.common.buffer.TupleSourceNotFoundException;
-import com.metamatrix.common.log.LogManager;
-import com.metamatrix.common.util.PropertiesUtils;
-import com.metamatrix.core.log.MessageLevel;
-import com.metamatrix.core.util.Assertion;
-import com.metamatrix.dqp.util.LogConstants;
-import com.metamatrix.query.execution.QueryExecPlugin;
-/**
- * This class stores batches in files on disk in a specified directory. Every
- * tuple source gets it's own set of files, named b_<i>id</i>_<i>num</i> where <i>id</i> is the
- * unique id value of the TupleSourceID, and <i>num</i> is a file counter for the files
- * associated with the TupleSourceID. Batches are stored random access into the
- * file, typically but not necessarily in order. An in memory data structure stores
- * info about all open files and where the batches are in the file (file pointers and
- * lengths).
- */
-public class FileStorageManager implements StorageManager {
-
- private static final String FILE_PREFIX = "b_"; //$NON-NLS-1$
-
- // Initialization
- private int maxOpenFiles = 10;
- private long maxFileSize = 2L * 1024L * 1024L * 1024L; // 2GB
- private String directory;
- private File dirFile;
-
- // State
- private Map<TupleSourceID, TupleSourceInfo> tupleSourceMap = new HashMap<TupleSourceID, TupleSourceInfo>();
- private Map<File, RandomAccessFile> fileCache = Collections.synchronizedMap(new LinkedHashMap<File, RandomAccessFile>() {
- @Override
- protected boolean removeEldestEntry(
- java.util.Map.Entry<File, RandomAccessFile> eldest) {
- if (this.size() > maxOpenFiles) {
- try {
- eldest.getValue().close();
- } catch (IOException e) {
- }
- return true;
- }
- return false;
- }
- });
-
- /**
- * Initialize with properties
- * @param props Initialization properties
- * @see com.metamatrix.common.buffer.BufferManagerPropertyNames#BUFFER_STORAGE_DIRECTORY
- * @see com.metamatrix.common.buffer.BufferManagerPropertyNames#MAX_OPEN_FILES
- * @see com.metamatrix.common.buffer.BufferManagerPropertyNames#MAX_FILE_SIZE
- */
- public void initialize(Properties props) throws MetaMatrixComponentException {
- this.directory = props.getProperty(BufferManagerPropertyNames.BUFFER_STORAGE_DIRECTORY);
- if(this.directory == null) {
- throw new MetaMatrixComponentException(QueryExecPlugin.Util.getString("FileStoreageManager.no_directory")); //$NON-NLS-1$
- }
-
- dirFile = new File(this.directory);
- if(dirFile.exists()) {
- if(! dirFile.isDirectory()) {
- throw new MetaMatrixComponentException(QueryExecPlugin.Util.getString("FileStoreageManager.not_a_directory", dirFile.getAbsoluteFile())); //$NON-NLS-1$
-
- }
- } else if(! dirFile.mkdirs()) {
- throw new MetaMatrixComponentException(QueryExecPlugin.Util.getString("FileStoreageManager.error_creating", dirFile.getAbsoluteFile())); //$NON-NLS-1$
- }
-
- // Set up max number of open file descriptors
- maxOpenFiles = PropertiesUtils.getIntProperty(props, BufferManagerPropertyNames.MAX_OPEN_FILES, 10);
-
- // Set the max file size
- maxFileSize = PropertiesUtils.getIntProperty(props, BufferManagerPropertyNames.MAX_FILE_SIZE, 2048) * 1024L * 1024L; // Multiply by 1MB
- }
-
- /**
- * Look up tuple source info and possibly create. First the file map is used to find an
- * existing file info. If the info is found it is returned. If not, then
- * a TupleSourceInfo is created according to shouldCreate flag
- * @param sourceID Source identifier
- * @param shouldCreate true if this method should create info related to the tuple source ID if it does not exist.
- * @return All the tuple source info or null if shouldCreate == false and no tuple source info was found
- */
- private TupleSourceInfo getTupleSourceInfo(TupleSourceID sourceID, boolean shouldCreate) {
-
- // Try to find in cache
- synchronized(tupleSourceMap) {
- TupleSourceInfo info = tupleSourceMap.get(sourceID);
- if(info == null && shouldCreate) {
- info = new TupleSourceInfo();
- tupleSourceMap.put(sourceID, info);
- }
- return info;
- }
- }
-
- /**
- * Creates a new file to the specified TupleSourceID
- * @param sourceID The TupleSourceID
- * @param fileNumber a number uniquely identifying this file within the set of tuple source files.
- * @return the newly created file with the name b_<i>sourceID</i>_<i>fileNumber</i>
- * @throws MetaMatrixComponentException if a file with this name already exists in the directory, or if it cannot be created
- * @since 4.2
- */
- private File createFile(TupleSourceID sourceID, int fileNumber) throws MetaMatrixComponentException {
- File storageFile = new File(this.directory, FILE_PREFIX + sourceID.getIDValue() + "_" + fileNumber); //$NON-NLS-1$
- if (LogManager.isMessageToBeRecorded(LogConstants.CTX_STORAGE_MGR, MessageLevel.DETAIL)) {
- LogManager.logDetail(LogConstants.CTX_STORAGE_MGR, "Creating temporary storage area file " + storageFile.getAbsoluteFile()); //$NON-NLS-1$
- }
- try {
- boolean created = storageFile.createNewFile();
- if(!created) {
- throw new MetaMatrixComponentException(QueryExecPlugin.Util.getString("FileStoreageManager.file_exists", storageFile.getAbsoluteFile())); //$NON-NLS-1$
- }
- } catch(IOException e) {
- throw new MetaMatrixComponentException(e, QueryExecPlugin.Util.getString("FileStoreageManager.error_creating", storageFile.getAbsoluteFile())); //$NON-NLS-1$
- }
- return storageFile;
- }
-
- /**
- * Add a batch to the storage manager. Because we don't implement remove, it's
- * possible that this batch already exists, in which case nothing is done.
- * @param sourceID Source identifier
- * @param batch Batch to add
- */
- public void addBatch(TupleSourceID sourceID, TupleBatch batch, String[] types)
- throws MetaMatrixComponentException {
-
- // Defect 13342 - addBatch method now creates spill files if the total bytes exceeds the max file size limit
- TupleSourceInfo tsInfo = getTupleSourceInfo(sourceID, true);
- synchronized (tsInfo) {
- if (tsInfo.isRemoved) {
- return;
- }
- Integer batchKey = new Integer(batch.getBeginRow());
- if (tsInfo.tupleBatchPointers.containsKey(batchKey)) {
- return;
- }
- byte[] bytes = convertToBytes(batch, types);
- if (bytes.length > maxFileSize) {
- LogManager.logWarning(LogConstants.CTX_STORAGE_MGR, "Detected an attempt to save a batch (" + sourceID + ", begin=" + batch.getBeginRow()+ ", size=" + bytes.length + ") larger than the buffer max file size setting of " + maxFileSize + " bytes. The buffer manager will ignore the max file size setting for this batch, and create a buffer file dedicated to this batch. It may be necessary to reduce the buffer batch setting or increase the buffer max file size setting."); //$NON-NLS-1$//$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ //$NON-NLS-5$
- }
- FileInfo fileInfo = tsInfo.getMostRecentlyCreatedFile();
- if (fileInfo == null ||
- (/*fileInfo.file.length() != 0 && */fileInfo.file.length() + bytes.length > maxFileSize)) {
- // Create and add
- fileInfo = new FileInfo(createFile(sourceID, tsInfo.storageFiles.size()));
- tsInfo.storageFiles.add(fileInfo);
- }
- long pointer = 0;
-
- if (LogManager.isMessageToBeRecorded(LogConstants.CTX_STORAGE_MGR, MessageLevel.DETAIL)) {
- LogManager.logDetail(LogConstants.CTX_STORAGE_MGR, "Adding batch to storage area file " + fileInfo.file.getAbsoluteFile() + " [ sourceID: " + sourceID + "batch: " + batch + " ]"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
- }
- try {
- // Get access to the file and remember whether we had to open it or not
- fileInfo.open();
- RandomAccessFile fileAccess = fileInfo.getAccess();
-
- // Store the batch in the file
- pointer = fileAccess.length();
- fileAccess.setLength(pointer + bytes.length);
- fileAccess.seek(pointer);
- fileAccess.write(bytes);
- } catch(IOException e) {
- throw new MetaMatrixComponentException(e, QueryExecPlugin.Util.getString("FileStoreageManager.error_reading", fileInfo.file.getAbsoluteFile())); //$NON-NLS-1$
- } finally {
- fileInfo.close();
- }
-
- // Update the pointers
- tsInfo.tupleBatchPointers.put(batchKey, new PointerInfo(fileInfo, pointer, bytes.length));
- }
- }
-
- /**
- * Convert from an object to a byte array
- * @param object Object to convert
- * @return Byte array
- */
- private byte[] convertToBytes(TupleBatch batch, String[] types) throws MetaMatrixComponentException {
- ObjectOutputStream oos = null;
- try {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- oos = new ObjectOutputStream(baos);
-
- batch.setDataTypes(types);
- batch.writeExternal(oos);
- oos.flush();
- return baos.toByteArray();
-
- } catch(IOException e) {
- throw new MetaMatrixComponentException(e, QueryExecPlugin.Util.getString("FileStorageManager.batch_error")); //$NON-NLS-1$
- } finally {
- if(oos != null) {
- try {
- oos.close();
- } catch(IOException e) {
- }
- }
- }
- }
-
- /**
- * Get a batch from the storage manager based on the beginRow.
- * @param sourceID Source identifier
- * @param beginRow Beginning row of batch to retrieve
- * @return Batch retrieved
- */
- public TupleBatch getBatch(TupleSourceID sourceID, int beginRow, String[] types)
- throws TupleSourceNotFoundException, MetaMatrixComponentException {
-
- TupleSourceInfo info = getTupleSourceInfo(sourceID, false);
- if(info == null) {
- throw new TupleSourceNotFoundException(QueryExecPlugin.Util.getString("BufferManagerImpl.tuple_source_not_found", sourceID)); //$NON-NLS-1$
- }
-
- byte[] bytes = null;
- synchronized(info) {
- if(info.isRemoved) {
- throw new TupleSourceNotFoundException(QueryExecPlugin.Util.getString("BufferManagerImpl.tuple_source_not_found", sourceID)); //$NON-NLS-1$
- }
- // Find pointer
- PointerInfo pointerInfo = info.tupleBatchPointers.get(new Integer(beginRow));
- Assertion.isNotNull(pointerInfo);
-
- FileInfo fileInfo = pointerInfo.fileInfo;
- // Get access to the file
- RandomAccessFile fileAccess = null;
- try {
- fileInfo.open();
- fileAccess = fileInfo.getAccess();
- fileAccess.seek(pointerInfo.pointer);
- bytes = new byte[pointerInfo.length];
- fileAccess.readFully(bytes);
- ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
- ObjectInputStream ois = new ObjectInputStream(bais);
-
- TupleBatch batch = new TupleBatch();
- batch.setDataTypes(types);
- batch.readExternal(ois);
- return batch;
- } catch(IOException e) {
- throw new MetaMatrixComponentException(e, QueryExecPlugin.Util.getString("FileStoreageManager.error_reading", fileInfo.file.getAbsoluteFile())); //$NON-NLS-1$
- } catch (ClassNotFoundException e) {
- throw new MetaMatrixComponentException(e, QueryExecPlugin.Util.getString("FileStoreageManager.error_reading", fileInfo.file.getAbsoluteFile())); //$NON-NLS-1$
- } finally {
- fileInfo.close();
- }
- }
- }
-
- /**
- * Remove all batches for a sourceID. Before removal, the file is closed.
- * @param sourceID Tuple source ID
- */
- public void removeBatches(TupleSourceID sourceID) throws MetaMatrixComponentException {
- TupleSourceInfo info = null;
- // Remove info from the file map
- synchronized(tupleSourceMap) {
- info = tupleSourceMap.remove(sourceID);
- }
-
- // Didn't find a file
- if(info == null) {
- return;
- }
-
- if (LogManager.isMessageToBeRecorded(LogConstants.CTX_STORAGE_MGR, MessageLevel.DETAIL)) {
- LogManager.logDetail(LogConstants.CTX_STORAGE_MGR, "Removing storage for " + sourceID); //$NON-NLS-1$
- }
-
- synchronized(info) {
- // If open, close the file and decrement the open file counter
- for (int i = 0; i < info.storageFiles.size(); i++) {
- FileInfo fileInfo = info.storageFiles.get(i);
- fileInfo.delete();
- }
- // Delete the file and mark info as being removed
- info.isRemoved = true;
- }
- }
-
- /**
- * This method removes all storage area files by walking through the file info
- * map and closing and removing each file.
- */
- public synchronized void shutdown() {
-
- LogManager.logDetail(LogConstants.CTX_STORAGE_MGR, "Removing all storage area files "); //$NON-NLS-1$
-
- List<TupleSourceID> ids = null;
- synchronized (tupleSourceMap) {
- ids = new LinkedList<TupleSourceID>(tupleSourceMap.keySet());
- }
-
- for (TupleSourceID key : ids) {
- try {
- removeBatches(key);
- } catch (MetaMatrixComponentException e) {
- LogManager.logWarning(LogConstants.CTX_STORAGE_MGR, e, "Shutdown failed while removing batches for tuple source: " + key); //$NON-NLS-1$
- }
- }
-
- tupleSourceMap = null;
- }
-
- public int getOpenFiles() {
- return this.fileCache.size();
- }
-
- private class FileInfo {
- private File file;
- private RandomAccessFile fileData; // may be null if not open
-
- public FileInfo(File file) {
- this.file = file;
- }
-
- public void open() throws FileNotFoundException {
- if(this.fileData == null) {
- this.fileData = fileCache.remove(this.file);
- if (this.fileData == null) {
- this.fileData = new RandomAccessFile(file, "rw"); //$NON-NLS-1$
- }
- }
- }
-
- public RandomAccessFile getAccess() {
- return this.fileData;
- }
-
- public void close() {
- if(this.fileData != null) {
- fileCache.put(file, this.fileData);
- this.fileData = null;
- }
- }
-
- public void delete() {
- if (this.fileData == null) {
- this.fileData = fileCache.remove(this.file);
- }
- if (this.fileData != null) {
- try {
- this.fileData.close();
- } catch(Exception e) {
- }
- this.fileData = null;
- }
- file.delete();
- }
-
- public String toString() {
- return "FileInfo<" + file.getName() + ", has fileData = " + (fileData != null) + ">"; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
- }
- }
-
- private static class PointerInfo {
- FileInfo fileInfo;
- public long pointer;
- public int length;
-
- public PointerInfo(FileInfo fileInfo, long pointer, int length) {
- this.fileInfo = fileInfo;
- this.pointer = pointer;
- this.length = length;
- }
- }
-
- private static class TupleSourceInfo {
- Map<Integer, PointerInfo> tupleBatchPointers = new HashMap<Integer, PointerInfo>();
- List<FileInfo> storageFiles = new ArrayList<FileInfo>(2); // Stores all the FileInfos for this tupleSource
- private boolean isRemoved = false;
-
- FileInfo getMostRecentlyCreatedFile() {
- if (storageFiles.isEmpty()) {
- return null;
- }
- return storageFiles.get(storageFiles.size() - 1);
- }
- }
-
-}
Copied: trunk/engine/src/main/java/com/metamatrix/dqp/internal/datamgr (from rev 1659, trunk/client/src/main/java/com/metamatrix/dqp/internal/datamgr)
Modified: trunk/engine/src/main/java/com/metamatrix/query/eval/Evaluator.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/eval/Evaluator.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/eval/Evaluator.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -22,13 +22,12 @@
package com.metamatrix.query.eval;
-import java.math.BigDecimal;
import java.sql.SQLException;
import java.util.Collections;
-import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
@@ -319,21 +318,13 @@
} else if (criteria instanceof DependentSetCriteria){
ContextReference ref = (ContextReference)criteria;
ValueIteratorSource vis = (ValueIteratorSource)getContext(criteria).getVariableContext().getGlobalValue(ref.getContextSymbol());
- HashSet<Object> values;
+ Set<Object> values;
try {
values = vis.getCachedSet(ref.getValueExpression());
} catch (MetaMatrixProcessingException e) {
throw new CriteriaEvaluationException(e, e.getMessage());
}
if (values != null) {
- if (leftValue instanceof BigDecimal) {
- for (Object object : values) {
- if (compareValues(leftValue, object) == 0) {
- return true;
- }
- }
- return false;
- }
return values.contains(leftValue);
}
//there are too many values to justify a linear search or holding
Modified: trunk/engine/src/main/java/com/metamatrix/query/optimizer/proc/ProcedurePlanner.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/optimizer/proc/ProcedurePlanner.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/optimizer/proc/ProcedurePlanner.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -41,9 +41,9 @@
import com.metamatrix.query.processor.proc.AssignmentInstruction;
import com.metamatrix.query.processor.proc.BreakInstruction;
import com.metamatrix.query.processor.proc.ContinueInstruction;
+import com.metamatrix.query.processor.proc.CreateCursorResultSetInstruction;
import com.metamatrix.query.processor.proc.ErrorInstruction;
import com.metamatrix.query.processor.proc.ExecDynamicSqlInstruction;
-import com.metamatrix.query.processor.proc.ExecSqlInstruction;
import com.metamatrix.query.processor.proc.IfInstruction;
import com.metamatrix.query.processor.proc.LoopInstruction;
import com.metamatrix.query.processor.proc.ProcedurePlan;
@@ -52,9 +52,7 @@
import com.metamatrix.query.processor.program.ProgramInstruction;
import com.metamatrix.query.sql.lang.Command;
import com.metamatrix.query.sql.lang.DynamicCommand;
-import com.metamatrix.query.sql.lang.Into;
import com.metamatrix.query.sql.lang.ProcedureContainer;
-import com.metamatrix.query.sql.lang.Query;
import com.metamatrix.query.sql.lang.TranslatableProcedureContainer;
import com.metamatrix.query.sql.proc.AssignmentStatement;
import com.metamatrix.query.sql.proc.Block;
@@ -65,7 +63,6 @@
import com.metamatrix.query.sql.proc.Statement;
import com.metamatrix.query.sql.proc.WhileStatement;
import com.metamatrix.query.sql.symbol.Expression;
-import com.metamatrix.query.sql.symbol.GroupSymbol;
import com.metamatrix.query.sql.visitor.CommandCollectorVisitor;
import com.metamatrix.query.util.CommandContext;
@@ -246,19 +243,12 @@
{
CommandStatement cmdStmt = (CommandStatement) statement;
Command command = cmdStmt.getCommand();
- GroupSymbol intoGroup = null;
- if(command instanceof Query){
- Into into = ((Query)command).getInto();
- if(into != null){
- intoGroup = into.getGroup();
- }
- }
ProcessorPlan commandPlan = cmdStmt.getCommand().getProcessorPlan();
if (command.getType() == Command.TYPE_DYNAMIC){
instruction = new ExecDynamicSqlInstruction(parentProcCommand,((DynamicCommand)command), metadata, idGenerator, capFinder );
}else{
- instruction = new ExecSqlInstruction(commandPlan, intoGroup);
+ instruction = new CreateCursorResultSetInstruction(CreateCursorResultSetInstruction.RS_NAME, commandPlan);
}
if(debug) {
Modified: trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/RelationalPlanner.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/RelationalPlanner.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/optimizer/relational/RelationalPlanner.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -470,7 +470,7 @@
if (c == null) {
c = QueryResolver.expandCommand(container, metadata, analysisRecord);
if (c != null) {
- Request.validateWithVisitor(new ValidationVisitor(), metadata, c, true);
+ Request.validateWithVisitor(new ValidationVisitor(), metadata, c);
metadata.addToMetadataCache(container.getGroup().getMetadataID(), cacheString, c.clone());
}
} else {
@@ -903,7 +903,7 @@
//parse, resolve, validate
result = convertToSubquery(qnode, metadata);
QueryResolver.resolveCommand(result, Collections.EMPTY_MAP, metadata, analysisRecord);
- Request.validateWithVisitor(new ValidationVisitor(), metadata, result, true);
+ Request.validateWithVisitor(new ValidationVisitor(), metadata, result);
metadata.addToMetadataCache(virtualGroup.getMetadataID(), "transformation/" + cacheString, result.clone()); //$NON-NLS-1$
}
return QueryRewriter.rewrite(result, metadata, context);
Copied: trunk/engine/src/main/java/com/metamatrix/query/processor/BatchCollector.java (from rev 1686, trunk/engine/src/main/java/com/metamatrix/query/processor/relational/BatchCollector.java)
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/processor/BatchCollector.java (rev 0)
+++ trunk/engine/src/main/java/com/metamatrix/query/processor/BatchCollector.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -0,0 +1,124 @@
+/*
+ * JBoss, Home of Professional Open Source.
+ * See the COPYRIGHT.txt file distributed with this work for information
+ * regarding copyright ownership. Some portions may be licensed
+ * to Red Hat, Inc. under one or more contributor license agreements.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301 USA.
+ */
+
+package com.metamatrix.query.processor;
+
+import java.util.List;
+
+import com.metamatrix.api.exception.MetaMatrixComponentException;
+import com.metamatrix.api.exception.MetaMatrixProcessingException;
+import com.metamatrix.common.buffer.BlockedException;
+import com.metamatrix.common.buffer.TupleBatch;
+import com.metamatrix.common.buffer.TupleBuffer;
+
+public class BatchCollector {
+
+ public interface BatchProducer {
+ /**
+ * Get a batch of results or possibly an Exception.
+ * @return Batch of results
+ * @throws BlockedException indicating next batch is not available yet
+ * @throws MetaMatrixComponentException for non-business rule exception
+ * @throws MetaMatrixProcessingException for business rule exception, related
+ * to user input or modeling
+ */
+ TupleBatch nextBatch() throws BlockedException, MetaMatrixComponentException, MetaMatrixProcessingException;
+
+ /**
+ * Get list of resolved elements describing output columns for this plan.
+ * @return List of SingleElementSymbol
+ */
+ List getOutputElements();
+ }
+
+ public interface BatchHandler {
+ void batchProduced(TupleBatch batch) throws MetaMatrixProcessingException, MetaMatrixComponentException;
+ }
+
+ private BatchProducer sourceNode;
+ private BatchHandler batchHandler;
+
+ private boolean done = false;
+ private TupleBuffer buffer;
+ private boolean collectedAny;
+
+ public BatchCollector(BatchProducer sourceNode, TupleBuffer buffer) {
+ this.sourceNode = sourceNode;
+ this.buffer = buffer;
+ }
+
+ public TupleBuffer collectTuples() throws MetaMatrixComponentException, MetaMatrixProcessingException {
+ TupleBatch batch = null;
+ while(!done) {
+ batch = sourceNode.nextBatch();
+
+ flushBatch(batch);
+
+ // Check for termination condition
+ if(batch.getTerminationFlag()) {
+ done = true;
+ buffer.close();
+ break;
+ }
+ }
+ return buffer;
+ }
+
+ public TupleBuffer getTupleBuffer() {
+ return buffer;
+ }
+
+ /**
+ * Flush the batch by giving it to the buffer manager.
+ */
+ private void flushBatch(TupleBatch batch) throws MetaMatrixComponentException, MetaMatrixProcessingException {
+ // Add batch
+ if(batch.getRowCount() > 0) {
+ for (List tuple : batch.getAllTuples()) {
+ buffer.addTuple(tuple);
+ }
+ collectedAny = true;
+ }
+ if (this.batchHandler != null && (batch.getRowCount() > 0 || batch.getTerminationFlag())) {
+ this.batchHandler.batchProduced(batch);
+ }
+ }
+
+ public void setBatchHandler(BatchHandler batchHandler) {
+ this.batchHandler = batchHandler;
+ }
+
+ public boolean collectedAny() {
+ boolean result = collectedAny;
+ collectedAny = false;
+ return result;
+ }
+
+ public int getRowCount() {
+ return buffer.getRowCount();
+ }
+
+ public boolean isDone() {
+ return done;
+ }
+
+}
Copied: trunk/engine/src/main/java/com/metamatrix/query/processor/BatchIterator.java (from rev 1686, trunk/engine/src/main/java/com/metamatrix/query/processor/relational/BatchIterator.java)
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/processor/BatchIterator.java (rev 0)
+++ trunk/engine/src/main/java/com/metamatrix/query/processor/BatchIterator.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -0,0 +1,119 @@
+/*
+ * JBoss, Home of Professional Open Source.
+ * See the COPYRIGHT.txt file distributed with this work for information
+ * regarding copyright ownership. Some portions may be licensed
+ * to Red Hat, Inc. under one or more contributor license agreements.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301 USA.
+ */
+
+package com.metamatrix.query.processor;
+
+import java.util.List;
+
+import com.metamatrix.api.exception.MetaMatrixComponentException;
+import com.metamatrix.api.exception.MetaMatrixProcessingException;
+import com.metamatrix.common.buffer.IndexedTupleSource;
+import com.metamatrix.common.buffer.TupleBatch;
+import com.metamatrix.query.processor.BatchCollector.BatchProducer;
+import com.metamatrix.query.sql.symbol.SingleElementSymbol;
+
+public class BatchIterator implements
+ IndexedTupleSource {
+
+ private final BatchProducer source;
+
+ public BatchIterator(BatchProducer source) {
+ this.source = source;
+ }
+
+ private boolean done;
+ private int currentRow = 1;
+ private TupleBatch currentBatch;
+ private List currentTuple;
+
+ public boolean hasNext() throws MetaMatrixComponentException,
+ MetaMatrixProcessingException {
+ if (done) {
+ return false;
+ }
+ while (currentTuple == null) {
+ if (currentBatch == null) {
+ currentBatch = this.source.nextBatch();
+ }
+
+ if (currentBatch.getEndRow() >= currentRow) {
+ this.currentTuple = currentBatch.getTuple(currentRow++);
+ } else {
+ done = currentBatch.getTerminationFlag();
+ currentBatch = null;
+ if (done) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public void closeSource() throws MetaMatrixComponentException {
+
+ }
+
+ @Override
+ public List<SingleElementSymbol> getSchema() {
+ return source.getOutputElements();
+ }
+
+ @Override
+ public List<?> nextTuple() throws MetaMatrixComponentException,
+ MetaMatrixProcessingException {
+ if (currentTuple == null && !hasNext()) {
+ return null;
+ }
+ List result = currentTuple;
+ currentTuple = null;
+ return result;
+ }
+
+ public void reset() {
+ throw new UnsupportedOperationException();
+ }
+
+ public void mark() {
+ //does nothing
+ }
+
+ @Override
+ public int getCurrentIndex() {
+ return currentRow;
+ }
+
+ public void setPosition(int position) {
+ if (position == this.currentRow) {
+ return;
+ }
+ if (position < this.currentRow && (this.currentBatch == null || position < this.currentBatch.getBeginRow())) {
+ throw new UnsupportedOperationException("Backwards positioning is not allowed"); //$NON-NLS-1$
+ }
+ this.currentRow = position;
+ this.currentTuple = null;
+ if (currentBatch.getEndRow() < currentRow) {
+ this.currentBatch = null;
+ }
+ }
+
+}
Added: trunk/engine/src/main/java/com/metamatrix/query/processor/CollectionTupleSource.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/processor/CollectionTupleSource.java (rev 0)
+++ trunk/engine/src/main/java/com/metamatrix/query/processor/CollectionTupleSource.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -0,0 +1,70 @@
+/*
+ * JBoss, Home of Professional Open Source.
+ * See the COPYRIGHT.txt file distributed with this work for information
+ * regarding copyright ownership. Some portions may be licensed
+ * to Red Hat, Inc. under one or more contributor license agreements.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301 USA.
+ */
+
+package com.metamatrix.query.processor;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+
+import com.metamatrix.common.buffer.TupleSource;
+import com.metamatrix.query.sql.lang.Command;
+import com.metamatrix.query.sql.symbol.SingleElementSymbol;
+
+public class CollectionTupleSource implements TupleSource {
+
+ private Iterator<? extends List<?>> tuples;
+ private List<SingleElementSymbol> schema;
+
+ public static CollectionTupleSource createUpdateCountTupleSource(int count) {
+ return new CollectionTupleSource(Arrays.asList(Arrays.asList(count)).iterator(), Command.getUpdateCommandSymbol());
+ }
+
+ public static CollectionTupleSource createNullTupleSource(List<SingleElementSymbol> schema) {
+ return new CollectionTupleSource(new ArrayList<List<Object>>(0).iterator(), schema);
+ }
+
+ public CollectionTupleSource(Iterator<? extends List<?>> tuples,
+ List<SingleElementSymbol> schema) {
+ this.tuples = tuples;
+ this.schema = schema;
+ }
+
+ @Override
+ public List<?> nextTuple() {
+ if (tuples.hasNext()) {
+ return tuples.next();
+ }
+ return null;
+ }
+
+ @Override
+ public List<SingleElementSymbol> getSchema() {
+ return schema;
+ }
+
+ @Override
+ public void closeSource() {
+
+ }
+}
\ No newline at end of file
Property changes on: trunk/engine/src/main/java/com/metamatrix/query/processor/CollectionTupleSource.java
___________________________________________________________________
Name: svn:mime-type
+ text/plain
Deleted: trunk/engine/src/main/java/com/metamatrix/query/processor/NullTupleSource.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/processor/NullTupleSource.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/processor/NullTupleSource.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -1,79 +0,0 @@
-/*
- * JBoss, Home of Professional Open Source.
- * See the COPYRIGHT.txt file distributed with this work for information
- * regarding copyright ownership. Some portions may be licensed
- * to Red Hat, Inc. under one or more contributor license agreements.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301 USA.
- */
-
-package com.metamatrix.query.processor;
-
-import java.util.List;
-
-import com.metamatrix.common.buffer.TupleSource;
-
-/**
- * <p>This class can be used to represent null tuples or no data for an
- * atomic query execution. Example usage is in implementation of partial
- * results feature where we have to return null tuples if there is an error
- * in the execution of atomic query.</p>
- */
-
-public class NullTupleSource implements TupleSource {
-
- private List elements; // elementSymbols containing null tuples
-
- /**
- * <p>Constructor that takes the list of elementSymbols
- * that have null rows.</p>
- * @param elements List of elementSymbols that this source contains.
- */
- public NullTupleSource(List elements) {
- this.elements = elements;
- }
-
- /**
- * @see TupleSource#getSchema()
- */
- public List getSchema() {
- return this.elements;
- }
-
- /**
- * <p>Open the tuple source to read tuples. This method does nothing.</p>
- */
- public void openSource() {
- // do nothing
- }
-
- /**
- * <p>This method returns a null indicating, that there are no
- * rows in the source.</p>
- * @return list containing the values of the next tuple.
- */
- public List nextTuple() {
- return null;
- }
-
- /**
- * <p>Close the tuple source after reading tuples. This method does nothing.</p>
- */
- public void closeSource() {
- // do nothing
- }
-
-} // END CLASS
Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/ProcessorPlan.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/processor/ProcessorPlan.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/processor/ProcessorPlan.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -28,6 +28,7 @@
import com.metamatrix.api.exception.MetaMatrixComponentException;
import com.metamatrix.api.exception.MetaMatrixProcessingException;
import com.metamatrix.common.buffer.*;
+import com.metamatrix.query.processor.BatchCollector.BatchProducer;
import com.metamatrix.query.util.CommandContext;
/**
@@ -43,7 +44,7 @@
* the call to {@link #close}.
* </p>
*/
-public interface ProcessorPlan extends Cloneable, Describable {
+public interface ProcessorPlan extends Cloneable, Describable, BatchProducer {
/**
* Initialize the plan with some required pieces of data for making
Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/QueryProcessor.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/processor/QueryProcessor.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/processor/QueryProcessor.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -30,43 +30,33 @@
import com.metamatrix.common.buffer.BlockedException;
import com.metamatrix.common.buffer.BufferManager;
import com.metamatrix.common.buffer.TupleBatch;
-import com.metamatrix.common.buffer.TupleSourceID;
-import com.metamatrix.common.buffer.TupleSourceNotFoundException;
-import com.metamatrix.common.buffer.BufferManager.TupleSourceStatus;
import com.metamatrix.common.buffer.BufferManager.TupleSourceType;
import com.metamatrix.common.log.LogManager;
-import com.metamatrix.core.MetaMatrixCoreException;
import com.metamatrix.core.log.MessageLevel;
+import com.metamatrix.core.util.Assertion;
import com.metamatrix.dqp.util.LogConstants;
import com.metamatrix.query.execution.QueryExecPlugin;
+import com.metamatrix.query.processor.BatchCollector.BatchProducer;
import com.metamatrix.query.util.CommandContext;
-/**
- * TODO: combine overlapping responsibilities with RequestWorkItem
- */
-public class QueryProcessor {
+public class QueryProcessor implements BatchProducer {
public interface ProcessorFactory {
QueryProcessor createQueryProcessor(String query, String recursionGroup, CommandContext commandContext) throws MetaMatrixProcessingException, MetaMatrixComponentException;
}
- public interface BatchHandler {
- void batchProduced(TupleBatch batch) throws MetaMatrixProcessingException, MetaMatrixComponentException;
- }
-
- private TupleSourceID resultsID;
private CommandContext context;
private ProcessorDataManager dataMgr;
- private BufferManager bufferMgr;
+ private BufferManager bufferMgr;
private ProcessorPlan processPlan;
private boolean initialized = false;
/** Flag that marks whether the request has been canceled. */
private volatile boolean requestCanceled = false;
- private volatile boolean requestClosed = false;
private static final int DEFAULT_WAIT = 50;
private boolean processorClosed = false;
- private BatchHandler batchHandler;
private volatile int highestRow;
+
+ private boolean nonBlocking = false;
/**
* Construct a processor with all necessary information to process.
@@ -79,12 +69,10 @@
public QueryProcessor(ProcessorPlan plan, CommandContext context, BufferManager bufferMgr, ProcessorDataManager dataMgr) throws MetaMatrixComponentException {
this.context = context;
this.dataMgr = dataMgr;
- this.bufferMgr = bufferMgr;
this.processPlan = plan;
-
+ this.bufferMgr = bufferMgr;
// Add data manager to all nodes in tree
this.processPlan.initialize(context, this.dataMgr, bufferMgr);
- this.resultsID = bufferMgr.createTupleSource(processPlan.getOutputElements(), context.getConnectionID(), TupleSourceType.PROCESSOR);
}
public CommandContext getContext() {
@@ -95,123 +83,98 @@
return this.context.getProcessorID();
}
- public TupleSourceID getResultsID() {
- return resultsID;
- }
-
public ProcessorPlan getProcessorPlan() {
return this.processPlan;
}
- private void initialize()
- throws BlockedException, MetaMatrixComponentException, MetaMatrixProcessingException {
-
- // initialize if necessary
- if(! initialized) {
- // Open the top node for reading
- processPlan.open();
-
- initialized = true;
- }
- }
-
- /**
- * Process fully blocking
- * @throws MetaMatrixCoreException
- */
- public void process()
+ public TupleBatch nextBatch()
throws BlockedException, MetaMatrixProcessingException, MetaMatrixComponentException {
-
- while(true) {
- try {
- if (process(Integer.MAX_VALUE)) {
- break;
- }
- } catch(BlockedException e) {
- try {
- Thread.sleep(DEFAULT_WAIT);
- } catch (InterruptedException err) {
- throw new MetaMatrixComponentException(err);
- }
- }
- }
+
+ while (true) {
+ try {
+ return nextBatchDirect();
+ } catch (BlockedException e) {
+ if (nonBlocking) {
+ try {
+ Thread.sleep(DEFAULT_WAIT);
+ } catch (InterruptedException err) {
+ throw new MetaMatrixComponentException(err);
+ }
+ continue;
+ }
+ throw e;
+ }
+ }
}
-
- /**
- * Process until time expires or a {@link BlockedException} occurs
- * @param time
- * @return
- */
- public boolean process(long time)
+
+ private TupleBatch nextBatchDirect()
throws BlockedException, MetaMatrixProcessingException, MetaMatrixComponentException {
boolean done = false;
+ TupleBatch result = null;
try {
- // Will only initialize the first time process() is called
- initialize();
+ // initialize if necessary
+ if(! initialized) {
+ // Open the top node for reading
+ processPlan.open();
- long start = System.currentTimeMillis();
-
- while(System.currentTimeMillis() - start < time && !requestClosed && !processorClosed) {
- checkState();
-
- TupleBatch batch = processPlan.nextBatch();
- flushBatch(batch);
-
- if(batch.getTerminationFlag()) {
- done = true;
- break;
- }
- }
- } catch (BlockedException e) {
- throw e;
- } catch (MetaMatrixException e) {
- try {
- closeProcessing();
- } catch (MetaMatrixException e1){
- LogManager.logDetail(LogConstants.CTX_DQP, e1, "Error closing processor"); //$NON-NLS-1$
- }
- if (e instanceof MetaMatrixProcessingException) {
- throw (MetaMatrixProcessingException)e;
- }
- if (e instanceof MetaMatrixComponentException) {
- throw (MetaMatrixComponentException)e;
- }
- throw new MetaMatrixComponentException(e);
- } finally {
- //iff this is the root command context release any pinned (Unclosed tuplesources)
- if (this.context.getParent() == null) {
- bufferMgr.releasePinnedBatches();
- }
- }
-
- if(done || requestClosed) {
+ initialized = true;
+ }
+
+ long currentTime = System.currentTimeMillis();
+ Assertion.assertTrue(!processorClosed);
+ while(currentTime < context.getTimeSliceEnd()) {
+ if (requestCanceled) {
+ throw new MetaMatrixProcessingException(QueryExecPlugin.Util.getString("QueryProcessor.request_cancelled", getProcessID())); //$NON-NLS-1$
+ }
+ if (currentTime > context.getTimeoutEnd()) {
+ throw new MetaMatrixProcessingException("Query timed out"); //$NON-NLS-1$
+ }
+ result = processPlan.nextBatch();
+ this.highestRow = result.getEndRow();
+
+ if(result.getTerminationFlag()) {
+ done = true;
+ break;
+ }
+
+ if (result.getRowCount() > 0) {
+ break;
+ }
+
+ }
+ } catch (BlockedException e) {
+ throw e;
+ } catch (MetaMatrixException e) {
+ try {
+ closeProcessing();
+ } catch (MetaMatrixException e1){
+ LogManager.logDetail(LogConstants.CTX_DQP, e1, "Error closing processor"); //$NON-NLS-1$
+ }
+ if (e instanceof MetaMatrixProcessingException) {
+ throw (MetaMatrixProcessingException)e;
+ }
+ if (e instanceof MetaMatrixComponentException) {
+ throw (MetaMatrixComponentException)e;
+ }
+ throw new MetaMatrixComponentException(e);
+ }
+ if(done) {
closeProcessing();
- return true;
}
- return processorClosed;
+ if (result == null) {
+ throw BlockedException.INSTANCE; //expired timeslice
+ }
+ return result;
}
-
+
+
/**
- * Flush the batch by giving it to the buffer manager.
- */
- private void flushBatch(TupleBatch batch) throws MetaMatrixComponentException, MetaMatrixProcessingException {
- if(batch.getRowCount() > 0) {
- this.bufferMgr.addTupleBatch(this.resultsID, batch);
- this.highestRow = batch.getEndRow();
- }
- if (this.batchHandler != null && (batch.getRowCount() > 0 || batch.getTerminationFlag())) {
- this.batchHandler.batchProduced(batch);
- }
- }
-
- /**
* Close processing and clean everything up. Should only be called by the same thread that called process.
* @throws MetaMatrixComponentException
- * @throws TupleSourceNotFoundException
*/
- public void closeProcessing() throws TupleSourceNotFoundException, MetaMatrixComponentException {
+ public void closeProcessing() throws MetaMatrixComponentException {
if (processorClosed) {
return;
}
@@ -220,23 +183,12 @@
}
processorClosed = true;
- try {
- processPlan.close();
- } finally {
- // Mark tuple source FULL
- this.bufferMgr.setStatus(this.resultsID, TupleSourceStatus.FULL);
- }
+ processPlan.close();
}
- /**
- * Checks whether the request has been canceled of an error has occurred, and throws an exception if either of these conditions is true.
- * @throws MetaMatrixException if the request has been canceled or an error occurred during processing.
- * @since 4.2
- */
- private void checkState() throws MetaMatrixException {
- if (requestCanceled) {
- throw new MetaMatrixProcessingException(QueryExecPlugin.Util.getString("QueryProcessor.request_cancelled", getProcessID())); //$NON-NLS-1$
- }
+ @Override
+ public List getOutputElements() {
+ return this.processPlan.getOutputElements();
}
public List<Exception> getAndClearWarnings() {
@@ -250,18 +202,15 @@
this.requestCanceled = true;
}
- /**
- * Asynch graceful shutdown of the QueryProcessor
- */
- public void requestClosed() {
- this.requestClosed = true;
- }
-
- public void setBatchHandler(BatchHandler batchHandler) {
- this.batchHandler = batchHandler;
- }
-
public int getHighestRow() {
return highestRow;
}
+
+ public BatchCollector createBatchCollector() throws MetaMatrixComponentException {
+ return new BatchCollector(this, this.bufferMgr.createTupleBuffer(this.processPlan.getOutputElements(), context.getConnectionID(), TupleSourceType.PROCESSOR));
+ }
+
+ public void setNonBlocking(boolean nonBlocking) {
+ this.nonBlocking = nonBlocking;
+ }
}
Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/dynamic/SqlEval.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/processor/dynamic/SqlEval.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/processor/dynamic/SqlEval.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -25,7 +25,6 @@
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -44,13 +43,11 @@
import com.metamatrix.api.exception.query.ExpressionEvaluationException;
import com.metamatrix.api.exception.query.QueryParserException;
import com.metamatrix.common.buffer.BlockedException;
-import com.metamatrix.common.buffer.BufferManager;
import com.metamatrix.common.buffer.TupleSource;
-import com.metamatrix.common.buffer.TupleSourceID;
-import com.metamatrix.common.buffer.TupleSourceNotFoundException;
import com.metamatrix.common.types.DataTypeManager;
import com.metamatrix.common.types.XMLType;
import com.metamatrix.query.eval.Evaluator;
+import com.metamatrix.query.processor.BatchIterator;
import com.metamatrix.query.processor.ProcessorDataManager;
import com.metamatrix.query.processor.QueryProcessor;
import com.metamatrix.query.sql.symbol.ElementSymbol;
@@ -69,9 +66,8 @@
private static final String NO_RESULTS_DOCUMENT = "<?xml version=\"1.0\" encoding=\"UTF-8\"?><results/>"; //$NON-NLS-1$
- private BufferManager bufferMgr;
private CommandContext context;
- private ArrayList<TupleSourceID> openTupleList;
+ private ArrayList<QueryProcessor> processors;
private String parentGroup;
private Map<String, Expression> params;
private ProcessorDataManager dataManager;
@@ -120,8 +116,7 @@
}
}
- public SqlEval(BufferManager bufferMgr, ProcessorDataManager dataManager, CommandContext context, String parentGroup, Map<String, Expression> params) {
- this.bufferMgr = bufferMgr;
+ public SqlEval(ProcessorDataManager dataManager, CommandContext context, String parentGroup, Map<String, Expression> params) {
this.dataManager = dataManager;
this.context = context;
this.parentGroup = parentGroup;
@@ -136,17 +131,13 @@
throws QueryParserException, MetaMatrixProcessingException, MetaMatrixComponentException {
QueryProcessor processor = context.getQueryProcessorFactory().createQueryProcessor(sql, parentGroup, context);
- // keep track of all the tuple sources opened during the
- // xquery process;
- if (openTupleList == null) {
- openTupleList = new ArrayList<TupleSourceID>();
+ processor.setNonBlocking(true);
+ if (processors == null) {
+ processors = new ArrayList<QueryProcessor>();
}
- openTupleList.add(processor.getResultsID());
+ processors.add(processor);
+ TupleSource src = new BatchIterator(processor);
- processor.process();
-
- TupleSourceID tsID = processor.getResultsID();
- TupleSource src = this.bufferMgr.getTupleSource(tsID);
String[] columns = elementNames(src.getSchema());
Class[] types= elementTypes(src.getSchema());
boolean xml = false;
@@ -209,16 +200,10 @@
* Closes any resources opened during the evaluation
*/
public void close() throws MetaMatrixComponentException {
- if (openTupleList != null && !openTupleList.isEmpty()) {
- for (Iterator i = openTupleList.iterator(); i.hasNext();) {
- TupleSourceID id = (TupleSourceID)i.next();
- try {
- this.bufferMgr.removeTupleSource(id);
- } catch (TupleSourceNotFoundException e) {
- // ignore and go on..
- }
- }
+ if (processors != null) {
+ for (QueryProcessor processor : processors) {
+ processor.closeProcessing();
+ }
}
-
}
}
Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/proc/CreateCursorResultSetInstruction.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/processor/proc/CreateCursorResultSetInstruction.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/processor/proc/CreateCursorResultSetInstruction.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -39,6 +39,9 @@
/**
*/
public class CreateCursorResultSetInstruction extends ProgramInstruction {
+
+ public static final String RS_NAME = "EXECSQL_INSTRUCTION"; //$NON-NLS-1$
+
protected String rsName;
protected ProcessorPlan plan;
Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/proc/ExecDynamicSqlInstruction.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/processor/proc/ExecDynamicSqlInstruction.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/processor/proc/ExecDynamicSqlInstruction.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -196,8 +196,7 @@
.createNonRecordingRecord(), procEnv
.getContext());
- ExecSqlInstruction inst = new ExecSqlInstruction(commandPlan,
- dynamicCommand.getIntoGroup());
+ CreateCursorResultSetInstruction inst = new CreateCursorResultSetInstruction(CreateCursorResultSetInstruction.RS_NAME, commandPlan);
dynamicProgram = new Program();
dynamicProgram.addInstruction(inst);
@@ -332,7 +331,7 @@
public Map getDescriptionProperties() {
Map props = new HashMap();
props.put(PROP_TYPE, "SQL"); //$NON-NLS-1$
- props.put(PROP_SQL, ExecSqlInstruction.RS_NAME);
+ props.put(PROP_SQL, CreateCursorResultSetInstruction.RS_NAME);
if (dynamicCommand.getIntoGroup() != null) {
props.put(PROP_GROUP, dynamicCommand.getIntoGroup().toString());
}
Deleted: trunk/engine/src/main/java/com/metamatrix/query/processor/proc/ExecSqlInstruction.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/processor/proc/ExecSqlInstruction.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/processor/proc/ExecSqlInstruction.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -1,123 +0,0 @@
-/*
- * JBoss, Home of Professional Open Source.
- * See the COPYRIGHT.txt file distributed with this work for information
- * regarding copyright ownership. Some portions may be licensed
- * to Red Hat, Inc. under one or more contributor license agreements.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301 USA.
- */
-
-package com.metamatrix.query.processor.proc;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import com.metamatrix.api.exception.MetaMatrixComponentException;
-import com.metamatrix.api.exception.MetaMatrixProcessingException;
-import com.metamatrix.common.buffer.BlockedException;
-import com.metamatrix.query.processor.ProcessorPlan;
-import com.metamatrix.query.processor.program.ProgramInstruction;
-import com.metamatrix.query.sql.symbol.GroupSymbol;
-
-/**
- * <p> Executes a SQL statement, and remove its results from the buffer manager. Executing this
- * instruction does not modify the values of any of the variables, hence it's results are not
- * important so they are removed immediately.</p>
- */
-public class ExecSqlInstruction extends ProgramInstruction {
-
- public static final String RS_NAME = "EXECSQL_INSTRUCTION"; //$NON-NLS-1$
-
- // processor plan for the command on the CommandStatement
- private ProcessorPlan commandPlan;
- //group for into if it is Select Into
- private GroupSymbol intoGroup;
-
- /**
- * Constructor for ExecSqlInstruction.
- * @param command Object (such as a ProcessorPlan) that is executed when this instruction
- * is processed.
- */
- public ExecSqlInstruction(ProcessorPlan commandPlan, GroupSymbol intoGroup) {
- this.commandPlan = commandPlan;
- this.intoGroup = intoGroup;
- }
-
- /**
- * <p>Processing this instruction executes the ProcessorPlan for the command on the
- * CommandStatement of the update procedure language. Executing this plan does not effect
- * the values of any of the variables defined as part of the update procedure and hence
- * the results of the ProcessPlan execution need not be stored for further processing. The
- * results are removed from the buffer manager immediately after execution. The program
- * counter is incremented after execution of the plan.</p>
- * @throws BlockedException if this processing the plan throws a currentVarContext
- */
- public void process(ProcedurePlan procEnv)
- throws BlockedException, MetaMatrixComponentException, MetaMatrixProcessingException {
-
- if(intoGroup != null && intoGroup.isTempGroupSymbol()){
- procEnv.executePlan(commandPlan, intoGroup.getName());
- }else{
- procEnv.executePlan(commandPlan, RS_NAME);
- }
-
- // We don't close the result here because this ExecSQLInstruction may be the last one in the procedure,
- // in which case the containing ProcedurePlan.close will manage the lifecycle of the last tuple source.
- // Those tuple sources that are not the last will be closed by the ProcedureEnvironment.executePlan method before the next
- // instruction is executed.
- }
-
- public ProcessorPlan getProcessorPlan(){ //Defect 13291 - made public to support changes to ProcedurePlan
- return this.commandPlan;
- }
-
- /**
- * Returns a deep clone
- */
- public Object clone(){
- ExecSqlInstruction clone = new ExecSqlInstruction((ProcessorPlan)commandPlan.clone(), intoGroup);
- return clone;
- }
-
- public String toString() {
- return "ExecSQLInstruction"; //$NON-NLS-1$
- }
-
- public Map getDescriptionProperties() {
- Map props = new HashMap();
- props.put(PROP_TYPE, "SQL"); //$NON-NLS-1$
- props.put(PROP_SQL, RS_NAME);
- props.put(PROP_PROGRAM, this.commandPlan.getDescriptionProperties());
- if(intoGroup != null) {
- props.put(PROP_GROUP, intoGroup.toString());
- }
- return props;
- }
-
- /**
- * @see com.metamatrix.query.processor.program.ProgramInstruction#getChildPlans()
- * @since 4.2
- */
- public Collection getChildPlans() {
- List plans = new ArrayList(1);
- plans.add(this.commandPlan);
- return plans;
- }
-
-}
Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/proc/ProcedurePlan.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/processor/proc/ProcedurePlan.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/processor/proc/ProcedurePlan.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -39,18 +39,18 @@
import com.metamatrix.api.exception.query.QueryValidatorException;
import com.metamatrix.common.buffer.BlockedException;
import com.metamatrix.common.buffer.BufferManager;
+import com.metamatrix.common.buffer.IndexedTupleSource;
import com.metamatrix.common.buffer.TupleBatch;
import com.metamatrix.common.buffer.TupleSource;
-import com.metamatrix.common.buffer.TupleSourceID;
-import com.metamatrix.common.buffer.TupleSourceNotFoundException;
import com.metamatrix.common.log.LogManager;
import com.metamatrix.dqp.util.LogConstants;
import com.metamatrix.query.execution.QueryExecPlugin;
import com.metamatrix.query.metadata.QueryMetadataInterface;
import com.metamatrix.query.metadata.SupportConstants;
import com.metamatrix.query.processor.BaseProcessorPlan;
+import com.metamatrix.query.processor.BatchIterator;
+import com.metamatrix.query.processor.CollectionTupleSource;
import com.metamatrix.query.processor.DescribableUtil;
-import com.metamatrix.query.processor.NullTupleSource;
import com.metamatrix.query.processor.ProcessorDataManager;
import com.metamatrix.query.processor.ProcessorPlan;
import com.metamatrix.query.processor.QueryProcessor;
@@ -73,10 +73,10 @@
*/
public class ProcedurePlan extends BaseProcessorPlan {
- private class CursorState {
- TupleSourceID tsID;
- TupleSource ts;
- List<?> currentRow;
+ public static class CursorState {
+ private QueryProcessor processor;
+ private IndexedTupleSource ts;
+ private List<?> currentRow;
}
private Program originalProgram;
@@ -88,8 +88,7 @@
private int batchSize;
private boolean done = false;
- private QueryProcessor internalProcessor;
- private TupleSourceID internalResultID;
+ private CursorState currentState;
// Temp state for final results
private TupleSource finalTupleSource;
@@ -157,8 +156,7 @@
lastTupleSource = null;
done = false;
- internalProcessor = null;
- internalResultID = null;
+ currentState = null;
finalTupleSource = null;
beginBatch = 1;
@@ -295,7 +293,7 @@
}
if(lastTupleSource == null){
- return new NullTupleSource(null);
+ return CollectionTupleSource.createNullTupleSource(null);
}
return lastTupleSource;
}
@@ -305,14 +303,9 @@
if (!this.cursorStates.isEmpty()) {
List<String> cursors = new ArrayList<String>(this.cursorStates.keySet());
for (String rsName : cursors) {
- try {
- removeResults(rsName);
- } catch (MetaMatrixComponentException e) {
- LogManager.logDetail(LogConstants.CTX_DQP, e, "Exception closing procedure plan cursor"); //$NON-NLS-1$
- }
+ removeResults(rsName);
}
}
- internalResultID = null;
if(getTempTableStore()!=null) {
getTempTableStore().removeTempTables();
}
@@ -409,7 +402,7 @@
if(rowCount == null) {
rowCount = new Integer(NO_ROWS_UPDATED);
}
- return new UpdateCountTupleSource((Integer)rowCount);
+ return CollectionTupleSource.createUpdateCountTupleSource((Integer)rowCount);
}
/**
@@ -423,43 +416,35 @@
return this.currentVarContext;
}
- public void executePlan(ProcessorPlan command, String rsName)
+ public CursorState executePlan(ProcessorPlan command, String rsName)
throws MetaMatrixComponentException, MetaMatrixProcessingException {
- boolean isExecSQLInstruction = rsName.equals(ExecSqlInstruction.RS_NAME);
- // Defect 14544: Close all non-final ExecSqlInstruction tuple sources before creating a new source.
- // This guarantees that the tuple source will be removed predictably from the buffer manager.
- if (isExecSQLInstruction) {
- removeResults(ExecSqlInstruction.RS_NAME);
- }
-
- if(this.internalProcessor == null){
- //this may not be the first time the plan is being run
- command.reset();
+
+ CursorState state = this.cursorStates.get(rsName.toUpperCase());
+ if (state == null) {
+ if (this.currentState == null) {
+ //this may not be the first time the plan is being run
+ command.reset();
+
+ CommandContext subContext = (CommandContext) getContext().clone();
+ subContext.setVariableContext(this.currentVarContext);
+ subContext.setTempTableStore(getTempTableStore());
+ state = new CursorState();
+ state.processor = new QueryProcessor(command, subContext, this.bufferMgr, this.dataMgr);
+ state.ts = new BatchIterator(state.processor);
+
+ //keep a reference to the tuple source
+ //it may be the last one
+ this.lastTupleSource = state.ts;
+
+ this.currentState = state;
+ }
+ //force execution to the first batch
+ this.currentState.ts.hasNext();
- // Run query processor on command
- CommandContext subContext = (CommandContext) getContext().clone();
- subContext.setVariableContext(this.currentVarContext);
- subContext.setTempTableStore(getTempTableStore());
- internalProcessor = new QueryProcessor(command, subContext, this.bufferMgr, this.dataMgr);
- this.internalResultID = this.internalProcessor.getResultsID();
+ this.cursorStates.put(rsName.toUpperCase(), this.currentState);
+ this.currentState = null;
}
- this.internalProcessor.process(Integer.MAX_VALUE); //TODO: put a better value here
-
- // didn't throw processor blocked, so must be done
- TupleSource source = this.bufferMgr.getTupleSource(this.internalResultID);
-
- this.internalProcessor = null;
-
- CursorState cursorState = new CursorState();
- cursorState.tsID = this.internalResultID;
- cursorState.ts = source;
- this.cursorStates.put(rsName.toUpperCase(), cursorState);
-
- if(isExecSQLInstruction){
- //keep a reference to the tuple source
- //it may be the last one
- this.lastTupleSource = source;
- }
+ return state;
}
/**
@@ -561,15 +546,8 @@
public void removeResults(String rsName) throws MetaMatrixComponentException {
String rsKey = rsName.toUpperCase();
CursorState state = this.cursorStates.remove(rsKey);
- if(state != null) {
- try {
- this.bufferMgr.removeTupleSource(state.tsID);
- } catch (TupleSourceNotFoundException e) {
- throw new MetaMatrixComponentException(e, ErrorMessageKeys.PROCESSOR_0021, QueryExecPlugin.Util.getString(ErrorMessageKeys.PROCESSOR_0021, (String)null));
- } catch (MetaMatrixComponentException e) {
- throw new MetaMatrixComponentException(e, ErrorMessageKeys.PROCESSOR_0022, QueryExecPlugin.Util.getString(ErrorMessageKeys.PROCESSOR_0022, (String) null));
- }
- LogManager.logTrace(LogConstants.CTX_DQP, new Object[]{"removed tuple source", state.tsID, "for result set"}); //$NON-NLS-1$ //$NON-NLS-2$
+ if (state != null) {
+ state.processor.closeProcessing();
}
}
Deleted: trunk/engine/src/main/java/com/metamatrix/query/processor/proc/UpdateCountTupleSource.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/processor/proc/UpdateCountTupleSource.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/processor/proc/UpdateCountTupleSource.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -1,61 +0,0 @@
-/*
- * JBoss, Home of Professional Open Source.
- * See the COPYRIGHT.txt file distributed with this work for information
- * regarding copyright ownership. Some portions may be licensed
- * to Red Hat, Inc. under one or more contributor license agreements.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301 USA.
- */
-
-package com.metamatrix.query.processor.proc;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import com.metamatrix.api.exception.MetaMatrixComponentException;
-import com.metamatrix.common.buffer.TupleSource;
-import com.metamatrix.query.sql.lang.Command;
-
-public class UpdateCountTupleSource implements
- TupleSource {
-
- private boolean hasNextTuple = true;
- private List nextTuple;
-
- public UpdateCountTupleSource(List tuple) {
- nextTuple = tuple;
- }
-
- public UpdateCountTupleSource(int count) {
- nextTuple = new ArrayList(1);
- nextTuple.add(new Integer(count));
- }
-
- public List getSchema() {
- return Command.getUpdateCommandSymbol();
- }
-
- public List nextTuple() throws MetaMatrixComponentException {
- if (hasNextTuple) {
- hasNextTuple = false;
- return nextTuple;
- }
- return null;
- }
-
- public void closeSource() throws MetaMatrixComponentException {
- }
-}
\ No newline at end of file
Deleted: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/BatchCollector.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/BatchCollector.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/BatchCollector.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -1,107 +0,0 @@
-/*
- * JBoss, Home of Professional Open Source.
- * See the COPYRIGHT.txt file distributed with this work for information
- * regarding copyright ownership. Some portions may be licensed
- * to Red Hat, Inc. under one or more contributor license agreements.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301 USA.
- */
-
-package com.metamatrix.query.processor.relational;
-
-import java.util.List;
-
-import com.metamatrix.api.exception.MetaMatrixComponentException;
-import com.metamatrix.api.exception.MetaMatrixProcessingException;
-import com.metamatrix.common.buffer.TupleBatch;
-import com.metamatrix.common.buffer.TupleSourceID;
-import com.metamatrix.common.buffer.TupleSourceNotFoundException;
-import com.metamatrix.common.buffer.BufferManager.TupleSourceStatus;
-import com.metamatrix.common.buffer.BufferManager.TupleSourceType;
-
-public class BatchCollector {
-
- private RelationalNode sourceNode;
-
- private boolean done = false;
- private TupleSourceID tsID;
- private int rowCount = 0;
- private boolean collectedAny;
-
- public BatchCollector(RelationalNode sourceNode) throws MetaMatrixComponentException {
- this.sourceNode = sourceNode;
- List sourceElements = sourceNode.getElements();
- tsID = sourceNode.getBufferManager().createTupleSource(sourceElements, sourceNode.getConnectionID(), TupleSourceType.PROCESSOR);
- }
-
- public TupleSourceID collectTuples() throws MetaMatrixComponentException, MetaMatrixProcessingException {
- return collectTuples(null);
- }
-
- public TupleSourceID collectTuples(TupleBatch batch) throws MetaMatrixComponentException, MetaMatrixProcessingException {
- while(!done) {
- if (batch == null) {
- batch = sourceNode.nextBatch();
- }
-
- // Add batch
- if(batch.getRowCount() > 0) {
- this.rowCount = batch.getEndRow();
- sourceNode.getBufferManager().addTupleBatch(tsID, batch);
- collectedAny = true;
- }
-
- // Check for termination condition - batch ending with null row
- if(batch.getTerminationFlag()) {
- break;
- }
-
- batch = null;
- }
- if (!done) {
- sourceNode.getBufferManager().setStatus(tsID, TupleSourceStatus.FULL);
- done = true;
- }
- return tsID;
- }
-
- public boolean collectedAny() {
- boolean result = collectedAny;
- collectedAny = false;
- return result;
- }
-
- public int getRowCount() {
- return rowCount;
- }
-
- public void close() throws MetaMatrixComponentException {
- try {
- this.sourceNode.getBufferManager().removeTupleSource(tsID);
- } catch (TupleSourceNotFoundException err) {
- //ignore
- }
- }
-
- public TupleSourceID getTupleSourceID() {
- return this.tsID;
- }
-
- public boolean isDone() {
- return done;
- }
-
-}
Deleted: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/BatchIterator.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/BatchIterator.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/BatchIterator.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -1,123 +0,0 @@
-/*
- * JBoss, Home of Professional Open Source.
- * See the COPYRIGHT.txt file distributed with this work for information
- * regarding copyright ownership. Some portions may be licensed
- * to Red Hat, Inc. under one or more contributor license agreements.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301 USA.
- */
-
-package com.metamatrix.query.processor.relational;
-
-import java.util.List;
-
-import com.metamatrix.api.exception.MetaMatrixComponentException;
-import com.metamatrix.api.exception.MetaMatrixProcessingException;
-import com.metamatrix.common.buffer.IndexedTupleSource;
-import com.metamatrix.common.buffer.TupleBatch;
-import com.metamatrix.query.sql.symbol.SingleElementSymbol;
-
-final class BatchIterator implements
- IndexedTupleSource {
-
- private final RelationalNode source;
-
- BatchIterator(RelationalNode source) {
- this.source = source;
- }
-
- private boolean done;
- private int currentRow = 1;
- private TupleBatch currentBatch;
- private List currentTuple;
-
- public boolean hasNext() throws MetaMatrixComponentException,
- MetaMatrixProcessingException {
- if (done) {
- return false;
- }
- while (currentTuple == null) {
- if (currentBatch == null) {
- currentBatch = this.source.nextBatch();
- }
-
- if (currentBatch.getEndRow() >= currentRow) {
- this.currentTuple = currentBatch.getTuple(currentRow++);
- } else {
- done = currentBatch.getTerminationFlag();
- currentBatch = null;
- if (done) {
- return false;
- }
- }
- }
- return true;
- }
-
- @Override
- public void closeSource() throws MetaMatrixComponentException {
-
- }
-
- @Override
- public List<SingleElementSymbol> getSchema() {
- return source.getElements();
- }
-
- @Override
- public List<?> nextTuple() throws MetaMatrixComponentException,
- MetaMatrixProcessingException {
- if (currentTuple == null && !hasNext()) {
- return null;
- }
- List result = currentTuple;
- currentTuple = null;
- return result;
- }
-
- public void reset() {
- throw new UnsupportedOperationException();
- }
-
- public void mark() {
- //does nothing
- }
-
- @Override
- public int getCurrentIndex() {
- return currentRow;
- }
-
- public void setPosition(int position) {
- if (position == this.currentRow) {
- return;
- }
- if (position < this.currentRow && (this.currentBatch == null || position < this.currentBatch.getBeginRow())) {
- throw new UnsupportedOperationException("Backwards positioning is not allowed"); //$NON-NLS-1$
- }
- this.currentRow = position;
- this.currentTuple = null;
- if (currentBatch.getEndRow() < currentRow) {
- this.currentBatch = null;
- }
- }
-
- @Override
- public TupleBatch getBatch() throws MetaMatrixComponentException {
- return currentBatch;
- }
-
-}
Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentCriteriaProcessor.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentCriteriaProcessor.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentCriteriaProcessor.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -34,9 +34,10 @@
import java.util.Map;
import com.metamatrix.api.exception.MetaMatrixComponentException;
+import com.metamatrix.api.exception.MetaMatrixProcessingException;
import com.metamatrix.api.exception.query.ExpressionEvaluationException;
import com.metamatrix.common.buffer.BlockedException;
-import com.metamatrix.common.buffer.TupleSourceNotFoundException;
+import com.metamatrix.query.processor.relational.SortUtility.Mode;
import com.metamatrix.query.rewriter.QueryRewriter;
import com.metamatrix.query.sql.lang.AbstractSetCriteria;
import com.metamatrix.query.sql.lang.CollectionValueIterator;
@@ -77,7 +78,7 @@
}
public void sort() throws BlockedException,
- MetaMatrixComponentException {
+ MetaMatrixComponentException, MetaMatrixProcessingException {
if (dvs == null) {
if (sortUtility == null) {
List<Expression> sortSymbols = new ArrayList<Expression>(dependentSetStates.size());
@@ -87,9 +88,9 @@
sortSymbols.add(dependentSetStates.get(i).valueExpression);
}
DependentValueSource originalVs = (DependentValueSource)dependentNode.getContext().getVariableContext().getGlobalValue(valueSource);
- this.sortUtility = new SortUtility(originalVs.getTupleSourceID(), sortSymbols, sortDirection, true, dependentNode.getBufferManager(), dependentNode.getConnectionID());
+ this.sortUtility = new SortUtility(originalVs.getTupleBuffer().createIndexedTupleSource(), sortSymbols, sortDirection, Mode.DUP_REMOVE, dependentNode.getBufferManager(), dependentNode.getConnectionID());
}
- dvs = new DependentValueSource(sortUtility.sort(), dependentNode.getBufferManager());
+ dvs = new DependentValueSource(sortUtility.sort(), dependentNode.getBufferManager().getProcessorBatchSize() / 2);
for (SetState setState : dependentSetStates) {
setState.valueIterator = dvs.getValueIterator(setState.valueExpression);
}
@@ -99,10 +100,7 @@
public void close() throws MetaMatrixComponentException {
if (dvs != null) {
sortUtility = null;
- try {
- dependentNode.getBufferManager().removeTupleSource(dvs.getTupleSourceID());
- } catch (TupleSourceNotFoundException e) {
- }
+ dvs.getTupleBuffer().remove();
dvs = null;
}
}
@@ -189,7 +187,7 @@
}
}
- public Criteria prepareCriteria() throws MetaMatrixComponentException {
+ public Criteria prepareCriteria() throws MetaMatrixComponentException, MetaMatrixProcessingException {
if (phase == SORT) {
for (TupleState state : dependentState.values()) {
Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentProcedureExecutionNode.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentProcedureExecutionNode.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentProcedureExecutionNode.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -89,7 +89,6 @@
}
/**
- * @throws TupleSourceNotFoundException
* @see com.metamatrix.query.processor.relational.PlanExecutionNode#prepareNextCommand()
*/
protected boolean prepareNextCommand() throws BlockedException,
Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentValueSource.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentValueSource.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DependentValueSource.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -25,15 +25,17 @@
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
import com.metamatrix.api.exception.MetaMatrixComponentException;
import com.metamatrix.api.exception.MetaMatrixProcessingException;
-import com.metamatrix.common.buffer.BufferManager;
import com.metamatrix.common.buffer.IndexedTupleSource;
-import com.metamatrix.common.buffer.TupleSourceID;
-import com.metamatrix.common.buffer.TupleSourceNotFoundException;
+import com.metamatrix.common.buffer.TupleBuffer;
+import com.metamatrix.common.types.DataTypeManager;
import com.metamatrix.core.util.Assertion;
import com.metamatrix.query.sql.symbol.Expression;
+import com.metamatrix.query.sql.symbol.SingleElementSymbol;
import com.metamatrix.query.sql.util.ValueIterator;
import com.metamatrix.query.sql.util.ValueIteratorSource;
@@ -43,31 +45,25 @@
public class DependentValueSource implements
ValueIteratorSource {
- private TupleSourceID tupleSourceID;
- private BufferManager bm;
- private Map<Expression, HashSet<Object>> cachedSets;
+ private TupleBuffer tupleSourceID;
+ private int maxSetSize;
+ private Map<Expression, Set<Object>> cachedSets;
- public DependentValueSource(TupleSourceID tupleSourceID, BufferManager bm) {
+ public DependentValueSource(TupleBuffer tupleSourceID, int maxSetSize) {
this.tupleSourceID = tupleSourceID;
- this.bm = bm;
+ this.maxSetSize = maxSetSize;
}
- public TupleSourceID getTupleSourceID() {
+ public TupleBuffer getTupleBuffer() {
return tupleSourceID;
}
/**
* @throws MetaMatrixComponentException
- * @throws TupleSourceNotFoundException
* @see com.metamatrix.query.sql.util.ValueIteratorSource#getValueIterator(com.metamatrix.query.sql.symbol.Expression)
*/
public ValueIterator getValueIterator(Expression valueExpression) throws MetaMatrixComponentException {
- IndexedTupleSource its;
- try {
- its = bm.getTupleSource(tupleSourceID);
- } catch (TupleSourceNotFoundException e) {
- throw new MetaMatrixComponentException(e);
- }
+ IndexedTupleSource its = tupleSourceID.createIndexedTupleSource();
int index = 0;
if (valueExpression != null) {
index = its.getSchema().indexOf(valueExpression);
@@ -76,27 +72,26 @@
return new TupleSourceValueIterator(its, index);
}
- public HashSet<Object> getCachedSet(Expression valueExpression) throws MetaMatrixComponentException, MetaMatrixProcessingException {
- HashSet<Object> result = null;
+ public Set<Object> getCachedSet(Expression valueExpression) throws MetaMatrixComponentException, MetaMatrixProcessingException {
+ Set<Object> result = null;
if (cachedSets != null) {
result = cachedSets.get(valueExpression);
}
if (result == null) {
- IndexedTupleSource its;
- try {
- if (bm.getRowCount(tupleSourceID) > bm.getProcessorBatchSize() / 2) {
- return null;
- }
- its = bm.getTupleSource(tupleSourceID);
- } catch (TupleSourceNotFoundException e) {
- throw new MetaMatrixComponentException(e);
- }
+ if (tupleSourceID.getRowCount() > maxSetSize) {
+ return null;
+ }
+ IndexedTupleSource its = tupleSourceID.createIndexedTupleSource();
int index = 0;
if (valueExpression != null) {
index = its.getSchema().indexOf(valueExpression);
}
Assertion.assertTrue(index != -1);
- result = new HashSet<Object>();
+ if (((SingleElementSymbol)its.getSchema().get(index)).getType() == DataTypeManager.DefaultDataClasses.BIG_DECIMAL) {
+ result = new TreeSet<Object>();
+ } else {
+ result = new HashSet<Object>();
+ }
while (its.hasNext()) {
Object value = its.nextTuple().get(index);
if (value != null) {
@@ -105,7 +100,7 @@
}
its.closeSource();
if (cachedSets == null) {
- cachedSets = new HashMap<Expression, HashSet<Object>>();
+ cachedSets = new HashMap<Expression, Set<Object>>();
}
cachedSets.put(valueExpression, result);
}
Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DuplicateFilter.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DuplicateFilter.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/DuplicateFilter.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -30,11 +30,11 @@
import com.metamatrix.api.exception.query.ExpressionEvaluationException;
import com.metamatrix.api.exception.query.FunctionExecutionException;
import com.metamatrix.common.buffer.BufferManager;
+import com.metamatrix.common.buffer.TupleBuffer;
import com.metamatrix.common.buffer.TupleSource;
-import com.metamatrix.common.buffer.TupleSourceID;
-import com.metamatrix.common.buffer.TupleSourceNotFoundException;
import com.metamatrix.common.buffer.BufferManager.TupleSourceType;
import com.metamatrix.query.function.aggregate.AggregateFunction;
+import com.metamatrix.query.processor.relational.SortUtility.Mode;
import com.metamatrix.query.sql.lang.OrderBy;
import com.metamatrix.query.sql.symbol.ElementSymbol;
@@ -52,15 +52,13 @@
private List sortTypes;
// Temporary state - should be reset
- private TupleSourceID collectionID = null;
- private SortUtility sortUtility = null;
- private TupleSourceID sortedID = null;
- private TupleCollector tupleCollector;
+ private TupleBuffer collectionBuffer;
+ private SortUtility sortUtility;
/**
* Constructor for DuplicateFilter.
*/
- public DuplicateFilter(AggregateFunction proxy, BufferManager mgr, String groupName, int batchSize) {
+ public DuplicateFilter(AggregateFunction proxy, BufferManager mgr, String groupName) {
super();
this.proxy = proxy;
@@ -89,12 +87,16 @@
public void reset() {
this.proxy.reset();
+ close();
+ }
- this.collectionID = null;
- this.tupleCollector = null;
+ private void close() {
+ if (this.collectionBuffer != null) {
+ collectionBuffer.remove();
+ }
+ this.collectionBuffer = null;
this.sortUtility = null;
- this.sortedID = null;
- }
+ }
/**
* @see com.metamatrix.query.function.aggregate.AggregateFunction#addInput(Object)
@@ -102,18 +104,13 @@
public void addInput(Object input)
throws FunctionExecutionException, ExpressionEvaluationException, MetaMatrixComponentException {
- try {
- if(collectionID == null) {
- collectionID = mgr.createTupleSource(elements, groupName, TupleSourceType.PROCESSOR);
- this.tupleCollector = new TupleCollector(collectionID, mgr);
- }
-
- List row = new ArrayList(1);
- row.add(input);
- this.tupleCollector.addTuple(row);
- } catch(TupleSourceNotFoundException e) {
- throw new MetaMatrixComponentException(e, e.getMessage());
+ if(collectionBuffer == null) {
+ collectionBuffer = mgr.createTupleBuffer(elements, groupName, TupleSourceType.PROCESSOR);
}
+
+ List row = new ArrayList(1);
+ row.add(input);
+ this.collectionBuffer.addTuple(row);
}
/**
@@ -123,32 +120,27 @@
public Object getResult()
throws MetaMatrixComponentException, MetaMatrixProcessingException {
- try {
- if(collectionID != null) {
- this.tupleCollector.close();
+ if(collectionBuffer != null) {
+ this.collectionBuffer.close();
- // Sort
- sortUtility = new SortUtility(collectionID, elements, sortTypes, true, mgr, groupName);
- this.sortedID = sortUtility.sort();
-
- // Add all input to proxy
- TupleSource sortedSource = mgr.getTupleSource(sortedID);
- while(true) {
- List tuple = sortedSource.nextTuple();
- if(tuple == null) {
- break;
- }
- this.proxy.addInput(tuple.get(0));
- }
- sortedSource.closeSource();
-
- // Clean up
- mgr.removeTupleSource(collectionID);
- mgr.removeTupleSource(sortedID);
+ // Sort
+ sortUtility = new SortUtility(collectionBuffer.createIndexedTupleSource(), elements, sortTypes, Mode.DUP_REMOVE, mgr, groupName);
+ TupleBuffer sorted = sortUtility.sort();
+ try {
+ // Add all input to proxy
+ TupleSource sortedSource = sorted.createIndexedTupleSource();
+ while(true) {
+ List tuple = sortedSource.nextTuple();
+ if(tuple == null) {
+ break;
+ }
+ this.proxy.addInput(tuple.get(0));
+ }
+ } finally {
+ sorted.remove();
}
-
- } catch(TupleSourceNotFoundException e) {
- throw new MetaMatrixComponentException(e, e.getMessage());
+
+ close();
}
// Return
Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/GroupingNode.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/GroupingNode.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/GroupingNode.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -32,12 +32,9 @@
import com.metamatrix.api.exception.MetaMatrixProcessingException;
import com.metamatrix.api.exception.query.ExpressionEvaluationException;
import com.metamatrix.common.buffer.BlockedException;
-import com.metamatrix.common.buffer.IndexedTupleSource;
import com.metamatrix.common.buffer.TupleBatch;
-import com.metamatrix.common.buffer.TupleSourceID;
-import com.metamatrix.common.buffer.TupleSourceNotFoundException;
-import com.metamatrix.common.buffer.BufferManager.TupleSourceStatus;
-import com.metamatrix.common.buffer.BufferManager.TupleSourceType;
+import com.metamatrix.common.buffer.TupleBuffer;
+import com.metamatrix.common.buffer.TupleSource;
import com.metamatrix.query.eval.Evaluator;
import com.metamatrix.query.function.aggregate.AggregateFunction;
import com.metamatrix.query.function.aggregate.Avg;
@@ -65,21 +62,15 @@
private int phase = COLLECTION;
private Map elementMap; // Map of incoming symbol to index in source elements
private List collectedExpressions; // Collected Expressions
- private TupleBatch sourceBatch; // Current batch loaded from the source, if blocked
- private int sourceRow; // Current row index in batch, if blocked
- private List[] collectedRows; // Rows evaluated and collected
- private TupleSourceID collectionID; // Tuple source collecting the endpoint of collection phase
- private int rowCount; // Row count of incoming rows
// Sort phase
private SortUtility sortUtility;
- private TupleSourceID sortedID;
+ private TupleBuffer sortedID;
+ private TupleSource groupTupleSource;
// Group phase
private Map expressionMap; // Index map for all collected expressions (Expression -> index in collectedExpressions)
private AggregateFunction[] functions;
- private IndexedTupleSource groupTupleSource;
- private int groupBegin = 1;
private List lastRow;
private List currentGroupTuple;
@@ -97,19 +88,12 @@
phase = COLLECTION;
elementMap = null;
collectedExpressions = null;
- sourceBatch = null;
- sourceRow = -1;
- collectedRows = null;
- collectionID = null;
- rowCount = 0;
sortUtility = null;
sortedID = null;
expressionMap = null;
functions = null;
- groupBegin = 1;
- groupTupleSource = null;
lastRow = null;
currentGroupTuple = null;
}
@@ -147,8 +131,6 @@
// Determine expressions to build (all grouping expressions + expressions used by aggregates)
collectExpressions();
- this.collectionID = getBufferManager().createTupleSource(collectedExpressions, getConnectionID(), TupleSourceType.PROCESSOR);
-
initializeFunctionAccumulators();
}
@@ -213,7 +195,7 @@
}
if(aggSymbol.isDistinct()) {
- functions[i] = new DuplicateFilter(functions[i], getBufferManager(), getConnectionID(), getBatchSize());
+ functions[i] = new DuplicateFilter(functions[i], getBufferManager(), getConnectionID());
}
functions[i] = new NullFilter(functions[i]);
@@ -233,107 +215,108 @@
public TupleBatch nextBatchDirect()
throws BlockedException, MetaMatrixComponentException, MetaMatrixProcessingException {
- try {
- // Take inputs, evaluate expressions, and build initial tuple source
- if(this.phase == COLLECTION) {
- collectionPhase();
- }
+ // Take inputs, evaluate expressions, and build initial tuple source
+ if(this.phase == COLLECTION) {
+ collectionPhase();
+ }
- // If necessary, sort to determine groups (if no group cols, no need to sort)
- if(this.phase == SORT) {
- sortPhase();
- }
-
- // Walk through the sorted results and for each group, emit a row
- if(this.phase == GROUP) {
- return groupPhase();
- }
-
- TupleBatch terminationBatch = new TupleBatch(1, Collections.EMPTY_LIST);
- terminationBatch.setTerminationFlag(true);
- return terminationBatch;
- } catch(TupleSourceNotFoundException e) {
- throw new MetaMatrixComponentException(e, e.getMessage());
+ // If necessary, sort to determine groups (if no group cols, no need to sort)
+ if(this.phase == SORT) {
+ sortPhase();
}
+ // Walk through the sorted results and for each group, emit a row
+ if(this.phase == GROUP) {
+ return groupPhase();
+ }
+
+ TupleBatch terminationBatch = new TupleBatch(1, Collections.EMPTY_LIST);
+ terminationBatch.setTerminationFlag(true);
+ return terminationBatch;
}
+
+ public TupleSource getCollectionTupleSource() {
+
+ final RelationalNode sourceNode = this.getChildren()[0];
+
+ return new TupleSource() {
+ private TupleBatch sourceBatch; // Current batch loaded from the source, if blocked
+ private int sourceRow = 1;
- private void collectionPhase() throws BlockedException, TupleSourceNotFoundException, MetaMatrixComponentException, MetaMatrixProcessingException {
- RelationalNode sourceNode = this.getChildren()[0];
+ @Override
+ public List<?> nextTuple() throws MetaMatrixComponentException,
+ MetaMatrixProcessingException {
+ while (true) {
+ if(sourceBatch == null) {
+ // Read next batch
+ sourceBatch = sourceNode.nextBatch();
+ }
+
+ if(sourceBatch.getRowCount() > 0 && sourceRow <= sourceBatch.getEndRow()) {
+ // Evaluate expressions needed for grouping
+ List tuple = sourceBatch.getTuple(sourceRow);
+
+ int columns = collectedExpressions.size();
+ List exprTuple = new ArrayList(columns);
+ for(int col = 0; col<columns; col++) {
+ // The following call may throw BlockedException, but all state to this point
+ // is saved in class variables so we can start over on building this tuple
+ Object value = new Evaluator(elementMap, getDataManager(), getContext()).evaluate((Expression) collectedExpressions.get(col), tuple);
+ exprTuple.add(value);
+ }
+ sourceRow++;
+ return exprTuple;
+ }
+
+ // Check for termination condition
+ if(sourceBatch.getTerminationFlag()) {
+ sourceBatch = null;
+ return null;
+ }
+ sourceBatch = null;
+ }
+ }
+
+ @Override
+ public List<SingleElementSymbol> getSchema() {
+ return collectedExpressions;
+ }
+
+ @Override
+ public void closeSource() throws MetaMatrixComponentException {
+
+ }
+ };
+
+ }
- while(true) {
- if(this.sourceBatch == null) {
- // Read next batch
- this.sourceBatch = sourceNode.nextBatch();
- }
-
- if(this.sourceBatch.getRowCount() > 0) {
- this.rowCount = this.sourceBatch.getEndRow();
- this.sourceRow = this.sourceBatch.getBeginRow();
- this.collectedRows = new List[this.sourceBatch.getRowCount()];
- // Evaluate expressions needed for grouping
- for(int row=this.sourceRow; row<=this.sourceBatch.getEndRow(); row++) {
- List tuple = this.sourceBatch.getTuple(row);
-
- int columns = this.collectedExpressions.size();
- List exprTuple = new ArrayList(columns);
- for(int col = 0; col<columns; col++) {
- // The following call may throw BlockedException, but all state to this point
- // is saved in class variables so we can start over on building this tuple
- Object value = new Evaluator(this.elementMap, getDataManager(), getContext()).evaluate((Expression) this.collectedExpressions.get(col), tuple);
- exprTuple.add(value);
- }
-
- collectedRows[row-this.sourceBatch.getBeginRow()] = exprTuple;
- }
-
- // Add collected batch
- TupleBatch exprBatch = new TupleBatch(this.sourceBatch.getBeginRow(), collectedRows);
- getBufferManager().addTupleBatch(collectionID, exprBatch);
- }
- // Clear and setup for next loop
- this.sourceRow = -1;
- this.collectedRows = null;
-
- // Check for termination condition
- if(this.sourceBatch.getTerminationFlag()) {
- this.sourceBatch = null;
- break;
- }
-
- this.sourceBatch = null;
- }
-
- this.getBufferManager().setStatus(this.collectionID, TupleSourceStatus.FULL);
-
- if(this.sortElements == null || this.rowCount == 0) {
+ private void collectionPhase() {
+ if(this.sortElements == null) {
// No need to sort
- this.sortedID = this.collectionID;
- this.collectionID = null;
+ this.groupTupleSource = getCollectionTupleSource();
this.phase = GROUP;
} else {
- this.sortUtility = new SortUtility(collectionID, sortElements,
+ this.sortUtility = new SortUtility(getCollectionTupleSource(), sortElements,
sortTypes, removeDuplicates?Mode.DUP_REMOVE_SORT:Mode.SORT, getBufferManager(),
- getConnectionID(), removeDuplicates);
+ getConnectionID());
this.phase = SORT;
}
}
private void sortPhase() throws BlockedException, MetaMatrixComponentException, MetaMatrixProcessingException {
this.sortedID = this.sortUtility.sort();
- this.rowCount = this.getBufferManager().getFinalRowCount(this.sortedID);
+ this.groupTupleSource = this.sortedID.createIndexedTupleSource();
this.phase = GROUP;
}
- private TupleBatch groupPhase() throws BlockedException, TupleSourceNotFoundException, MetaMatrixComponentException, MetaMatrixProcessingException {
- if (this.groupTupleSource == null) {
- this.groupTupleSource = this.getBufferManager().getTupleSource(sortedID);
- }
-
- while(groupBegin <= rowCount) {
+ private TupleBatch groupPhase() throws BlockedException, MetaMatrixComponentException, MetaMatrixProcessingException {
+ while(true) {
if (currentGroupTuple == null) {
currentGroupTuple = this.groupTupleSource.nextTuple();
+ if (currentGroupTuple == null) {
+ break;
+ }
}
if(lastRow == null) {
@@ -365,10 +348,8 @@
// Update function accumulators with new row - can throw blocked exception
updateAggregates(currentGroupTuple);
currentGroupTuple = null;
- groupBegin++;
}
- this.groupTupleSource.closeSource();
- if(rowCount != 0 || sortElements == null) {
+ if(lastRow != null || sortElements == null) {
// Close last group
List row = new ArrayList(functions.length);
for(int i=0; i<functions.length; i++) {
@@ -431,16 +412,6 @@
public void close() throws MetaMatrixComponentException {
if (!isClosed()) {
super.close();
- try {
- if(this.collectionID != null) {
- getBufferManager().removeTupleSource(collectionID);
- }
- if(this.sortedID != null) {
- getBufferManager().removeTupleSource(sortedID);
- }
- } catch(TupleSourceNotFoundException e) {
- throw new MetaMatrixComponentException(e, e.getMessage());
- }
}
}
Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/JoinNode.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/JoinNode.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/JoinNode.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -35,8 +35,7 @@
import com.metamatrix.common.buffer.BlockedException;
import com.metamatrix.common.buffer.BufferManager;
import com.metamatrix.common.buffer.TupleBatch;
-import com.metamatrix.common.buffer.TupleSourceID;
-import com.metamatrix.common.buffer.TupleSourceNotFoundException;
+import com.metamatrix.common.buffer.TupleBuffer;
import com.metamatrix.query.processor.ProcessorDataManager;
import com.metamatrix.query.sql.LanguageObject;
import com.metamatrix.query.sql.lang.Criteria;
@@ -54,7 +53,7 @@
NESTED_LOOP
}
- private enum State { LOAD_LEFT, LOAD_RIGHT, POST_LOAD_LEFT, POST_LOAD_RIGHT, EXECUTE }
+ private enum State { LOAD_LEFT, LOAD_RIGHT, EXECUTE }
private State state = State.LOAD_LEFT;
private boolean leftOpened;
@@ -191,23 +190,20 @@
}
if (state == State.LOAD_RIGHT) {
if (isDependent() && !this.rightOpened) {
- TupleSourceID tsID = this.joinStrategy.leftSource.getTupleSourceID();
- this.getContext().getVariableContext().setGlobalValue(this.dependentValueSource, new DependentValueSource(tsID, this.getBufferManager()));
+ TupleBuffer tsID = this.joinStrategy.leftSource.getTupleBuffer();
+ this.getContext().getVariableContext().setGlobalValue(this.dependentValueSource, new DependentValueSource(tsID, this.getBufferManager().getProcessorBatchSize() / 2));
//open the right side now that the tuples have been collected
this.getChildren()[1].open();
this.rightOpened = true;
}
this.joinStrategy.loadRight();
- state = State.POST_LOAD_LEFT;
+ //force buffering based upon join type - may have already happened in the strategy load methods
+ this.joinStrategy.rightSource.getTupleBuffer();
+ if (joinType == JoinType.JOIN_FULL_OUTER) {
+ this.joinStrategy.leftSource.getTupleBuffer();
+ }
+ state = State.EXECUTE;
}
- if (state == State.POST_LOAD_LEFT) {
- this.joinStrategy.postLoadLeft();
- state = State.POST_LOAD_RIGHT;
- }
- if (state == State.POST_LOAD_RIGHT) {
- this.joinStrategy.postLoadRight();
- state = State.EXECUTE;
- }
while(true) {
if(super.isBatchFull()) {
@@ -299,12 +295,7 @@
public void close()
throws MetaMatrixComponentException {
super.close();
-
- try {
- joinStrategy.close();
- } catch (TupleSourceNotFoundException err) {
- //ignore
- }
+ joinStrategy.close();
if (this.isDependent()) {
this.getContext().getVariableContext().setGlobalValue(this.dependentValueSource, null);
}
Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/JoinStrategy.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/JoinStrategy.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/JoinStrategy.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -28,7 +28,6 @@
import com.metamatrix.api.exception.MetaMatrixComponentException;
import com.metamatrix.api.exception.MetaMatrixProcessingException;
import com.metamatrix.api.exception.query.CriteriaEvaluationException;
-import com.metamatrix.common.buffer.TupleSourceNotFoundException;
public abstract class JoinStrategy {
@@ -36,7 +35,7 @@
protected SourceState leftSource;
protected SourceState rightSource;
- public void close() throws TupleSourceNotFoundException, MetaMatrixComponentException {
+ public void close() {
try {
if (leftSource != null) {
leftSource.close();
@@ -53,8 +52,7 @@
}
}
- public void initialize(JoinNode joinNode)
- throws MetaMatrixComponentException {
+ public void initialize(JoinNode joinNode) {
this.joinNode = joinNode;
this.leftSource = new SourceState(joinNode.getChildren()[0], joinNode.getLeftExpressions());
this.leftSource.markDistinct(this.joinNode.isLeftDistinct());
@@ -63,19 +61,11 @@
}
protected void loadLeft() throws MetaMatrixComponentException, MetaMatrixProcessingException {
- this.leftSource.collectTuples();
}
- protected void postLoadLeft() throws MetaMatrixComponentException, MetaMatrixProcessingException {
- }
-
protected void loadRight() throws MetaMatrixComponentException, MetaMatrixProcessingException {
- this.rightSource.collectTuples();
}
- protected void postLoadRight() throws MetaMatrixComponentException, MetaMatrixProcessingException {
- }
-
/**
* Output a combined, projected tuple based on tuple parts from the left and right.
* @param leftTuple Left tuple part
Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/MergeJoinStrategy.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/MergeJoinStrategy.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/MergeJoinStrategy.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -28,8 +28,6 @@
import com.metamatrix.api.exception.MetaMatrixComponentException;
import com.metamatrix.api.exception.MetaMatrixProcessingException;
import com.metamatrix.api.exception.query.CriteriaEvaluationException;
-import com.metamatrix.common.buffer.BlockedOnMemoryException;
-import com.metamatrix.common.buffer.TupleSourceNotFoundException;
import com.metamatrix.query.processor.relational.SortUtility.Mode;
import com.metamatrix.query.sql.lang.JoinType;
import com.metamatrix.query.sql.lang.OrderBy;
@@ -116,7 +114,7 @@
* @see com.metamatrix.query.processor.relational.JoinStrategy#initialize(com.metamatrix.query.processor.relational.JoinNode)
*/
@Override
- public void initialize(JoinNode joinNode) throws MetaMatrixComponentException {
+ public void initialize(JoinNode joinNode) {
super.initialize(joinNode);
this.outerState = this.leftSource;
this.innerState = this.rightSource;
@@ -134,8 +132,7 @@
* @see com.metamatrix.query.processor.relational.JoinStrategy#close()
*/
@Override
- public void close() throws TupleSourceNotFoundException,
- MetaMatrixComponentException {
+ public void close() {
super.close();
this.outerState = null;
this.innerState = null;
@@ -331,27 +328,15 @@
return 0;
}
- /**
- * @see com.metamatrix.query.processor.relational.JoinStrategy#loadLeft()
- */
@Override
protected void loadLeft() throws MetaMatrixComponentException,
- MetaMatrixProcessingException {
- if (sortLeft == SortOption.ALREADY_SORTED && !this.joinNode.isDependent() && !JoinType.JOIN_FULL_OUTER.equals(joinNode.getJoinType())) {
- return; // don't buffer
- }
- super.loadLeft();
- }
-
- @Override
- protected void postLoadLeft() throws MetaMatrixComponentException,
MetaMatrixProcessingException {
if (this.processingSortLeft == SortOption.SORT || this.processingSortLeft == SortOption.SORT_DISTINCT) {
if (this.leftSort == null) {
List expressions = this.joinNode.getLeftExpressions();
- this.leftSort = new SortUtility(this.leftSource.getTupleSourceID(),
+ this.leftSort = new SortUtility(this.leftSource.getIterator(),
expressions, Collections.nCopies(expressions.size(), OrderBy.ASC), processingSortLeft == SortOption.SORT_DISTINCT?Mode.DUP_REMOVE_SORT:Mode.SORT,
- this.joinNode.getBufferManager(), this.joinNode.getConnectionID(), true);
+ this.joinNode.getBufferManager(), this.joinNode.getConnectionID());
this.leftSource.markDistinct(processingSortLeft == SortOption.SORT_DISTINCT && expressions.size() == this.leftSource.getOuterVals().size());
}
this.leftSource.setTupleSource(leftSort.sort());
@@ -360,19 +345,14 @@
}
@Override
- protected void postLoadRight() throws MetaMatrixComponentException,
+ protected void loadRight() throws MetaMatrixComponentException,
MetaMatrixProcessingException {
- sortRight();
- }
-
- protected void sortRight() throws MetaMatrixComponentException,
- TupleSourceNotFoundException, BlockedOnMemoryException {
if (this.processingSortRight == SortOption.SORT || this.processingSortRight == SortOption.SORT_DISTINCT) {
if (this.rightSort == null) {
List expressions = this.joinNode.getRightExpressions();
- this.rightSort = new SortUtility(this.rightSource.getTupleSourceID(),
+ this.rightSort = new SortUtility(this.rightSource.getIterator(),
expressions, Collections.nCopies(expressions.size(), OrderBy.ASC), processingSortRight == SortOption.SORT_DISTINCT?Mode.DUP_REMOVE_SORT:Mode.SORT,
- this.joinNode.getBufferManager(), this.joinNode.getConnectionID(), true);
+ this.joinNode.getBufferManager(), this.joinNode.getConnectionID());
this.rightSource.markDistinct(processingSortRight == SortOption.SORT_DISTINCT && expressions.size() == this.rightSource.getOuterVals().size());
}
this.rightSource.setTupleSource(rightSort.sort());
Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/NestedLoopJoinStrategy.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/NestedLoopJoinStrategy.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/NestedLoopJoinStrategy.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -24,6 +24,9 @@
import java.util.List;
+import com.metamatrix.api.exception.MetaMatrixComponentException;
+import com.metamatrix.api.exception.MetaMatrixProcessingException;
+
/**
* Nested loop is currently implemented as a degenerate case of merge join.
*
Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/PartitionedSortJoin.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/PartitionedSortJoin.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/PartitionedSortJoin.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -29,12 +29,9 @@
import com.metamatrix.api.exception.MetaMatrixComponentException;
import com.metamatrix.api.exception.MetaMatrixProcessingException;
import com.metamatrix.api.exception.query.CriteriaEvaluationException;
-import com.metamatrix.common.buffer.BlockedOnMemoryException;
import com.metamatrix.common.buffer.IndexedTupleSource;
-import com.metamatrix.common.buffer.MemoryNotAvailableException;
import com.metamatrix.common.buffer.TupleBatch;
-import com.metamatrix.common.buffer.TupleSourceID;
-import com.metamatrix.common.buffer.TupleSourceNotFoundException;
+import com.metamatrix.common.buffer.TupleBuffer;
/**
* Extends the basic fully sorted merge join to check for conditions necessary
@@ -59,8 +56,7 @@
private List[] endTuples;
private List<Boolean> overlap = new ArrayList<Boolean>();
private List<Integer> endRows = new ArrayList<Integer>();
- private List<TupleSourceID> partitionIds = new ArrayList<TupleSourceID>();
- private List<TupleCollector> tupleCollectors = new ArrayList<TupleCollector>();
+ private List<TupleBuffer> partitionIds = new ArrayList<TupleBuffer>();
private int currentPartition;
private IndexedTupleSource currentSource;
private SourceState sortedSource;
@@ -75,21 +71,15 @@
}
@Override
- public void close() throws TupleSourceNotFoundException,
- MetaMatrixComponentException {
+ public void close() {
super.close();
- for (TupleSourceID tupleSourceID : this.partitionIds) {
- try {
- this.joinNode.getBufferManager().removeTupleSource(tupleSourceID);
- } catch (TupleSourceNotFoundException e) {
-
- }
+ for (TupleBuffer tupleSourceID : this.partitionIds) {
+ tupleSourceID.remove();
}
this.endTuples = null;
this.overlap.clear();
this.endRows.clear();
this.partitionIds.clear();
- this.tupleCollectors.clear();
this.currentSource = null;
this.sortedSource = null;
this.partitionedSource = null;
@@ -97,8 +87,7 @@
}
@Override
- public void initialize(JoinNode joinNode)
- throws MetaMatrixComponentException {
+ public void initialize(JoinNode joinNode) {
super.initialize(joinNode);
this.currentPartition = 0;
this.partitioned = false;
@@ -106,8 +95,7 @@
this.matchEnd = -1;
}
- //TODO: save partial work or combine with the sort operation
- public void computeBatchBounds(SourceState state) throws TupleSourceNotFoundException, MetaMatrixComponentException {
+ public void computeBatchBounds(SourceState state) throws MetaMatrixComponentException, MetaMatrixProcessingException {
if (endTuples != null) {
return;
}
@@ -115,22 +103,16 @@
ArrayList<List<?>> bounds = new ArrayList<List<?>>();
int beginRow = 1;
while (beginRow <= state.getRowCount()) {
- TupleBatch batch = null;
- try {
- batch = this.joinNode.getBufferManager().pinTupleBatch(state.getTupleSourceID(), beginRow);
- if (batch.getRowCount() == 0) {
- break;
- }
- beginRow = batch.getEndRow() + 1;
- this.joinNode.getBufferManager().unpinTupleBatch(state.getTupleSourceID(), batch.getBeginRow());
- if (!bounds.isEmpty()) {
- overlap.add(comp.compare(bounds.get(bounds.size() - 1), batch.getTuple(batch.getBeginRow())) == 0);
- }
- bounds.add(batch.getTuple(batch.getEndRow()));
- endRows.add(batch.getEndRow());
- } catch (MemoryNotAvailableException e) {
- throw BlockedOnMemoryException.INSTANCE;
- }
+ TupleBatch batch = state.getTupleBuffer().getBatch(beginRow);
+ if (batch.getRowCount() == 0) {
+ break;
+ }
+ beginRow = batch.getEndRow() + 1;
+ if (!bounds.isEmpty()) {
+ overlap.add(comp.compare(bounds.get(bounds.size() - 1), batch.getTuple(batch.getBeginRow())) == 0);
+ }
+ bounds.add(batch.getTuple(batch.getEndRow()));
+ endRows.add(batch.getEndRow());
}
this.endTuples = bounds.toArray(new List[bounds.size()]);
}
@@ -139,13 +121,13 @@
protected void loadLeft() throws MetaMatrixComponentException,
MetaMatrixProcessingException {
//always buffer to determine row counts
- this.leftSource.collectTuples();
+ this.leftSource.getTupleBuffer();
}
@Override
protected void loadRight() throws MetaMatrixComponentException,
MetaMatrixProcessingException {
- super.loadRight();
+ this.rightSource.getTupleBuffer();
int maxRows = this.joinNode.getBatchSize() * MAX_PARTITIONS;
if (processingSortRight == SortOption.SORT
&& this.leftSource.getRowCount() < maxRows
@@ -156,42 +138,30 @@
&& this.rightSource.getRowCount() * 4 < this.leftSource.getRowCount()) {
this.processingSortLeft = SortOption.PARTITION;
}
- sortRight();
+ super.loadRight(); //sort right if needed
if (this.processingSortLeft == SortOption.PARTITION) {
computeBatchBounds(this.rightSource);
this.sortedSource = this.rightSource;
this.partitionedSource = this.leftSource;
}
- }
-
- @Override
- protected void postLoadLeft() throws MetaMatrixComponentException,
- MetaMatrixProcessingException {
- super.postLoadLeft();
-
+ super.loadLeft(); //sort left if needed
if (this.processingSortRight == SortOption.PARTITION) {
computeBatchBounds(this.leftSource);
this.sortedSource = this.leftSource;
this.partitionedSource = this.rightSource;
}
-
if (this.processingSortLeft == SortOption.PARTITION) {
partitionSource(true);
- }
- }
-
- @Override
- protected void postLoadRight() throws MetaMatrixComponentException,
- MetaMatrixProcessingException {
- if (this.processingSortRight == SortOption.PARTITION) {
+ }
+ if (this.processingSortRight == SortOption.PARTITION) {
partitionSource(false);
if (this.processingSortRight == SortOption.SORT) {
//degrade to a merge join
- sortRight();
+ super.loadRight();
}
- }
+ }
}
-
+
private void partitionSource(boolean left) throws MetaMatrixComponentException,
MetaMatrixProcessingException {
if (partitioned) {
@@ -206,14 +176,13 @@
return;
}
if (endTuples.length < 2) {
- partitionIds.add(this.partitionedSource.getTupleSourceID());
+ partitionIds.add(this.partitionedSource.getTupleBuffer());
} else {
if (partitionIds.isEmpty()) {
for (int i = 0; i < endTuples.length; i++) {
- TupleCollector tc = new TupleCollector(this.partitionedSource.createSourceTupleSource(), this.joinNode.getBufferManager());
+ TupleBuffer tc = this.partitionedSource.createSourceTupleBuffer();
tc.setBatchSize(Math.max(1, this.joinNode.getBatchSize()/4));
- this.tupleCollectors.add(tc);
- this.partitionIds.add(tc.getTupleSourceID());
+ this.partitionIds.add(tc);
}
}
while (this.partitionedSource.getIterator().hasNext()) {
@@ -222,14 +191,14 @@
if (index < 0) {
index = -index - 1;
}
- if (index > this.tupleCollectors.size() -1) {
+ if (index > this.partitionIds.size() -1) {
continue;
}
while (index > 0 && this.overlap.get(index - 1)
&& compare(tuple, this.endTuples[index - 1], this.partitionedSource.getExpressionIndexes(), this.sortedSource.getExpressionIndexes()) == 0) {
index--;
}
- this.tupleCollectors.get(index).addTuple(tuple);
+ this.partitionIds.get(index).addTuple(tuple);
}
this.partitionedSource.getIterator().setPosition(1);
}
@@ -247,16 +216,15 @@
}
while (currentPartition < partitionIds.size()) {
if (currentSource == null) {
- if (!this.tupleCollectors.isEmpty()) {
- this.tupleCollectors.get(currentPartition).close();
+ if (!this.partitionIds.isEmpty()) {
+ this.partitionIds.get(currentPartition).close();
}
- currentSource = this.joinNode.getBufferManager().getTupleSource(partitionIds.get(currentPartition));
+ currentSource = partitionIds.get(currentPartition).createIndexedTupleSource();
}
int beginIndex = currentPartition>0?endRows.get(currentPartition - 1)+1:1;
- this.sortedSource.getIterator().setPosition(beginIndex);
- List[] batch = this.sortedSource.getIterator().getBatch().getAllTuples();
+ List[] batch = this.sortedSource.getTupleBuffer().getBatch(beginIndex).getAllTuples();
while (partitionedTuple != null || currentSource.hasNext()) {
if (partitionedTuple == null) {
@@ -283,7 +251,7 @@
}
}
if (matchEnd == batch.length - 1 && currentPartition < overlap.size() && overlap.get(currentPartition)) {
- this.tupleCollectors.get(currentPartition + 1).addTuple(partitionedTuple);
+ this.partitionIds.get(currentPartition + 1).addTuple(partitionedTuple);
}
}
while (matchBegin <= matchEnd) {
Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/RelationalNode.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/RelationalNode.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/RelationalNode.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -44,6 +44,7 @@
import com.metamatrix.query.processor.DescribableUtil;
import com.metamatrix.query.processor.ProcessorDataManager;
import com.metamatrix.query.processor.ProcessorPlan;
+import com.metamatrix.query.processor.BatchCollector.BatchProducer;
import com.metamatrix.query.sql.LanguageObject;
import com.metamatrix.query.sql.lang.SubqueryContainer;
import com.metamatrix.query.sql.symbol.AliasSymbol;
@@ -52,7 +53,7 @@
import com.metamatrix.query.util.CommandContext;
import com.metamatrix.query.util.ErrorMessageKeys;
-public abstract class RelationalNode implements Cloneable, Describable{
+public abstract class RelationalNode implements Cloneable, Describable, BatchProducer {
// External context and state
private CommandContext context;
@@ -155,6 +156,11 @@
public void setElements(List elements) {
this.elements = elements;
}
+
+ @Override
+ public List getOutputElements() {
+ return getElements();
+ }
public List getElements() {
return this.elements;
Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SortNode.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SortNode.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SortNode.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -30,11 +30,9 @@
import com.metamatrix.api.exception.MetaMatrixComponentException;
import com.metamatrix.api.exception.MetaMatrixProcessingException;
import com.metamatrix.common.buffer.BlockedException;
-import com.metamatrix.common.buffer.BlockedOnMemoryException;
-import com.metamatrix.common.buffer.MemoryNotAvailableException;
import com.metamatrix.common.buffer.TupleBatch;
-import com.metamatrix.common.buffer.TupleSourceID;
-import com.metamatrix.common.buffer.TupleSourceNotFoundException;
+import com.metamatrix.common.buffer.TupleBuffer;
+import com.metamatrix.query.processor.BatchIterator;
import com.metamatrix.query.processor.relational.SortUtility.Mode;
import com.metamatrix.query.sql.lang.OrderBy;
@@ -45,13 +43,11 @@
private Mode mode = Mode.SORT;
private SortUtility sortUtility;
- private int phase = COLLECTION;
- private TupleSourceID outputID;
+ private int phase = SORT;
+ private TupleBuffer outputID;
private int rowCount = -1;
private int outputBeginRow = 1;
- private BatchCollector collector;
- private static final int COLLECTION = 1;
private static final int SORT = 2;
private static final int OUTPUT = 3;
@@ -62,11 +58,10 @@
public void reset() {
super.reset();
sortUtility = null;
- phase = COLLECTION;
+ phase = SORT;
outputID = null;
rowCount = -1;
outputBeginRow = 1;
- this.collector = null;
}
public void setSortElements(List sortElements, List<Boolean> sortTypes) {
@@ -90,56 +85,33 @@
throws MetaMatrixComponentException, MetaMatrixProcessingException {
super.open();
- this.collector = new BatchCollector(this.getChildren()[0]);
}
public TupleBatch nextBatchDirect()
throws BlockedException, MetaMatrixComponentException, MetaMatrixProcessingException {
-
- try {
-
- if(this.phase == COLLECTION) {
- collectionPhase();
- }
-
- if(this.phase == SORT) {
- sortPhase();
- }
-
- return outputPhase();
- } catch(TupleSourceNotFoundException e) {
- throw new MetaMatrixComponentException(e, e.getMessage());
+ if(this.phase == SORT) {
+ sortPhase();
}
+
+ return outputPhase();
}
- private void collectionPhase() throws BlockedException, TupleSourceNotFoundException, MetaMatrixComponentException, MetaMatrixProcessingException {
- try {
- collector.collectTuples();
- } catch (BlockedOnMemoryException e) {
- throw e;
- } catch (BlockedException e) {
- if (mode != Mode.DUP_REMOVE || !collector.collectedAny()) {
- throw e;
- }
- }
- if (this.sortUtility == null) {
- this.sortUtility = new SortUtility(collector.getTupleSourceID(), sortElements,
+ private void sortPhase() throws BlockedException, MetaMatrixComponentException, MetaMatrixProcessingException {
+ if (this.sortUtility == null) {
+ this.sortUtility = new SortUtility(new BatchIterator(getChildren()[0]), sortElements,
sortTypes, this.mode, getBufferManager(),
- getConnectionID(), true);
+ getConnectionID());
}
- this.phase = SORT;
- }
-
- private void sortPhase() throws BlockedException, MetaMatrixComponentException {
this.outputID = this.sortUtility.sort();
this.phase = OUTPUT;
}
- private TupleBatch outputPhase() throws BlockedException, TupleSourceNotFoundException, MetaMatrixComponentException {
+ private TupleBatch outputPhase() throws BlockedException, MetaMatrixComponentException {
if (this.rowCount == -1) {
- this.rowCount = getBufferManager().getFinalRowCount(outputID);
- if (this.rowCount == -1) {
- this.phase = this.collector.isDone()?SORT:COLLECTION;
+ if (this.outputID.isFinal()) {
+ this.rowCount = this.outputID.getRowCount();
+ } else {
+ this.phase = SORT;
}
}
if(this.rowCount == 0 || (this.rowCount != -1 && this.outputBeginRow > this.rowCount)) {
@@ -148,23 +120,17 @@
return terminationBatch;
}
int beginPinned = this.outputBeginRow;
- try {
- TupleBatch outputBatch = getBufferManager().pinTupleBatch(outputID, beginPinned);
-
- this.outputBeginRow += outputBatch.getRowCount();
+ TupleBatch outputBatch = this.outputID.getBatch(beginPinned);
+
+ this.outputBeginRow += outputBatch.getRowCount();
- outputBatch = removeUnrelatedColumns(outputBatch);
+ outputBatch = removeUnrelatedColumns(outputBatch);
- if(rowCount != -1 && outputBeginRow > rowCount) {
- outputBatch.setTerminationFlag(true);
- }
-
- return outputBatch;
- } catch(MemoryNotAvailableException e) {
- throw BlockedOnMemoryException.INSTANCE;
- } finally {
- getBufferManager().unpinTupleBatch(outputID, beginPinned);
+ if(rowCount != -1 && outputBeginRow > rowCount) {
+ outputBatch.setTerminationFlag(true);
}
+
+ return outputBatch;
}
private TupleBatch removeUnrelatedColumns(TupleBatch outputBatch) {
@@ -182,17 +148,9 @@
public void close() throws MetaMatrixComponentException {
if (!isClosed()) {
super.close();
- try {
- if(this.collector != null) {
- this.collector.close();
- this.collector = null;
- }
- if(this.outputID != null) {
- getBufferManager().removeTupleSource(outputID);
- this.outputID = null;
- }
- } catch(TupleSourceNotFoundException e) {
- throw new MetaMatrixComponentException(e, e.getMessage());
+ if(this.outputID != null) {
+ this.outputID.remove();
+ this.outputID = null;
}
}
}
Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SortUtility.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SortUtility.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SortUtility.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -23,417 +23,268 @@
package com.metamatrix.query.processor.relational;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import com.metamatrix.api.exception.MetaMatrixComponentException;
import com.metamatrix.api.exception.MetaMatrixProcessingException;
-import com.metamatrix.common.buffer.BlockedOnMemoryException;
+import com.metamatrix.common.buffer.BlockedException;
import com.metamatrix.common.buffer.BufferManager;
import com.metamatrix.common.buffer.IndexedTupleSource;
-import com.metamatrix.common.buffer.MemoryNotAvailableException;
-import com.metamatrix.common.buffer.TupleBatch;
-import com.metamatrix.common.buffer.TupleSourceID;
-import com.metamatrix.common.buffer.TupleSourceNotFoundException;
-import com.metamatrix.common.buffer.BufferManager.TupleSourceStatus;
+import com.metamatrix.common.buffer.TupleBuffer;
+import com.metamatrix.common.buffer.TupleSource;
import com.metamatrix.common.buffer.BufferManager.TupleSourceType;
import com.metamatrix.core.util.Assertion;
import com.metamatrix.query.sql.lang.OrderBy;
import com.metamatrix.query.sql.symbol.SingleElementSymbol;
/**
+ * Implements several modes of a multi-pass sort.
*/
public class SortUtility {
public enum Mode {
SORT,
- DUP_REMOVE,
+ /** Removes duplicates, no sort elements need to be specified.
+ * may perform additional passes as new batches become available
+ */
+ DUP_REMOVE,
+ /** Removes duplicates, but guarantees order based upon the sort elements.
+ */
DUP_REMOVE_SORT
}
+
+ /**
+ * state holder for the merge algorithm
+ */
+ private class SortedSublist implements Comparable<SortedSublist> {
+ List<?> tuple;
+ int index;
+ boolean duplicate;
+ IndexedTupleSource its;
+
+ @Override
+ public int compareTo(SortedSublist o) {
+ return comparator.compare(this.tuple, o.tuple);
+ }
+ }
//constructor state
- private TupleSourceID sourceID;
- protected List sortElements;
- protected List<Boolean> sortTypes;
+ private TupleSource sourceID;
private Mode mode;
protected BufferManager bufferManager;
private String groupName;
- private boolean useAllColumns;
-
- //init state
- private int batchSize;
protected List schema;
- protected int[] sortCols;
private ListNestedSortComparator comparator;
- private TupleSourceID outputID;
- private IndexedTupleSource outTs;
+ private TupleBuffer output;
private boolean doneReading;
- private int sortPhaseRow = 1;
private int phase = INITIAL_SORT;
- protected List<TupleSourceID> activeTupleIDs = new ArrayList<TupleSourceID>();
- private List<TupleBatch> workingBatches;
+ protected List<TupleBuffer> activeTupleBuffers = new ArrayList<TupleBuffer>();
private int masterSortIndex;
- private TupleSourceID mergedID;
- private TupleSourceID tempOutId;
// Phase constants for readability
private static final int INITIAL_SORT = 1;
private static final int MERGE = 2;
private static final int DONE = 3;
- public SortUtility(TupleSourceID sourceID, List sortElements, List<Boolean> sortTypes, boolean removeDups, BufferManager bufferMgr,
- String groupName) {
- this(sourceID, sortElements, sortTypes, removeDups?Mode.DUP_REMOVE_SORT:Mode.SORT, bufferMgr, groupName, false);
- }
-
- public SortUtility(TupleSourceID sourceID, List sortElements, List<Boolean> sortTypes, Mode mode, BufferManager bufferMgr,
- String groupName, boolean useAllColumns) {
+ public SortUtility(TupleSource sourceID, List sortElements, List<Boolean> sortTypes, Mode mode, BufferManager bufferMgr,
+ String groupName) {
this.sourceID = sourceID;
- this.sortElements = sortElements;
- this.sortTypes = sortTypes;
this.mode = mode;
this.bufferManager = bufferMgr;
this.groupName = groupName;
- this.useAllColumns = useAllColumns;
+ this.schema = this.sourceID.getSchema();
+ int distinctIndex = sortElements != null? sortElements.size() - 1:0;
+ if (mode != Mode.SORT) {
+ if (sortElements != null && sortElements.size() < schema.size()) {
+ sortElements = new ArrayList(sortElements);
+ List toAdd = new ArrayList(schema);
+ toAdd.removeAll(sortElements);
+ sortElements.addAll(toAdd);
+ sortTypes = new ArrayList<Boolean>(sortTypes);
+ sortTypes.addAll(Collections.nCopies(sortElements.size() - sortTypes.size(), OrderBy.ASC));
+ } else {
+ sortElements = this.schema;
+ sortTypes = Collections.nCopies(sortElements.size(), OrderBy.ASC);
+ }
+ }
+
+ int[] cols = new int[sortElements.size()];
+
+ Iterator iter = sortElements.iterator();
+
+ for (int i = 0; i < cols.length; i++) {
+ SingleElementSymbol elem = (SingleElementSymbol)iter.next();
+
+ cols[i] = schema.indexOf(elem);
+ Assertion.assertTrue(cols[i] != -1);
+ }
+ this.comparator = new ListNestedSortComparator(cols, sortTypes);
+ this.comparator.setDistinctIndex(distinctIndex);
}
public boolean isDone() {
return this.doneReading && this.phase == DONE;
}
- public TupleSourceID sort()
- throws BlockedOnMemoryException, MetaMatrixComponentException {
+ public TupleBuffer sort()
+ throws MetaMatrixComponentException, MetaMatrixProcessingException {
- try {
- // One time setup
- if(this.schema == null) {
- initialize();
- }
-
- if(this.phase == INITIAL_SORT) {
- initialSort();
- }
-
- if(this.phase == MERGE) {
- try {
- mergePhase();
- } finally {
- if (this.mergedID != null) {
- this.bufferManager.removeTupleSource(mergedID);
- this.mergedID = null;
- }
- if (this.tempOutId != null) {
- this.bufferManager.removeTupleSource(tempOutId);
- this.tempOutId = null;
- }
- if (workingBatches != null) {
- for (int i = 0; i < workingBatches.size(); i++) {
- TupleBatch tupleBatch = workingBatches.get(i);
- if (tupleBatch != null) {
- unpinWorkingBatch(i, tupleBatch);
- }
- }
- }
- workingBatches = null;
- }
- }
- if (this.outputID != null) {
- return this.outputID;
- }
- return this.activeTupleIDs.get(0);
- } catch(TupleSourceNotFoundException e) {
- throw new MetaMatrixComponentException(e, e.getMessage());
+ if(this.phase == INITIAL_SORT) {
+ initialSort();
}
-
+
+ if(this.phase == MERGE) {
+ mergePhase();
+ }
+ if (this.output != null) {
+ return this.output;
+ }
+ return this.activeTupleBuffers.get(0);
}
- private TupleSourceID createTupleSource() throws MetaMatrixComponentException {
- return bufferManager.createTupleSource(this.schema, this.groupName, TupleSourceType.PROCESSOR);
+ private TupleBuffer createTupleBuffer() throws MetaMatrixComponentException {
+ return bufferManager.createTupleBuffer(this.schema, this.groupName, TupleSourceType.PROCESSOR);
}
-
- protected void initialSort() throws BlockedOnMemoryException, TupleSourceNotFoundException, MetaMatrixComponentException {
+
+ /**
+ * creates sort sublists stored in tuplebuffers
+ */
+ protected void initialSort() throws MetaMatrixComponentException, MetaMatrixProcessingException {
while(!doneReading) {
- ArrayList<int[]> pinned = new ArrayList<int[]>(); // of int[2] representing begin, end
List<List<Object>> workingTuples = new ArrayList<List<Object>>();
- // Load data until out of memory
- while(!doneReading) {
+ while(!doneReading) { //TODO: limit rows
try {
- // Load and pin batch
- TupleBatch batch = bufferManager.pinTupleBatch(sourceID, sortPhaseRow);
-
- if (batch.getRowCount() == 0) {
- if (bufferManager.getStatus(sourceID) == TupleSourceStatus.FULL) {
- doneReading = true;
- }
- break;
- }
- // Remember pinned rows
- pinned.add(new int[] { sortPhaseRow, batch.getEndRow() });
-
- addTuples(workingTuples, batch);
-
- sortPhaseRow += batch.getRowCount();
- } catch(MemoryNotAvailableException e) {
- break;
- }
+ List<?> tuple = sourceID.nextTuple();
+
+ if (tuple == null) {
+ doneReading = true;
+ break;
+ }
+
+ addTuple(workingTuples, tuple);
+ } catch(BlockedException e) {
+ if (workingTuples.isEmpty() && (mode != Mode.DUP_REMOVE || activeTupleBuffers.isEmpty())) {
+ throw e; //block if no work can be performed
+ }
+ break;
+ }
}
if(workingTuples.isEmpty()) {
- break;
+ break;
}
- TupleSourceID activeID = createTupleSource();
- activeTupleIDs.add(activeID);
- int sortedThisPass = workingTuples.size();
+ TupleBuffer activeID = createTupleBuffer();
+ activeTupleBuffers.add(activeID);
if (this.mode == Mode.SORT) {
//perform a stable sort
Collections.sort(workingTuples, comparator);
}
- int writeBegin = 1;
- while(writeBegin <= sortedThisPass) {
- int writeEnd = Math.min(sortedThisPass, writeBegin + batchSize - 1);
-
- TupleBatch writeBatch = new TupleBatch(writeBegin, workingTuples.subList(writeBegin-1, writeEnd));
- bufferManager.addTupleBatch(activeID, writeBatch);
- writeBegin += writeBatch.getRowCount();
- }
-
- // Clean up - unpin rows
- for (int[] bounds : pinned) {
- bufferManager.unpinTupleBatch(sourceID, bounds[0]);
- }
+ for (List<Object> list : workingTuples) {
+ activeID.addTuple(list);
+ }
}
-
- if (!doneReading && (mode != Mode.DUP_REMOVE || this.activeTupleIDs.isEmpty())) {
- throw BlockedOnMemoryException.INSTANCE;
- }
- if (this.activeTupleIDs.isEmpty()) {
- activeTupleIDs.add(createTupleSource());
+ if (this.activeTupleBuffers.isEmpty()) {
+ activeTupleBuffers.add(createTupleBuffer());
}
- // Clean up
this.phase = MERGE;
}
- protected void addTuples(List workingTuples, TupleBatch batch) {
+ protected void addTuple(List workingTuples, List tuple) {
if (this.mode == Mode.SORT) {
- workingTuples.addAll(Arrays.asList(batch.getAllTuples()));
+ workingTuples.add(tuple);
return;
}
- for (List<Object> list : batch.getAllTuples()) {
- int index = Collections.binarySearch(workingTuples, list, comparator);
- if (index >= 0) {
- continue;
- }
- workingTuples.add(-index - 1, list);
+ int index = Collections.binarySearch(workingTuples, tuple, comparator);
+ if (index >= 0) {
+ return; //it's already there
}
+ workingTuples.add(-index - 1, tuple);
}
-
- protected void mergePhase() throws BlockedOnMemoryException, MetaMatrixComponentException, TupleSourceNotFoundException {
- appendOutput();
- TupleCollector tempCollector = null;
- while(this.activeTupleIDs.size() > 1) {
- // Load and pin batch from sorted sublists while memory available
- this.workingBatches = new ArrayList<TupleBatch>(activeTupleIDs.size());
+
+ protected void mergePhase() throws MetaMatrixComponentException, MetaMatrixProcessingException {
+ while(this.activeTupleBuffers.size() > 1) {
+ ArrayList<SortedSublist> workingTuples = new ArrayList<SortedSublist>(activeTupleBuffers.size());
+
+ TupleBuffer merged = createTupleBuffer();
+
int sortedIndex = 0;
- for(; sortedIndex<activeTupleIDs.size(); sortedIndex++) {
- TupleSourceID activeID = activeTupleIDs.get(sortedIndex);
- try {
- TupleBatch sortedBatch = bufferManager.pinTupleBatch(activeID, 1);
- workingBatches.add(sortedBatch);
- } catch(MemoryNotAvailableException e) {
- break;
- }
+ for(; sortedIndex<activeTupleBuffers.size(); sortedIndex++) { //TODO: limit activeTupleIDs
+ TupleBuffer activeID = activeTupleBuffers.get(sortedIndex);
+ SortedSublist sortedSublist = new SortedSublist();
+ sortedSublist.its = activeID.createIndexedTupleSource();
+ sortedSublist.index = sortedIndex;
+ addWorkingTuple(workingTuples, sortedSublist);
}
- //if we cannot make progress, just block for now
- if (workingBatches.size() < 2) {
- throw BlockedOnMemoryException.INSTANCE;
- }
-
- // Initialize pointers into working batches
- int[] workingPointers = new int[workingBatches.size()];
- Arrays.fill(workingPointers, 1);
-
- mergedID = createTupleSource();
-
- // Merge from working sorted batches
- TupleCollector collector = new TupleCollector(mergedID, this.bufferManager);
- while(true) {
- // Find least valued row among working batches
- List<?> currentRow = null;
- int chosenBatchIndex = -1;
- TupleBatch chosenBatch = null;
- for(int i=0; i<workingBatches.size(); i++) {
- TupleBatch batch = workingBatches.get(i);
- if(batch == null) {
- continue;
+ // iteratively process the lowest tuple
+ while (workingTuples.size() > 0) {
+ SortedSublist sortedSublist = workingTuples.remove(0);
+ if (!sortedSublist.duplicate) {
+ merged.addTuple(sortedSublist.tuple);
+ if (this.output != null && sortedSublist.index != masterSortIndex && sortedIndex > masterSortIndex) {
+ this.output.addTuple(sortedSublist.tuple);
}
- List<?> testRow = batch.getTuple(workingPointers[i]);
- int compare = -1;
- if (currentRow != null) {
- compare = comparator.compare(testRow, currentRow);
- }
- if(compare < 0) {
- // Found lower row
- currentRow = testRow;
- chosenBatchIndex = i;
- chosenBatch = batch;
- } else if (compare == 0 && this.mode != Mode.SORT) {
- incrementWorkingBatch(i, workingPointers, batch);
- }
- }
+ }
+ addWorkingTuple(workingTuples, sortedSublist);
+ }
- // Check for termination condition - all batches must have been null
- if(currentRow == null) {
- break;
- }
- // Output the row and update pointers
- collector.addTuple(currentRow);
- if (this.outputID != null && chosenBatchIndex != masterSortIndex && sortedIndex > masterSortIndex) {
- if (tempCollector == null) {
- tempOutId = createTupleSource();
- tempCollector = new TupleCollector(tempOutId, this.bufferManager);
- }
- tempCollector.addTuple(currentRow);
- }
- incrementWorkingBatch(chosenBatchIndex, workingPointers, chosenBatch);
- }
-
- // Save without closing
- collector.saveBatch();
-
// Remove merged sublists
for(int i=0; i<sortedIndex; i++) {
- TupleSourceID id = activeTupleIDs.remove(0);
- if (!id.equals(this.outputID)) {
- bufferManager.removeTupleSource(id);
+ TupleBuffer id = activeTupleBuffers.remove(0);
+ if (id != this.output) {
+ id.remove();
}
}
- this.activeTupleIDs.add(mergedID);
- this.mergedID = null;
+ this.activeTupleBuffers.add(merged);
masterSortIndex = masterSortIndex - sortedIndex + 1;
if (masterSortIndex < 0) {
- masterSortIndex = this.activeTupleIDs.size() - 1;
+ masterSortIndex = this.activeTupleBuffers.size() - 1;
}
- }
+ }
- if (tempCollector != null) {
- tempCollector.close();
- this.outTs = this.bufferManager.getTupleSource(tempOutId);
- appendOutput();
- }
-
// Close sorted source (all others have been removed)
if (doneReading) {
- bufferManager.setStatus(activeTupleIDs.get(0), TupleSourceStatus.FULL);
- if (this.outputID != null) {
- bufferManager.setStatus(outputID, TupleSourceStatus.FULL);
+ activeTupleBuffers.get(0).close();
+ if (this.output != null) {
+ this.output.close();
}
this.phase = DONE;
return;
}
Assertion.assertTrue(mode == Mode.DUP_REMOVE);
- if (this.outputID == null) {
- this.outputID = activeTupleIDs.get(0);
+ if (this.output == null) {
+ this.output = activeTupleBuffers.get(0);
}
this.phase = INITIAL_SORT;
}
- private void appendOutput() throws TupleSourceNotFoundException,
- MetaMatrixComponentException {
- if (this.outTs != null) {
- //transfer the new dup removed tuples to the output id
- TupleCollector tc = new TupleCollector(outputID, this.bufferManager);
- try {
- try {
- while (outTs.hasNext()) {
- tc.addTuple(outTs.nextTuple());
- }
- } catch (MetaMatrixProcessingException e) {
- throw new MetaMatrixComponentException(e);
- }
- } finally {
- tc.saveBatch();
- }
- outTs.closeSource();
- outTs = null;
+ private void addWorkingTuple(ArrayList<SortedSublist> workingTuples, SortedSublist sortedSublist) throws MetaMatrixComponentException, MetaMatrixProcessingException {
+ sortedSublist.tuple = null;
+ try {
+ sortedSublist.tuple = sortedSublist.its.nextTuple();
+ } catch (BlockedException e) {
+ //intermediate sources aren't closed
+ }
+ if (sortedSublist.tuple == null) {
+ return; // done with this sublist
}
- }
+ sortedSublist.duplicate = false;
+ int index = Collections.binarySearch(workingTuples, sortedSublist);
+ if (index >= 0) {
+ sortedSublist.duplicate = mode != Mode.SORT;
+ workingTuples.add(index, sortedSublist);
+ } else {
+ workingTuples.add(-index - 1, sortedSublist);
+ }
+ }
- /**
- * Increment the working batch at batchIndex. The currentBatch is the currentBatch
- * for that batchIndex, which we already happen to have. Return whether the batch
- * was changed or not. True = changed.
- */
- private void incrementWorkingBatch(int batchIndex, int[] workingPointers, TupleBatch currentBatch) throws BlockedOnMemoryException, TupleSourceNotFoundException, MetaMatrixComponentException {
- workingPointers[batchIndex] += 1;
- if(workingPointers[batchIndex] > currentBatch.getEndRow()) {
- TupleSourceID tsID = unpinWorkingBatch(batchIndex, currentBatch);
-
- int beginRow = workingPointers[batchIndex];
-
- try {
- TupleBatch newBatch = bufferManager.pinTupleBatch(tsID, beginRow);
- if(newBatch.getRowCount() == 0) {
- // Done with this working batch
- workingBatches.set(batchIndex, null);
- } else {
- workingBatches.set(batchIndex, newBatch);
- }
- } catch(MemoryNotAvailableException e) {
- throw BlockedOnMemoryException.INSTANCE;
- }
- }
- }
-
- private TupleSourceID unpinWorkingBatch(int batchIndex,
- TupleBatch currentBatch) throws TupleSourceNotFoundException,
- MetaMatrixComponentException {
- TupleSourceID tsID = activeTupleIDs.get(batchIndex);
- int lastBeginRow = currentBatch.getBeginRow();
- bufferManager.unpinTupleBatch(tsID, lastBeginRow);
- return tsID;
- }
-
- private void initialize() throws TupleSourceNotFoundException, MetaMatrixComponentException {
- this.schema = this.bufferManager.getTupleSchema(this.sourceID);
- this.batchSize = bufferManager.getProcessorBatchSize();
- int distinctIndex = sortElements != null? sortElements.size() - 1:0;
- if (useAllColumns && mode != Mode.SORT) {
- if (this.sortElements != null) {
- this.sortElements = new ArrayList(this.sortElements);
- List toAdd = new ArrayList(schema);
- toAdd.removeAll(this.sortElements);
- this.sortElements.addAll(toAdd);
- this.sortTypes = new ArrayList<Boolean>(this.sortTypes);
- this.sortTypes.addAll(Collections.nCopies(this.sortElements.size() - this.sortTypes.size(), OrderBy.ASC));
- } else {
- this.sortElements = this.schema;
- this.sortTypes = Collections.nCopies(this.sortElements.size(), OrderBy.ASC);
- }
- }
-
- int[] cols = new int[sortElements.size()];
-
- Iterator iter = sortElements.iterator();
-
- for (int i = 0; i < cols.length; i++) {
- SingleElementSymbol elem = (SingleElementSymbol)iter.next();
-
- cols[i] = schema.indexOf(elem);
- Assertion.assertTrue(cols[i] != -1);
- }
- this.sortCols = cols;
- this.comparator = new ListNestedSortComparator(sortCols, sortTypes);
- this.comparator.setDistinctIndex(distinctIndex);
- }
-
public boolean isDistinct() {
return this.comparator.isDistinct();
}
Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SourceState.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SourceState.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SourceState.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -28,16 +28,16 @@
import com.metamatrix.api.exception.MetaMatrixComponentException;
import com.metamatrix.api.exception.MetaMatrixProcessingException;
import com.metamatrix.common.buffer.IndexedTupleSource;
-import com.metamatrix.common.buffer.TupleSourceID;
-import com.metamatrix.common.buffer.TupleSourceNotFoundException;
+import com.metamatrix.common.buffer.TupleBuffer;
import com.metamatrix.common.buffer.BufferManager.TupleSourceType;
+import com.metamatrix.query.processor.BatchCollector;
+import com.metamatrix.query.processor.BatchIterator;
class SourceState {
private RelationalNode source;
private BatchCollector collector;
- private IndexedTupleSource tupleSource;
- private TupleSourceID tsID;
+ private TupleBuffer buffer;
private List<Object> outerVals;
private IndexedTupleSource iterator;
private int[] expressionIndexes;
@@ -45,6 +45,8 @@
private int maxProbeMatch = 1;
private boolean distinct;
+ private boolean canBuffer = true;
+
public SourceState(RelationalNode source, List expressions) {
this.source = source;
List elements = source.getElements();
@@ -64,8 +66,8 @@
return indecies;
}
- TupleSourceID createSourceTupleSource() throws MetaMatrixComponentException {
- return this.source.getBufferManager().createTupleSource(source.getElements(), source.getConnectionID(), TupleSourceType.PROCESSOR);
+ TupleBuffer createSourceTupleBuffer() throws MetaMatrixComponentException {
+ return this.source.getBufferManager().createTupleBuffer(source.getElements(), source.getConnectionID(), TupleSourceType.PROCESSOR);
}
public List saveNext() throws MetaMatrixComponentException, MetaMatrixProcessingException {
@@ -79,52 +81,35 @@
this.currentTuple = null;
}
- public void close() throws TupleSourceNotFoundException, MetaMatrixComponentException {
+ public void close() {
closeTupleSource();
}
- private void closeTupleSource() throws TupleSourceNotFoundException,
- MetaMatrixComponentException {
- if (this.tsID != null) {
- this.source.getBufferManager().removeTupleSource(tsID);
- this.tsID = null;
+ private void closeTupleSource() {
+ if (this.buffer != null) {
+ this.buffer.remove();
+ this.buffer = null;
}
+ if (this.iterator != null) {
+ this.iterator = null;
+ }
}
- public int getRowCount() {
- return this.collector.getRowCount();
+ public int getRowCount() throws MetaMatrixComponentException, MetaMatrixProcessingException {
+ return this.getTupleBuffer().getRowCount();
}
- /**
- * Collect the underlying batches into a tuple source. Subsequent calls will return that tuple source
- */
- public TupleSourceID collectTuples() throws MetaMatrixComponentException,
- MetaMatrixProcessingException {
- if (collector == null) {
- collector = new BatchCollector(source);
- }
- TupleSourceID result = collector.collectTuples();
- if (this.tsID == null) {
- setTupleSource(result);
- }
- return result;
+ void setTupleSource(TupleBuffer result) {
+ closeTupleSource();
+ this.buffer = result;
}
-
- void setTupleSource(TupleSourceID result) throws MetaMatrixComponentException, TupleSourceNotFoundException {
- closeTupleSource();
- this.tsID = result;
- try {
- this.tupleSource = source.getBufferManager().getTupleSource(result);
- } catch (TupleSourceNotFoundException e) {
- throw new MetaMatrixComponentException(e, e.getMessage());
- }
- }
IndexedTupleSource getIterator() {
if (this.iterator == null) {
- if (this.tupleSource != null) {
- iterator = this.tupleSource;
+ if (this.buffer != null) {
+ iterator = buffer.createIndexedTupleSource();
} else {
+ canBuffer = false;
// return a TupleBatch tuplesource iterator
iterator = new BatchIterator(this.source);
}
@@ -152,8 +137,18 @@
return maxProbeMatch;
}
- public TupleSourceID getTupleSourceID() {
- return this.tsID;
+ public TupleBuffer getTupleBuffer() throws MetaMatrixComponentException, MetaMatrixProcessingException {
+ if (this.buffer == null) {
+ if (!canBuffer) {
+ throw new AssertionError("cannot buffer the source"); //$NON-NLS-1$
+ }
+ if (collector == null) {
+ collector = new BatchCollector(source, createSourceTupleBuffer());
+ }
+ TupleBuffer result = collector.collectTuples();
+ setTupleSource(result);
+ }
+ return this.buffer;
}
public boolean isDistinct() {
Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SubqueryAwareEvaluator.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SubqueryAwareEvaluator.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/SubqueryAwareEvaluator.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -30,9 +30,8 @@
import com.metamatrix.api.exception.MetaMatrixProcessingException;
import com.metamatrix.common.buffer.BlockedException;
import com.metamatrix.common.buffer.BufferManager;
-import com.metamatrix.common.buffer.TupleSourceID;
-import com.metamatrix.common.buffer.TupleSourceNotFoundException;
import com.metamatrix.query.eval.Evaluator;
+import com.metamatrix.query.processor.BatchCollector;
import com.metamatrix.query.processor.ProcessorDataManager;
import com.metamatrix.query.processor.ProcessorPlan;
import com.metamatrix.query.processor.QueryProcessor;
@@ -52,6 +51,7 @@
public class SubqueryState {
QueryProcessor processor;
+ BatchCollector collector;
boolean done;
List<?> tuple;
ProcessorPlan plan;
@@ -60,13 +60,8 @@
if (processor == null) {
return;
}
- try {
- processor.closeProcessing();
- TupleSourceID id = processor.getResultsID();
- manager.removeTupleSource(id);
- } catch (TupleSourceNotFoundException e) {
-
- }
+ processor.closeProcessing();
+ collector.getTupleBuffer().close();
processor = null;
this.done = false;
}
@@ -125,12 +120,12 @@
}
state.processor.getContext().pushVariableContext(currentContext);
}
+ state.collector = state.processor.createBatchCollector();
}
- state.processor.process(Integer.MAX_VALUE);
state.done = true;
state.processor.getProcessorPlan().reset();
}
- return new DependentValueSource(state.processor.getResultsID(), this.manager).getValueIterator(ref.getValueExpression());
+ return new DependentValueSource(state.collector.collectTuples(), this.manager.getProcessorBatchSize() / 2).getValueIterator(ref.getValueExpression());
}
}
Deleted: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/TupleCollector.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/TupleCollector.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/TupleCollector.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -1,112 +0,0 @@
-/*
- * JBoss, Home of Professional Open Source.
- * See the COPYRIGHT.txt file distributed with this work for information
- * regarding copyright ownership. Some portions may be licensed
- * to Red Hat, Inc. under one or more contributor license agreements.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301 USA.
- */
-
-package com.metamatrix.query.processor.relational;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import com.metamatrix.api.exception.MetaMatrixComponentException;
-import com.metamatrix.common.buffer.BufferManager;
-import com.metamatrix.common.buffer.TupleBatch;
-import com.metamatrix.common.buffer.TupleSourceID;
-import com.metamatrix.common.buffer.TupleSourceNotFoundException;
-import com.metamatrix.common.buffer.BufferManager.TupleSourceStatus;
-
-public class TupleCollector {
-
- private TupleSourceID tsid;
- private BufferManager bm;
-
- private int batchSize;
- private ArrayList<List<?>> batch;
- private int index;
-
- public TupleCollector(TupleSourceID tsid, BufferManager bm) throws TupleSourceNotFoundException, MetaMatrixComponentException {
- this.tsid = tsid;
- this.batchSize = bm.getProcessorBatchSize();
- this.bm = bm;
- this.index = bm.getRowCount(tsid) + 1;
- }
-
- public TupleCollector(BufferManager bm) {
- this.batchSize = bm.getProcessorBatchSize();
- this.bm = bm;
- this.index = 1;
- }
-
- public void setBatchSize(int batchSize) {
- this.batchSize = batchSize;
- }
-
- public void setTupleSourceID(TupleSourceID tsid) {
- this.tsid = tsid;
- }
-
- public boolean isEmpty() {
- return this.batch == null || this.batch.isEmpty();
- }
-
- public TupleSourceID getTupleSourceID() {
- return tsid;
- }
-
- public ArrayList<List<?>> getBatch() {
- if (batch == null) {
- batch = new ArrayList<List<?>>(batchSize/4);
- }
- return batch;
- }
-
- public void addTuple(List<?> tuple) throws TupleSourceNotFoundException, MetaMatrixComponentException {
- if (batch == null) {
- batch = new ArrayList<List<?>>(batchSize/4);
- }
- batch.add(tuple);
- if (batch.size() == batchSize) {
- saveBatch();
- }
- }
-
- public void saveBatch() throws TupleSourceNotFoundException,
- MetaMatrixComponentException {
- if (batch == null || batch.isEmpty()) {
- return;
- }
- int batchIndex = 0;
- while(batchIndex < batch.size()) {
- int writeEnd = Math.min(batch.size(), batchIndex + batchSize);
-
- TupleBatch writeBatch = new TupleBatch(index, batch.subList(batchIndex, writeEnd));
- bm.addTupleBatch(tsid, writeBatch);
- index += writeBatch.getRowCount();
- batchIndex += writeBatch.getRowCount();
- }
- batch = null;
- }
-
- public void close() throws TupleSourceNotFoundException, MetaMatrixComponentException {
- saveBatch();
- this.bm.setStatus(this.tsid, TupleSourceStatus.FULL);
- }
-
-}
Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/relational/UnionAllNode.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/processor/relational/UnionAllNode.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/processor/relational/UnionAllNode.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -28,7 +28,6 @@
import com.metamatrix.api.exception.MetaMatrixComponentException;
import com.metamatrix.api.exception.MetaMatrixProcessingException;
import com.metamatrix.common.buffer.BlockedException;
-import com.metamatrix.common.buffer.BlockedOnMemoryException;
import com.metamatrix.common.buffer.TupleBatch;
public class UnionAllNode extends RelationalNode {
@@ -81,9 +80,6 @@
sourceDone[i] = true;
activeSources--;
}
-
- } catch (BlockedOnMemoryException e) {
- throw e;
} catch(BlockedException e) {
if(i<children.length-1 && hasDependentProcedureExecutionNode(children[0])){
throw e;
Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/xml/RelationalPlanExecutor.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/processor/xml/RelationalPlanExecutor.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/processor/xml/RelationalPlanExecutor.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -29,11 +29,11 @@
import com.metamatrix.api.exception.MetaMatrixProcessingException;
import com.metamatrix.common.buffer.BlockedException;
import com.metamatrix.common.buffer.BufferManager;
-import com.metamatrix.common.buffer.TupleSource;
-import com.metamatrix.common.buffer.TupleSourceNotFoundException;
+import com.metamatrix.common.buffer.IndexedTupleSource;
import com.metamatrix.common.log.LogManager;
import com.metamatrix.query.execution.QueryExecPlugin;
import com.metamatrix.query.mapping.xml.ResultSetInfo;
+import com.metamatrix.query.processor.BatchIterator;
import com.metamatrix.query.processor.ProcessorDataManager;
import com.metamatrix.query.processor.ProcessorPlan;
import com.metamatrix.query.processor.QueryProcessor;
@@ -60,7 +60,7 @@
// flag to denote the end of rows
boolean endOfRows = false;
// results after the execution bucket.
- TupleSource tupleSource;
+ IndexedTupleSource tupleSource;
// cached current row of results.
List currentRow;
int currentRowNumber = 0;
@@ -90,20 +90,15 @@
* @see com.metamatrix.query.processor.xml.PlanExecutor#execute(java.util.Map)
*/
public void execute(Map referenceValues) throws MetaMatrixComponentException, BlockedException, MetaMatrixProcessingException {
- try {
- // set the dynamic reference values
- setReferenceValues(referenceValues);
-
- // execute
- this.internalProcessor.process(Integer.MAX_VALUE); //TODO: but an actual value here
- } catch (MetaMatrixComponentException e) {
- throw e;
- } catch (MetaMatrixProcessingException e) {
- throw e;
+ if (this.tupleSource == null) {
+ setReferenceValues(referenceValues);
+ this.tupleSource = new BatchIterator(internalProcessor);
}
+ //force execution
+ this.tupleSource.hasNext();
}
- void setReferenceValues(Map<ElementSymbol, Object> referencesValues) throws MetaMatrixComponentException {
+ void setReferenceValues(Map<ElementSymbol, Object> referencesValues) {
if (referencesValues == null || referencesValues.isEmpty()) {
return;
}
@@ -118,14 +113,6 @@
* @throws MetaMatrixComponentException
*/
public List nextRow() throws MetaMatrixComponentException, MetaMatrixProcessingException {
- if (this.tupleSource == null) {
- try {
- this.tupleSource = this.bufferMgr.getTupleSource(this.internalProcessor.getResultsID());
- } catch (TupleSourceNotFoundException e) {
- throw new MetaMatrixComponentException(e, QueryExecPlugin.Util.getString("tuple_not_found", this.resultInfo.getResultSetName())); //$NON-NLS-1$
- }
- }
-
// if we not already closed the tuple source; look for more results.
if (!endOfRows) {
@@ -166,12 +153,8 @@
* Close the executor and release all the resources.
*/
public void close() throws MetaMatrixComponentException {
- try {
- this.bufferMgr.removeTupleSource(this.internalProcessor.getResultsID());
- } catch (TupleSourceNotFoundException e) {
-
- }
- LogManager.logTrace(LogConstants.CTX_XML_PLAN, new Object[]{"removed tuple source", this.internalProcessor.getResultsID(), "for result set", this.internalProcessor.getResultsID()}); //$NON-NLS-1$ //$NON-NLS-2$
+ this.internalProcessor.closeProcessing();
+ LogManager.logTrace(LogConstants.CTX_XML_PLAN, new Object[]{"closed executor", resultInfo.getResultSetName()}); //$NON-NLS-1$
}
}
Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/xml/XMLPlan.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/processor/xml/XMLPlan.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/processor/xml/XMLPlan.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -62,17 +62,16 @@
import com.metamatrix.api.exception.MetaMatrixException;
import com.metamatrix.api.exception.MetaMatrixProcessingException;
import com.metamatrix.common.buffer.BlockedException;
-import com.metamatrix.common.buffer.BlockedOnMemoryException;
import com.metamatrix.common.buffer.BufferManager;
import com.metamatrix.common.buffer.TupleBatch;
-import com.metamatrix.common.buffer.TupleSourceID;
-import com.metamatrix.common.buffer.BufferManager.TupleSourceStatus;
+import com.metamatrix.common.buffer.TupleBuffer;
import com.metamatrix.common.lob.LobChunk;
import com.metamatrix.common.log.LogManager;
import com.metamatrix.common.types.DataTypeManager;
import com.metamatrix.common.types.SQLXMLImpl;
import com.metamatrix.common.types.Streamable;
import com.metamatrix.common.types.XMLType;
+import com.metamatrix.core.util.Assertion;
import com.metamatrix.query.execution.QueryExecPlugin;
import com.metamatrix.query.processor.BaseProcessorPlan;
import com.metamatrix.query.processor.DescribableUtil;
@@ -108,8 +107,7 @@
// is document in progress currently?
boolean docInProgress = false;
- TupleSourceID docInProgressTupleSourceId = null;
- int chunkPosition = 0;
+ private TupleBuffer docInProgressTupleSourceId;
// Post-processing
private String styleSheet;
@@ -177,25 +175,22 @@
public TupleBatch nextBatch()
throws MetaMatrixComponentException, MetaMatrixProcessingException, BlockedException {
- // If finished, processXML();
while(true){
- List rows = processXML();
-
- // we have no more to send
- if (rows == null){
- TupleBatch batch = new TupleBatch(nextBatchCount++, Collections.EMPTY_LIST);
+ LobChunk chunk = getNextXMLChunk(this.chunkSize);
+
+ if (chunk == null) {
+ Assertion.assertTrue(!this.docInProgress);
+ TupleBatch batch = new TupleBatch(nextBatchCount++, Collections.EMPTY_LIST);
batch.setTerminationFlag(true);
return batch;
- }
+ }
+
+ XMLType doc = processXML(chunk);
- // we got one more to send.
- List listOfRows = new ArrayList();
- listOfRows.add(rows);
-
- TupleBatch batch = new TupleBatch(nextBatchCount++, listOfRows);
- // when true; multiple doc return fails.
- batch.setTerminationFlag(false);
- return batch;
+ if (doc != null) {
+ TupleBatch batch = new TupleBatch(nextBatchCount++, Arrays.asList(Arrays.asList(doc)));
+ return batch;
+ }
}
}
@@ -217,16 +212,10 @@
* many XML documents, if the root of the document model has
* a mapping class (i.e. result set) associated with it.</p>
*
- * @return List a single tuple that contains the ValueID
- * of the XML document for the first document chunk, or empty row
- * for the rest of the document chunk; or will return null when
- * there are no more documents to return
*/
- private List processXML()
- throws MetaMatrixComponentException, MetaMatrixProcessingException, BlockedException {
+ private XMLType processXML(LobChunk chunk)
+ throws MetaMatrixComponentException, BlockedException {
-
-
boolean postProcess = (this.styleSheet != null || this.shouldValidate);
// Note that we need to stream the document right away, as multiple documents
// results are generated from the same "env" object. So the trick here is either
@@ -234,13 +223,8 @@
// right away so that we give way to next batch call. (this due to bad design of xml model)
// also note that there could be "inst" object alive but no more chunks; that is reason
// for "if" null block
- LobChunk chunk = getNextXMLChunk(this.chunkSize);
XMLType xml = null;
- if (chunk == null) {
- return null;
- }
-
// if the chunk size is less than one chunk unit then send this as string based xml.
if (!this.docInProgress && chunk.isLast()) {
xml = new XMLType(new SQLXMLImpl(new String(chunk.getBytes(), Charset.forName("UTF-16")), getProperties())); //$NON-NLS-1$
@@ -251,29 +235,26 @@
if (!this.docInProgress) {
this.docInProgress = true;
this.docInProgressTupleSourceId = XMLUtil.createXMLTupleSource(this.bufferMgr, this.getContext().getConnectionID());
- this.chunkPosition = 1;
}
// now save the chunk of data to the buffer manager and move on.
- this.bufferMgr.addStreamablePart(this.docInProgressTupleSourceId, chunk, chunkPosition++);
+ this.docInProgressTupleSourceId.addTuple(Arrays.asList(chunk));
// now document is finished, so create a xml object and return to the client.
if (chunk.isLast()) {
- this.bufferMgr.setStatus(this.docInProgressTupleSourceId, TupleSourceStatus.FULL);
+ this.docInProgressTupleSourceId.close();
// we want this to be naturally feed by chunks whether inside
// or out side the processor
- xml = new XMLType(XMLUtil.getFromBufferManager(bufferMgr, this.docInProgressTupleSourceId, getProperties()));
- this.bufferMgr.setPersistentTupleSource(this.docInProgressTupleSourceId, xml);
+ xml = new XMLType(XMLUtil.getFromBufferManager(this.docInProgressTupleSourceId, getProperties()));
+ this.docInProgressTupleSourceId.setContainingLobReference(xml);
//reset current document state.
this.docInProgress = false;
this.docInProgressTupleSourceId = null;
- this.chunkPosition = 0;
+ } else {
+ return null; //need to continue processing
}
- else {
- throw BlockedOnMemoryException.INSTANCE;
- }
}
// check to see if we need to do any post validation on the document.
@@ -281,9 +262,7 @@
xml = postProcessDocument(xml, getProperties());
}
- List row = new ArrayList(1);
- row.add(xml);
- return row;
+ return xml;
}
Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/xml/XMLUtil.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/processor/xml/XMLUtil.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/processor/xml/XMLUtil.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -27,22 +27,26 @@
import java.sql.SQLException;
import java.sql.SQLXML;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import java.util.Properties;
import com.metamatrix.api.exception.MetaMatrixComponentException;
+import com.metamatrix.api.exception.MetaMatrixProcessingException;
import com.metamatrix.common.buffer.BufferManager;
-import com.metamatrix.common.buffer.BufferManagerLobChunkStream;
-import com.metamatrix.common.buffer.TupleSourceID;
-import com.metamatrix.common.buffer.TupleSourceNotFoundException;
-import com.metamatrix.common.buffer.BufferManager.TupleSourceStatus;
+import com.metamatrix.common.buffer.IndexedTupleSource;
+import com.metamatrix.common.buffer.TupleBuffer;
import com.metamatrix.common.buffer.BufferManager.TupleSourceType;
import com.metamatrix.common.lob.ByteLobChunkStream;
import com.metamatrix.common.lob.LobChunk;
import com.metamatrix.common.lob.LobChunkInputStream;
+import com.metamatrix.common.lob.LobChunkProducer;
+import com.metamatrix.common.log.LogManager;
import com.metamatrix.common.types.DataTypeManager;
import com.metamatrix.common.types.SQLXMLImpl;
import com.metamatrix.common.types.XMLReaderFactory;
+import com.metamatrix.dqp.DQPPlugin;
+import com.metamatrix.dqp.util.LogConstants;
import com.metamatrix.query.sql.symbol.ElementSymbol;
@@ -55,18 +59,17 @@
* This method saves the given XML object to the buffer manager's disk process
* and returns the id which is saved under.
*/
- public static TupleSourceID saveToBufferManager(BufferManager bufferMgr, String tupleGroupName, SQLXML srcXML, int chunkSize)
+ public static TupleBuffer saveToBufferManager(BufferManager bufferMgr, String tupleGroupName, SQLXML srcXML, int chunkSize)
throws MetaMatrixComponentException {
try{
// first persist the XML to the Buffer Manager
- TupleSourceID sourceId = createXMLTupleSource(bufferMgr, tupleGroupName);
+ TupleBuffer sourceId = createXMLTupleSource(bufferMgr, tupleGroupName);
// since this object possibly will be streamed we need another streamable
// wrapper layer.
ByteLobChunkStream lobStream = new ByteLobChunkStream(srcXML.getBinaryStream(), chunkSize);
// read all the globs and Store into storage manager
- int batchCount = 1;
LobChunk lobChunk = null;
do {
try {
@@ -74,15 +77,13 @@
} catch (IOException e) {
throw new MetaMatrixComponentException(e);
}
- bufferMgr.addStreamablePart(sourceId, lobChunk, batchCount++);
+ sourceId.addTuple(Arrays.asList(lobChunk));
} while (!lobChunk.isLast());
lobStream.close();
- bufferMgr.setStatus(sourceId, TupleSourceStatus.FULL);
+ sourceId.close();
return sourceId;
- } catch (TupleSourceNotFoundException e) {
- throw new MetaMatrixComponentException(e);
} catch(SQLException e) {
throw new MetaMatrixComponentException(e);
} catch(IOException e) {
@@ -94,15 +95,16 @@
* This will reconstruct the XML object from the buffer manager from given
* buffer manager id.
*/
- public static SQLXML getFromBufferManager(final BufferManager bufferMgr, final TupleSourceID sourceId, Properties props) {
- return new SQLXMLImpl(new BufferMangerXMLReaderFactory(bufferMgr, sourceId), props);
+ public static SQLXML getFromBufferManager(final TupleBuffer sourceId, Properties props) {
+ return new SQLXMLImpl(new BufferMangerXMLReaderFactory(sourceId), props);
}
/**
* Creates a buffer manager's id for XML based data.
*/
- public static TupleSourceID createXMLTupleSource(BufferManager bufferMgr, String tupleGroupName) throws MetaMatrixComponentException {
- TupleSourceID sourceID = bufferMgr.createTupleSource(getOutputElements(), tupleGroupName, TupleSourceType.PROCESSOR);
+ public static TupleBuffer createXMLTupleSource(BufferManager bufferMgr, String tupleGroupName) throws MetaMatrixComponentException {
+ TupleBuffer sourceID = bufferMgr.createTupleBuffer(getOutputElements(), tupleGroupName, TupleSourceType.PROCESSOR);
+ sourceID.setBatchSize(1);
return sourceID;
}
@@ -119,16 +121,42 @@
* so that XML can be streamed by the clients more than once.
*/
static class BufferMangerXMLReaderFactory implements XMLReaderFactory {
- BufferManager bufferMgr;
- TupleSourceID sourceId;
+ private TupleBuffer tb;
- BufferMangerXMLReaderFactory(BufferManager mgr, TupleSourceID id){
- this.bufferMgr = mgr;
- this.sourceId = id;
+ BufferMangerXMLReaderFactory(TupleBuffer tb){
+ this.tb = tb;
}
- public Reader getReader() {
- return new LobChunkInputStream(new BufferManagerLobChunkStream(this.sourceId.getStringID(), this.bufferMgr)).getUTF16Reader();
+ public Reader getReader() {
+ final IndexedTupleSource its = tb.createIndexedTupleSource();
+ LobChunkProducer producer = new LobChunkProducer() {
+ public LobChunk getNextChunk() throws IOException {
+ try {
+ List<?> tuple = its.nextTuple();
+ if (tuple == null) {
+ return null;
+ }
+ return (LobChunk)tuple.get(0);
+ } catch (MetaMatrixProcessingException e) {
+ String msg = DQPPlugin.Util.getString("BufferManagerLobChunkStream.no_tuple_source", new Object[] {tb}); //$NON-NLS-1$
+ LogManager.logWarning(LogConstants.CTX_BUFFER_MGR, e, msg);
+ throw new IOException(msg);
+ } catch (MetaMatrixComponentException e) {
+ String msg = DQPPlugin.Util.getString("BufferManagerLobChunkStream.error_processing", new Object[] {tb}); //$NON-NLS-1$
+ LogManager.logWarning(LogConstants.CTX_BUFFER_MGR, e, msg);
+ throw new IOException(msg);
+ }
+ }
+
+ /**
+ * @see com.metamatrix.common.lob.LobChunkProducer#close()
+ */
+ public void close() throws IOException {
+ // we could remove the buffer tuple here but, this is just a stream, so we need to delete
+ // that when we close th eplan.
+ }
+ };
+ return new LobChunkInputStream(producer).getUTF16Reader();
}
}
}
\ No newline at end of file
Modified: trunk/engine/src/main/java/com/metamatrix/query/processor/xquery/XQueryPlan.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/processor/xquery/XQueryPlan.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/processor/xquery/XQueryPlan.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -36,8 +36,7 @@
import com.metamatrix.common.buffer.BlockedException;
import com.metamatrix.common.buffer.BufferManager;
import com.metamatrix.common.buffer.TupleBatch;
-import com.metamatrix.common.buffer.TupleSourceID;
-import com.metamatrix.common.buffer.TupleSourceNotFoundException;
+import com.metamatrix.common.buffer.TupleBuffer;
import com.metamatrix.common.types.DataTypeManager;
import com.metamatrix.common.types.Streamable;
import com.metamatrix.common.types.XMLType;
@@ -120,7 +119,7 @@
XQueryExpression expr = this.xQuery.getCompiledXQuery();
expr.setXMLFormat(xmlFormat);
- SqlEval sqlEval = new SqlEval(bufferMgr, this.dataManager, getContext(), this.xQuery.getProcedureGroup(), this.xQuery.getVariables());
+ SqlEval sqlEval = new SqlEval(this.dataManager, getContext(), this.xQuery.getProcedureGroup(), this.xQuery.getVariables());
try {
SQLXML xml = expr.evaluateXQuery(sqlEval);
TupleBatch batch = packResultsIntoBatch(xml);
@@ -148,29 +147,25 @@
* @return
*/
private TupleBatch packResultsIntoBatch(SQLXML srcXML) throws MetaMatrixComponentException{
- try {
- List rows = new ArrayList(1);
- List row = new ArrayList(1);
+ List rows = new ArrayList(1);
+ List row = new ArrayList(1);
- TupleSourceID savedId = XMLUtil.saveToBufferManager(this.bufferMgr, this.getContext().getConnectionID(), srcXML, this.chunkSize);
+ TupleBuffer savedId = XMLUtil.saveToBufferManager(this.bufferMgr, this.getContext().getConnectionID(), srcXML, this.chunkSize);
- //for large documents use the buffermanager version instead
- if (this.bufferMgr.getFinalRowCount(savedId) > 1) {
- srcXML = XMLUtil.getFromBufferManager(this.bufferMgr, savedId, getFormatProperties());
- }
-
- XMLType xml = new XMLType(srcXML);
- this.bufferMgr.setPersistentTupleSource(savedId, xml);
-
- // now build the top batch with information from the saved one.
- row.add(xml);
- rows.add(row);
- TupleBatch batch = new TupleBatch(1, rows);
- batch.setTerminationFlag(true);
- return batch;
- } catch (TupleSourceNotFoundException e) {
- throw new MetaMatrixComponentException(e);
+ //for large documents use the buffermanager version instead
+ if (savedId.getRowCount() > 1) {
+ srcXML = XMLUtil.getFromBufferManager(savedId, getFormatProperties());
}
+
+ XMLType xml = new XMLType(srcXML);
+ savedId.setContainingLobReference(xml);
+
+ // now build the top batch with information from the saved one.
+ row.add(xml);
+ rows.add(row);
+ TupleBatch batch = new TupleBatch(1, rows);
+ batch.setTerminationFlag(true);
+ return batch;
}
/**
Modified: trunk/engine/src/main/java/com/metamatrix/query/sql/lang/Command.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/sql/lang/Command.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/sql/lang/Command.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -319,18 +319,9 @@
*/
public abstract int updatingModelCount(QueryMetadataInterface metadata) throws MetaMatrixComponentException;
-
protected int getSubCommandsUpdatingModelCount(QueryMetadataInterface metadata) throws MetaMatrixComponentException{
- return getSubCommandsUpdatingModelCount(this, metadata);
- }
-
- public static int getSubCommandsUpdatingModelCount(Command object, QueryMetadataInterface metadata) throws MetaMatrixComponentException {
- List<Command> subCommands = CommandCollectorVisitor.getCommands(object);
- int numCommands = subCommands.size();
+ List<Command> subCommands = CommandCollectorVisitor.getCommands(this);
for (Command command : subCommands) {
- if (numCommands == 1) {
- return command.updatingModelCount(metadata);
- }
if (command.updatingModelCount(metadata) > 0) {
return 2; //require a transaction if there is more than 1 statement
}
Modified: trunk/engine/src/main/java/com/metamatrix/query/sql/lang/Query.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/sql/lang/Query.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/sql/lang/Query.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -429,7 +429,14 @@
if(this.getInto() != null){
return false;
}
+ if (isXML) {
+ return true;
+ }
List projectedSymbols = getProjectedSymbols();
+ return areResultsCachable(projectedSymbols);
+ }
+
+ static boolean areResultsCachable(List projectedSymbols) {
for(int i=0; i<projectedSymbols.size(); i++){
SingleElementSymbol projectedSymbol = (SingleElementSymbol)projectedSymbols.get(i);
if(DataTypeManager.isLOB(projectedSymbol.getType())) {
@@ -440,7 +447,7 @@
}
public int updatingModelCount(QueryMetadataInterface metadata) throws MetaMatrixComponentException{
- if(isXML) {
+ if(isXML) { //TODO: this is incorrect
return 0;
}
@@ -448,13 +455,7 @@
return 2;
}
- //any subcommand performing an update requires a transaction
- //since we may not be able to roll it back other wise
- int count = getSubCommandsUpdatingModelCount(metadata);
- if (count > 0) {
- return 2;
- }
- return 0;
+ return getSubCommandsUpdatingModelCount(metadata);
}
/**
Modified: trunk/engine/src/main/java/com/metamatrix/query/sql/lang/StoredProcedure.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/sql/lang/StoredProcedure.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/sql/lang/StoredProcedure.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -335,7 +335,7 @@
* @see com.metamatrix.query.sql.lang.Command#areResultsCachable()
*/
public boolean areResultsCachable() {
- return true;
+ return Query.areResultsCachable(getProjectedSymbols());
}
/**
Modified: trunk/engine/src/main/java/com/metamatrix/query/sql/lang/XQuery.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/sql/lang/XQuery.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/sql/lang/XQuery.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -147,7 +147,7 @@
* @see com.metamatrix.query.sql.lang.Command#areResultsCachable()
*/
public boolean areResultsCachable() {
- return false;
+ return true;
}
public int updatingModelCount(QueryMetadataInterface metadata) throws MetaMatrixComponentException {
Modified: trunk/engine/src/main/java/com/metamatrix/query/sql/proc/CreateUpdateProcedureCommand.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/sql/proc/CreateUpdateProcedureCommand.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/sql/proc/CreateUpdateProcedureCommand.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -294,7 +294,7 @@
return 0;
}
Command lastCommand = null;
- Statement statement = (Statement)block.getStatements().get(block.getStatements().size() - 1);
+ Statement statement = block.getStatements().get(block.getStatements().size() - 1);
if (statement instanceof CommandStatement) {
CommandStatement cmdStatement = (CommandStatement)statement;
lastCommand = cmdStatement.getCommand();
Modified: trunk/engine/src/main/java/com/metamatrix/query/sql/util/ValueIteratorSource.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/sql/util/ValueIteratorSource.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/sql/util/ValueIteratorSource.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -22,11 +22,10 @@
package com.metamatrix.query.sql.util;
-import java.util.HashSet;
+import java.util.Set;
import com.metamatrix.api.exception.MetaMatrixComponentException;
import com.metamatrix.api.exception.MetaMatrixProcessingException;
-import com.metamatrix.common.buffer.TupleSourceNotFoundException;
import com.metamatrix.query.sql.symbol.Expression;
@@ -45,11 +44,10 @@
* @param valueExpression The expression we are retrieving an iterator for
* @return ValueIterator if ready, null otherwise
* @throws MetaMatrixComponentException
- * @throws TupleSourceNotFoundException
* @since 5.0.1
*/
ValueIterator getValueIterator(Expression valueExpression) throws MetaMatrixComponentException;
- HashSet<Object> getCachedSet(Expression valueExpression) throws MetaMatrixComponentException, MetaMatrixProcessingException;
+ Set<Object> getCachedSet(Expression valueExpression) throws MetaMatrixComponentException, MetaMatrixProcessingException;
}
Modified: trunk/engine/src/main/java/com/metamatrix/query/tempdata/TempTableStore.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/tempdata/TempTableStore.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/tempdata/TempTableStore.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -26,8 +26,8 @@
import com.metamatrix.api.exception.MetaMatrixComponentException;
import com.metamatrix.api.exception.MetaMatrixProcessingException;
+import com.metamatrix.common.buffer.TupleBuffer;
import com.metamatrix.common.buffer.TupleSource;
-import com.metamatrix.common.buffer.TupleSourceID;
import com.metamatrix.query.metadata.TempMetadataStore;
import com.metamatrix.query.sql.lang.Command;
@@ -48,5 +48,5 @@
public void removeTempTableByName(String tempTableName) throws MetaMatrixComponentException;
- public TupleSourceID getTupleSourceID(String tempTableName);
+ public TupleBuffer getTupleSourceID(String tempTableName);
}
Modified: trunk/engine/src/main/java/com/metamatrix/query/tempdata/TempTableStoreImpl.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/tempdata/TempTableStoreImpl.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/tempdata/TempTableStoreImpl.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -37,20 +37,15 @@
import com.metamatrix.api.exception.query.QueryProcessingException;
import com.metamatrix.common.buffer.BlockedException;
import com.metamatrix.common.buffer.BufferManager;
-import com.metamatrix.common.buffer.TupleBatch;
+import com.metamatrix.common.buffer.TupleBuffer;
import com.metamatrix.common.buffer.TupleSource;
-import com.metamatrix.common.buffer.TupleSourceID;
-import com.metamatrix.common.buffer.TupleSourceNotFoundException;
-import com.metamatrix.common.buffer.BufferManager.TupleSourceStatus;
import com.metamatrix.common.buffer.BufferManager.TupleSourceType;
-import com.metamatrix.core.util.Assertion;
import com.metamatrix.query.eval.Evaluator;
import com.metamatrix.query.execution.QueryExecPlugin;
import com.metamatrix.query.metadata.TempMetadataAdapter;
import com.metamatrix.query.metadata.TempMetadataStore;
-import com.metamatrix.query.processor.proc.UpdateCountTupleSource;
+import com.metamatrix.query.processor.CollectionTupleSource;
import com.metamatrix.query.processor.relational.RelationalNode;
-import com.metamatrix.query.processor.relational.TupleCollector;
import com.metamatrix.query.resolver.util.ResolverUtil;
import com.metamatrix.query.sql.lang.Command;
import com.metamatrix.query.sql.lang.Create;
@@ -74,24 +69,22 @@
private abstract class UpdateTupleSource implements TupleSource {
private final String groupKey;
- private final TupleSourceID tsId;
+ private final TupleBuffer oldBuffer;
private final TupleSource ts;
protected final Map lookup;
- private final TupleCollector tc;
- private final TupleSourceID newTs;
+ private final TupleBuffer newBuffer;
protected final Evaluator eval;
private final Criteria crit;
protected int updateCount = 0;
private boolean done;
- private UpdateTupleSource(String groupKey, TupleSourceID tsId, Criteria crit) throws TupleSourceNotFoundException, MetaMatrixComponentException {
+ private UpdateTupleSource(String groupKey, TupleBuffer tsId, Criteria crit) throws MetaMatrixComponentException {
this.groupKey = groupKey;
- this.tsId = tsId;
- this.ts = buffer.getTupleSource(tsId);
- List columns = buffer.getTupleSchema(tsId);
+ this.oldBuffer = tsId;
+ this.ts = tsId.createIndexedTupleSource();
+ List columns = tsId.getSchema();
this.lookup = RelationalNode.createLookupMap(columns);
- this.newTs = buffer.createTupleSource(columns, sessionID, TupleSourceType.PROCESSOR);
- this.tc = new TupleCollector(newTs, buffer);
+ this.newBuffer = buffer.createTupleBuffer(columns, sessionID, TupleSourceType.PROCESSOR);
this.eval = new Evaluator(lookup, null, null);
this.crit = crit;
}
@@ -112,23 +105,15 @@
tupleFailed(tuple);
}
}
- tc.close();
- groupToTupleSourceID.put(groupKey, newTs);
- try {
- buffer.removeTupleSource(tsId);
- }catch(TupleSourceNotFoundException e) {
-
- }
+ newBuffer.close();
+ groupToTupleSourceID.put(groupKey, newBuffer);
+ oldBuffer.remove();
done = true;
return Arrays.asList(updateCount);
}
protected void addTuple(List<?> tuple) throws MetaMatrixComponentException {
- try {
- tc.addTuple(tuple);
- } catch (TupleSourceNotFoundException e) {
- throw new MetaMatrixComponentException(e);
- }
+ newBuffer.addTuple(tuple);
}
protected abstract void tuplePassed(List<?> tuple) throws ExpressionEvaluationException, BlockedException, MetaMatrixComponentException;
@@ -148,7 +133,7 @@
private BufferManager buffer;
private TempMetadataStore tempMetadataStore = new TempMetadataStore();
- private Map<String, TupleSourceID> groupToTupleSourceID = new HashMap<String, TupleSourceID>();
+ private Map<String, TupleBuffer> groupToTupleSourceID = new HashMap<String, TupleBuffer>();
private String sessionID;
private TempTableStore parentTempTableStore;
@@ -169,23 +154,16 @@
//add metadata
tempMetadataStore.addTempGroup(tempTableName, columns, false, true);
//create tuple source
- TupleSourceID tsId = buffer.createTupleSource(columns, sessionID, TupleSourceType.PROCESSOR);
- try {
- buffer.setStatus(tsId, TupleSourceStatus.FULL);
- }catch(TupleSourceNotFoundException e) {
- //should never happen because the TupleSourceID is just created
- Assertion.failed("Could not find local tuple source for inserting into temp table."); //$NON-NLS-1$
- }
- groupToTupleSourceID.put(tempTableName, tsId);
+ TupleBuffer tupleBuffer = buffer.createTupleBuffer(columns, sessionID, TupleSourceType.PROCESSOR);
+ tupleBuffer.setFinal(true); //final, but not closed so that we can append on insert
+ groupToTupleSourceID.put(tempTableName, tupleBuffer);
}
public void removeTempTableByName(String tempTableName) throws MetaMatrixComponentException {
tempMetadataStore.removeTempGroup(tempTableName);
- TupleSourceID tsId = this.groupToTupleSourceID.remove(tempTableName);
+ TupleBuffer tsId = this.groupToTupleSourceID.remove(tempTableName);
if(tsId != null) {
- try {
- buffer.removeTupleSource(tsId);
- }catch(TupleSourceNotFoundException e) {}
+ tsId.remove();
}
}
@@ -200,12 +178,8 @@
if (!group.isTempGroupSymbol()) {
return null;
}
- TupleSourceID tsId = getTupleSourceID(group.getNonCorrelationName().toUpperCase(), command);
- try {
- return buffer.getTupleSource(tsId);
- }catch(TupleSourceNotFoundException e) {
- throw new MetaMatrixComponentException(e);
- }
+ TupleBuffer tsId = getTupleSourceID(group.getNonCorrelationName().toUpperCase(), command);
+ return tsId.createIndexedTupleSource();
}
if (command instanceof ProcedureContainer) {
GroupSymbol group = ((ProcedureContainer)command).getGroup();
@@ -213,65 +187,61 @@
return null;
}
final String groupKey = group.getNonCorrelationName().toUpperCase();
- final TupleSourceID tsId = getTupleSourceID(groupKey, command);
+ final TupleBuffer tsId = getTupleSourceID(groupKey, command);
if (command instanceof Insert) {
return addTuple((Insert)command, tsId);
}
- try {
- if (command instanceof Update) {
- final Update update = (Update)command;
- final Criteria crit = update.getCriteria();
- return new UpdateTupleSource(groupKey, tsId, crit) {
- @Override
- protected void tuplePassed(List<?> tuple)
- throws ExpressionEvaluationException,
- BlockedException, MetaMatrixComponentException {
- List<Object> newTuple = new ArrayList<Object>(tuple);
- for (Map.Entry<ElementSymbol, Expression> entry : update.getChangeList().getClauseMap().entrySet()) {
- newTuple.set((Integer)lookup.get(entry.getKey()), eval.evaluate(entry.getValue(), tuple));
- }
- updateCount++;
- addTuple(newTuple);
+ if (command instanceof Update) {
+ final Update update = (Update)command;
+ final Criteria crit = update.getCriteria();
+ return new UpdateTupleSource(groupKey, tsId, crit) {
+ @Override
+ protected void tuplePassed(List<?> tuple)
+ throws ExpressionEvaluationException,
+ BlockedException, MetaMatrixComponentException {
+ List<Object> newTuple = new ArrayList<Object>(tuple);
+ for (Map.Entry<ElementSymbol, Expression> entry : update.getChangeList().getClauseMap().entrySet()) {
+ newTuple.set((Integer)lookup.get(entry.getKey()), eval.evaluate(entry.getValue(), tuple));
}
-
- protected void tupleFailed(java.util.List<?> tuple) throws MetaMatrixComponentException {
- addTuple(tuple);
- }
- };
- }
- if (command instanceof Delete) {
- final Delete delete = (Delete)command;
- final Criteria crit = delete.getCriteria();
- if (crit == null) {
- int rows = buffer.getRowCount(tsId);
- addTempTable(groupKey, buffer.getTupleSchema(tsId), true);
- return new UpdateCountTupleSource(rows);
- }
- return new UpdateTupleSource(groupKey, tsId, crit) {
- @Override
- protected void tuplePassed(List<?> tuple)
- throws ExpressionEvaluationException,
- BlockedException, MetaMatrixComponentException {
- updateCount++;
- }
-
- protected void tupleFailed(java.util.List<?> tuple) throws MetaMatrixComponentException {
- addTuple(tuple);
- }
- };
- }
- } catch (TupleSourceNotFoundException e) {
- throw new MetaMatrixComponentException(e);
+ updateCount++;
+ addTuple(newTuple);
+ }
+
+ protected void tupleFailed(java.util.List<?> tuple) throws MetaMatrixComponentException {
+ addTuple(tuple);
+ }
+ };
}
+ if (command instanceof Delete) {
+ final Delete delete = (Delete)command;
+ final Criteria crit = delete.getCriteria();
+ if (crit == null) {
+ int rows = tsId.getRowCount();
+ addTempTable(groupKey, tsId.getSchema(), true);
+ return CollectionTupleSource.createUpdateCountTupleSource(rows);
+ }
+ return new UpdateTupleSource(groupKey, tsId, crit) {
+ @Override
+ protected void tuplePassed(List<?> tuple)
+ throws ExpressionEvaluationException,
+ BlockedException, MetaMatrixComponentException {
+ updateCount++;
+ }
+
+ protected void tupleFailed(java.util.List<?> tuple) throws MetaMatrixComponentException {
+ addTuple(tuple);
+ }
+ };
+ }
}
if (command instanceof Create) {
addTempTable(((Create)command).getTable().getName().toUpperCase(), ((Create)command).getColumns(), false);
- return new UpdateCountTupleSource(0);
+ return CollectionTupleSource.createUpdateCountTupleSource(0);
}
if (command instanceof Drop) {
String tempTableName = ((Drop)command).getTable().getName().toUpperCase();
removeTempTableByName(tempTableName);
- return new UpdateCountTupleSource(0);
+ return CollectionTupleSource.createUpdateCountTupleSource(0);
}
return null;
}
@@ -282,8 +252,8 @@
}
}
- private TupleSourceID getTupleSourceID(String tempTableID, Command command) throws MetaMatrixComponentException, QueryProcessingException{
- TupleSourceID tsID = groupToTupleSourceID.get(tempTableID);
+ private TupleBuffer getTupleSourceID(String tempTableID, Command command) throws MetaMatrixComponentException, QueryProcessingException{
+ TupleBuffer tsID = groupToTupleSourceID.get(tempTableID);
if(tsID != null) {
return tsID;
}
@@ -317,31 +287,24 @@
return groupToTupleSourceID.get(tempTableID);
}
- private TupleSource addTuple(Insert insert, TupleSourceID tsId) throws MetaMatrixComponentException, ExpressionEvaluationException {
+ private TupleSource addTuple(Insert insert, TupleBuffer tsId) throws MetaMatrixComponentException, ExpressionEvaluationException {
GroupSymbol group = insert.getGroup();
int tuplesAdded = 0;
try {
- int rowCount = buffer.getRowCount(tsId);
- TupleBatch tupleBatch;
List<ElementSymbol> elements = ResolverUtil.resolveElementsInGroup(group, new TempMetadataAdapter(null, tempMetadataStore));
List<List<Object>> tuples = getBulkRows(insert, elements);
- tuplesAdded = tuples.size();
-
- // Buffer manager has 1 based index for the tuple sources
- tupleBatch = new TupleBatch((++rowCount), tuples);
+ for (List<Object> list : tuples) {
+ tsId.addTuple(list);
+ }
- buffer.addTupleBatch(tsId, tupleBatch);
- } catch (TupleSourceNotFoundException err) {
- throw new MetaMatrixComponentException(err);
- } catch (BlockedException err) {
- throw new MetaMatrixComponentException(err);
+ tuplesAdded = tuples.size();
} catch (QueryMetadataException err) {
throw new MetaMatrixComponentException(err);
}
- return new UpdateCountTupleSource(tuplesAdded);
+ return CollectionTupleSource.createUpdateCountTupleSource(tuplesAdded);
}
public static List<List<Object>> getBulkRows(Insert insert, List<ElementSymbol> elements) throws ExpressionEvaluationException, BlockedException, MetaMatrixComponentException {
@@ -403,7 +366,7 @@
return new HashSet<String>(this.groupToTupleSourceID.keySet());
}
- public TupleSourceID getTupleSourceID(String tempTableName) {
+ public TupleBuffer getTupleSourceID(String tempTableName) {
return groupToTupleSourceID.get(tempTableName.toUpperCase());
}
}
Modified: trunk/engine/src/main/java/com/metamatrix/query/util/CommandContext.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/query/util/CommandContext.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/com/metamatrix/query/util/CommandContext.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -49,57 +49,61 @@
* framework.
*/
public class CommandContext implements Cloneable {
+
+ private static class GlobalState {
+ /** Uniquely identify the command being processed */
+ private Object processorID;
+
+ /** Identify a group of related commands, which typically get cleaned up together */
+ private String connectionID;
- /** Uniquely identify the command being processed */
- private Object processorID;
-
- /** Identify a group of related commands, which typically get cleaned up together */
- private String connectionID;
+ private int processorBatchSize = BufferConfig.DEFAULT_PROCESSOR_BATCH_SIZE;
+
+ private int connectorBatchSize = BufferConfig.DEFAULT_CONNECTOR_BATCH_SIZE;
- private int processorBatchSize = BufferConfig.DEFAULT_PROCESSOR_BATCH_SIZE;
-
- private int connectorBatchSize = BufferConfig.DEFAULT_CONNECTOR_BATCH_SIZE;
+ private String userName;
+
+ private Serializable commandPayload;
+
+ private String vdbName;
+
+ private String vdbVersion;
+
+ private Properties environmentProperties;
+
+ /** Indicate whether data should be dumped for debugging purposes while processing the query */
+ private boolean processDebug;
+
+ /** Indicate whether statistics should be collected for relational node processing*/
+ private boolean collectNodeStatistics;
+
+ private int streamingBatchSize;
+
+ private Random random = null;
+
+ private SecurityFunctionEvaluator securityFunctionEvaluator;
+
+ private TimeZone timezone = TimeZone.getDefault();
+
+ private PlanToProcessConverter planToProcessConverter;
+
+ private QueryProcessor.ProcessorFactory queryProcessorFactory;
+
+ private boolean sessionFunctionEvaluated;
+
+ private Set<String> groups;
+
+ private long timeSliceEnd = Long.MAX_VALUE;
+
+ private long timeoutEnd = Long.MAX_VALUE;
+ }
+
+ private GlobalState globalState = new GlobalState();
- private String userName;
-
- private Serializable commandPayload;
-
- private String vdbName;
-
- private String vdbVersion;
-
- private Properties environmentProperties;
-
- /** Indicate whether data should be dumped for debugging purposes while processing the query */
- private boolean processDebug;
-
- /** Indicate whether statistics should be collected for relational node processing*/
- private boolean collectNodeStatistics;
-
- private int streamingBatchSize;
-
- private Random random = null;
-
- private Stack<String> recursionStack = null;
-
- private SecurityFunctionEvaluator securityFunctionEvaluator;
-
- private Object tempTableStore;
-
- private TimeZone timezone = TimeZone.getDefault();
-
- private PlanToProcessConverter planToProcessConverter;
-
- private QueryProcessor.ProcessorFactory queryProcessorFactory;
-
private VariableContext variableContext = new VariableContext();
-
- private CommandContext parent;
-
- private boolean sessionFunctionEvaluated;
-
- private Set<String> groups;
-
+ private Object tempTableStore;
+ private Stack<String> recursionStack;
+
/**
* Construct a new context.
* @param collectNodeStatistics TODO
@@ -128,66 +132,30 @@
}
- protected CommandContext(CommandContext context) {
- setConnectionID(context.connectionID);
-
- // Reuse existing processor ID - may be overridden
- setProcessorID(context.processorID);
-
- setUserName(context.userName);
- setCommandPayload(context.commandPayload);
- setVdbName(context.vdbName);
- setVdbVersion(context.vdbVersion);
- setEnvironmentProperties(context.environmentProperties);
- setProcessDebug(context.processDebug);
- setProcessorBatchSize(context.processorBatchSize);
- setConnectorBatchSize(context.connectorBatchSize);
- setRandom(context.random);
- if (context.recursionStack != null) {
- this.recursionStack = (Stack)context.recursionStack.clone();
- }
- this.setSecurityFunctionEvaluator(context.getSecurityFunctionEvaluator());
- this.planToProcessConverter = context.planToProcessConverter;
- this.queryProcessorFactory = context.queryProcessorFactory;
- this.variableContext = context.variableContext;
- this.parent = context;
- }
-
public CommandContext() {
}
- public CommandContext getParent() {
- return parent;
- }
-
public boolean isSessionFunctionEvaluated() {
- if (parent != null) {
- return parent.isSessionFunctionEvaluated();
- }
- return sessionFunctionEvaluated;
+ return globalState.sessionFunctionEvaluated;
}
public void setSessionFunctionEvaluated(boolean sessionFunctionEvaluated) {
- if (parent != null) {
- parent.setSessionFunctionEvaluated(sessionFunctionEvaluated);
- } else {
- this.sessionFunctionEvaluated = sessionFunctionEvaluated;
- }
+ globalState.sessionFunctionEvaluated = sessionFunctionEvaluated;
}
/**
* @return
*/
public Object getProcessorID() {
- return processorID;
+ return globalState.processorID;
}
public boolean getProcessDebug() {
- return this.processDebug;
+ return globalState.processDebug;
}
public void setProcessDebug(boolean processDebug) {
- this.processDebug = processDebug;
+ globalState.processDebug = processDebug;
}
/**
@@ -195,43 +163,49 @@
*/
public void setProcessorID(Object object) {
ArgCheck.isNotNull(object);
- processorID = object;
+ globalState.processorID = object;
}
public Object clone() {
- return new CommandContext(this);
+ CommandContext clone = new CommandContext();
+ clone.globalState = this.globalState;
+ clone.variableContext = this.variableContext;
+ if (this.recursionStack != null) {
+ clone.recursionStack = (Stack<String>)this.recursionStack.clone();
+ }
+ return clone;
}
public String toString() {
- return "CommandContext: " + processorID; //$NON-NLS-1$
+ return "CommandContext: " + globalState.processorID; //$NON-NLS-1$
}
/**
* @return String
*/
public String getConnectionID() {
- return connectionID;
+ return globalState.connectionID;
}
/**
* @return String
*/
public String getUserName() {
- return userName;
+ return globalState.userName;
}
/**
* @return String
*/
public String getVdbName() {
- return vdbName;
+ return globalState.vdbName;
}
/**
* @return String
*/
public String getVdbVersion() {
- return vdbVersion;
+ return globalState.vdbVersion;
}
/**
@@ -239,7 +213,7 @@
* @param connectionID The connectionID to set
*/
public void setConnectionID(String connectionID) {
- this.connectionID = connectionID;
+ this.globalState.connectionID = connectionID;
}
/**
@@ -247,7 +221,7 @@
* @param userName The userName to set
*/
public void setUserName(String userName) {
- this.userName = userName;
+ this.globalState.userName = userName;
}
/**
@@ -255,7 +229,7 @@
* @param vdbName The vdbName to set
*/
public void setVdbName(String vdbName) {
- this.vdbName = vdbName;
+ this.globalState.vdbName = vdbName;
}
/**
@@ -263,22 +237,22 @@
* @param vdbVersion The vdbVersion to set
*/
public void setVdbVersion(String vdbVersion) {
- this.vdbVersion = vdbVersion;
+ this.globalState.vdbVersion = vdbVersion;
}
public Properties getEnvironmentProperties() {
- return environmentProperties;
+ return globalState.environmentProperties;
}
public void setEnvironmentProperties(Properties properties) {
- environmentProperties = properties;
+ globalState.environmentProperties = properties;
}
public Serializable getCommandPayload() {
- return this.commandPayload;
+ return this.globalState.commandPayload;
}
public void setCommandPayload(Serializable commandPayload) {
- this.commandPayload = commandPayload;
+ this.globalState.commandPayload = commandPayload;
}
/**
@@ -286,64 +260,58 @@
* @since 4.2
*/
public void setCollectNodeStatistics(boolean collectNodeStatistics) {
- this.collectNodeStatistics = collectNodeStatistics;
+ this.globalState.collectNodeStatistics = collectNodeStatistics;
}
public boolean getCollectNodeStatistics() {
- return this.collectNodeStatistics;
+ return this.globalState.collectNodeStatistics;
}
public int getStreamingBatchSize() {
- return streamingBatchSize;
+ return globalState.streamingBatchSize;
}
public void setStreamingBatchSize(int streamingBatchSize) {
- this.streamingBatchSize = streamingBatchSize;
+ this.globalState.streamingBatchSize = streamingBatchSize;
}
public int getConnectorBatchSize() {
- return this.connectorBatchSize;
+ return this.globalState.connectorBatchSize;
}
public void setConnectorBatchSize(int connectorBatchSize) {
- this.connectorBatchSize = connectorBatchSize;
+ this.globalState.connectorBatchSize = connectorBatchSize;
}
public int getProcessorBatchSize() {
- return this.processorBatchSize;
+ return this.globalState.processorBatchSize;
}
public void setProcessorBatchSize(int processorBatchSize) {
- this.processorBatchSize = processorBatchSize;
+ this.globalState.processorBatchSize = processorBatchSize;
}
public double getNextRand() {
- if (parent != null) {
- return parent.getNextRand();
- }
- if (random == null) {
- random = new Random();
+ if (globalState.random == null) {
+ globalState.random = new Random();
}
- return random.nextDouble();
+ return globalState.random.nextDouble();
}
public double getNextRand(long seed) {
- if (parent != null) {
- return parent.getNextRand(seed);
- }
- if (random == null) {
- random = new Random();
+ if (globalState.random == null) {
+ globalState.random = new Random();
}
- random.setSeed(seed);
- return random.nextDouble();
+ globalState.random.setSeed(seed);
+ return globalState.random.nextDouble();
}
void setRandom(Random random) {
- this.random = random;
+ this.globalState.random = random;
}
public void pushCall(String value) throws QueryProcessingException {
@@ -373,14 +341,14 @@
* @return Returns the securityFunctionEvaluator.
*/
public SecurityFunctionEvaluator getSecurityFunctionEvaluator() {
- return this.securityFunctionEvaluator;
+ return this.globalState.securityFunctionEvaluator;
}
/**
* @param securityFunctionEvaluator The securityFunctionEvaluator to set.
*/
public void setSecurityFunctionEvaluator(SecurityFunctionEvaluator securityFunctionEvaluator) {
- this.securityFunctionEvaluator = securityFunctionEvaluator;
+ this.globalState.securityFunctionEvaluator = securityFunctionEvaluator;
}
public Object getTempTableStore() {
@@ -392,23 +360,23 @@
}
public TimeZone getServerTimeZone() {
- return timezone;
+ return globalState.timezone;
}
public void setPlanToProcessConverter(PlanToProcessConverter planToProcessConverter) {
- this.planToProcessConverter = planToProcessConverter;
+ this.globalState.planToProcessConverter = planToProcessConverter;
}
public PlanToProcessConverter getPlanToProcessConverter() {
- return planToProcessConverter;
+ return globalState.planToProcessConverter;
}
public QueryProcessor.ProcessorFactory getQueryProcessorFactory() {
- return this.queryProcessorFactory;
+ return this.globalState.queryProcessorFactory;
}
public void setQueryProcessorFactory(QueryProcessor.ProcessorFactory queryProcessorFactory) {
- this.queryProcessorFactory = queryProcessorFactory;
+ this.globalState.queryProcessorFactory = queryProcessorFactory;
}
public VariableContext getVariableContext() {
@@ -436,11 +404,27 @@
}
public Set<String> getGroups() {
- return groups;
+ return globalState.groups;
}
public void setGroups(Set<String> groups) {
- this.groups = groups;
+ this.globalState.groups = groups;
}
+ public long getTimeSliceEnd() {
+ return globalState.timeSliceEnd;
+ }
+
+ public long getTimeoutEnd() {
+ return globalState.timeoutEnd;
+ }
+
+ public void setTimeSliceEnd(long timeSliceEnd) {
+ globalState.timeSliceEnd = timeSliceEnd;
+ }
+
+ public void setTimeoutEnd(long timeoutEnd) {
+ globalState.timeoutEnd = timeoutEnd;
+ }
+
}
Deleted: trunk/engine/src/main/java/org/teiid/dqp/internal/cache/CacheID.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/dqp/internal/cache/CacheID.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/org/teiid/dqp/internal/cache/CacheID.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -1,78 +0,0 @@
-/*
- * JBoss, Home of Professional Open Source.
- * See the COPYRIGHT.txt file distributed with this work for information
- * regarding copyright ownership. Some portions may be licensed
- * to Red Hat, Inc. under one or more contributor license agreements.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301 USA.
- */
-
-package org.teiid.dqp.internal.cache;
-
-import java.io.Serializable;
-import java.util.List;
-
-import com.metamatrix.core.util.HashCodeUtil;
-
-public class CacheID implements Serializable {
- private String scopeID;
- private String command;
- private int hashCode;
- private List preparedStatementValues;
-
- public CacheID(String scopeID, String command){
- this(scopeID, command, null);
- }
-
- public CacheID(String scopeID, String command, List preparedStatementValue){
- this.scopeID = scopeID;
- this.command = command;
- this.preparedStatementValues = preparedStatementValue;
- hashCode = HashCodeUtil.expHashCode(HashCodeUtil.hashCode(HashCodeUtil.hashCode(0, scopeID), command), preparedStatementValues);
- }
-
- public String getCommand() {
- return command;
- }
-
- public boolean equals(Object obj){
- if(obj == this) {
- return true;
- } else if(! (obj instanceof CacheID)) {
- return false;
- } else {
- CacheID that = (CacheID)obj;
- return this.scopeID.equals(that.scopeID)
- && this.command.equals(that.command)
- && compareParamValues(preparedStatementValues, that.preparedStatementValues);
- }
- }
-
- private boolean compareParamValues(List thisPreparedStatementValues, List thatPreparedStatementValues) {
- if(thisPreparedStatementValues == null && thatPreparedStatementValues == null){
- return true;
- }
- if(thisPreparedStatementValues == null || thatPreparedStatementValues == null){
- return false;
- }
- return thisPreparedStatementValues.equals(thatPreparedStatementValues);
- }
-
- public int hashCode() {
- return hashCode;
- }
-
-}
Deleted: trunk/engine/src/main/java/org/teiid/dqp/internal/cache/CacheResults.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/dqp/internal/cache/CacheResults.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/org/teiid/dqp/internal/cache/CacheResults.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -1,125 +0,0 @@
-/*
- * JBoss, Home of Professional Open Source.
- * See the COPYRIGHT.txt file distributed with this work for information
- * regarding copyright ownership. Some portions may be licensed
- * to Red Hat, Inc. under one or more contributor license agreements.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301 USA.
- */
-
-package org.teiid.dqp.internal.cache;
-
-import java.io.Serializable;
-import java.util.Arrays;
-import java.util.List;
-
-import com.metamatrix.core.MetaMatrixRuntimeException;
-import com.metamatrix.dqp.DQPPlugin;
-import com.metamatrix.query.analysis.AnalysisRecord;
-import com.metamatrix.query.sql.lang.Command;
-import com.metamatrix.query.sql.symbol.SingleElementSymbol;
-
-public class CacheResults implements Serializable {
- private List[] results;
- private AnalysisRecord analysisRecord;
- private Command command;
-
- private boolean isFinal;
- private int firstRow = 1;
- //size of this results in memory
- private long size= -1;
-
- public CacheResults(List[] results, int firstRow, boolean isFinal){
- this.results = results;
- this.firstRow = firstRow;
- this.isFinal = isFinal;
- }
-
- public int getFirstRow() {
- return firstRow;
- }
-
- public boolean isFinal() {
- return isFinal;
- }
-
- public List<?>[] getResults() {
- return results;
- }
-
- public List<SingleElementSymbol> getElements() {
- if (command == null) {
- return null;
- }
- return command.getProjectedSymbols();
- }
-
- public Command getCommand() {
- return command;
- }
-
- public void setCommand(Command command) {
- this.command = command;
- }
-
- public long getSize() {
- return size;
- }
-
- public void setSize(long size) {
- this.size = size;
- }
-
- public AnalysisRecord getAnalysisRecord() {
- return analysisRecord;
- }
-
- public void setAnalysisRecord(AnalysisRecord analysisRecord) {
- this.analysisRecord = analysisRecord;
- }
-
- public int getFinalRow() {
- if (this.isFinal) {
- return results.length;
- }
- return -1;
- }
-
- //add the results to the existing one, this is used
- //when building the batched results
- boolean addResults(CacheResults cacheResults){
- if (this.firstRow + results.length != cacheResults.getFirstRow()) {
- throw new MetaMatrixRuntimeException(DQPPlugin.Util.getString("ResultSetCache.1"));//$NON-NLS-1$
- }
- this.size += cacheResults.size;
- List[] batchResults = cacheResults.getResults();
- if (results == null) {
- this.results = batchResults;
- } else if (batchResults.length > 0){
- this.results = Arrays.copyOf(this.results, this.results.length + batchResults.length);
- System.arraycopy(batchResults, 0, this.results, this.results.length - batchResults.length, batchResults.length);
- }
-
- if(cacheResults.isFinal()){
- this.command = cacheResults.getCommand();
- this.analysisRecord = cacheResults.getAnalysisRecord();
- this.firstRow = 1;
- this.isFinal = true;
- }
- return true;
- }
-
-}
Deleted: trunk/engine/src/main/java/org/teiid/dqp/internal/cache/ResultSetCache.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/dqp/internal/cache/ResultSetCache.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/org/teiid/dqp/internal/cache/ResultSetCache.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -1,232 +0,0 @@
-/*
- * JBoss, Home of Professional Open Source.
- * See the COPYRIGHT.txt file distributed with this work for information
- * regarding copyright ownership. Some portions may be licensed
- * to Red Hat, Inc. under one or more contributor license agreements.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301 USA.
- */
-
-package org.teiid.dqp.internal.cache;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.teiid.dqp.internal.process.DQPWorkContext;
-
-import com.metamatrix.cache.Cache;
-import com.metamatrix.cache.CacheConfiguration;
-import com.metamatrix.cache.CacheFactory;
-import com.metamatrix.cache.Cache.Type;
-import com.metamatrix.cache.CacheConfiguration.Policy;
-import com.metamatrix.common.buffer.TupleBatch;
-import com.metamatrix.common.buffer.impl.BufferManagerImpl;
-import com.metamatrix.common.buffer.impl.SizeUtility;
-import com.metamatrix.core.util.HashCodeUtil;
-
-/**
- * Used to cache ResultSet based on the exact match of sql string.
- */
-public class ResultSetCache {
-
- static class TempKey {
- CacheID cacheID;
- Object requestID;
-
- public TempKey(CacheID cacheID, Object requestID) {
- this.cacheID = cacheID;
- this.requestID = requestID;
- }
- @Override
- public int hashCode() {
- return HashCodeUtil.hashCode(cacheID.hashCode(), requestID.hashCode());
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (!(obj instanceof TempKey)) {
- return false;
- }
- TempKey other = (TempKey) obj;
- return (cacheID.equals(other.cacheID) && requestID.equals(other.requestID));
- }
- }
-
- //configurable parameters
- public static final String RS_CACHE_MAX_ENTRIES = "maxEntries"; //$NON-NLS-1$
- public static final String RS_CACHE_MAX_SIZE = "maxEntrySize"; //$NON-NLS-1$
- public static final String RS_CACHE_MAX_AGE = "maxAge"; //$NON-NLS-1$
- public static final String RS_CACHE_SCOPE = "scope"; //$NON-NLS-1$
-
- //constants
- public static final String RS_CACHE_SCOPE_VDB = "vdb"; //$NON-NLS-1$
- public static final String RS_CACHE_SCOPE_CONN = "connection"; //$NON-NLS-1$
- private static final char DELIMITOR = '.';
-
- private Cache<CacheID, CacheResults> cache;
- private String scope = RS_CACHE_SCOPE_VDB;
- private Map<TempKey, CacheResults> tempBatchResults = new HashMap<TempKey, CacheResults>();
- private int maxEntrySize = 100 * 1024; //bytes
- private int maxAge = 60 * 60; // seconds
- private int maxEntries = 1024;
-
- public void start(CacheFactory cacheFactory) {
- this.cache = cacheFactory.get(Type.RESULTSET, new CacheConfiguration(Policy.MRU, maxAge, maxEntries));
- }
-
- public void setMaxEntries(int maxEntries) {
- this.maxEntries = maxEntries;
- }
-
- public void setMaxEntrySize(int maxEntrySize) {
- this.maxEntrySize = maxEntrySize;
- }
-
- public void setMaxAge(int maxAge) {
- if (maxAge <= 0) {
- this.maxAge = Integer.MAX_VALUE;
- }
- this.maxAge = Math.max(1, maxAge / 1000);
- }
-
- public void setScope(String scope) {
- this.scope = scope;
- }
-
- //interval is 1 based.
- public final CacheResults getResults(CacheID cacheID, int[] interval){
- CacheResults cacheResults = null;
- cacheResults = cache.get(cacheID);
- if(cacheResults == null){
- return null;
- }
- if (interval == null) {
- return cacheResults;
- }
- int firstRow = interval[0] - 1;
- int resultSize = cacheResults.getResults().length;
- int finalRow = resultSize - 1;
- int lastRow = Math.min(finalRow, interval[1] - 1);
- if(resultSize == 0 || (firstRow ==0 && lastRow == finalRow)){
- //the whole results
- return cacheResults;
- }
- int batchSize = lastRow - firstRow + 1;
- List<?>[] resultsPart = new List[batchSize];
- System.arraycopy(cacheResults.getResults(), firstRow, resultsPart, 0, batchSize);
- CacheResults newCacheResults = new CacheResults(resultsPart, firstRow + 1, lastRow == finalRow);
- newCacheResults.setCommand(cacheResults.getCommand());
- newCacheResults.setAnalysisRecord(cacheResults.getAnalysisRecord());
- return newCacheResults;
- }
-
- public boolean hasResults(CacheID cacheID){
- boolean hasResults = cache.get(cacheID) != null;
- return hasResults;
- }
-
- /**
- * if valueID is not null, it is lob (XML document) chunk
- * @return true if the result was cachable
- */
- public boolean setResults(CacheID cacheID, CacheResults cacheResults, Object requestID){
- List<?>[] results = cacheResults.getResults();
- if(cacheResults.getSize() == TupleBatch.UNKNOWN_SIZE){
- cacheResults.setSize(SizeUtility.getBatchSize(BufferManagerImpl.getTypeNames(cacheResults.getElements()), results));
- }
-
- TempKey key = new TempKey(cacheID, requestID);
- //do not cache if it is over cache limit
- if(isOverCacheLimit(cacheResults.getSize())){
- removeTempResults(key);
- return false;
- }
-
- synchronized(tempBatchResults){
- CacheResults savedResults = tempBatchResults.get(key);
- if(savedResults == null){
- if (tempBatchResults.size() >= maxEntries) {
- return false;
- }
- savedResults = cacheResults;
- tempBatchResults.put(key, cacheResults);
- } else if(!savedResults.addResults(cacheResults)){
- removeTempResults(key);
- return false;
- }
-
- //do not cache if it is over cache limit
- if(isOverCacheLimit(savedResults.getSize())){
- removeTempResults(key);
- return false;
- }
-
- if(savedResults.isFinal()){
- tempBatchResults.remove(cacheID);
- cache.put(cacheID, savedResults);
- }
- }
-
- return true;
- }
-
- private boolean isOverCacheLimit(long sizeToAdd) {
- if(maxEntrySize == 0 || sizeToAdd == 0){
- return false;
- }
- return sizeToAdd > maxEntrySize;
- }
-
- public void removeTempResults(CacheID cacheID, Object requestID){
- removeTempResults(new TempKey(cacheID, requestID));
- }
-
- public void removeTempResults(TempKey cacheID){
- synchronized(tempBatchResults){
- tempBatchResults.remove(cacheID);
- }
- }
-
- public void clear(){
- cache.clear();
- synchronized(tempBatchResults){
- tempBatchResults.clear();
- }
- }
-
- public void shutDown(){
- clear();
- }
-
- public String getCacheScope(){
- return scope;
- }
-
- public CacheID createCacheID(DQPWorkContext workContext, String command, List<?> parameterValues){
- String scopeID = null;
- if(RS_CACHE_SCOPE_VDB.equalsIgnoreCase(getCacheScope())){
- scopeID = workContext.getVdbName() + DELIMITOR + workContext.getVdbVersion();
- }else{
- scopeID = workContext.getConnectionID();
- }
- return new CacheID(scopeID, command, parameterValues);
- }
-
-}
Copied: trunk/engine/src/main/java/org/teiid/dqp/internal/process/CacheResults.java (from rev 1666, trunk/engine/src/main/java/org/teiid/dqp/internal/cache/CacheResults.java)
===================================================================
--- trunk/engine/src/main/java/org/teiid/dqp/internal/process/CacheResults.java (rev 0)
+++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/CacheResults.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -0,0 +1,68 @@
+/*
+ * JBoss, Home of Professional Open Source.
+ * See the COPYRIGHT.txt file distributed with this work for information
+ * regarding copyright ownership. Some portions may be licensed
+ * to Red Hat, Inc. under one or more contributor license agreements.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301 USA.
+ */
+
+package org.teiid.dqp.internal.process;
+
+import java.io.Serializable;
+import java.util.List;
+
+import com.metamatrix.common.buffer.TupleSourceID;
+import com.metamatrix.query.analysis.AnalysisRecord;
+import com.metamatrix.query.sql.lang.Command;
+import com.metamatrix.query.sql.symbol.SingleElementSymbol;
+
+public class CacheResults implements Serializable {
+ private TupleSourceID results;
+ private AnalysisRecord analysisRecord;
+ private Command command;
+
+ public CacheResults(TupleSourceID results){
+ this.results = results;
+ }
+
+ public TupleSourceID getResults() {
+ return results;
+ }
+
+ public List<SingleElementSymbol> getElements() {
+ if (command == null) {
+ return null;
+ }
+ return command.getProjectedSymbols();
+ }
+
+ public Command getCommand() {
+ return command;
+ }
+
+ public void setCommand(Command command) {
+ this.command = command;
+ }
+
+ public AnalysisRecord getAnalysisRecord() {
+ return analysisRecord;
+ }
+
+ public void setAnalysisRecord(AnalysisRecord analysisRecord) {
+ this.analysisRecord = analysisRecord;
+ }
+}
Deleted: trunk/engine/src/main/java/org/teiid/dqp/internal/process/CachedRequestWorkItem.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/dqp/internal/process/CachedRequestWorkItem.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/CachedRequestWorkItem.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -1,102 +0,0 @@
-/*
- * JBoss, Home of Professional Open Source.
- * See the COPYRIGHT.txt file distributed with this work for information
- * regarding copyright ownership. Some portions may be licensed
- * to Red Hat, Inc. under one or more contributor license agreements.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301 USA.
- */
-
-package org.teiid.dqp.internal.process;
-
-import java.util.List;
-
-import javax.transaction.SystemException;
-
-import org.teiid.dqp.internal.cache.CacheResults;
-
-import com.metamatrix.api.exception.MetaMatrixComponentException;
-import com.metamatrix.api.exception.MetaMatrixProcessingException;
-import com.metamatrix.common.buffer.BlockedException;
-import com.metamatrix.common.buffer.BlockedOnMemoryException;
-import com.metamatrix.common.buffer.TupleBatch;
-import com.metamatrix.common.buffer.TupleSourceNotFoundException;
-import com.metamatrix.common.comm.api.ResultsReceiver;
-import com.metamatrix.core.MetaMatrixCoreException;
-import com.metamatrix.dqp.DQPPlugin;
-import com.metamatrix.dqp.message.RequestID;
-import com.metamatrix.dqp.message.RequestMessage;
-import com.metamatrix.dqp.message.ResultsMessage;
-import com.metamatrix.query.sql.lang.Command;
-
-public class CachedRequestWorkItem extends RequestWorkItem {
-
- public CachedRequestWorkItem(DQPCore dqpCore, RequestMessage requestMsg, Request request, ResultsReceiver<ResultsMessage> receiver, RequestID requestID, DQPWorkContext workContext, Command originalCommand) {
- super(dqpCore, requestMsg, request, receiver, requestID, workContext);
- this.originalCommand = originalCommand;
- }
-
- @Override
- protected void processNew() throws MetaMatrixComponentException, MetaMatrixProcessingException {
- request.initMetadata();
- request.validateAccess(originalCommand);
- this.request = null;
- }
-
- @Override
- protected void processMore() throws SystemException, BlockedException,
- MetaMatrixCoreException {
- //do nothing
- }
-
-
- @Override
- protected void sendResultsIfNeeded(TupleBatch batch)
- throws BlockedOnMemoryException, MetaMatrixComponentException,
- TupleSourceNotFoundException {
- synchronized (this.resultsCursor) {
- if (!this.resultsCursor.resultsRequested) {
- return;
- }
- }
- CacheResults cResult = rsCache.getResults(cid, new int[] {this.resultsCursor.begin, this.resultsCursor.end});
- if (cResult == null) {
- throw new MetaMatrixComponentException(DQPPlugin.Util.getString("CachedRequestWorkItem.not_available")); //$NON-NLS-1$
- }
- List results[] = cResult.getResults();
- int firstRow = cResult.getFirstRow();
-
- ResultsMessage response = createResultsMessage(requestMsg, results, cResult.getElements(), cResult.getAnalysisRecord());
- response.setFirstRow(firstRow);
- response.setLastRow(firstRow + results.length - 1);
-
- boolean isFinal = cResult.isFinal();
- if(isFinal){
- response.setFinalRow(cResult.getFinalRow());
- }
-
- this.resultsCursor.resultsSent();
- this.resultsReceiver.receiveResults(response);
- }
-
- @Override
- protected void attemptClose() {
- this.isClosed = true;
- dqpCore.logMMCommand(this, false, false, -1);
- this.dqpCore.removeRequest(this);
- }
-
-}
\ No newline at end of file
Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/process/DQPCore.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/dqp/internal/process/DQPCore.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/DQPCore.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -39,10 +39,7 @@
import javax.transaction.xa.Xid;
import org.teiid.connector.xa.api.TransactionContext;
-import org.teiid.dqp.internal.cache.CacheID;
-import org.teiid.dqp.internal.cache.CacheResults;
import org.teiid.dqp.internal.cache.DQPContextCache;
-import org.teiid.dqp.internal.cache.ResultSetCache;
import com.google.inject.Inject;
import com.google.inject.Singleton;
@@ -149,7 +146,7 @@
private PreparedPlanCache prepPlanCache;
private TransactionService transactionService;
private MetadataService metadataService;
- private ResultSetCache rsCache;
+ //private ResultSetCache rsCache;
// Query worker pool for processing plans
private WorkerPool processWorkerPool;
@@ -266,21 +263,10 @@
state.tempTableStoreImpl, workContext,
chunkSize);
- RequestWorkItem workItem = null;
-
ResultsFuture<ResultsMessage> resultsFuture = new ResultsFuture<ResultsMessage>();
-
- if(this.rsCache != null && requestMsg.useResultSetCache()) {
- CacheID cID = this.rsCache.createCacheID(workContext, requestMsg.getCommandString(), requestMsg.getParameterValues());
- CacheResults cr = this.rsCache.getResults(cID, new int[]{1, 1});
- if (cr != null) {
- workItem = new CachedRequestWorkItem(this, requestMsg, request, resultsFuture.getResultsReceiver(), requestID, workContext, cr.getCommand());
- }
- }
- if (workItem == null) {
- workItem = new RequestWorkItem(this, requestMsg, request, resultsFuture.getResultsReceiver(), requestID, workContext);
- }
-
+
+ RequestWorkItem workItem = new RequestWorkItem(this, requestMsg, request, resultsFuture.getResultsReceiver(), requestID, workContext);
+
logMMCommand(workItem, true, false, 0); //TODO: there is no transaction at this point
addRequest(requestID, workItem, state);
@@ -423,7 +409,7 @@
// cleanup the buffer manager
try {
- bufferManager.removeTupleSources(sessionId);
+ bufferManager.removeTupleBuffers(sessionId);
} catch (Exception e) {
LogManager.logWarning(LogConstants.CTX_DQP, e, "Failed to remove buffered tuples for connection " + sessionId); //$NON-NLS-1$
}
@@ -461,7 +447,7 @@
return markCancelled;
}
- public ResultsFuture<?> closeRequest(long requestId) throws MetaMatrixProcessingException {
+ public ResultsFuture<?> closeRequest(long requestId) throws MetaMatrixProcessingException, MetaMatrixComponentException {
DQPWorkContext workContext = DQPWorkContext.getWorkContext();
closeRequest(workContext.getRequestID(requestId));
return null;
@@ -470,8 +456,9 @@
/**
* Close the request with given ID
* @param requestID
+ * @throws MetaMatrixComponentException
*/
- void closeRequest(RequestID requestID) {
+ void closeRequest(RequestID requestID) throws MetaMatrixComponentException {
if (LogManager.isMessageToBeRecorded(LogConstants.CTX_DQP, MessageLevel.DETAIL)) {
LogManager.logDetail(LogConstants.CTX_DQP, "closeQuery for requestID=" + requestID); //$NON-NLS-1$
}
@@ -496,9 +483,9 @@
public void clearResultSetCache() {
//clear cache in server
- if(rsCache != null){
+ /*if(rsCache != null){
rsCache.clear();
- }
+ }*/
}
void logMMCommand(RequestWorkItem workItem, boolean isBegin, boolean isCancel, int rowCount) {
@@ -569,9 +556,9 @@
return transactionService;
}
- ResultSetCache getRsCache() {
+ /*ResultSetCache getRsCache() {
return rsCache;
- }
+ }*/
int getProcessorTimeSlice() {
return this.processorTimeslice;
@@ -632,11 +619,11 @@
this.chunkSize = PropertiesUtils.getIntProperty(props, DQPEmbeddedProperties.STREAMING_BATCH_SIZE, 10) * 1024;
//result set cache
- if(PropertiesUtils.getBooleanProperty(props, DQPEmbeddedProperties.USE_RESULTSET_CACHE, false)){
+ /*if(PropertiesUtils.getBooleanProperty(props, DQPEmbeddedProperties.USE_RESULTSET_CACHE, false)){
this.rsCache = new ResultSetCache();
PropertiesUtils.setBeanProperties(this.rsCache, props, "ResultSetCache"); //$NON-NLS-1$
this.rsCache.start(cacheFactory);
- }
+ }*/
//prepared plan cache
int maxSizeTotal = PropertiesUtils.getIntProperty(props, DQPEmbeddedProperties.MAX_PLAN_CACHE_SIZE, PreparedPlanCache.DEFAULT_MAX_SIZE_TOTAL);
Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/process/DataTierManagerImpl.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/dqp/internal/process/DataTierManagerImpl.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/DataTierManagerImpl.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -54,7 +54,6 @@
import com.metamatrix.common.buffer.TupleBatch;
import com.metamatrix.common.buffer.TupleSource;
import com.metamatrix.common.comm.api.ResultsReceiver;
-import com.metamatrix.common.log.LogManager;
import com.metamatrix.common.vdb.api.ModelInfo;
import com.metamatrix.core.CoreConstants;
import com.metamatrix.core.util.Assertion;
@@ -70,8 +69,8 @@
import com.metamatrix.dqp.service.DataService;
import com.metamatrix.dqp.service.MetadataService;
import com.metamatrix.dqp.service.VDBService;
-import com.metamatrix.dqp.util.LogConstants;
import com.metamatrix.metadata.runtime.api.MetadataSourceUtil;
+import com.metamatrix.query.processor.CollectionTupleSource;
import com.metamatrix.query.processor.ProcessorDataManager;
import com.metamatrix.query.processor.QueryProcessor;
import com.metamatrix.query.sql.ReservedWords;
@@ -80,7 +79,6 @@
import com.metamatrix.query.sql.lang.StoredProcedure;
import com.metamatrix.query.sql.lang.UnaryFromClause;
import com.metamatrix.query.sql.symbol.GroupSymbol;
-import com.metamatrix.query.sql.symbol.SingleElementSymbol;
import com.metamatrix.query.util.CommandContext;
public class DataTierManagerImpl implements ProcessorDataManager {
@@ -105,37 +103,6 @@
GETVDBRESOURCEPATHS
}
- private class CollectionTupleSource implements TupleSource {
-
- private Iterator<List<Object>> tuples;
- private List<SingleElementSymbol> schema;
-
- public CollectionTupleSource(Iterator<List<Object>> tuples,
- List<SingleElementSymbol> schema) {
- this.tuples = tuples;
- this.schema = schema;
- }
-
- @Override
- public List<?> nextTuple() throws MetaMatrixComponentException,
- MetaMatrixProcessingException {
- if (tuples.hasNext()) {
- return tuples.next();
- }
- return null;
- }
-
- @Override
- public List<SingleElementSymbol> getSchema() {
- return schema;
- }
-
- @Override
- public void closeSource() throws MetaMatrixComponentException {
-
- }
- }
-
// Resources
private DQPCore requestMgr;
private DataService dataService;
@@ -294,11 +261,11 @@
break;
case REFERENCEKEYCOLUMNS:
for (ForeignKey key : table.getForeignKeys()) {
- int postition = 0;
+ short postition = 0;
for (Column column : key.getColumns()) {
Table pkTable = key.getPrimaryKey().getTable();
rows.add(Arrays.asList(vdbName, pkTable.getSchema().getName(), pkTable.getName(), key.getPrimaryKey().getColumns().get(postition).getName(), vdbName, schema.getName(), table.getName(), column.getName(),
- ++postition, (short)DatabaseMetaData.importedKeyNoAction, (short)DatabaseMetaData.importedKeyNoAction, key.getName(), key.getPrimaryKey().getName(), (short)DatabaseMetaData.importedKeyInitiallyDeferred));
+ ++postition, DatabaseMetaData.importedKeyNoAction, DatabaseMetaData.importedKeyNoAction, key.getName(), key.getPrimaryKey().getName(), DatabaseMetaData.importedKeyInitiallyDeferred));
}
}
break;
@@ -467,19 +434,16 @@
final CacheKey codeRequestId = this.codeTableCache.createCacheRequest(codeTableName, returnElementName, keyElementName, context);
boolean success = false;
- QueryProcessor processor = null;
try {
- processor = context.getQueryProcessorFactory().createQueryProcessor(query, codeTableName.toUpperCase(), context);
-
- processor.setBatchHandler(new QueryProcessor.BatchHandler() {
- @Override
- public void batchProduced(TupleBatch batch) throws MetaMatrixProcessingException {
- codeTableCache.loadTable(codeRequestId, batch.getAllTuples());
+ QueryProcessor processor = context.getQueryProcessorFactory().createQueryProcessor(query, codeTableName.toUpperCase(), context);
+ processor.setNonBlocking(true); //process lookup as fully blocking
+ while (true) {
+ TupleBatch batch = processor.nextBatch();
+ codeTableCache.loadTable(codeRequestId, batch.getAllTuples());
+ if (batch.getTerminationFlag()) {
+ break;
}
- });
-
- //process lookup as fully blocking
- processor.process();
+ }
success = true;
} finally {
Collection requests = null;
@@ -489,13 +453,6 @@
requests = codeTableCache.errorLoadingCache(codeRequestId);
}
notifyWaitingCodeTableRequests(requests);
- if (processor != null) {
- try {
- this.bufferService.getBufferManager().removeTupleSource(processor.getResultsID());
- } catch (MetaMatrixComponentException e1) {
- LogManager.logDetail(LogConstants.CTX_DQP, "Exception closing code table request"); //$NON-NLS-1$
- }
- }
}
}
Deleted: trunk/engine/src/main/java/org/teiid/dqp/internal/process/LobChunkStream.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/dqp/internal/process/LobChunkStream.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/LobChunkStream.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -1,83 +0,0 @@
-/*
- * JBoss, Home of Professional Open Source.
- * See the COPYRIGHT.txt file distributed with this work for information
- * regarding copyright ownership. Some portions may be licensed
- * to Red Hat, Inc. under one or more contributor license agreements.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301 USA.
- */
-
-package org.teiid.dqp.internal.process;
-
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.sql.SQLException;
-
-import com.metamatrix.common.lob.ByteLobChunkStream;
-import com.metamatrix.common.lob.LobChunk;
-import com.metamatrix.common.lob.LobChunkProducer;
-import com.metamatrix.common.lob.ReaderInputStream;
-import com.metamatrix.common.types.BlobType;
-import com.metamatrix.common.types.ClobType;
-import com.metamatrix.common.types.Streamable;
-import com.metamatrix.common.types.XMLType;
-
-/**
- * A Lob Stream builder class. Given the Lob object this object can build
- * an object which can produce a stream lob chunks, which can be sent to the
- * client one by one.
- */
-class LobChunkStream implements LobChunkProducer {
-
- LobChunkProducer internalStream = null;
-
- public LobChunkStream(Streamable<?> streamable, int chunkSize)
- throws IOException {
-
- try {
- if (streamable instanceof XMLType) {
- XMLType xml = (XMLType)streamable;
- this.internalStream = new ByteLobChunkStream(new ReaderInputStream(xml.getCharacterStream(), Charset.forName("UTF-16")), chunkSize); //$NON-NLS-1$
- }
- else if (streamable instanceof ClobType) {
- ClobType clob = (ClobType)streamable;
- this.internalStream = new ByteLobChunkStream(new ReaderInputStream(clob.getCharacterStream(), Charset.forName("UTF-16")), chunkSize); //$NON-NLS-1$
- }
- else if (streamable instanceof BlobType) {
- BlobType blob = (BlobType)streamable;
- this.internalStream = new ByteLobChunkStream(blob.getBinaryStream(), chunkSize);
- }
- } catch(SQLException e) {
- IOException ex = new IOException();
- ex.initCause(e);
- throw ex;
- }
- }
-
- /**
- * @see com.metamatrix.common.lob.LobChunkProducer#getNextChunk(int)
- */
- public LobChunk getNextChunk() throws IOException {
- return internalStream.getNextChunk();
- }
-
- /**
- * @see com.metamatrix.common.lob.LobChunkProducer#close()
- */
- public void close() throws IOException {
- internalStream.close();
- }
-}
Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/process/LobWorkItem.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/dqp/internal/process/LobWorkItem.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/LobWorkItem.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -23,14 +23,19 @@
package org.teiid.dqp.internal.process;
import java.io.IOException;
+import java.nio.charset.Charset;
+import java.sql.SQLException;
import com.metamatrix.api.exception.MetaMatrixComponentException;
-import com.metamatrix.common.buffer.BlockedOnMemoryException;
-import com.metamatrix.common.buffer.TupleSourceNotFoundException;
import com.metamatrix.common.comm.api.ResultsReceiver;
+import com.metamatrix.common.lob.ByteLobChunkStream;
import com.metamatrix.common.lob.LobChunk;
+import com.metamatrix.common.lob.ReaderInputStream;
import com.metamatrix.common.log.LogManager;
+import com.metamatrix.common.types.BlobType;
+import com.metamatrix.common.types.ClobType;
import com.metamatrix.common.types.Streamable;
+import com.metamatrix.common.types.XMLType;
import com.metamatrix.core.util.Assertion;
import com.metamatrix.dqp.DQPPlugin;
import com.metamatrix.dqp.message.RequestID;
@@ -46,7 +51,7 @@
/* private work item state */
private String streamId;
- private LobChunkStream stream;
+ private ByteLobChunkStream stream;
private int streamRequestId;
private ResultsReceiver<LobChunk> resultsReceiver;
@@ -74,13 +79,6 @@
// now get the chunk from stream
chunk = stream.getNextChunk();
shouldClose = chunk.isLast();
- } catch (BlockedOnMemoryException e) {
- LogManager.logDetail(LogConstants.CTX_DQP, new Object[] {"Reenqueueing LOB chunk request due to lack of available memory ###########", requestID}); //$NON-NLS-1$
- this.dqpCore.addWork(this);
- return;
- } catch (TupleSourceNotFoundException e) {
- LogManager.logWarning(LogConstants.CTX_DQP, e, DQPPlugin.Util.getString("BufferManagerLobChunkStream.no_tuple_source", streamId)); //$NON-NLS-1$
- ex = e;
} catch (MetaMatrixComponentException e) {
LogManager.logWarning(LogConstants.CTX_DQP, e, DQPPlugin.Util.getString("ProcessWorker.LobError")); //$NON-NLS-1$
ex = e;
@@ -114,13 +112,29 @@
* Create a object which can create a sequence of LobChunk objects on a given
* LOB object
*/
- private LobChunkStream createLobStream(String referenceStreamId)
- throws BlockedOnMemoryException, MetaMatrixComponentException, IOException, TupleSourceNotFoundException {
+ private ByteLobChunkStream createLobStream(String referenceStreamId)
+ throws MetaMatrixComponentException, IOException {
// get the reference object in the buffer manager, and try to stream off
// the original sources.
- Streamable<?> streamable = dqpCore.getBufferManager().getStreamable(parent.resultsID, referenceStreamId);
- return new LobChunkStream(streamable, chunkSize);
+ Streamable<?> streamable = parent.resultsBuffer.getLobReference(referenceStreamId);
+
+ try {
+ if (streamable instanceof XMLType) {
+ XMLType xml = (XMLType)streamable;
+ return new ByteLobChunkStream(new ReaderInputStream(xml.getCharacterStream(), Charset.forName("UTF-16")), chunkSize); //$NON-NLS-1$
+ }
+ else if (streamable instanceof ClobType) {
+ ClobType clob = (ClobType)streamable;
+ return new ByteLobChunkStream(new ReaderInputStream(clob.getCharacterStream(), Charset.forName("UTF-16")), chunkSize); //$NON-NLS-1$
+ }
+ BlobType blob = (BlobType)streamable;
+ return new ByteLobChunkStream(blob.getBinaryStream(), chunkSize);
+ } catch(SQLException e) {
+ IOException ex = new IOException();
+ ex.initCause(e);
+ throw ex;
+ }
}
synchronized void setResultsReceiver(ResultsReceiver<LobChunk> resultsReceiver) {
Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/process/PreparedPlanCache.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/dqp/internal/process/PreparedPlanCache.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/PreparedPlanCache.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -131,7 +131,6 @@
static class PreparedPlan{
private ProcessorPlan plan;
private Command command;
- private Command rewritenCommand;
private List<Reference> refs;
private AnalysisRecord analysisRecord;
@@ -191,14 +190,6 @@
refs = refsValue;
}
- public void setRewritenCommand(Command rewritenCommand) {
- this.rewritenCommand = rewritenCommand;
- }
-
- public Command getRewritenCommand() {
- return rewritenCommand;
- }
-
}
//for testing purpose
Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/process/PreparedStatementRequest.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/dqp/internal/process/PreparedStatementRequest.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/PreparedStatementRequest.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -28,6 +28,7 @@
import java.util.List;
import org.teiid.dqp.internal.process.PreparedPlanCache.CacheID;
+import org.teiid.dqp.internal.process.PreparedPlanCache.PreparedPlan;
import com.metamatrix.api.exception.MetaMatrixComponentException;
import com.metamatrix.api.exception.query.ExpressionEvaluationException;
@@ -62,7 +63,7 @@
*/
public class PreparedStatementRequest extends Request {
private PreparedPlanCache prepPlanCache;
- private PreparedPlanCache.PreparedPlan prepPlan;
+ private PreparedPlan prepPlan;
public PreparedStatementRequest(PreparedPlanCache prepPlanCache) {
this.prepPlanCache = prepPlanCache;
@@ -129,20 +130,20 @@
* @throws QueryPlannerException
* @see org.teiid.dqp.internal.process.Request#generatePlan()
*/
- protected Command generatePlan() throws QueryPlannerException, QueryParserException, QueryResolverException, QueryValidatorException, MetaMatrixComponentException {
+ protected void generatePlan() throws QueryPlannerException, QueryParserException, QueryResolverException, QueryValidatorException, MetaMatrixComponentException {
String sqlQuery = requestMsg.getCommands()[0];
CacheID id = new PreparedPlanCache.CacheID(this.workContext, Request.createParseInfo(this.requestMsg), sqlQuery);
prepPlan = prepPlanCache.getPreparedPlan(id);
if (prepPlan == null) {
//if prepared plan does not exist, create one
- prepPlan = new PreparedPlanCache.PreparedPlan();
+ prepPlan = new PreparedPlan();
LogManager.logTrace(LogConstants.CTX_DQP, new Object[] { "Query does not exist in cache: ", sqlQuery}); //$NON-NLS-1$
}
ProcessorPlan cachedPlan = prepPlan.getPlan();
if (cachedPlan == null) {
- prepPlan.setRewritenCommand(super.generatePlan());
+ super.generatePlan();
if (!this.addedLimit) { //TODO: this is a little problematic
prepPlan.setCommand(this.userCommand);
// Defect 13751: Clone the plan in its current state (i.e. before processing) so that it can be used for later queries
@@ -168,7 +169,6 @@
PreparedStatementRequest.resolveParameterValues(params, values, this.context);
}
- return prepPlan.getRewritenCommand();
}
/**
@@ -191,7 +191,7 @@
throw new QueryValidatorException("No batch values sent for prepared batch update"); //$NON-NLS-1$
}
boolean supportPreparedBatchUpdate = false;
- if (this.processPlan instanceof RelationalPlan && this.prepPlan.getRewritenCommand().getSubCommands().isEmpty()) {
+ if (this.processPlan instanceof RelationalPlan) {
RelationalPlan rPlan = (RelationalPlan)this.processPlan;
if (rPlan.getRootNode() instanceof AccessNode) {
AccessNode aNode = (AccessNode)rPlan.getRootNode();
@@ -218,7 +218,7 @@
}
continue;
}
- Command c = (Command)this.prepPlan.getRewritenCommand().clone();
+ Command c = (Command)this.prepPlan.getCommand().clone();
commands.add(c);
c.setProcessorPlan((ProcessorPlan)this.processPlan.clone());
}
Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/process/Request.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/dqp/internal/process/Request.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/Request.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -303,7 +303,7 @@
// Create generic sql validation visitor
AbstractValidationVisitor visitor = new ValidationVisitor();
- validateWithVisitor(visitor, metadata, command, false);
+ validateWithVisitor(visitor, metadata, command);
}
private Command parseCommand() throws QueryParserException {
@@ -330,8 +330,7 @@
public static void validateWithVisitor(
AbstractValidationVisitor visitor,
QueryMetadataInterface metadata,
- Command command,
- boolean validateOnlyEmbedded)
+ Command command)
throws QueryValidatorException, MetaMatrixComponentException {
// Validate with visitor
@@ -422,7 +421,7 @@
* @throws QueryResolverException
* @throws QueryValidatorException
*/
- protected Command generatePlan() throws MetaMatrixComponentException, QueryPlannerException, QueryParserException, QueryResolverException, QueryValidatorException {
+ protected void generatePlan() throws MetaMatrixComponentException, QueryPlannerException, QueryParserException, QueryResolverException, QueryValidatorException {
Command command = parseCommand();
List<Reference> references = ReferenceCollectorVisitor.getReferences(command);
@@ -486,7 +485,6 @@
String msg = DQPPlugin.Util.getString("DQPCore.Unknown_query_metadata_exception_while_registering_query__{0}.", params); //$NON-NLS-1$
throw new QueryPlannerException(e, msg);
}
- return command;
}
private void setSchemasForXMLPlan(Command command, QueryMetadataInterface metadata)
@@ -555,13 +553,13 @@
initMetadata();
- Command processingCommand = generatePlan();
+ generatePlan();
validateAccess(userCommand);
setSchemasForXMLPlan(userCommand, metadata);
- createProcessor(processingCommand);
+ createProcessor(userCommand);
}
public QueryProcessor createQueryProcessor(String query, String recursionGroup, CommandContext commandContext) throws MetaMatrixProcessingException, MetaMatrixComponentException {
@@ -601,6 +599,6 @@
// See if entitlement checking is turned on
AuthorizationValidationVisitor visitor = new AuthorizationValidationVisitor(this.workContext.getConnectionID(), authSvc, this.vdbService, this.vdbName, this.vdbVersion);
- validateWithVisitor(visitor, this.metadata, command, true);
+ validateWithVisitor(visitor, this.metadata, command);
}
}
Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -36,21 +36,14 @@
import javax.transaction.SystemException;
import org.teiid.connector.xa.api.TransactionContext;
-import org.teiid.dqp.internal.cache.CacheID;
-import org.teiid.dqp.internal.cache.CacheResults;
-import org.teiid.dqp.internal.cache.ResultSetCache;
import com.metamatrix.api.exception.MetaMatrixComponentException;
import com.metamatrix.api.exception.MetaMatrixException;
import com.metamatrix.api.exception.MetaMatrixProcessingException;
import com.metamatrix.common.buffer.BlockedException;
-import com.metamatrix.common.buffer.BlockedOnMemoryException;
import com.metamatrix.common.buffer.BufferManager;
-import com.metamatrix.common.buffer.MemoryNotAvailableException;
import com.metamatrix.common.buffer.TupleBatch;
-import com.metamatrix.common.buffer.TupleSourceID;
-import com.metamatrix.common.buffer.TupleSourceNotFoundException;
-import com.metamatrix.common.buffer.BufferManager.TupleSourceStatus;
+import com.metamatrix.common.buffer.TupleBuffer;
import com.metamatrix.common.comm.api.ResultsReceiver;
import com.metamatrix.common.lob.LobChunk;
import com.metamatrix.common.log.LogManager;
@@ -71,8 +64,9 @@
import com.metamatrix.query.analysis.AnalysisRecord;
import com.metamatrix.query.analysis.QueryAnnotation;
import com.metamatrix.query.execution.QueryExecPlugin;
+import com.metamatrix.query.processor.BatchCollector;
import com.metamatrix.query.processor.QueryProcessor;
-import com.metamatrix.query.processor.QueryProcessor.BatchHandler;
+import com.metamatrix.query.processor.BatchCollector.BatchHandler;
import com.metamatrix.query.sql.lang.Command;
import com.metamatrix.query.sql.lang.Option;
import com.metamatrix.query.sql.lang.SPParameter;
@@ -125,8 +119,8 @@
protected Request request; //provides the processing plan, held on a temporary basis
final private BufferManager bufferMgr;
final private int processorTimeslice;
- protected ResultSetCache rsCache;
- protected CacheID cid;
+ //protected ResultSetCache rsCache;
+ //protected CacheID cid;
final private TransactionService transactionService;
final DQPWorkContext dqpWorkContext;
ResultsReceiver<ResultsMessage> resultsReceiver;
@@ -135,10 +129,11 @@
* obtained during new
*/
private volatile QueryProcessor processor;
+ private BatchCollector collector;
protected Command originalCommand;
private AnalysisRecord analysisRecord;
private TransactionContext transactionContext;
- protected TupleSourceID resultsID;
+ protected TupleBuffer resultsBuffer;
private Collection schemas; // These are schemas associated with XML results
private boolean returnsUpdateCount;
@@ -166,10 +161,10 @@
this.resultsCursor.requestResults(1, requestMsg.getFetchSize(), false);
this.bufferMgr = dqpCore.getBufferManager();
this.processorTimeslice = dqpCore.getProcessorTimeSlice();
- this.rsCache = dqpCore.getRsCache();
+ /*this.rsCache = dqpCore.getRsCache();
if (this.rsCache != null) {
this.cid = this.rsCache.createCacheID(workContext, requestMsg.getCommandString(), requestMsg.getParameterValues());
- }
+ }*/
this.transactionService = dqpCore.getTransactionServiceDirect();
this.dqpCore = dqpCore;
this.request = request;
@@ -208,17 +203,10 @@
this.state = ProcessingState.CLOSE;
}
}
- } catch (BlockedOnMemoryException e) {
- moreWork(false);
- LogManager.logDetail(LogConstants.CTX_DQP, "############# PW EXITING on", requestID, "- reenqueueing for more processing due to lack of available memory ###########"); //$NON-NLS-1$ //$NON-NLS-2$
} catch (BlockedException e) {
LogManager.logDetail(LogConstants.CTX_DQP, "############# PW EXITING on", requestID, "- processor blocked ###########"); //$NON-NLS-1$ //$NON-NLS-2$
} catch (Throwable e) {
LogManager.logDetail(LogConstants.CTX_DQP, e, "############# PW EXITING on", requestID, "- error occurred ###########"); //$NON-NLS-1$ //$NON-NLS-2$
- //if there is a cache, remove temp results if there is any
- if(this.rsCache != null){
- rsCache.removeTempResults(cid, requestID);
- }
if (!isCanceled()) {
logCommandError();
@@ -255,11 +243,19 @@
}
protected void processMore() throws SystemException, BlockedException, MetaMatrixCoreException {
+ this.processor.getContext().setTimeSliceEnd(System.currentTimeMillis() + this.processorTimeslice);
if (!doneProducingBatches) {
sendResultsIfNeeded(null);
- doneProducingBatches = processor.process(this.processorTimeslice);
+ collector.collectTuples();
+ doneProducingBatches = collector.isDone();
}
if (doneProducingBatches) {
+ /*if(rsCache != null && requestMsg.useResultSetCache() && originalCommand.areResultsCachable()){
+ boolean sessionScope = this.processor.getContext().isSessionFunctionEvaluated();
+ CacheResults cr = new CacheResults();
+ cr.setCommand(originalCommand);
+ cr.setAnalysisRecord(analysisRecord);
+ }*/
if (this.transactionState == TransactionState.ACTIVE) {
boolean end = true;
/*
@@ -296,28 +292,20 @@
* Any errors that occur will not make it to the client, instead we just log them here.
*/
protected void attemptClose() {
- if (this.resultsID != null) {
+ if (this.resultsBuffer != null) {
try {
- if (LogManager.isMessageToBeRecorded(LogConstants.CTX_DQP, MessageLevel.DETAIL)) {
- LogManager.logDetail(LogConstants.CTX_DQP, "Removing tuplesource for the request " + requestID); //$NON-NLS-1$
- }
- this.dqpCore.getBufferManager().removeTupleSource(resultsID);
- } catch(TupleSourceNotFoundException e) {
- // ignore
- } catch (MetaMatrixComponentException e) {
- LogManager.logDetail(LogConstants.CTX_DQP, e, e.getMessage());
- }
-
- try {
this.processor.closeProcessing();
- } catch (TupleSourceNotFoundException e) {
- // ignore
} catch (MetaMatrixComponentException e) {
if (LogManager.isMessageToBeRecorded(LogConstants.CTX_DQP, MessageLevel.DETAIL)) {
LogManager.logDetail(LogConstants.CTX_DQP, e, e.getMessage());
}
}
+ if (LogManager.isMessageToBeRecorded(LogConstants.CTX_DQP, MessageLevel.DETAIL)) {
+ LogManager.logDetail(LogConstants.CTX_DQP, "Removing tuplesource for the request " + requestID); //$NON-NLS-1$
+ }
+ resultsBuffer.remove();
+
for (DataTierTupleSource connectorRequest : this.connectorInfo.values()) {
try {
connectorRequest.fullyCloseSource();
@@ -328,7 +316,7 @@
}
}
- this.resultsID = null;
+ this.resultsBuffer = null;
}
if (this.transactionState == TransactionState.ACTIVE) {
@@ -364,22 +352,13 @@
request.processRequest();
originalCommand = request.userCommand;
processor = request.processor;
- processor.setBatchHandler(new BatchHandler() {
- public void batchProduced(TupleBatch batch) throws BlockedOnMemoryException, TupleSourceNotFoundException, MetaMatrixComponentException {
- //if there is a cache, and it is a query, save it
- if(rsCache != null && requestMsg.useResultSetCache() && originalCommand.areResultsCachable() && transactionState == TransactionState.NONE && !rsCache.hasResults(cid)){
- CacheResults cr = new CacheResults(batch.getAllTuples(), batch.getBeginRow(), !doneProducingBatches);
- cr.setCommand(originalCommand);
- cr.setSize(batch.getSize());
- cr.setAnalysisRecord(analysisRecord);
- if (!rsCache.setResults(cid, cr, requestID)) {
- rsCache = null; //disable caching if we are over size
- }
- }
+ collector = processor.createBatchCollector();
+ collector.setBatchHandler(new BatchHandler() {
+ public void batchProduced(TupleBatch batch) throws MetaMatrixComponentException {
sendResultsIfNeeded(batch);
}
});
- resultsID = processor.getResultsID();
+ resultsBuffer = collector.getTupleBuffer();
analysisRecord = request.analysisRecord;
schemas = request.schemas;
transactionContext = request.transactionContext;
@@ -389,7 +368,7 @@
Option option = originalCommand.getOption();
if (option != null && option.getPlanOnly()) {
doneProducingBatches = true;
- this.bufferMgr.setStatus(resultsID, TupleSourceStatus.FULL);
+ resultsBuffer.close();
}
if (analysisRecord.recordQueryPlan()) {
@@ -402,8 +381,7 @@
/**
* Send results if they have been requested. This should only be called from the processing thread.
*/
- protected void sendResultsIfNeeded(TupleBatch batch) throws BlockedOnMemoryException,
- MetaMatrixComponentException, TupleSourceNotFoundException {
+ protected void sendResultsIfNeeded(TupleBatch batch) throws MetaMatrixComponentException {
synchronized (resultsCursor) {
if (!this.resultsCursor.resultsRequested
@@ -414,76 +392,61 @@
}
if (LogManager.isMessageToBeRecorded(LogConstants.CTX_DQP, MessageLevel.DETAIL)) {
- LogManager.logDetail(LogConstants.CTX_DQP, "[RequestWorkItem.sendResultsIfNeeded] requestID: " + requestID + " resultsID: " + this.resultsID + " done: " + doneProducingBatches ); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
+ LogManager.logDetail(LogConstants.CTX_DQP, "[RequestWorkItem.sendResultsIfNeeded] requestID: " + requestID + " resultsID: " + this.resultsBuffer + " done: " + doneProducingBatches ); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
}
- boolean pinned = false;
-
- try {
- if (batch == null || batch.getBeginRow() > this.resultsCursor.begin) {
- batch = this.bufferMgr.pinTupleBatch(resultsID, resultsCursor.begin);
- pinned = true;
- //TODO: support fetching more than 1 batch
- int count = this.resultsCursor.end - this.resultsCursor.begin + 1;
- if (batch.getRowCount() > count) {
- int beginRow = Math.min(this.resultsCursor.begin, batch.getEndRow() - count + 1);
- int endRow = Math.min(beginRow + count - 1, batch.getEndRow());
- int firstOffset = beginRow - batch.getBeginRow();
- List[] memoryRows = batch.getAllTuples();
- List[] rows = new List[count];
- System.arraycopy(memoryRows, firstOffset, rows, 0, endRow - beginRow + 1);
- batch = new TupleBatch(beginRow, rows);
- }
- }
- int finalRowCount = doneProducingBatches?this.processor.getHighestRow():-1;
-
- ResultsMessage response = createResultsMessage(requestMsg, batch.getAllTuples(), this.processor.getProcessorPlan().getOutputElements(), analysisRecord);
- response.setFirstRow(batch.getBeginRow());
- response.setLastRow(batch.getEndRow());
- response.setUpdateResult(this.returnsUpdateCount);
- // set final row
- response.setFinalRow(finalRowCount);
+ if (batch == null || batch.getBeginRow() > this.resultsCursor.begin) {
+ batch = resultsBuffer.getBatch(resultsCursor.begin);
+ //TODO: support fetching more than 1 batch
+ int count = this.resultsCursor.end - this.resultsCursor.begin + 1;
+ if (batch.getRowCount() > count) {
+ int beginRow = Math.min(this.resultsCursor.begin, batch.getEndRow() - count + 1);
+ int endRow = Math.min(beginRow + count - 1, batch.getEndRow());
+ int firstOffset = beginRow - batch.getBeginRow();
+ List[] memoryRows = batch.getAllTuples();
+ List[] rows = new List[count];
+ System.arraycopy(memoryRows, firstOffset, rows, 0, endRow - beginRow + 1);
+ batch = new TupleBatch(beginRow, rows);
+ }
+ }
+ int finalRowCount = doneProducingBatches?this.processor.getHighestRow():-1;
+
+ ResultsMessage response = createResultsMessage(requestMsg, batch.getAllTuples(), this.processor.getProcessorPlan().getOutputElements(), analysisRecord);
+ response.setFirstRow(batch.getBeginRow());
+ response.setLastRow(batch.getEndRow());
+ response.setUpdateResult(this.returnsUpdateCount);
+ // set final row
+ response.setFinalRow(finalRowCount);
- // send any schemas associated with the results
- response.setSchemas(this.schemas);
-
- // send any warnings with the response object
- List<Throwable> responseWarnings = new ArrayList<Throwable>();
- List<Exception> currentWarnings = processor.getAndClearWarnings();
- if (currentWarnings != null) {
- responseWarnings.addAll(currentWarnings);
- }
- synchronized (warnings) {
- responseWarnings.addAll(this.warnings);
- this.warnings.clear();
- }
- response.setWarnings(responseWarnings);
-
- // If it is stored procedure, set parameters
- if (originalCommand instanceof StoredProcedure) {
- StoredProcedure proc = (StoredProcedure)originalCommand;
- if (proc.returnParameters()) {
- response.setParameters(getParameterInfo(proc));
- }
- }
-
- /*
- * mark the results sent at this point.
- * communication exceptions will be treated as non-recoverable
- */
- this.resultsCursor.resultsSent();
- this.resultsReceiver.receiveResults(response);
- } catch (MemoryNotAvailableException e) {
- throw BlockedOnMemoryException.INSTANCE;
- } finally {
- try {
- if (pinned) {
- this.bufferMgr.unpinTupleBatch(this.resultsID, batch.getBeginRow());
- }
- } catch (Exception e) {
- // ignore - nothing more we can do
- }
+ // send any schemas associated with the results
+ response.setSchemas(this.schemas);
+
+ // send any warnings with the response object
+ List<Throwable> responseWarnings = new ArrayList<Throwable>();
+ List<Exception> currentWarnings = processor.getAndClearWarnings();
+ if (currentWarnings != null) {
+ responseWarnings.addAll(currentWarnings);
+ }
+ synchronized (warnings) {
+ responseWarnings.addAll(this.warnings);
+ this.warnings.clear();
+ }
+ response.setWarnings(responseWarnings);
+
+ // If it is stored procedure, set parameters
+ if (originalCommand instanceof StoredProcedure) {
+ StoredProcedure proc = (StoredProcedure)originalCommand;
+ if (proc.returnParameters()) {
+ response.setParameters(getParameterInfo(proc));
+ }
}
+
+ /*
+ * mark the results sent at this point.
+ * communication exceptions will be treated as non-recoverable
+ */
+ this.resultsCursor.resultsSent();
+ this.resultsReceiver.receiveResults(response);
}
public static ResultsMessage createResultsMessage(RequestMessage message, List[] batch, List columnSymbols, AnalysisRecord analysisRecord) {
@@ -616,17 +579,17 @@
return false;
}
- public synchronized void requestClose() {
- if (this.state == ProcessingState.CLOSE || this.closeRequested) {
- if (LogManager.isMessageToBeRecorded(LogConstants.CTX_DQP, MessageLevel.DETAIL)) {
- LogManager.logDetail(LogConstants.CTX_DQP, "Request already closing" + requestID); //$NON-NLS-1$
- }
- return;
- }
+ public void requestClose() throws MetaMatrixComponentException {
+ synchronized (this) {
+ if (this.state == ProcessingState.CLOSE || this.closeRequested) {
+ if (LogManager.isMessageToBeRecorded(LogConstants.CTX_DQP, MessageLevel.DETAIL)) {
+ LogManager.logDetail(LogConstants.CTX_DQP, "Request already closing" + requestID); //$NON-NLS-1$
+ }
+ return;
+ }
+ }
this.closeRequested = true;
- if (this.processor != null) {
- this.processor.requestClosed();
- }
+ this.requestCancel(); //pending work should be canceled for fastest clean up
this.moreWork();
}
Modified: trunk/engine/src/test/java/com/metamatrix/common/buffer/BufferManagerFactory.java
===================================================================
--- trunk/engine/src/test/java/com/metamatrix/common/buffer/BufferManagerFactory.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/test/java/com/metamatrix/common/buffer/BufferManagerFactory.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -22,14 +22,11 @@
package com.metamatrix.common.buffer;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
import java.util.Properties;
import com.metamatrix.api.exception.MetaMatrixComponentException;
import com.metamatrix.common.buffer.impl.BufferManagerImpl;
-import com.metamatrix.query.execution.QueryExecPlugin;
+import com.metamatrix.common.buffer.impl.MemoryStorageManager;
/**
* <p>Factory for BufferManager instances. One method will get
@@ -44,75 +41,6 @@
*/
public class BufferManagerFactory {
- public static class MemoryStorageManager implements StorageManager {
-
- // TupleSourceID -> List<TupleBatch> (ordered by startRow)
- private Map<TupleSourceID, Map<Integer, TupleBatch>> storage = Collections.synchronizedMap(new HashMap<TupleSourceID, Map<Integer, TupleBatch>>());
-
- /**
- * @see StorageManager#initialize(Properties)
- */
- public void initialize(Properties props) throws MetaMatrixComponentException {
- }
-
- /**
- * @see StorageManager#addBatch(TupleSourceID, TupleBatch)
- */
- public void addBatch(TupleSourceID storageID, TupleBatch batch, String[] types)
- throws MetaMatrixComponentException {
-
- Map<Integer, TupleBatch> batches = null;
- synchronized(this.storage) {
- batches = storage.get(storageID);
- if(batches == null) {
- batches = new HashMap<Integer, TupleBatch>();
- this.storage.put(storageID, batches);
- }
- }
-
- synchronized(batches) {
- batches.put(batch.getBeginRow(), batch);
- }
- }
-
- /**
- * @see StorageManager#getBatch(TupleSourceID, int, int)
- */
- public TupleBatch getBatch(TupleSourceID storageID, int beginRow, String[] types)
- throws TupleSourceNotFoundException, MetaMatrixComponentException {
-
- Map<Integer, TupleBatch> batches = storage.get(storageID);
-
- if(batches == null) {
- throw new TupleSourceNotFoundException(QueryExecPlugin.Util.getString("BufferManagerImpl.tuple_source_not_found", storageID)); //$NON-NLS-1$
- }
-
- synchronized(batches) {
- TupleBatch batch = batches.get(beginRow);
- if(batch == null) {
- throw new MetaMatrixComponentException("unknown batch"); //$NON-NLS-1$
- }
- return batch;
- }
- }
-
- /**
- * @see StorageManager#removeStorageArea(TupleSourceID)
- */
- public void removeBatches(TupleSourceID storageID) throws MetaMatrixComponentException {
- storage.remove(storageID);
- }
-
- /**
- * @see StorageManager#shutdown()
- */
- public void shutdown() {
- this.storage.clear();
- this.storage = null;
- }
-
- }
-
private static BufferManager INSTANCE;
/**
@@ -128,7 +56,7 @@
props.setProperty(BufferManagerPropertyNames.SESSION_USE_PERCENTAGE, "100"); //$NON-NLS-1$
props.setProperty(BufferManagerPropertyNames.LOG_STATS_INTERVAL, "0"); //$NON-NLS-1$
props.setProperty(BufferManagerPropertyNames.MANAGEMENT_INTERVAL, "0"); //$NON-NLS-1$
- bufferMgr.initialize("local", props); //$NON-NLS-1$
+ bufferMgr.initialize(props);
// Add unmanaged memory storage manager
bufferMgr.setStorageManager(new MemoryStorageManager());
Modified: trunk/engine/src/test/java/com/metamatrix/common/buffer/storage/file/TestFileStorageManager.java
===================================================================
--- trunk/engine/src/test/java/com/metamatrix/common/buffer/storage/file/TestFileStorageManager.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/test/java/com/metamatrix/common/buffer/storage/file/TestFileStorageManager.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -36,11 +36,11 @@
import junit.framework.TestCase;
import com.metamatrix.api.exception.MetaMatrixComponentException;
-import com.metamatrix.api.exception.MetaMatrixException;
import com.metamatrix.common.buffer.BufferManagerPropertyNames;
import com.metamatrix.common.buffer.StorageManager;
import com.metamatrix.common.buffer.TupleBatch;
import com.metamatrix.common.buffer.TupleSourceID;
+import com.metamatrix.common.buffer.impl.FileStorageManager;
import com.metamatrix.core.util.UnitTestUtil;
/**
@@ -250,43 +250,37 @@
}
- public void testAddGetBatch1() {
+ public void testAddGetBatch1() throws Exception {
StorageManager sm = getStorageManager(null);
TupleSourceID tsID = new TupleSourceID("local,1:0"); //$NON-NLS-1$
TupleBatch batch = exampleBatch(1, 10);
- try {
- // Add one batch
- sm.addBatch(tsID, batch, null);
-
- // Get that batch
- TupleBatch actual = sm.getBatch(tsID, batch.getBeginRow(), null);
- helpCompareBatches(batch, actual);
-
- // Remove the batches
- sm.removeBatches(tsID);
-
- } catch(MetaMatrixException e) {
- fail("Unexpected exception of type " + e.getClass().getName() + ": " + e.getMessage()); //$NON-NLS-1$ //$NON-NLS-2$
- }
+ // Add one batch
+ sm.addBatch(tsID, batch, null);
+
+ // Get that batch
+ TupleBatch actual = sm.getBatch(tsID, batch.getBeginRow(), null);
+ helpCompareBatches(batch, actual);
+
+ // Remove the batches
+ sm.removeBatches(tsID);
}
- public void testAddGetBatch2() {
+ public void testAddGetBatch2() throws Exception {
StorageManager sm = getStorageManager(null);
TupleSourceID tsID = new TupleSourceID("local,1:0"); //$NON-NLS-1$
int numBatches = 20;
int batchSize = 100;
- try {
- // Add batches
- for(int i=0; i<numBatches; i++) {
- int begin = (i*batchSize)+1;
- int end = begin + batchSize - 1;
- TupleBatch batch = exampleBatch(begin, end);
- sm.addBatch(tsID, batch, null);
- }
+ // Add batches
+ for(int i=0; i<numBatches; i++) {
+ int begin = (i*batchSize)+1;
+ int end = begin + batchSize - 1;
+ TupleBatch batch = exampleBatch(begin, end);
+ sm.addBatch(tsID, batch, null);
+ }
- // Get batches
+ // Get batches
for(int i=0; i<numBatches; i++) {
int begin = (i*batchSize)+1;
int end = begin + batchSize - 1;
@@ -295,31 +289,21 @@
}
// Remove the batches
- sm.removeBatches(tsID);
-
- } catch(MetaMatrixException e) {
- fail("Unexpected exception of type " + e.getClass().getName() + ": " + e.getMessage()); //$NON-NLS-1$ //$NON-NLS-2$
- }
+ sm.removeBatches(tsID);
}
- public void testCreateNewDirectory() {
+ public void testCreateNewDirectory() throws Exception {
Properties resourceProps = new Properties();
String nonExistentDirectory = UnitTestUtil.getTestScratchPath() + File.separator + "GONZO"; //$NON-NLS-1$
resourceProps.put(BufferManagerPropertyNames.BUFFER_STORAGE_DIRECTORY, nonExistentDirectory);
StorageManager sm = new FileStorageManager();
- try {
- sm.initialize(resourceProps);
-
- File file = new File(nonExistentDirectory);
- assertTrue("Directory doesn't exist", file.exists()); //$NON-NLS-1$
- assertTrue("Directory was created as a file", file.isDirectory()); //$NON-NLS-1$
-
- } catch(Exception e) {
- e.printStackTrace();
- fail("Unexpected exception: " + e.getMessage()); //$NON-NLS-1$
- }
+ sm.initialize(resourceProps);
+
+ File file = new File(nonExistentDirectory);
+ assertTrue("Directory doesn't exist", file.exists()); //$NON-NLS-1$
+ assertTrue("Directory was created as a file", file.isDirectory()); //$NON-NLS-1$
}
public void testAddTwice() throws Exception {
@@ -400,9 +384,9 @@
assertTrue(storageFile1.exists());
assertTrue(storageFile2.exists());
assertTrue(storageFile3.exists());
- assertEquals(1048435, storageFile1.length());
- assertEquals(211806, storageFile2.length());
- assertEquals(2108106, storageFile3.length());
+ assertEquals(1048427, storageFile1.length());
+ assertEquals(211798, storageFile2.length());
+ assertEquals(2108098, storageFile3.length());
sm.removeBatches(tsID);
assertFalse(storageFile1.exists());
@@ -454,14 +438,8 @@
} catch(Throwable e) {
this.error = e;
} finally {
- // Remove all the batches
- //System.out.println(tsID.toString() + ": removing batches");
- try {
- sm.removeBatches(tsID);
- } catch(MetaMatrixComponentException e) {
- }
+ sm.removeBatches(tsID);
}
- //System.out.println(tsID.toString() + ": ending");
}
}
@@ -521,7 +499,6 @@
} else {
// Remove
if(added[batch]) {
- //System.out.println(tsID.toString() + ": removing batch " + batch);
added[batch] = false;
}
}
@@ -531,14 +508,8 @@
} catch(Throwable e) {
this.error = e;
} finally {
- // Remove all the batches
- //System.out.println(tsID.toString() + ": removing batches");
- try {
- sm.removeBatches(tsID);
- } catch(MetaMatrixComponentException e) {
- }
+ sm.removeBatches(tsID);
}
- //System.out.println(tsID.toString() + ": ending");
}
}
}
Modified: trunk/engine/src/test/java/com/metamatrix/query/optimizer/relational/rules/TestRuleAccessPatternValidation.java
===================================================================
--- trunk/engine/src/test/java/com/metamatrix/query/optimizer/relational/rules/TestRuleAccessPatternValidation.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/test/java/com/metamatrix/query/optimizer/relational/rules/TestRuleAccessPatternValidation.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -28,7 +28,6 @@
import java.util.Collection;
import org.junit.Test;
-import org.teiid.dqp.internal.process.TestPreparedStatement;
import com.metamatrix.api.exception.MetaMatrixComponentException;
import com.metamatrix.api.exception.query.QueryMetadataException;
Copied: trunk/engine/src/test/java/com/metamatrix/query/processor/TestCollectionTupleSource.java (from rev 1666, trunk/engine/src/test/java/com/metamatrix/query/processor/TestNullTupleSource.java)
===================================================================
--- trunk/engine/src/test/java/com/metamatrix/query/processor/TestCollectionTupleSource.java (rev 0)
+++ trunk/engine/src/test/java/com/metamatrix/query/processor/TestCollectionTupleSource.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -0,0 +1,70 @@
+/*
+ * JBoss, Home of Professional Open Source.
+ * See the COPYRIGHT.txt file distributed with this work for information
+ * regarding copyright ownership. Some portions may be licensed
+ * to Red Hat, Inc. under one or more contributor license agreements.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301 USA.
+ */
+
+package com.metamatrix.query.processor;
+
+import static org.junit.Assert.*;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.junit.Test;
+
+import com.metamatrix.query.sql.lang.Command;
+import com.metamatrix.query.sql.symbol.ElementSymbol;
+import com.metamatrix.query.sql.symbol.SingleElementSymbol;
+/**
+ */
+public class TestCollectionTupleSource {
+
+ @Test public void testNullSource() {
+ List<SingleElementSymbol> elements = new ArrayList<SingleElementSymbol>();
+ elements.add(new ElementSymbol("x")); //$NON-NLS-1$
+ elements.add(new ElementSymbol("y")); //$NON-NLS-1$
+ CollectionTupleSource nts = CollectionTupleSource.createNullTupleSource(elements);
+
+ // Check schema
+ assertEquals("Didn't get expected schema", elements, nts.getSchema()); //$NON-NLS-1$
+
+ // Walk it and get no data
+ List tuple = nts.nextTuple();
+ nts.closeSource();
+
+ assertEquals("Didn't get termination tuple for first tuple", null, tuple); //$NON-NLS-1$
+ }
+
+ @Test public void testUpdateCountSource() {
+ CollectionTupleSource nts = CollectionTupleSource.createUpdateCountTupleSource(5);
+
+ // Check schema
+ assertEquals("Didn't get expected schema", Command.getUpdateCommandSymbol(), nts.getSchema()); //$NON-NLS-1$
+
+ // Walk it and get no data
+ List tuple = nts.nextTuple();
+ nts.closeSource();
+
+ assertEquals("Didn't get termination tuple for first tuple", Arrays.asList(5), tuple); //$NON-NLS-1$
+ }
+
+
+}
Deleted: trunk/engine/src/test/java/com/metamatrix/query/processor/TestNullTupleSource.java
===================================================================
--- trunk/engine/src/test/java/com/metamatrix/query/processor/TestNullTupleSource.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/test/java/com/metamatrix/query/processor/TestNullTupleSource.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -1,61 +0,0 @@
-/*
- * JBoss, Home of Professional Open Source.
- * See the COPYRIGHT.txt file distributed with this work for information
- * regarding copyright ownership. Some portions may be licensed
- * to Red Hat, Inc. under one or more contributor license agreements.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301 USA.
- */
-
-package com.metamatrix.query.processor;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import junit.framework.TestCase;
-
-import com.metamatrix.query.sql.symbol.ElementSymbol;
-
-/**
- */
-public class TestNullTupleSource extends TestCase {
-
- /**
- * Constructor for TestNullTupleSource.
- * @param name
- */
- public TestNullTupleSource(String name) {
- super(name);
- }
-
- public void testSource() {
- List elements = new ArrayList();
- elements.add(new ElementSymbol("x")); //$NON-NLS-1$
- elements.add(new ElementSymbol("y")); //$NON-NLS-1$
- NullTupleSource nts = new NullTupleSource(elements);
-
- // Check schema
- assertEquals("Didn't get expected schema", elements, nts.getSchema()); //$NON-NLS-1$
-
- // Walk it and get no data
- nts.openSource();
- List tuple = nts.nextTuple();
- nts.closeSource();
-
- assertEquals("Didn't get termination tuple for first tuple", null, tuple); //$NON-NLS-1$
- }
-
-}
Modified: trunk/engine/src/test/java/com/metamatrix/query/processor/TestQueryProcessor.java
===================================================================
--- trunk/engine/src/test/java/com/metamatrix/query/processor/TestQueryProcessor.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/test/java/com/metamatrix/query/processor/TestQueryProcessor.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -30,13 +30,16 @@
import junit.framework.TestCase;
import com.metamatrix.api.exception.MetaMatrixComponentException;
+import com.metamatrix.api.exception.MetaMatrixProcessingException;
import com.metamatrix.common.buffer.BlockedException;
import com.metamatrix.common.buffer.BufferManager;
import com.metamatrix.common.buffer.BufferManagerFactory;
import com.metamatrix.common.buffer.TupleBatch;
+import com.metamatrix.common.buffer.TupleBuffer;
import com.metamatrix.common.buffer.TupleSource;
import com.metamatrix.common.buffer.TupleSourceNotFoundException;
import com.metamatrix.core.MetaMatrixCoreException;
+import com.metamatrix.query.processor.BatchCollector.BatchHandler;
import com.metamatrix.query.sql.symbol.ElementSymbol;
import com.metamatrix.query.util.CommandContext;
@@ -58,22 +61,21 @@
CommandContext context = new CommandContext("pid", "group", null, null, null); //$NON-NLS-1$ //$NON-NLS-2$
QueryProcessor processor = new QueryProcessor(plan, context, bufferMgr, dataManager);
-
+ BatchCollector collector = processor.createBatchCollector();
+ TupleBuffer tsID = null;
while(true) {
try {
- boolean done = processor.process(timeslice);
- if(done) {
- break;
- }
+ tsID = collector.collectTuples();
+ break;
} catch(BlockedException e) {
}
}
// Compare # of rows in actual and expected
- assertEquals("Did not get expected # of rows", expectedResults.length, bufferMgr.getFinalRowCount(processor.getResultsID())); //$NON-NLS-1$
+ assertEquals("Did not get expected # of rows", expectedResults.length, tsID.getRowCount()); //$NON-NLS-1$
// Compare actual with expected results
- TupleSource actual = bufferMgr.getTupleSource(processor.getResultsID());
+ TupleSource actual = tsID.createIndexedTupleSource();
if(expectedResults.length > 0) {
for(int i=0; i<expectedResults.length; i++) {
List actRecord = actual.nextTuple();
@@ -81,6 +83,7 @@
assertEquals("Did not match row at row index " + i, expRecord, actRecord); //$NON-NLS-1$
}
}
+ tsID.remove();
}
public void testNoResults() throws Exception {
@@ -138,58 +141,4 @@
helpTestProcessor(plan, 1000, expectedResults);
}
- public void testProcessWhenClientNeedsBatch() throws Exception {
- List elements = new ArrayList();
- elements.add(new ElementSymbol("a")); //$NON-NLS-1$
-
- HashSet blocked = new HashSet(Arrays.asList(new Integer[] { new Integer(2)}));
- int numBatches = 5;
- int batchRow = 1;
- int rowsPerBatch = 10;
- List[] expectedResults = new List[rowsPerBatch*(numBatches-blocked.size())];
- List batches = new ArrayList();
- for(int b=0; b<numBatches; b++) {
- if(blocked.contains(new Integer(b))) {
- batches.add(BlockedException.INSTANCE);
- } else {
- List[] rows = new List[rowsPerBatch];
- for(int i=0; i<rowsPerBatch; i++) {
- rows[i] = new ArrayList();
- rows[i].add(new Integer(batchRow));
- expectedResults[batchRow-1] = rows[i];
- batchRow++;
- }
-
- TupleBatch batch = new TupleBatch(batchRow-rows.length, rows);
- if(b == numBatches-1) {
- batch.setTerminationFlag(true);
- }
- batches.add(batch);
- }
- }
-
- final FakeProcessorPlan plan = new FakeProcessorPlan(elements, batches);
-
- BufferManager bufferMgr = BufferManagerFactory.getStandaloneBufferManager();
- FakeDataManager dataManager = new FakeDataManager();
-
- CommandContext context = new CommandContext("pid", "group", null, null, null); //$NON-NLS-1$ //$NON-NLS-2$
- final QueryProcessor processor = new QueryProcessor(plan, context, bufferMgr, dataManager);
-
- processor.setBatchHandler(new QueryProcessor.BatchHandler() {
-
- int count = 0;
-
- public void batchProduced(TupleBatch batch) throws TupleSourceNotFoundException, MetaMatrixComponentException {
-
- assertEquals(++count, plan.batchIndex);
- if (count == 2) {
- processor.closeProcessing();
- }
- }
- });
-
- // Give the processor plenty of time to process
- processor.process(30000);
- }
}
Modified: trunk/engine/src/test/java/com/metamatrix/query/processor/TestVirtualDepJoin.java
===================================================================
--- trunk/engine/src/test/java/com/metamatrix/query/processor/TestVirtualDepJoin.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/test/java/com/metamatrix/query/processor/TestVirtualDepJoin.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -39,9 +39,10 @@
import com.metamatrix.api.exception.MetaMatrixException;
import com.metamatrix.common.buffer.BufferManager;
import com.metamatrix.common.buffer.BufferManagerPropertyNames;
+import com.metamatrix.common.buffer.TupleBuffer;
import com.metamatrix.common.buffer.TupleSourceNotFoundException;
-import com.metamatrix.common.buffer.BufferManagerFactory.MemoryStorageManager;
import com.metamatrix.common.buffer.impl.BufferManagerImpl;
+import com.metamatrix.common.buffer.impl.MemoryStorageManager;
import com.metamatrix.common.types.DataTypeManager;
import com.metamatrix.query.mapping.relational.QueryNode;
import com.metamatrix.query.optimizer.TestOptimizer;
@@ -484,9 +485,11 @@
// Run query
BufferManager bufferMgr = createCustomBufferMgr(2);
QueryProcessor processor = new QueryProcessor(plan, context, bufferMgr, dataManager);
- processor.process();
+ processor.setNonBlocking(true);
+ BatchCollector collector = processor.createBatchCollector();
+ TupleBuffer id = collector.collectTuples();
- TestProcessor.examineResults((List[])expected.toArray(new List[expected.size()]), bufferMgr, processor.getResultsID());
+ TestProcessor.examineResults((List[])expected.toArray(new List[expected.size()]), bufferMgr, id);
}
private BufferManager createCustomBufferMgr(int batchSize) throws MetaMatrixComponentException {
@@ -498,7 +501,7 @@
props.setProperty(BufferManagerPropertyNames.MANAGEMENT_INTERVAL, "0"); //$NON-NLS-1$
props.setProperty(BufferManagerPropertyNames.PROCESSOR_BATCH_SIZE, String.valueOf(batchSize));
props.setProperty(BufferManagerPropertyNames.CONNECTOR_BATCH_SIZE, String.valueOf(batchSize));
- bufferMgr.initialize("local", props); //$NON-NLS-1$
+ bufferMgr.initialize(props);
// Add unmanaged memory storage manager
bufferMgr.setStorageManager(new MemoryStorageManager());
Modified: trunk/engine/src/test/java/com/metamatrix/query/processor/dynamic/TestSqlEval.java
===================================================================
--- trunk/engine/src/test/java/com/metamatrix/query/processor/dynamic/TestSqlEval.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/test/java/com/metamatrix/query/processor/dynamic/TestSqlEval.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -69,7 +69,7 @@
CommandContext cc = TestProcessor.createCommandContext();
cc.setQueryProcessorFactory(factory);
- SqlEval sqlEval = new SqlEval(bufferMgr, null, cc, null, Collections.EMPTY_MAP);
+ SqlEval sqlEval = new SqlEval(null, cc, null, Collections.EMPTY_MAP);
Source results = sqlEval.executeSQL(sql);
String result = toXMLString(results);
sqlEval.close();
Modified: trunk/engine/src/test/java/com/metamatrix/query/processor/proc/TestProcedureProcessor.java
===================================================================
--- trunk/engine/src/test/java/com/metamatrix/query/processor/proc/TestProcedureProcessor.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/test/java/com/metamatrix/query/processor/proc/TestProcedureProcessor.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -40,6 +40,7 @@
import com.metamatrix.api.exception.query.QueryValidatorException;
import com.metamatrix.common.buffer.BufferManager;
import com.metamatrix.common.buffer.BufferManagerFactory;
+import com.metamatrix.common.buffer.TupleBuffer;
import com.metamatrix.common.buffer.TupleSource;
import com.metamatrix.common.buffer.TupleSourceID;
import com.metamatrix.common.types.DataTypeManager;
@@ -54,6 +55,7 @@
import com.metamatrix.query.optimizer.capabilities.CapabilitiesFinder;
import com.metamatrix.query.optimizer.capabilities.DefaultCapabilitiesFinder;
import com.metamatrix.query.parser.QueryParser;
+import com.metamatrix.query.processor.BatchCollector;
import com.metamatrix.query.processor.FakeDataManager;
import com.metamatrix.query.processor.ProcessorDataManager;
import com.metamatrix.query.processor.ProcessorPlan;
@@ -117,16 +119,17 @@
context.getNextRand(0);
context.setProcessDebug(DEBUG);
QueryProcessor processor = new QueryProcessor(procPlan, context, bufferMgr, dataMgr);
- TupleSourceID tsID = processor.getResultsID();
- processor.process();
+ processor.setNonBlocking(true);
+ BatchCollector collector = processor.createBatchCollector();
+ TupleBuffer tsID = collector.collectTuples();
// Create QueryResults from TupleSource
- TupleSource ts = bufferMgr.getTupleSource(tsID);
-
+ TupleSource ts = tsID.createIndexedTupleSource();
+ int count = tsID.getRowCount();
+
if(DEBUG) {
- int count = bufferMgr.getFinalRowCount(tsID);
- System.out.println("\nResults:\n" + bufferMgr.getTupleSchema(tsID)); //$NON-NLS-1$
- TupleSource ts2 = bufferMgr.getTupleSource(tsID);
+ System.out.println("\nResults:\n" + tsID.getSchema()); //$NON-NLS-1$
+ TupleSource ts2 = tsID.createIndexedTupleSource();
for(int j=0; j<count; j++) {
System.out.println("" + j + ": " + ts2.nextTuple()); //$NON-NLS-1$ //$NON-NLS-2$
}
@@ -137,8 +140,6 @@
fail("Expected processing to fail"); //$NON-NLS-1$
}
- int count = bufferMgr.getFinalRowCount(tsID);
-
if (expectedResults == null) {
assertEquals("Incorrect number of rows: ", 1, count); //$NON-NLS-1$
List tuple = ts.nextTuple();
@@ -150,7 +151,7 @@
if (expectedResults != null) {
//Walk results and compare
- ts = bufferMgr.getTupleSource(tsID);
+ ts = tsID.createIndexedTupleSource();
assertEquals("Incorrect number of rows: ", expectedResults.length, count); //$NON-NLS-1$
for(int j=0; j<count; j++) {
List record = ts.nextTuple();
Modified: trunk/engine/src/test/java/com/metamatrix/query/processor/relational/NodeTestUtil.java
===================================================================
--- trunk/engine/src/test/java/com/metamatrix/query/processor/relational/NodeTestUtil.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/test/java/com/metamatrix/query/processor/relational/NodeTestUtil.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -23,19 +23,13 @@
package com.metamatrix.query.processor.relational;
import java.util.Properties;
-import java.util.Set;
import com.metamatrix.api.exception.MetaMatrixComponentException;
import com.metamatrix.common.buffer.BufferManager;
import com.metamatrix.common.buffer.BufferManagerPropertyNames;
-import com.metamatrix.common.buffer.MemoryNotAvailableException;
import com.metamatrix.common.buffer.StorageManager;
-import com.metamatrix.common.buffer.TupleBatch;
-import com.metamatrix.common.buffer.TupleSourceID;
-import com.metamatrix.common.buffer.TupleSourceNotFoundException;
-import com.metamatrix.common.buffer.BufferManagerFactory.MemoryStorageManager;
import com.metamatrix.common.buffer.impl.BufferManagerImpl;
-import com.metamatrix.core.MetaMatrixRuntimeException;
+import com.metamatrix.common.buffer.impl.MemoryStorageManager;
/**
@@ -63,13 +57,13 @@
}
static BufferManager createBufferManager(Properties bmProps) {
- BufferManagerImpl bufferManager = new TestableBufferManagerImpl();
+ BufferManagerImpl bufferManager = new BufferManagerImpl();
bmProps.setProperty(BufferManagerPropertyNames.MANAGEMENT_INTERVAL, "0"); //$NON-NLS-1$
try {
- bufferManager.initialize("local", bmProps); //$NON-NLS-1$
- } catch (MetaMatrixComponentException err) {
- throw new MetaMatrixRuntimeException(err);
- }
+ bufferManager.initialize(bmProps);
+ } catch (MetaMatrixComponentException e) {
+ throw new RuntimeException(e);
+ }
// Add storage managers
@@ -82,41 +76,4 @@
return new MemoryStorageManager();
}
- public static class TestableBufferManagerImpl extends BufferManagerImpl {
-
- private Set blockOn;
- private int pinCount = 0;
- private boolean wasBlocked;
-
- /**
- * @see com.metamatrix.common.buffer.impl.BufferManagerImpl#pinTupleBatch(com.metamatrix.common.buffer.TupleSourceID, int)
- */
- public TupleBatch pinTupleBatch(TupleSourceID tupleSourceID,
- int beginRow) throws TupleSourceNotFoundException,
- MemoryNotAvailableException,
- MetaMatrixComponentException {
- if (blockOn != null && blockOn.contains(new Integer(++pinCount))) {
- wasBlocked = true;
- throw new MemoryNotAvailableException();
- }
-
- wasBlocked = false;
-
- return super.pinTupleBatch(tupleSourceID, beginRow);
- }
-
-
- /**
- * @param blockOn The blockOn to set.
- */
- public void setBlockOn(Set blockOn) {
- this.blockOn = blockOn;
- }
-
- public boolean wasBlocked() {
- return wasBlocked;
- }
-
- }
-
}
Modified: trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestDuplicateFilter.java
===================================================================
--- trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestDuplicateFilter.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestDuplicateFilter.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -46,7 +46,7 @@
public void helpTestDuplicateFilter(Object[] input, Class dataType, int expected) throws MetaMatrixComponentException, MetaMatrixProcessingException {
BufferManager mgr = BufferManagerFactory.getStandaloneBufferManager();
- DuplicateFilter filter = new DuplicateFilter(new Count(), mgr, "test", mgr.getProcessorBatchSize()); //$NON-NLS-1$
+ DuplicateFilter filter = new DuplicateFilter(new Count(), mgr, "test"); //$NON-NLS-1$
filter.initialize(dataType, dataType);
filter.reset();
Modified: trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestSelectNode.java
===================================================================
--- trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestSelectNode.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestSelectNode.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -39,6 +39,7 @@
import com.metamatrix.common.types.DataTypeManager;
import com.metamatrix.query.function.FunctionDescriptor;
import com.metamatrix.query.function.FunctionLibraryManager;
+import com.metamatrix.query.processor.BatchIterator;
import com.metamatrix.query.processor.FakeDataManager;
import com.metamatrix.query.processor.ProcessorDataManager;
import com.metamatrix.query.sql.lang.CompareCriteria;
Modified: trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestSortNode.java
===================================================================
--- trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestSortNode.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/test/java/com/metamatrix/query/processor/relational/TestSortNode.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -36,16 +36,13 @@
import com.metamatrix.api.exception.MetaMatrixComponentException;
import com.metamatrix.api.exception.MetaMatrixProcessingException;
import com.metamatrix.common.buffer.BlockedException;
-import com.metamatrix.common.buffer.BlockedOnMemoryException;
import com.metamatrix.common.buffer.BufferManager;
import com.metamatrix.common.buffer.BufferManagerFactory;
import com.metamatrix.common.buffer.TupleBatch;
+import com.metamatrix.common.buffer.TupleBuffer;
import com.metamatrix.common.buffer.TupleSource;
-import com.metamatrix.common.buffer.TupleSourceID;
import com.metamatrix.common.buffer.BufferManager.TupleSourceType;
-import com.metamatrix.common.buffer.impl.SizeUtility;
import com.metamatrix.common.types.DataTypeManager;
-import com.metamatrix.query.processor.relational.NodeTestUtil.TestableBufferManagerImpl;
import com.metamatrix.query.processor.relational.SortUtility.Mode;
import com.metamatrix.query.sql.lang.OrderBy;
import com.metamatrix.query.sql.symbol.ElementSymbol;
@@ -54,15 +51,9 @@
public class TestSortNode {
public static final int BATCH_SIZE = 100;
- public static final int INT_BATCH_SIZE = TestSortNode.getIntBatchSize(); //the size of 100 integers
- private void helpTestSort(long bytesInMemory, List elements, List[] data, List sortElements, List sortTypes, List[] expected, Set blockOn, Mode mode) throws MetaMatrixComponentException, MetaMatrixProcessingException {
- BufferManager mgr = NodeTestUtil.getTestBufferManager(bytesInMemory, BATCH_SIZE, BATCH_SIZE);
- TestableBufferManagerImpl impl = (TestableBufferManagerImpl) mgr;
- impl.setBlockOn(blockOn);
- impl.getConfig().setTotalAvailableMemory(bytesInMemory);
- impl.getConfig().setGroupUsePercentage(100);
- impl.getConfig().setManagementInterval(0);
+ private void helpTestSort(List elements, List[] data, List sortElements, List sortTypes, List[] expected, Set blockOn, Mode mode) throws MetaMatrixComponentException, MetaMatrixProcessingException {
+ BufferManager mgr = NodeTestUtil.getTestBufferManager(100, BATCH_SIZE, BATCH_SIZE);
CommandContext context = new CommandContext ("pid", "test", null, null, null); //$NON-NLS-1$ //$NON-NLS-2$
FakeRelationalNode dataNode = new FakeRelationalNode(2, data);
@@ -86,36 +77,20 @@
int currentRow = 1;
while(true) {
- try {
- TupleBatch batch = sortNode.nextBatch();
- if (mode != Mode.DUP_REMOVE) {
- for(int row = currentRow; row <= batch.getEndRow(); row++) {
- assertEquals("Rows don't match at " + row, expected[row-1], batch.getTuple(row)); //$NON-NLS-1$
- }
+ TupleBatch batch = sortNode.nextBatch();
+ if (mode != Mode.DUP_REMOVE) {
+ for(int row = currentRow; row <= batch.getEndRow(); row++) {
+ assertEquals("Rows don't match at " + row, expected[row-1], batch.getTuple(row)); //$NON-NLS-1$
}
- currentRow += batch.getRowCount();
- if(batch.getTerminationFlag()) {
- break;
- }
- } catch (BlockedOnMemoryException e) {
- if (!impl.wasBlocked()) {
- throw new BlockedOnMemoryException();
- }
}
+ currentRow += batch.getRowCount();
+ if(batch.getTerminationFlag()) {
+ break;
+ }
}
assertEquals(expected.length, currentRow - 1);
}
- public static int getIntBatchSize() {
- List[] expected = new List[BATCH_SIZE];
- Arrays.fill(expected, Arrays.asList(1));
-
- String[] types = { "integer" }; //$NON-NLS-1$
-
- int size = (int)SizeUtility.getBatchSize( types, expected );
- return size;
- }
-
/*
* 1 batch all in memory
*/
@@ -158,7 +133,7 @@
if (i > 0) {
blockedOn.add(new Integer(i));
}
- helpTestSort(INT_BATCH_SIZE*2, elements, data, sortElements, sortTypes, expected, blockedOn, mode);
+ helpTestSort(elements, data, sortElements, sortTypes, expected, blockedOn, mode);
}
}
@@ -209,7 +184,7 @@
blockedOn.add(new Integer(batches*(i+1)));
}
//5 batches in memory out of 10 total
- helpTestSort(INT_BATCH_SIZE * inMemoryBatches, elements, data, sortElements, sortTypes, expected, blockedOn, mode);
+ helpTestSort(elements, data, sortElements, sortTypes, expected, blockedOn, mode);
}
}
}
@@ -305,7 +280,7 @@
helpTestBiggerSort(10, 5);
}
- @Test(expected=BlockedOnMemoryException.class) public void testBiggerSortLowMemory() throws Exception {
+ @Test public void testBiggerSortLowMemory() throws Exception {
helpTestBiggerSort(5, 1);
}
@@ -322,11 +297,11 @@
ElementSymbol es1 = new ElementSymbol("e1"); //$NON-NLS-1$
es1.setType(DataTypeManager.DefaultDataClasses.INTEGER);
BufferManager bm = BufferManagerFactory.getStandaloneBufferManager();
- TupleSourceID tsid = bm.createTupleSource(Arrays.asList(es1), "test", TupleSourceType.PROCESSOR); //$NON-NLS-1$
- bm.addTupleBatch(tsid, new TupleBatch(1, new List[] {Arrays.asList(1)}));
- SortUtility su = new SortUtility(tsid, Arrays.asList(es1), Arrays.asList(Boolean.TRUE), Mode.DUP_REMOVE, bm, "test", true); //$NON-NLS-1$
- TupleSourceID out = su.sort();
- TupleSource ts = bm.getTupleSource(out);
+ TupleBuffer tsid = bm.createTupleBuffer(Arrays.asList(es1), "test", TupleSourceType.PROCESSOR); //$NON-NLS-1$
+ tsid.addTuple(Arrays.asList(1));
+ SortUtility su = new SortUtility(tsid.createIndexedTupleSource(), Arrays.asList(es1), Arrays.asList(Boolean.TRUE), Mode.DUP_REMOVE, bm, "test"); //$NON-NLS-1$
+ TupleBuffer out = su.sort();
+ TupleSource ts = out.createIndexedTupleSource();
assertEquals(Arrays.asList(1), ts.nextTuple());
try {
ts.nextTuple();
@@ -334,7 +309,7 @@
} catch (BlockedException e) {
}
- bm.addTupleBatch(tsid, new TupleBatch(2, new List[] {Arrays.asList(2)}));
+ tsid.addTuple(Arrays.asList(2));
su.sort();
assertEquals(Arrays.asList(2), ts.nextTuple());
}
Modified: trunk/engine/src/test/java/com/metamatrix/query/processor/xml/TestXMLProcessor.java
===================================================================
--- trunk/engine/src/test/java/com/metamatrix/query/processor/xml/TestXMLProcessor.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/test/java/com/metamatrix/query/processor/xml/TestXMLProcessor.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -42,9 +42,9 @@
import com.metamatrix.api.exception.query.QueryPlannerException;
import com.metamatrix.api.exception.query.QueryResolverException;
import com.metamatrix.api.exception.query.QueryValidatorException;
-import com.metamatrix.common.buffer.BlockedException;
import com.metamatrix.common.buffer.BufferManager;
import com.metamatrix.common.buffer.BufferManagerFactory;
+import com.metamatrix.common.buffer.TupleBuffer;
import com.metamatrix.common.buffer.TupleSource;
import com.metamatrix.common.types.DataTypeManager;
import com.metamatrix.common.types.XMLType;
@@ -71,6 +71,7 @@
import com.metamatrix.query.optimizer.capabilities.SourceCapabilities.Capability;
import com.metamatrix.query.optimizer.xml.TestXMLPlanner;
import com.metamatrix.query.parser.QueryParser;
+import com.metamatrix.query.processor.BatchCollector;
import com.metamatrix.query.processor.FakeDataManager;
import com.metamatrix.query.processor.QueryProcessor;
import com.metamatrix.query.resolver.QueryResolver;
@@ -2964,20 +2965,11 @@
CommandContext context = new CommandContext("pID", "TestConn", "testUser", null, null); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
context.setProcessDebug(DEBUG);
QueryProcessor processor = new QueryProcessor(plan, context, bufferMgr, dataMgr);
-
- while(true) {
- try {
- processor.process();
- break;
- } catch(BlockedException e) {
- // retry
- }
- }
+ processor.setNonBlocking(true);
+ BatchCollector collector = processor.createBatchCollector();
+ TupleBuffer id = collector.collectTuples();
- //int count = bufferMgr.getFinalRowCount(tsID);
- //assertEquals("Incorrect number of records: ", 1, count); //$NON-NLS-1$
-
- TupleSource ts = bufferMgr.getTupleSource(processor.getResultsID());
+ TupleSource ts = id.createIndexedTupleSource();
List row = ts.nextTuple();
assertEquals("Incorrect number of columns: ", 1, row.size()); //$NON-NLS-1$
@@ -2985,7 +2977,7 @@
XMLType result = (XMLType)row.get(0);
String actualDoc = result.getString();
- bufferMgr.removeTupleSource(processor.getResultsID());
+ id.remove();
if(DEBUG) {
System.out.println("expectedDoc = \n" + expectedDoc); //$NON-NLS-1$
@@ -3009,7 +3001,9 @@
BufferManager bufferMgr = BufferManagerFactory.getStandaloneBufferManager();
CommandContext context = new CommandContext("pID", null, null, null, null); //$NON-NLS-1$
QueryProcessor processor = new QueryProcessor(plan, context, bufferMgr, dataMgr);
- processor.process();
+ processor.setNonBlocking(true);
+ BatchCollector collector = processor.createBatchCollector();
+ collector.collectTuples();
} catch (Exception e){
if (expectedException.isInstance(e)){
expected = e;
@@ -3053,12 +3047,13 @@
BufferManager bufferMgr = BufferManagerFactory.getStandaloneBufferManager();
CommandContext context = new CommandContext("pID", null, null, null, null); //$NON-NLS-1$
QueryProcessor processor = new QueryProcessor(plan, context, bufferMgr, dataMgr);
- processor.process();
+ processor.setNonBlocking(true);
+ BatchCollector collector = processor.createBatchCollector();
+ TupleBuffer tupleBuffer = collector.collectTuples();
+ int count = tupleBuffer.getRowCount();
+ assertEquals("Incorrect number of records: ", expectedDocs.length, count); //$NON-NLS-1$
- int count = bufferMgr.getFinalRowCount(processor.getResultsID());
- assertEquals("Incorrect number of records: ", expectedDocs.length, count); //$NON-NLS-1$
-
- TupleSource ts = bufferMgr.getTupleSource(processor.getResultsID());
+ TupleSource ts = tupleBuffer.createIndexedTupleSource();
for (int i=0; i<expectedDocs.length; i++){
List row = ts.nextTuple();
if(row.isEmpty()){
@@ -3071,7 +3066,7 @@
//assertEquals("XML doc result # " + i +" mismatch: ", expectedDocs[i], actualDoc); //$NON-NLS-1$ //$NON-NLS-2$
compareDocuments(expectedDocs[i], actualDoc);
}
- bufferMgr.removeTupleSource(processor.getResultsID());
+ tupleBuffer.remove();
}
// =============================================================================================
@@ -6060,10 +6055,11 @@
BufferManager bufferMgr = BufferManagerFactory.getStandaloneBufferManager();
CommandContext context = new CommandContext("pID", null, null, null, null); //$NON-NLS-1$
QueryProcessor processor = new QueryProcessor(plan, context, bufferMgr, dataMgr);
-
+ processor.setNonBlocking(true);
+ BatchCollector collector = processor.createBatchCollector();
MetaMatrixComponentException failOnDefaultException = null;
try{
- processor.process();
+ collector.collectTuples();
} catch (MetaMatrixComponentException e){
failOnDefaultException = e;
}
Deleted: trunk/engine/src/test/java/org/teiid/dqp/internal/cache/TestCacheID.java
===================================================================
--- trunk/engine/src/test/java/org/teiid/dqp/internal/cache/TestCacheID.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/test/java/org/teiid/dqp/internal/cache/TestCacheID.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -1,107 +0,0 @@
-/*
- * JBoss, Home of Professional Open Source.
- * See the COPYRIGHT.txt file distributed with this work for information
- * regarding copyright ownership. Some portions may be licensed
- * to Red Hat, Inc. under one or more contributor license agreements.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301 USA.
- */
-
-package org.teiid.dqp.internal.cache;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.teiid.dqp.internal.cache.CacheID;
-
-import junit.framework.TestCase;
-
-public class TestCacheID extends TestCase{
- public TestCacheID(final String name) {
- super(name);
- }
-
- public void testIDEquals1(){
- CacheID id1 = new CacheID("12345", "select * from table1"); //$NON-NLS-1$//$NON-NLS-2$
- CacheID id2 = new CacheID("12345", "select * from table1"); //$NON-NLS-1$//$NON-NLS-2$
- assertEquals(id1, id2);
- }
-
- public void testIDEquals2(){
- List variables1 = new ArrayList();
- variables1.add(new Integer(6));
- variables1.add("aaa"); //$NON-NLS-1$
- CacheID id1 = new CacheID("12345", "select * from table1",variables1); //$NON-NLS-1$//$NON-NLS-2$
- List variables2 = new ArrayList();
- variables2.add(new Integer(6));
- variables2.add("aaa"); //$NON-NLS-1$
- CacheID id2 = new CacheID("12345", "select * from table1",variables2); //$NON-NLS-1$//$NON-NLS-2$
- assertEquals(id1, id2);
- }
-
- public void testIDEquals3(){
- CacheID id1 = new CacheID("12345", "select * from table1", null); //$NON-NLS-1$//$NON-NLS-2$
- CacheID id2 = new CacheID("12345", "select * from table1"); //$NON-NLS-1$//$NON-NLS-2$
- assertEquals(id1, id2);
- }
-
- public void testIDNotEquals1(){
- CacheID id1 = new CacheID("12345", "select * from table1"); //$NON-NLS-1$//$NON-NLS-2$
- CacheID id2 = new CacheID("2345", "select * from table1"); //$NON-NLS-1$//$NON-NLS-2$
- assertTrue(!id1.equals(id2));
- }
-
- public void testIDNotEquals2(){
- CacheID id1 = new CacheID("12345", "select * from table1"); //$NON-NLS-1$//$NON-NLS-2$
- CacheID id2 = new CacheID("12345", "select * from table2"); //$NON-NLS-1$//$NON-NLS-2$
- assertTrue(!id1.equals(id2));
- }
-
- public void testIDNotEquals3(){
- List variables1 = new ArrayList();
- variables1.add(new Integer(6));
- variables1.add("aba"); //$NON-NLS-1$
- CacheID id1 = new CacheID("12345", "select * from table1",variables1); //$NON-NLS-1$//$NON-NLS-2$
- List variables2 = new ArrayList();
- variables2.add(new Integer(6));
- variables2.add("aaa"); //$NON-NLS-1$
- CacheID id2 = new CacheID("12345", "select * from table1",variables2); //$NON-NLS-1$//$NON-NLS-2$
- assertTrue(!id1.equals(id2));
- }
-
- public void testIDNotEquals4(){
- List variables1 = null;
- CacheID id1 = new CacheID("12345", "select * from table1",variables1); //$NON-NLS-1$//$NON-NLS-2$
- List variables2 = new ArrayList();
- variables2.add(new Integer(6));
- variables2.add("aaa"); //$NON-NLS-1$
- CacheID id2 = new CacheID("12345", "select * from table1",variables2); //$NON-NLS-1$//$NON-NLS-2$
- assertTrue(!id1.equals(id2));
- }
-
- public void testIDNotEquals5(){
- List variables1 = new ArrayList();
- variables1.add(new Integer(6));
- variables1.add("aaa"); //$NON-NLS-1$
- variables1.add("aaa"); //$NON-NLS-1$
- CacheID id1 = new CacheID("12345", "select * from table1",variables1); //$NON-NLS-1$//$NON-NLS-2$
- List variables2 = new ArrayList();
- variables2.add(new Integer(6));
- variables2.add("aaa"); //$NON-NLS-1$
- CacheID id2 = new CacheID("12345", "select * from table1",variables2); //$NON-NLS-1$//$NON-NLS-2$
- assertTrue(!id1.equals(id2));
- }
-}
Deleted: trunk/engine/src/test/java/org/teiid/dqp/internal/cache/TestResultSetCache.java
===================================================================
--- trunk/engine/src/test/java/org/teiid/dqp/internal/cache/TestResultSetCache.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/test/java/org/teiid/dqp/internal/cache/TestResultSetCache.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -1,190 +0,0 @@
-/*
- * JBoss, Home of Professional Open Source.
- * See the COPYRIGHT.txt file distributed with this work for information
- * regarding copyright ownership. Some portions may be licensed
- * to Red Hat, Inc. under one or more contributor license agreements.
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
- * 02110-1301 USA.
- */
-
-package org.teiid.dqp.internal.cache;
-
-import static org.junit.Assert.*;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Properties;
-
-import org.junit.Test;
-
-import com.metamatrix.cache.FakeCache;
-import com.metamatrix.common.buffer.impl.SizeUtility;
-import com.metamatrix.core.MetaMatrixRuntimeException;
-
-public class TestResultSetCache {
-
- @Test public void testSetAndGetResultsForSession() throws Exception{
- ResultSetCache cache = new ResultSetCache();
- cache.setScope(ResultSetCache.RS_CACHE_SCOPE_CONN);
- cache.start(new FakeCache.FakeCacheFactory());
- CacheID id1 = new CacheID("12345", "select * from table1"); //$NON-NLS-1$//$NON-NLS-2$
- List[] result1 = new List[]{new ArrayList()};
- cache.setResults(id1, new CacheResults(result1, 1, true ), "req1"); //$NON-NLS-1$
- CacheID id2 = new CacheID("12346", "select * from table2"); //$NON-NLS-1$//$NON-NLS-2$
- List[] result2 = new List[]{new ArrayList()};
- cache.setResults(id2, new CacheResults(result2, 1, true), "req2" ); //$NON-NLS-1$
- assertEquals(result1, cache.getResults(id1, new int[]{1, 500}).getResults());
- assertEquals(result2, cache.getResults(id2, new int[]{1, 500}).getResults());
- }
-
- @Test public void testSetAndGetResultsForVDB() throws Exception{
- ResultSetCache cache = new ResultSetCache();
- cache.start(new FakeCache.FakeCacheFactory());
- CacheID id1 = new CacheID("vdb1", "select * from table1"); //$NON-NLS-1$//$NON-NLS-2$
- List[] result1 = new List[]{new ArrayList()};
- cache.setResults(id1, new CacheResults(result1, 1, true) , "req1"); //$NON-NLS-1$
- CacheID id2 = new CacheID("vdb2", "select * from table2"); //$NON-NLS-1$//$NON-NLS-2$
- List[] result2 = new List[]{new ArrayList()};
- cache.setResults(id2, new CacheResults(result2, 1, true ), "req2"); //$NON-NLS-1$
- assertEquals(cache.getResults(id1, new int[]{1, 500}).getResults(), result1);
- assertEquals(cache.getResults(id2, new int[]{1, 500}).getResults(), result2);
- }
-
- @Test public void testClearAllCache() throws Exception{
- ResultSetCache cache = new ResultSetCache();
- cache.start(new FakeCache.FakeCacheFactory());
- CacheID id1 = new CacheID("vdb1", "select * from table1"); //$NON-NLS-1$//$NON-NLS-2$
- List[] result1 = new List[]{new ArrayList()};
- cache.setResults(id1, new CacheResults(result1, 1, true), "req1" ); //$NON-NLS-1$
- CacheID id2 = new CacheID("vdb2", "select * from table2"); //$NON-NLS-1$//$NON-NLS-2$
- List[] result2 = new List[]{new ArrayList()};
- cache.setResults(id2, new CacheResults(result2, 1, true ), "req2"); //$NON-NLS-1$
- cache.clear();
- assertNull(cache.getResults(id1, new int[]{1, 500}));
- }
-
- @Test public void testSetAndGetResultsForSession1() throws Exception{
- ResultSetCache cache = new ResultSetCache();
- cache.setScope(ResultSetCache.RS_CACHE_SCOPE_CONN);
- cache.start(new FakeCache.FakeCacheFactory());
- CacheID id1 = new CacheID("12345", "select * from table1"); //$NON-NLS-1$//$NON-NLS-2$
- List row1 = new ArrayList();
- row1.add("1"); //$NON-NLS-1$
- List row2 = new ArrayList();
- row2.add("2"); //$NON-NLS-1$
- List row3 = new ArrayList();
- row3.add("3"); //$NON-NLS-1$
- List row4 = new ArrayList();
- row4.add("4"); //$NON-NLS-1$
- List[] result1 = new List[]{row1, row2, row3, row4};
- cache.setResults(id1, new CacheResults(result1, 1, true ), "req1"); //$NON-NLS-1$
- assertEquals(cache.getResults(id1, new int[]{1, 500}).getResults(), result1);
- assertEquals(cache.getResults(id1, new int[]{1, 2}).getResults()[0], row1);
- }
-
- @Test public void testSetAndGetResultsForSession2() throws Exception{
- ResultSetCache cache = new ResultSetCache();
- cache.setScope(ResultSetCache.RS_CACHE_SCOPE_CONN);
- cache.start(new FakeCache.FakeCacheFactory());
- CacheID id1 = new CacheID("12345", "select * from table1"); //$NON-NLS-1$//$NON-NLS-2$
- List row1 = new ArrayList();
- row1.add("1"); //$NON-NLS-1$
- List row2 = new ArrayList();
- row2.add("2"); //$NON-NLS-1$
- List row3 = new ArrayList();
- row3.add("3"); //$NON-NLS-1$
- List row4 = new ArrayList();
- row4.add("4"); //$NON-NLS-1$
- List[] result1 = new List[]{row1, row2, row3, row4};
- cache.setResults(id1, new CacheResults(result1, 1, false) , "req1"); //$NON-NLS-1$
- List row5 = new ArrayList();
- row5.add("5"); //$NON-NLS-1$
- List[] result2 = new List[]{row5};
- cache.setResults(id1, new CacheResults(result2, 5, true), "req1" ); //$NON-NLS-1$
- assertEquals(cache.getResults(id1, new int[]{1, 500}).getResults()[4], row5);
- assertEquals(cache.getResults(id1, new int[]{1, 2}).getResults()[0], row1);
- }
-
- @Test public void testMaxSize() throws Exception{
- ResultSetCache cache = new ResultSetCache();
- cache.setMaxEntrySize(600000);
- cache.start(new FakeCache.FakeCacheFactory());
- CacheID id1 = new CacheID("vdb1", "select * from table1"); //$NON-NLS-1$//$NON-NLS-2$
- CacheResults result1 = createResults(500000, 1, 1, true);
- CacheID id2 = new CacheID("vdb2", "select * from table2"); //$NON-NLS-1$//$NON-NLS-2$
- CacheResults result2 = createResults(500000, 1, 1, true);
- CacheID id3 = new CacheID("vdb1", "select * from table3"); //$NON-NLS-1$//$NON-NLS-2$
- CacheResults result3 = createResults(700000, 1, 1, true);
- //add two results
- cache.setResults(id1, result1, "req1" ); //$NON-NLS-1$
- cache.setResults(id2, result2, "req2" ); //$NON-NLS-1$
- //use the first one
- assertNotNull(cache.getResults(id1, new int[]{1, 500}));
- //add one more result. Because the max size is 1mb, the
- //third one can not be added
- cache.setResults(id3, result3, "req3" ); //$NON-NLS-1$
- assertNull(cache.getResults(id3, new int[]{1, 500}));
- //reduced the size and try again. The second one should
- //be removed
- result3 = createResults(1000, 1, 1, true);
- cache.setResults(id3, result3, "req4" ); //$NON-NLS-1$
- assertNotNull(cache.getResults(id3, new int[]{1, 500}));
- }
-
- private CacheResults createResults(int size, int firstRow, int rowCnt, boolean isFinal){
- List[] results = new List[rowCnt];
- for(int i=0; i<rowCnt; i++){
- results[i] = new ArrayList();
- }
- CacheResults cr = new CacheResults(results, firstRow, isFinal);
- cr.setSize(size);
- return cr;
- }
-
- @Test public void testComputeSize() throws Exception{
- int cnt = 1000000;
- List[] results = new List[cnt];
- List row = new ArrayList();
- row.add("ajwuiotbn0w49yunq9 tjfvwioprkpo23bltplql;galkg"); //$NON-NLS-1$
- row.add(new Long(1242534776));
- row.add(new Double(24235.5476884));
- for(int i=0; i<cnt; i++){
- results[i] = row;
- }
- Properties props = new Properties();
- props.setProperty(ResultSetCache.RS_CACHE_MAX_AGE, "0"); //$NON-NLS-1$
- props.setProperty(ResultSetCache.RS_CACHE_MAX_SIZE, "0"); //$NON-NLS-1$
- props.setProperty(ResultSetCache.RS_CACHE_SCOPE, ResultSetCache.RS_CACHE_SCOPE_CONN);
- ResultSetCache cache = new ResultSetCache();
- CacheID id1 = new CacheID("vdb1", "select * from table1"); //$NON-NLS-1$//$NON-NLS-2$
- CacheResults result = new CacheResults(results, 1, true);
- cache.setResults(id1, result, "req1" ); //$NON-NLS-1$
-
- int size = (SizeUtility.IS_64BIT ? 304000016 : 260000016);
- assertEquals(size, result.getSize());
- }
-
- @Test(expected=MetaMatrixRuntimeException.class) public void testBatchNotContiguous() throws Exception{
- ResultSetCache cache = new ResultSetCache();
- cache.start(new FakeCache.FakeCacheFactory());
- CacheID id1 = new CacheID("vdb1", "select * from table1"); //$NON-NLS-1$//$NON-NLS-2$
- CacheResults result1 = createResults(1000, 1, 100, false);
- CacheResults result3 = createResults(2000, 102, 200, true);
- cache.setResults(id1, result1 , "req1"); //$NON-NLS-1$
- cache.setResults(id1, result3, "req1" ); //$NON-NLS-1$
- }
-
-}
Modified: trunk/engine/src/test/java/org/teiid/dqp/internal/process/TestPreparedPlanCache.java
===================================================================
--- trunk/engine/src/test/java/org/teiid/dqp/internal/process/TestPreparedPlanCache.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/test/java/org/teiid/dqp/internal/process/TestPreparedPlanCache.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -22,16 +22,16 @@
package org.teiid.dqp.internal.process;
+import static org.junit.Assert.*;
+
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
-import static org.junit.Assert.*;
-
import org.junit.BeforeClass;
import org.junit.Test;
-import org.teiid.dqp.internal.process.PreparedPlanCache;
import org.teiid.dqp.internal.process.PreparedPlanCache.CacheID;
+import org.teiid.dqp.internal.process.PreparedPlanCache.PreparedPlan;
import com.metamatrix.api.exception.query.QueryParserException;
import com.metamatrix.query.analysis.AnalysisRecord;
@@ -59,12 +59,12 @@
@Test public void testPutPreparedPlan(){
PreparedPlanCache cache = new PreparedPlanCache();
- CacheID id = new PreparedPlanCache.CacheID(token, pi, EXAMPLE_QUERY + 1);
+ CacheID id = new CacheID(token, pi, EXAMPLE_QUERY + 1);
//No PreparedPlan at the begining
assertNull(cache.getPreparedPlan(id));
//create one
- cache.putPreparedPlan(id, true, new PreparedPlanCache.PreparedPlan());
+ cache.putPreparedPlan(id, true, new PreparedPlan());
//should have one now
assertNotNull("Unable to get prepared plan from cache", cache.getPreparedPlan(id)); //$NON-NLS-1$
}
@@ -75,7 +75,7 @@
helpPutPreparedPlans(cache, token2, 0, 15);
//read an entry for session2 (token2)
- PreparedPlanCache.PreparedPlan pPlan = cache.getPreparedPlan(new PreparedPlanCache.CacheID(token2, pi, EXAMPLE_QUERY + 12));
+ PreparedPlan pPlan = cache.getPreparedPlan(new CacheID(token2, pi, EXAMPLE_QUERY + 12));
assertNotNull("Unable to get prepared plan from cache", pPlan); //$NON-NLS-1$
assertEquals("Error getting plan from cache", new RelationalPlan(new ProjectNode(12)).toString(), pPlan.getPlan().toString()); //$NON-NLS-1$
assertEquals("Error getting command from cache", EXAMPLE_QUERY + 12, pPlan.getCommand().toString()); //$NON-NLS-1$
@@ -90,24 +90,24 @@
helpPutPreparedPlans(cache, token, 1, 1);
helpPutPreparedPlans(cache, token2, 1, 1);
//should have one
- assertNotNull("Unable to get prepared plan from cache for token", cache.getPreparedPlan(new PreparedPlanCache.CacheID(token, pi, EXAMPLE_QUERY + 1))); //$NON-NLS-1$
+ assertNotNull("Unable to get prepared plan from cache for token", cache.getPreparedPlan(new CacheID(token, pi, EXAMPLE_QUERY + 1))); //$NON-NLS-1$
cache.clearAll();
//should not exist for token
- assertNull("Failed remove from cache", cache.getPreparedPlan(new PreparedPlanCache.CacheID(token, pi, EXAMPLE_QUERY + 1))); //$NON-NLS-1$
+ assertNull("Failed remove from cache", cache.getPreparedPlan(new CacheID(token, pi, EXAMPLE_QUERY + 1))); //$NON-NLS-1$
//should not exist for token2
- assertNull("Unable to get prepared plan from cache for token2", cache.getPreparedPlan(new PreparedPlanCache.CacheID(token2, pi, EXAMPLE_QUERY + 1))); //$NON-NLS-1$
+ assertNull("Unable to get prepared plan from cache for token2", cache.getPreparedPlan(new CacheID(token2, pi, EXAMPLE_QUERY + 1))); //$NON-NLS-1$
}
@Test public void testMaxSize(){
PreparedPlanCache cache = new PreparedPlanCache(100);
helpPutPreparedPlans(cache, token, 0, 101);
//the first one should be gone because the max size is 100
- assertNull(cache.getPreparedPlan(new PreparedPlanCache.CacheID(token, pi, EXAMPLE_QUERY + 0)));
+ assertNull(cache.getPreparedPlan(new CacheID(token, pi, EXAMPLE_QUERY + 0)));
- assertNotNull(cache.getPreparedPlan(new PreparedPlanCache.CacheID(token, pi, EXAMPLE_QUERY + 12)));
+ assertNotNull(cache.getPreparedPlan(new CacheID(token, pi, EXAMPLE_QUERY + 12)));
helpPutPreparedPlans(cache, token, 102, 50);
//"sql12" should still be there based on lru policy
- assertNotNull(cache.getPreparedPlan(new PreparedPlanCache.CacheID(token, pi, EXAMPLE_QUERY + 12)));
+ assertNotNull(cache.getPreparedPlan(new CacheID(token, pi, EXAMPLE_QUERY + 12)));
helpPutPreparedPlans(cache, token2, 0, 121);
helpPutPreparedPlans(cache, token, 0, 50);
@@ -121,12 +121,12 @@
// Add 1 plan and verify it is not in the cache
helpPutPreparedPlans(cache, token, 0, 1);
- assertNull(cache.getPreparedPlan(new PreparedPlanCache.CacheID(token, pi, EXAMPLE_QUERY + 0)));
+ assertNull(cache.getPreparedPlan(new CacheID(token, pi, EXAMPLE_QUERY + 0)));
assertEquals(0, cache.getSpaceUsed());
// Add another plan and verify it is not in the cache
helpPutPreparedPlans(cache, token, 1, 1);
- assertNull(cache.getPreparedPlan(new PreparedPlanCache.CacheID(token, pi, EXAMPLE_QUERY + 1)));
+ assertNull(cache.getPreparedPlan(new CacheID(token, pi, EXAMPLE_QUERY + 1)));
assertEquals(0, cache.getSpaceUsed());
}
@@ -148,9 +148,9 @@
} catch (QueryParserException e) {
throw new RuntimeException(e);
}
- CacheID id = new PreparedPlanCache.CacheID(session, pi, dummy.toString());
+ CacheID id = new CacheID(session, pi, dummy.toString());
- PreparedPlanCache.PreparedPlan pPlan = new PreparedPlanCache.PreparedPlan();
+ PreparedPlan pPlan = new PreparedPlan();
cache.putPreparedPlan(id, true, pPlan);
pPlan.setCommand(dummy);
pPlan.setPlan(new RelationalPlan(new ProjectNode(i)));
Modified: trunk/engine/src/test/java/org/teiid/dqp/internal/process/TestPreparedStatement.java
===================================================================
--- trunk/engine/src/test/java/org/teiid/dqp/internal/process/TestPreparedStatement.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/test/java/org/teiid/dqp/internal/process/TestPreparedStatement.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -84,21 +84,19 @@
}
static public void helpTestProcessing(String preparedSql, List values, List[] expected, ProcessorDataManager dataManager, CapabilitiesFinder capFinder, QueryMetadataInterface metadata, TestablePreparedPlanCache prepPlanCache, boolean callableStatement, boolean isSessionSpecific, boolean isAlreadyCached) throws Exception {
- TestablePreparedPlanCache pPlanCache = null;
- CapabilitiesFinder cFinder = null;
- ProcessorDataManager dManager = null;
-
if ( dataManager == null ) {
// Construct data manager with data
- dManager = new FakeDataManager();
- TestProcessor.sampleData1((FakeDataManager)dManager);
- } else dManager = dataManager;
+ dataManager = new FakeDataManager();
+ TestProcessor.sampleData1((FakeDataManager)dataManager);
+ }
- if ( capFinder == null ) cFinder = new DefaultCapabilitiesFinder();
- else cFinder = capFinder;
+ if ( capFinder == null ) {
+ capFinder = new DefaultCapabilitiesFinder();
+ }
- if ( prepPlanCache == null ) pPlanCache = new TestablePreparedPlanCache();
- else pPlanCache = prepPlanCache;
+ if ( prepPlanCache == null ) {
+ prepPlanCache = new TestablePreparedPlanCache();
+ }
// expected cache hit count
int exHitCount = -1;
@@ -109,28 +107,28 @@
* get the plan twice. Otherwise, we want it to be 1.
*/
if ( isAlreadyCached ) {
- exHitCount = pPlanCache.hitCount + 2;
+ exHitCount = prepPlanCache.hitCount + 2;
} else {
- exHitCount = pPlanCache.hitCount + 1;
+ exHitCount = prepPlanCache.hitCount + 1;
}
//Create plan or used cache plan if isPlanCached
- PreparedStatementRequest plan = TestPreparedStatement.helpGetProcessorPlan(preparedSql, values, cFinder, metadata, pPlanCache, SESSION_ID, callableStatement, false);
+ PreparedStatementRequest plan = TestPreparedStatement.helpGetProcessorPlan(preparedSql, values, capFinder, metadata, prepPlanCache, SESSION_ID, callableStatement, false);
// Run query
TestProcessor.doProcess(plan.processPlan, dataManager, expected, plan.context);
//test cached plan
- plan = TestPreparedStatement.helpGetProcessorPlan(preparedSql, values, cFinder, metadata, pPlanCache, SESSION_ID, callableStatement, false);
+ plan = TestPreparedStatement.helpGetProcessorPlan(preparedSql, values, capFinder, metadata, prepPlanCache, SESSION_ID, callableStatement, false);
//make sure the plan is only created once
- assertEquals("should reuse the plan", exHitCount, pPlanCache.hitCount); //$NON-NLS-1$
+ assertEquals("should reuse the plan", exHitCount, prepPlanCache.hitCount); //$NON-NLS-1$
// Run query again
TestProcessor.doProcess(plan.processPlan, dataManager, expected, plan.context);
//get the plan again with a new connection
- assertNotNull(TestPreparedStatement.helpGetProcessorPlan(preparedSql, values, cFinder, metadata, pPlanCache, 7, callableStatement, false));
+ assertNotNull(TestPreparedStatement.helpGetProcessorPlan(preparedSql, values, capFinder, metadata, prepPlanCache, 7, callableStatement, false));
/*
* If the command is not specific to a session we expect
@@ -139,7 +137,7 @@
* created and the hit count will be unchanged.
*/
if ( !isSessionSpecific ) exHitCount++;
- assertEquals(exHitCount, pPlanCache.hitCount);
+ assertEquals(exHitCount, prepPlanCache.hitCount);
}
@Test public void testWhere() throws Exception {
Modified: trunk/engine/src/test/java/org/teiid/dqp/internal/process/validator/TestAuthorizationValidationVisitor.java
===================================================================
--- trunk/engine/src/test/java/org/teiid/dqp/internal/process/validator/TestAuthorizationValidationVisitor.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/engine/src/test/java/org/teiid/dqp/internal/process/validator/TestAuthorizationValidationVisitor.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -251,7 +251,7 @@
String sql = "select lookup('pm1.g1', 'e1', 'e2', 1)"; //$NON-NLS-1$
Command command = QueryParser.getQueryParser().parseCommand(sql);
- Request.validateWithVisitor(mvvv, FakeMetadataFactory.example1Cached(), command, true);
+ Request.validateWithVisitor(mvvv, FakeMetadataFactory.example1Cached(), command);
}
public void testLookupVisibility() throws Exception {
Modified: trunk/runtime/src/main/java/com/metamatrix/dqp/embedded/services/EmbeddedBufferService.java
===================================================================
--- trunk/runtime/src/main/java/com/metamatrix/dqp/embedded/services/EmbeddedBufferService.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/runtime/src/main/java/com/metamatrix/dqp/embedded/services/EmbeddedBufferService.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -37,7 +37,8 @@
import com.metamatrix.common.buffer.BufferManagerPropertyNames;
import com.metamatrix.common.buffer.StorageManager;
import com.metamatrix.common.buffer.impl.BufferManagerImpl;
-import com.metamatrix.common.buffer.storage.file.FileStorageManager;
+import com.metamatrix.common.buffer.impl.FileStorageManager;
+import com.metamatrix.common.buffer.impl.MemoryStorageManager;
import com.metamatrix.dqp.embedded.DQPEmbeddedPlugin;
import com.metamatrix.dqp.service.BufferService;
import com.metamatrix.dqp.service.ConfigurationService;
@@ -113,7 +114,7 @@
// Construct and initialize the buffer manager
this.bufferMgr = new BufferManagerImpl();
- this.bufferMgr.initialize("local", bufferProps); //$NON-NLS-1$
+ this.bufferMgr.initialize(bufferProps);
// If necessary, add disk storage manager
if(useDisk) {
@@ -125,9 +126,11 @@
fsm.initialize(fsmProps);
this.bufferMgr.setStorageManager(fsm);
- // start the file storrage manager in clean state
+ // start the file storage manager in clean state
// wise FileStorageManager is smart enough to clen up after itself
cleanDirectory(bufferDir);
+ } else {
+ this.bufferMgr.setStorageManager(new MemoryStorageManager());
}
} catch(MetaMatrixComponentException e) {
@@ -141,7 +144,7 @@
* @see com.metamatrix.common.application.ApplicationService#stop()
*/
public void stopService() throws ApplicationLifecycleException {
- bufferMgr.stop();
+ bufferMgr.shutdown();
// Delete the buffer directory
if (bufferDir != null) {
Modified: trunk/test-integration/common/src/test/java/com/metamatrix/server/integration/BaseQueryTest.java
===================================================================
--- trunk/test-integration/common/src/test/java/com/metamatrix/server/integration/BaseQueryTest.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/test-integration/common/src/test/java/com/metamatrix/server/integration/BaseQueryTest.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -21,18 +21,13 @@
*/
package com.metamatrix.server.integration;
-import java.sql.SQLXML;
import java.util.List;
import java.util.Properties;
-import org.teiid.metadata.index.VDBMetadataFactory;
-
import junit.framework.TestCase;
-import com.metamatrix.common.buffer.BufferManager;
-import com.metamatrix.common.buffer.BufferManagerFactory;
-import com.metamatrix.common.buffer.TupleSource;
-import com.metamatrix.common.buffer.TupleSourceID;
+import org.teiid.metadata.index.VDBMetadataFactory;
+
import com.metamatrix.query.analysis.AnalysisRecord;
import com.metamatrix.query.metadata.QueryMetadataInterface;
import com.metamatrix.query.optimizer.QueryOptimizer;
@@ -40,7 +35,7 @@
import com.metamatrix.query.optimizer.capabilities.CapabilitiesFinder;
import com.metamatrix.query.processor.ProcessorDataManager;
import com.metamatrix.query.processor.ProcessorPlan;
-import com.metamatrix.query.processor.QueryProcessor;
+import com.metamatrix.query.processor.TestProcessor;
import com.metamatrix.query.sql.lang.Command;
import com.metamatrix.query.util.CommandContext;
@@ -61,9 +56,10 @@
public static QueryMetadataInterface createMetadata(String vdbFile, String systemVDBFile) {
return VDBMetadataFactory.getVDBMetadata(new String[] {vdbFile, systemVDBFile});
}
-
- public ProcessorPlan createPlan(QueryMetadataInterface metadata, String sql, CapabilitiesFinder capFinder, boolean debug) throws Exception {
-
+
+ protected void doProcess(QueryMetadataInterface metadata, String sql, CapabilitiesFinder capFinder, ProcessorDataManager dataManager, List[] expectedResults, boolean debug) throws Exception {
+ CommandContext context = createCommandContext();
+ context.setProcessDebug(debug);
Command command = TestOptimizer.helpGetCommand(sql, metadata, null);
// plan
@@ -76,80 +72,10 @@
System.out.println(analysisRecord.getDebugLog());
}
}
-
- return plan;
- }
-
- protected void doProcess(ProcessorPlan plan, ProcessorDataManager dataManager, List[] expectedResults, boolean debug) throws Exception {
- BufferManager bufferMgr = BufferManagerFactory.getStandaloneBufferManager();
- CommandContext context = createCommandContext();
- context.setProcessDebug(debug);
- QueryProcessor processor = new QueryProcessor(plan, context, bufferMgr, dataManager);
- TupleSourceID tsID = processor.getResultsID();
- processor.process();
- // Create QueryResults from TupleSource
- TupleSource ts = bufferMgr.getTupleSource(tsID);
- int count = bufferMgr.getFinalRowCount(tsID);
-
- if(debug) {
- System.out.println("\nResults:\n" + bufferMgr.getTupleSchema(tsID)); //$NON-NLS-1$
- TupleSource ts2 = bufferMgr.getTupleSource(tsID);
- for(int j=0; j<count; j++) {
- System.out.println("" + j + ": " + ts2.nextTuple()); //$NON-NLS-1$ //$NON-NLS-2$
- }
- ts2.closeSource();
- }
-
- // Compare actual to expected row count
- assertEquals("Did not get expected row count: ", expectedResults.length, count); //$NON-NLS-1$
-
- // Walk results and compare
- for(int i=0; i<count; i++) {
- List record = ts.nextTuple();
-
- Object value = record.get(0);
- if(value instanceof SQLXML) {
- record.set(0, ((SQLXML)value).getString());
- }
- assertEquals(expectedResults[i].get(0), record.get(0));
- }
- ts.closeSource();
-
- bufferMgr.removeTupleSource(tsID);
+ TestProcessor.doProcess(plan, dataManager, expectedResults, context);
}
- protected void doProcessNoResultsCheck(ProcessorPlan plan, ProcessorDataManager dataManager, int expectedRowCount, boolean debug) throws Exception {
- BufferManager bufferMgr = BufferManagerFactory.getStandaloneBufferManager();
- CommandContext context = createCommandContext();
- QueryProcessor processor = new QueryProcessor(plan, context, bufferMgr, dataManager);
- TupleSourceID tsID = processor.getResultsID();
- processor.process();
-
- // Create QueryResults from TupleSource
- TupleSource ts = bufferMgr.getTupleSource(tsID);
- int count = bufferMgr.getFinalRowCount(tsID);
-
- if(debug) {
- System.out.println("\nResults:\n" + bufferMgr.getTupleSchema(tsID)); //$NON-NLS-1$
- TupleSource ts2 = bufferMgr.getTupleSource(tsID);
- for(int j=0; j<count; j++) {
- System.out.println("" + j + ": " + ts2.nextTuple()); //$NON-NLS-1$ //$NON-NLS-2$
- }
- ts2.closeSource();
- }
-
- // Compare actual to expected row count
- assertEquals("Did not get expected row count: ", expectedRowCount, count); //$NON-NLS-1$
-
- // Walk results
- for(int i=0; i<count; i++) {
- ts.nextTuple();
- }
- ts.closeSource();
- bufferMgr.removeTupleSource(tsID);
- }
-
protected CommandContext createCommandContext() {
Properties props = new Properties();
//props.setProperty(ContextProperties.SOAP_HOST, "my.host.com"); //$NON-NLS-1$
@@ -158,8 +84,4 @@
return context;
}
- public void verifyQueryPlan(String[] expectedAtomic, ProcessorPlan plan, QueryMetadataInterface md, CapabilitiesFinder capFinder) throws Exception {
- TestOptimizer.checkAtomicQueries(expectedAtomic, plan, md, capFinder);
- }
-
}
Modified: trunk/test-integration/common/src/test/java/com/metamatrix/server/integration/TestTPCR.java
===================================================================
--- trunk/test-integration/common/src/test/java/com/metamatrix/server/integration/TestTPCR.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/test-integration/common/src/test/java/com/metamatrix/server/integration/TestTPCR.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -59,16 +59,6 @@
FakeCapabilitiesFinder finder = new FakeCapabilitiesFinder();
finder.addCapabilities("TPCR_Oracle_9i", CapabilitiesConverter.convertCapabilities(new OracleCapabilities())); //$NON-NLS-1$
- ProcessorPlan plan = createPlan(METADATA,
- "select l_orderkey, sum(l_extendedprice*(1-l_discount)) as revenue, o_orderdate, o_shippriority " + //$NON-NLS-1$
- "from customer, orders, lineitem " + //$NON-NLS-1$
- "where c_mktsegment = 'BUILDING' and c_custkey = o_custkey and l_orderkey = o_orderkey " + //$NON-NLS-1$
- "and o_orderdate < {ts'1995-03-15 00:00:00'} " + //$NON-NLS-1$
- "and l_shipdate > {ts'1995-03-15 00:00:00'} " + //$NON-NLS-1$
- "group by l_orderkey, o_orderdate, o_shippriority " + //$NON-NLS-1$
- "order by revenue desc, o_orderdate", //$NON-NLS-1$
- finder, DEBUG);
-
HardcodedDataManager dataMgr = new HardcodedDataManager();
List[] expected =
@@ -79,7 +69,15 @@
dataMgr.addData("SELECT g_2.l_orderkey AS c_0, SUM((g_2.l_extendedprice * (1 - g_2.l_discount))) AS c_1, g_1.o_orderdate AS c_2, g_1.o_shippriority AS c_3 FROM TPCR_Oracle_9i.CUSTOMER AS g_0, TPCR_Oracle_9i.ORDERS AS g_1, TPCR_Oracle_9i.LINEITEM AS g_2 WHERE (g_0.c_custkey = g_1.o_custkey) AND (g_2.l_orderkey = g_1.o_orderkey) AND (g_0.c_mktsegment = 'BUILDING') AND (g_1.o_orderdate < {d'1995-03-15'}) AND (g_2.l_shipdate > {ts'1995-03-15 00:00:00.0'}) GROUP BY g_2.l_orderkey, g_1.o_orderdate, g_1.o_shippriority ORDER BY c_1 DESC, c_2", //$NON-NLS-1$
expected);
- doProcess(plan, dataMgr, expected, DEBUG);
+ doProcess(METADATA,
+ "select l_orderkey, sum(l_extendedprice*(1-l_discount)) as revenue, o_orderdate, o_shippriority " + //$NON-NLS-1$
+ "from customer, orders, lineitem " + //$NON-NLS-1$
+ "where c_mktsegment = 'BUILDING' and c_custkey = o_custkey and l_orderkey = o_orderkey " + //$NON-NLS-1$
+ "and o_orderdate < {ts'1995-03-15 00:00:00'} " + //$NON-NLS-1$
+ "and l_shipdate > {ts'1995-03-15 00:00:00'} " + //$NON-NLS-1$
+ "group by l_orderkey, o_orderdate, o_shippriority " + //$NON-NLS-1$
+ "order by revenue desc, o_orderdate", //$NON-NLS-1$
+ finder, dataMgr, expected, DEBUG);
}
@@ -87,13 +85,6 @@
FakeCapabilitiesFinder finder = new FakeCapabilitiesFinder();
finder.addCapabilities("TPCR_Ora", CapabilitiesConverter.convertCapabilities(new OracleCapabilities())); //$NON-NLS-1$
- ProcessorPlan plan = createPlan(BaseQueryTest.createMetadata(UnitTestUtil.getTestDataPath()+"/tpcr/TPCR_3.vdb"), //$NON-NLS-1$
- "SELECT count (*) " + //$NON-NLS-1$
- "FROM TPCR_Ora.CUSTOMER LEFT OUTER JOIN TPCR_Ora.ORDERS ON C_CUSTKEY = O_CUSTKEY " + //$NON-NLS-1$
- "WHERE (O_ORDERKEY IS NULL) OR O_ORDERDATE < '1992-01-02 00:00:00' " + //$NON-NLS-1$
- "AND C_ACCTBAL > 0", //$NON-NLS-1$
- finder, DEBUG);
-
HardcodedDataManager dataMgr = new HardcodedDataManager();
List[] expected =
new List[] { Arrays.asList(new Object[] { new Integer(5) } ) };
@@ -101,7 +92,12 @@
dataMgr.addData("SELECT COUNT(*) FROM TPCR_Ora.CUSTOMER AS g_0 LEFT OUTER JOIN TPCR_Ora.ORDERS AS g_1 ON g_0.C_CUSTKEY = g_1.O_CUSTKEY WHERE (g_1.O_ORDERKEY IS NULL) OR ((g_1.O_ORDERDATE < {ts'1992-01-02 00:00:00.0'}) AND (g_0.C_ACCTBAL > 0))", //$NON-NLS-1$
expected);
- doProcess(plan, dataMgr, expected, DEBUG);
+ doProcess(BaseQueryTest.createMetadata(UnitTestUtil.getTestDataPath()+"/tpcr/TPCR_3.vdb"), //$NON-NLS-1$
+ "SELECT count (*) " + //$NON-NLS-1$
+ "FROM TPCR_Ora.CUSTOMER LEFT OUTER JOIN TPCR_Ora.ORDERS ON C_CUSTKEY = O_CUSTKEY " + //$NON-NLS-1$
+ "WHERE (O_ORDERKEY IS NULL) OR O_ORDERDATE < '1992-01-02 00:00:00' " + //$NON-NLS-1$
+ "AND C_ACCTBAL > 0", //$NON-NLS-1$
+ finder, dataMgr, expected, DEBUG);
}
@@ -122,13 +118,6 @@
finder.addCapabilities("TPCR_Ora", CapabilitiesConverter.convertCapabilities(new OracleCapabilities())); //$NON-NLS-1$
finder.addCapabilities("TPCR_SQLS", CapabilitiesConverter.convertCapabilities(new SqlServerCapabilities())); //$NON-NLS-1$
- ProcessorPlan plan = createPlan(BaseQueryTest.createMetadata(UnitTestUtil.getTestDataPath()+"/tpcr/TPCR_3.vdb"), //$NON-NLS-1$
- "SELECT C_CUSTKEY, C_NAME, C_ADDRESS, C_PHONE, C_ACCTBAL, O_ORDERKEY FROM TPCR_Ora.CUSTOMER " + //$NON-NLS-1$
- "LEFT OUTER JOIN TPCR_SQLS.ORDERS ON C_CUSTKEY = O_CUSTKEY " + //$NON-NLS-1$
- "AND O_ORDERDATE < {ts'1992-01-02 00:00:00.0'} " + //$NON-NLS-1$
- "WHERE (C_ACCTBAL > 50)", //$NON-NLS-1$
- finder, DEBUG);
-
HardcodedDataManager dataMgr = new HardcodedDataManager();
List[] oracleExpected =
@@ -149,7 +138,12 @@
Arrays.asList(new Object[] { new Long(5), "Bill", "101 Fake St.", "392839283", "21.12", new Integer(13) } ), //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
Arrays.asList(new Object[] { new Long(6), "Stu", "102 Fake St.", "385729385", "51.50", null } )}; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
- doProcess(plan, dataMgr, expected, DEBUG);
+ doProcess(BaseQueryTest.createMetadata(UnitTestUtil.getTestDataPath()+"/tpcr/TPCR_3.vdb"), //$NON-NLS-1$
+ "SELECT C_CUSTKEY, C_NAME, C_ADDRESS, C_PHONE, C_ACCTBAL, O_ORDERKEY FROM TPCR_Ora.CUSTOMER " + //$NON-NLS-1$
+ "LEFT OUTER JOIN TPCR_SQLS.ORDERS ON C_CUSTKEY = O_CUSTKEY " + //$NON-NLS-1$
+ "AND O_ORDERDATE < {ts'1992-01-02 00:00:00.0'} " + //$NON-NLS-1$
+ "WHERE (C_ACCTBAL > 50)", //$NON-NLS-1$
+ finder, dataMgr, expected, DEBUG);
}
@@ -164,14 +158,6 @@
finder.addCapabilities("TPCR_Ora", CapabilitiesConverter.convertCapabilities(new OracleCapabilities())); //$NON-NLS-1$
finder.addCapabilities("TPCR_SQLS", CapabilitiesConverter.convertCapabilities(new SqlServerCapabilities())); //$NON-NLS-1$
- ProcessorPlan plan = createPlan(BaseQueryTest.createMetadata(UnitTestUtil.getTestDataPath()+"/tpcr/TPCR_3.vdb"), //$NON-NLS-1$
- "SELECT C_CUSTKEY, C_NAME, C_ADDRESS, C_PHONE, C_ACCTBAL, O_ORDERKEY FROM TPCR_Ora.CUSTOMER " + //$NON-NLS-1$
- "LEFT OUTER JOIN " + //$NON-NLS-1$
- "(SELECT O_CUSTKEY, O_ORDERKEY FROM TPCR_SQLS.ORDERS WHERE O_ORDERDATE < {ts'1992-01-02 00:00:00.0'}) AS X " + //$NON-NLS-1$
- "ON C_CUSTKEY = O_CUSTKEY " + //$NON-NLS-1$
- "WHERE (C_ACCTBAL > 50)", //$NON-NLS-1$
- finder, DEBUG);
-
HardcodedDataManager dataMgr = new HardcodedDataManager();
List[] oracleExpected =
@@ -192,7 +178,13 @@
Arrays.asList(new Object[] { new Long(5), "Bill", "101 Fake St.", "392839283", "51.12", new Integer(13) } ), //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
Arrays.asList(new Object[] { new Long(6), "Stu", "102 Fake St.", "385729385", "51.50", null } )}; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
- doProcess(plan, dataMgr, expected, DEBUG);
+ doProcess(BaseQueryTest.createMetadata(UnitTestUtil.getTestDataPath()+"/tpcr/TPCR_3.vdb"), //$NON-NLS-1$
+ "SELECT C_CUSTKEY, C_NAME, C_ADDRESS, C_PHONE, C_ACCTBAL, O_ORDERKEY FROM TPCR_Ora.CUSTOMER " + //$NON-NLS-1$
+ "LEFT OUTER JOIN " + //$NON-NLS-1$
+ "(SELECT O_CUSTKEY, O_ORDERKEY FROM TPCR_SQLS.ORDERS WHERE O_ORDERDATE < {ts'1992-01-02 00:00:00.0'}) AS X " + //$NON-NLS-1$
+ "ON C_CUSTKEY = O_CUSTKEY " + //$NON-NLS-1$
+ "WHERE (C_ACCTBAL > 50)", //$NON-NLS-1$
+ finder, dataMgr, expected, DEBUG);
}
Modified: trunk/test-integration/common/src/test/java/com/metamatrix/server/integration/TestXMLTypeTranslations.java
===================================================================
--- trunk/test-integration/common/src/test/java/com/metamatrix/server/integration/TestXMLTypeTranslations.java 2009-12-22 17:45:11 UTC (rev 1697)
+++ trunk/test-integration/common/src/test/java/com/metamatrix/server/integration/TestXMLTypeTranslations.java 2009-12-22 19:36:54 UTC (rev 1698)
@@ -35,7 +35,6 @@
import com.metamatrix.query.metadata.QueryMetadataInterface;
import com.metamatrix.query.optimizer.capabilities.FakeCapabilitiesFinder;
import com.metamatrix.query.processor.HardcodedDataManager;
-import com.metamatrix.query.processor.ProcessorPlan;
import com.metamatrix.query.unittest.TimestampUtil;
public class TestXMLTypeTranslations extends BaseQueryTest {
@@ -55,11 +54,6 @@
String sql = "select * from testdocument.testDocument"; //$NON-NLS-1$
-
- ProcessorPlan plan = createPlan(metadata,
- sql,
- finder, DEBUG);
-
HardcodedDataManager dataMgr = new HardcodedDataManager();
Set models = new HashSet();
@@ -82,7 +76,9 @@
List[] expected = new List[] { Arrays.asList(new Object[] {"<?xml version=\"1.0\" encoding=\"UTF-8\"?><XSDTypesNS:test xmlns:xsd=\"http://www.w3.org/2001/XMLSchema\" xmlns:XSDTypesNS=\"http://www.metamatrix.com/XMLSchema/DataSets/XSDTypes\"><book><datetime>1903-04-04T05:06:10.0000002</datetime><double>-INF</double><float>INF</float><gday>---100</gday><gmonth>--100</gmonth><gmonthday>--04-04</gmonthday><gyear>0100</gyear><gyearmonth>1903-04</gyearmonth><string>1</string></book></XSDTypesNS:test>" })}; //$NON-NLS-1$
- doProcess(plan, dataMgr , expected, DEBUG);
+ doProcess(metadata,
+ sql,
+ finder, dataMgr , expected, DEBUG);
}
More information about the teiid-commits
mailing list