[teiid-commits] teiid SVN: r3507 - in branches/as7: adminshell/src/main/java/org/teiid/adminshell and 53 other directories.
teiid-commits at lists.jboss.org
teiid-commits at lists.jboss.org
Thu Sep 29 12:46:40 EDT 2011
Author: rareddy
Date: 2011-09-29 12:46:38 -0400 (Thu, 29 Sep 2011)
New Revision: 3507
Added:
branches/as7/engine/src/main/java/org/teiid/common/buffer/AutoCleanupUtil.java
branches/as7/engine/src/main/java/org/teiid/common/buffer/Cache.java
branches/as7/engine/src/main/java/org/teiid/common/buffer/CacheEntry.java
branches/as7/engine/src/main/java/org/teiid/common/buffer/LightWeightCopyOnWriteList.java
branches/as7/engine/src/main/java/org/teiid/common/buffer/Serializer.java
branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/BlockBitSetTree.java
branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/BlockClosedLongIntHashTable.java
branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/BlockManager.java
branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/ExtensibleBufferedOutputStream.java
branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/FileStoreCache.java
branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/SplittableStorageManager.java
branches/as7/engine/src/main/java/org/teiid/dqp/internal/process/SerializableTupleBatch.java
branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestBlockBitSetTree.java
branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestBlockClosedLongIntHashTable.java
branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestFileStoreCache.java
branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestSplittableStorageManager.java
Removed:
branches/as7/engine/src/test/java/org/teiid/core/
Modified:
branches/as7/
branches/as7/adminshell/src/main/java/org/teiid/adminshell/AdminShell.java
branches/as7/api/src/main/java/org/teiid/CommandContext.java
branches/as7/api/src/main/java/org/teiid/events/EventDistributor.java
branches/as7/build/kits/jboss-container/deploy/teiid/teiid-jboss-beans.xml
branches/as7/build/kits/jboss-container/teiid-releasenotes.html
branches/as7/client/src/main/java/org/teiid/client/BatchSerializer.java
branches/as7/client/src/main/java/org/teiid/client/ResultsMessage.java
branches/as7/client/src/main/java/org/teiid/jdbc/ConnectionImpl.java
branches/as7/client/src/main/java/org/teiid/jdbc/DatabaseMetaDataImpl.java
branches/as7/client/src/main/java/org/teiid/jdbc/EmbeddedProfile.java
branches/as7/client/src/main/java/org/teiid/jdbc/JDBCURL.java
branches/as7/client/src/main/java/org/teiid/jdbc/ResultSetImpl.java
branches/as7/client/src/main/java/org/teiid/jdbc/ResultSetMetaDataImpl.java
branches/as7/client/src/main/java/org/teiid/jdbc/SQLStates.java
branches/as7/client/src/main/java/org/teiid/jdbc/StatementImpl.java
branches/as7/client/src/main/java/org/teiid/jdbc/TeiidSQLException.java
branches/as7/client/src/main/java/org/teiid/net/socket/SocketServerConnectionFactory.java
branches/as7/client/src/test/java/org/teiid/client/TestBatchSerializer.java
branches/as7/client/src/test/java/org/teiid/jdbc/TestAllResultsImpl.java
branches/as7/client/src/test/java/org/teiid/jdbc/TestCallableStatement.java
branches/as7/client/src/test/java/org/teiid/jdbc/TestConnection.java
branches/as7/client/src/test/java/org/teiid/jdbc/TestSQLException.java
branches/as7/client/src/test/java/org/teiid/jdbc/TestStatement.java
branches/as7/client/src/test/java/org/teiid/jdbc/TestTeiidDriver.java
branches/as7/common-core/src/main/java/org/teiid/core/TeiidException.java
branches/as7/common-core/src/main/java/org/teiid/core/TeiidRuntimeException.java
branches/as7/common-core/src/main/java/org/teiid/core/types/BaseLob.java
branches/as7/common-core/src/main/java/org/teiid/core/types/BlobImpl.java
branches/as7/common-core/src/main/java/org/teiid/core/types/BlobType.java
branches/as7/common-core/src/main/java/org/teiid/core/types/ClobType.java
branches/as7/common-core/src/main/java/org/teiid/core/types/DataTypeManager.java
branches/as7/common-core/src/main/java/org/teiid/core/types/InputStreamFactory.java
branches/as7/common-core/src/main/java/org/teiid/core/types/SQLXMLImpl.java
branches/as7/common-core/src/main/java/org/teiid/core/types/Streamable.java
branches/as7/common-core/src/main/java/org/teiid/core/types/XMLType.java
branches/as7/common-core/src/main/resources/org/teiid/core/util/application.properties
branches/as7/common-core/src/test/java/org/teiid/core/TestMetaMatrixRuntimeException.java
branches/as7/common-core/src/test/java/org/teiid/core/types/TestBlobValue.java
branches/as7/common-core/src/test/java/org/teiid/core/types/TestClobValue.java
branches/as7/common-core/src/test/java/org/teiid/core/types/TestXMLValue.java
branches/as7/common-core/src/test/java/org/teiid/core/util/UnitTestUtil.java
branches/as7/documentation/caching-guide/src/main/docbook/en-US/content/hint-option.xml
branches/as7/engine/src/main/java/org/teiid/common/buffer/BatchManager.java
branches/as7/engine/src/main/java/org/teiid/common/buffer/BlockedException.java
branches/as7/engine/src/main/java/org/teiid/common/buffer/BufferManager.java
branches/as7/engine/src/main/java/org/teiid/common/buffer/FileStore.java
branches/as7/engine/src/main/java/org/teiid/common/buffer/FileStoreInputStreamFactory.java
branches/as7/engine/src/main/java/org/teiid/common/buffer/LobManager.java
branches/as7/engine/src/main/java/org/teiid/common/buffer/SPage.java
branches/as7/engine/src/main/java/org/teiid/common/buffer/STree.java
branches/as7/engine/src/main/java/org/teiid/common/buffer/TupleBatch.java
branches/as7/engine/src/main/java/org/teiid/common/buffer/TupleBrowser.java
branches/as7/engine/src/main/java/org/teiid/common/buffer/TupleBuffer.java
branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/BufferManagerImpl.java
branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/FileStorageManager.java
branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/MemoryStorageManager.java
branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/SizeUtility.java
branches/as7/engine/src/main/java/org/teiid/dqp/internal/datamgr/ConnectorWorkItem.java
branches/as7/engine/src/main/java/org/teiid/dqp/internal/process/DataTierTupleSource.java
branches/as7/engine/src/main/java/org/teiid/dqp/internal/process/Request.java
branches/as7/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java
branches/as7/engine/src/main/java/org/teiid/dqp/message/AtomicResultsMessage.java
branches/as7/engine/src/main/java/org/teiid/query/function/FunctionDescriptor.java
branches/as7/engine/src/main/java/org/teiid/query/function/FunctionTree.java
branches/as7/engine/src/main/java/org/teiid/query/function/source/SecuritySystemFunctions.java
branches/as7/engine/src/main/java/org/teiid/query/metadata/TransformationMetadata.java
branches/as7/engine/src/main/java/org/teiid/query/optimizer/relational/PlanToProcessConverter.java
branches/as7/engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java
branches/as7/engine/src/main/java/org/teiid/query/processor/QueryProcessor.java
branches/as7/engine/src/main/java/org/teiid/query/processor/proc/ProcedurePlan.java
branches/as7/engine/src/main/java/org/teiid/query/processor/relational/EnhancedSortMergeJoinStrategy.java
branches/as7/engine/src/main/java/org/teiid/query/processor/relational/NullNode.java
branches/as7/engine/src/main/java/org/teiid/query/processor/relational/RelationalNode.java
branches/as7/engine/src/main/java/org/teiid/query/processor/relational/SortUtility.java
branches/as7/engine/src/main/java/org/teiid/query/processor/xml/RelationalPlanExecutor.java
branches/as7/engine/src/main/java/org/teiid/query/sql/LanguageObject.java
branches/as7/engine/src/main/java/org/teiid/query/tempdata/GlobalTableStoreImpl.java
branches/as7/engine/src/main/java/org/teiid/query/tempdata/TempTable.java
branches/as7/engine/src/main/java/org/teiid/query/tempdata/TempTableDataManager.java
branches/as7/engine/src/main/java/org/teiid/query/tempdata/TempTableStore.java
branches/as7/engine/src/main/java/org/teiid/query/util/CommandContext.java
branches/as7/engine/src/main/resources/org/teiid/query/i18n.properties
branches/as7/engine/src/test/java/org/teiid/common/buffer/BufferManagerFactory.java
branches/as7/engine/src/test/java/org/teiid/common/buffer/TestLobManager.java
branches/as7/engine/src/test/java/org/teiid/common/buffer/TestSTree.java
branches/as7/engine/src/test/java/org/teiid/common/buffer/TestTupleBuffer.java
branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestFileStorageManager.java
branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestSizeUtility.java
branches/as7/engine/src/test/java/org/teiid/dqp/internal/process/TestCachedResults.java
branches/as7/engine/src/test/java/org/teiid/dqp/internal/process/TestDQPCore.java
branches/as7/engine/src/test/java/org/teiid/dqp/service/AutoGenDataService.java
branches/as7/engine/src/test/java/org/teiid/query/function/TestFunction.java
branches/as7/engine/src/test/java/org/teiid/query/function/TestFunctionLibrary.java
branches/as7/engine/src/test/java/org/teiid/query/function/TestFunctionTree.java
branches/as7/engine/src/test/java/org/teiid/query/function/TestResolvedFunctions.java
branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestAccessPatterns.java
branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestAggregatePushdown.java
branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestAnsiJoinPushdown.java
branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestBatchedUpdatePlanner.java
branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestDependentJoins.java
branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestExpressionsInGroupBy.java
branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestInlineView.java
branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestJoinOptimization.java
branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestJoinPushdownRestrictions.java
branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestLimit.java
branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestOptimizer.java
branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestOptionalJoins.java
branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestPartitionedJoinPlanning.java
branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestRuleMergeVirtual.java
branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestRuleRaiseNull.java
branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestRuleRemoveSorts.java
branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestSortOptimization.java
branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestStoredProcedurePlanning.java
branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestSubqueryPushdown.java
branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestUnionPlanning.java
branches/as7/engine/src/test/java/org/teiid/query/optimizer/relational/TestAliasGenerator.java
branches/as7/engine/src/test/java/org/teiid/query/processor/FakeDataManager.java
branches/as7/engine/src/test/java/org/teiid/query/processor/FakeDataStore.java
branches/as7/engine/src/test/java/org/teiid/query/processor/TestAggregateProcessing.java
branches/as7/engine/src/test/java/org/teiid/query/processor/TestDependentJoins.java
branches/as7/engine/src/test/java/org/teiid/query/processor/TestInsertProcessing.java
branches/as7/engine/src/test/java/org/teiid/query/processor/TestJoinWithFunction.java
branches/as7/engine/src/test/java/org/teiid/query/processor/TestOptionalJoins.java
branches/as7/engine/src/test/java/org/teiid/query/processor/TestOrderByProcessing.java
branches/as7/engine/src/test/java/org/teiid/query/processor/TestProcedureRelational.java
branches/as7/engine/src/test/java/org/teiid/query/processor/TestProcessor.java
branches/as7/engine/src/test/java/org/teiid/query/processor/TestSQLXMLProcessing.java
branches/as7/engine/src/test/java/org/teiid/query/processor/TestSecurityFunctions.java
branches/as7/engine/src/test/java/org/teiid/query/processor/TestSetProcessing.java
branches/as7/engine/src/test/java/org/teiid/query/processor/TestTempTables.java
branches/as7/engine/src/test/java/org/teiid/query/processor/TestTextTable.java
branches/as7/engine/src/test/java/org/teiid/query/processor/TestVirtualDepJoin.java
branches/as7/engine/src/test/java/org/teiid/query/processor/TestWithClauseProcessing.java
branches/as7/engine/src/test/java/org/teiid/query/processor/relational/TestJoinNode.java
branches/as7/engine/src/test/java/org/teiid/query/processor/relational/TestProjectIntoNode.java
branches/as7/engine/src/test/java/org/teiid/query/sql/symbol/TestElementSymbol.java
branches/as7/engine/src/test/resources/text/cdm_dos_win.txt
branches/as7/jboss-integration/pom.xml
branches/as7/metadata/src/main/java/org/teiid/internal/core/index/BlocksIndexInput.java
branches/as7/runtime/src/main/java/org/teiid/services/BufferServiceImpl.java
branches/as7/runtime/src/test/java/org/teiid/dqp/service/buffer/TestLocalBufferService.java
branches/as7/test-integration/common/src/test/java/org/teiid/systemmodel/TestMatViewReplication.java
branches/as7/test-integration/common/src/test/java/org/teiid/transport/TestODBCSocketTransport.java
branches/as7/test-integration/common/src/test/resources/TestODBCSocketTransport/testPk.expected
branches/as7/test-integration/common/src/test/resources/TestODBCSocketTransport/testPkPrepared.expected
branches/as7/test-integration/common/src/test/resources/TestSystemVirtualModel/testProcedureParams.expected
branches/as7/test-integration/common/src/test/resources/TestSystemVirtualModel/testProcedures.expected
Log:
merge -r 3451:3506 from trunk to as7
Property changes on: branches/as7
___________________________________________________________________
Modified: svn:mergeinfo
- /trunk:3188-3450
+ /trunk:3188-3450,3452-3506
Property changes on: branches/as7/adminshell/src/main/java/org/teiid/adminshell/AdminShell.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.1.x/adminshell/src/main/java/org/teiid/adminshell/AdminShell.java:2901-2940
/trunk/adminshell/src/main/java/org/teiid/adminshell/AdminShell.java:3188-3450
+ /branches/7.1.x/adminshell/src/main/java/org/teiid/adminshell/AdminShell.java:2901-2940
/trunk/adminshell/src/main/java/org/teiid/adminshell/AdminShell.java:3188-3450,3452-3506
Modified: branches/as7/api/src/main/java/org/teiid/CommandContext.java
===================================================================
--- branches/as7/api/src/main/java/org/teiid/CommandContext.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/api/src/main/java/org/teiid/CommandContext.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -87,7 +87,8 @@
/**
* Get the processor batch size set on the BufferManager
- * @return
+ * @return - the nominal batch size target. actual batch sizes will vary based
+ * upon the column types
*/
int getProcessorBatchSize();
Property changes on: branches/as7/api/src/main/java/org/teiid/events/EventDistributor.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/api/src/main/java/org/teiid/events/EventDistributor.java:3340-3349
/trunk/api/src/main/java/org/teiid/events/EventDistributor.java:3188-3450
+ /branches/7.4.x/api/src/main/java/org/teiid/events/EventDistributor.java:3340-3349
/trunk/api/src/main/java/org/teiid/events/EventDistributor.java:3188-3450,3452-3506
Modified: branches/as7/build/kits/jboss-container/deploy/teiid/teiid-jboss-beans.xml
===================================================================
--- branches/as7/build/kits/jboss-container/deploy/teiid/teiid-jboss-beans.xml 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/build/kits/jboss-container/deploy/teiid/teiid-jboss-beans.xml 2011-09-29 16:46:38 UTC (rev 3507)
@@ -43,7 +43,11 @@
<!-- Max storage space, in MB, to be used for buffer files (default 50G) -->
<property name="maxBufferSpace">51200</property>
<!-- Max open buffer files (default 64) -->
- <property name="maxOpenFiles">64</property>
+ <property name="maxOpenFiles">64</property>
+ <!-- Set to true to allow inlining of memory based and small lobs into results.
+ However inline lob values are not supported by pre-7.6 clients, so disable this
+ property if using older clients utilizing lobs. (default true) -->
+ <property name="inlineLobs">true</property>
</bean>
<bean name="CacheFactory" class="org.teiid.cache.jboss.ClusterableCacheFactory">
Modified: branches/as7/build/kits/jboss-container/teiid-releasenotes.html
===================================================================
--- branches/as7/build/kits/jboss-container/teiid-releasenotes.html 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/build/kits/jboss-container/teiid-releasenotes.html 2011-09-29 16:46:38 UTC (rev 3507)
@@ -30,6 +30,7 @@
<LI><B>File Enhancements</B> - the file translator can now optionally (via the ExceptionIfFileNotFound property) throw an exception if the path refers to a file that doesn't exist. The file resource adapter can be configured to map file names and can prevent parent path .. references. See the Admin Guide or the file-ds.xml template for more.
<LI><B>TEXTTABLE Enhancements</B> - TEXTTABLE can now parse fixed width files that do not use a row delimiter and can optionally produce fixed values that haven't been trimmed.
<LI><B>Temp table transactions</B> - Internal materialized views and temp table usage from a session and within procedures can take advantage of greater transaction support.
+ <LI><B>Buffering Improvements</B> - Added the ability to inline memory based or small lobs and added tracking of the memory held by soft references.
</UL>
<h2><a name="Compatibility">Compatibility Issues</a></h2>
@@ -39,9 +40,10 @@
<li>Support for using the FROM clause post item hints MAKEDEP/MAKENOTDEP has been deprecated. Use the pre item comment hint syntax instead, e.g. /*+ MAKEDEP */ tbl
</ul>
-<h4>from 7.4</h4>
+<h4>from 7.5</h4>
<ul>
- <li>Leave was added as a reserved word.
+ <li>Leave was added as a reserved word.
+ <li>Lob inlining is incompatible with clients older than 7.6. If a 7.6 server will have older clients that use lobs connect to it, then the BufferService property inlineLobs should be set to false in the teiid-jboss-beans.xml file.
</ul>
<h4>from 7.4</h4>
Modified: branches/as7/client/src/main/java/org/teiid/client/BatchSerializer.java
===================================================================
--- branches/as7/client/src/main/java/org/teiid/client/BatchSerializer.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/client/src/main/java/org/teiid/client/BatchSerializer.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -29,6 +29,7 @@
import java.math.BigInteger;
import java.sql.Time;
import java.sql.Timestamp;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
@@ -75,23 +76,23 @@
* @throws IOException
* @since 4.2
*/
- static void writeIsNullData(ObjectOutput out, int col, List[] batch) throws IOException {
- int numBytes = batch.length / 8, row = 0, currentByte = 0;
+ static void writeIsNullData(ObjectOutput out, int col, List<? extends List<?>> batch) throws IOException {
+ int numBytes = batch.size() / 8, row = 0, currentByte = 0;
for (int byteNum = 0; byteNum < numBytes; byteNum++, row+=8) {
- currentByte = (batch[row].get(col) == null) ? 0x80 : 0;
- if (batch[row+1].get(col) == null) currentByte |= 0x40;
- if (batch[row+2].get(col) == null) currentByte |= 0x20;
- if (batch[row+3].get(col) == null) currentByte |= 0x10;
- if (batch[row+4].get(col) == null) currentByte |= 0x08;
- if (batch[row+5].get(col) == null) currentByte |= 0x04;
- if (batch[row+6].get(col) == null) currentByte |= 0x02;
- if (batch[row+7].get(col) == null) currentByte |= 0x01;
+ currentByte = (batch.get(row).get(col) == null) ? 0x80 : 0;
+ if (batch.get(row+1).get(col) == null) currentByte |= 0x40;
+ if (batch.get(row+2).get(col) == null) currentByte |= 0x20;
+ if (batch.get(row+3).get(col) == null) currentByte |= 0x10;
+ if (batch.get(row+4).get(col) == null) currentByte |= 0x08;
+ if (batch.get(row+5).get(col) == null) currentByte |= 0x04;
+ if (batch.get(row+6).get(col) == null) currentByte |= 0x02;
+ if (batch.get(row+7).get(col) == null) currentByte |= 0x01;
out.write(currentByte);
}
- if (batch.length % 8 > 0) {
+ if (batch.size() % 8 > 0) {
currentByte = 0;
- for (int mask = 0x80; row < batch.length; row++, mask >>= 1) {
- if (batch[row].get(col) == null) currentByte |= mask;
+ for (int mask = 0x80; row < batch.size(); row++, mask >>= 1) {
+ if (batch.get(row).get(col) == null) currentByte |= mask;
}
out.write(currentByte);
}
@@ -126,22 +127,22 @@
* @since 4.2
*/
private static class ColumnSerializer {
- public void writeColumn(ObjectOutput out, int col, List[] batch) throws IOException {
+ public void writeColumn(ObjectOutput out, int col, List<? extends List<?>> batch) throws IOException {
writeIsNullData(out, col, batch);
Object obj = null;
- for (int i = 0; i < batch.length; i++) {
- obj = batch[i].get(col);
+ for (int i = 0; i < batch.size(); i++) {
+ obj = batch.get(i).get(col);
if (obj != null) {
writeObject(out, obj);
}
}
}
- public void readColumn(ObjectInput in, int col, List[] batch, byte[] isNull) throws IOException, ClassNotFoundException {
+ public void readColumn(ObjectInput in, int col, List<List<Object>> batch, byte[] isNull) throws IOException, ClassNotFoundException {
readIsNullData(in, isNull);
- for (int i = 0; i < batch.length; i++) {
+ for (int i = 0; i < batch.size(); i++) {
if (!isNullObject(isNull, i)) {
- batch[i].set(col, DataTypeManager.getCanonicalValue(readObject(in)));
+ batch.get(i).set(col, DataTypeManager.getCanonicalValue(readObject(in)));
}
}
}
@@ -203,13 +204,14 @@
/* This implementation compacts the isNull and boolean data for non-null values into a byte[]
* by using a 8 bit mask that is bit-shifted to mask each value.
*/
- public void writeColumn(ObjectOutput out, int col, List[] batch) throws IOException {
+ @Override
+ public void writeColumn(ObjectOutput out, int col, List<? extends List<?>> batch) throws IOException {
int currentByte = 0;
int mask = 0x80;
Object obj;
- for (int row = 0; row < batch.length; row++) {
+ for (int row = 0; row < batch.size(); row++) {
// Write the isNull value
- obj = batch[row].get(col);
+ obj = batch.get(row).get(col);
if (obj == null ) {
currentByte |= mask;
}
@@ -241,10 +243,13 @@
}
}
- public void readColumn(ObjectInput in, int col, List[] batch, byte[] isNull) throws IOException, ClassNotFoundException {
+ @Override
+ public void readColumn(ObjectInput in, int col,
+ List<List<Object>> batch, byte[] isNull) throws IOException,
+ ClassNotFoundException {
int currentByte = 0, mask = 0; // Initialize the mask so that it is reset in the loop
boolean isNullVal;
- for (int row = 0; row < batch.length; row++) {
+ for (int row = 0; row < batch.size(); row++) {
if (mask == 0) {
// If we used up the byte, read the next one, and reset the mask
currentByte = in.read();
@@ -257,7 +262,7 @@
currentByte = in.read();
mask = 0x80;
}
- batch[row].set(col, ((currentByte & mask) == 0) ? Boolean.FALSE : Boolean.TRUE);
+ batch.get(row).set(col, ((currentByte & mask) == 0) ? Boolean.FALSE : Boolean.TRUE);
mask >>= 1;
}
}
@@ -354,17 +359,12 @@
return cs;
}
- public static void writeBatch(ObjectOutput out, String[] types, List[] batch) throws IOException {
- // If there are no type hints, simply use the default mechanism to serialize
- if (types == null || types.length == 0) {
- out.writeObject(batch);
- return;
- }
+ public static void writeBatch(ObjectOutput out, String[] types, List<? extends List<?>> batch) throws IOException {
if (batch == null) {
out.writeInt(-1);
} else {
- out.writeInt(batch.length);
- if (batch.length > 0) {
+ out.writeInt(batch.size());
+ if (batch.size() > 0) {
int columns = types.length;
out.writeInt(columns);
for(int i = 0; i < columns; i++) {
@@ -374,8 +374,8 @@
} catch (ClassCastException e) {
Object obj = null;
String objectClass = null;
- objectSearch: for (int row = 0; row < batch.length; row++) {
- obj = batch[row].get(i);
+ objectSearch: for (int row = 0; row < batch.size(); row++) {
+ obj = batch.get(row).get(i);
if (obj != null) {
objectClass = obj.getClass().getName();
break objectSearch;
@@ -388,21 +388,17 @@
}
}
- public static List[] readBatch(ObjectInput in, String[] types) throws IOException, ClassNotFoundException {
- // If there are no type hints, use the default mechanism to deserialize
- if (types == null || types.length == 0) {
- return (List[])in.readObject();
- }
+ public static List<List<Object>> readBatch(ObjectInput in, String[] types) throws IOException, ClassNotFoundException {
int rows = in.readInt();
if (rows == 0) {
- return new List[0];
+ return new ArrayList<List<Object>>(0);
} else if (rows > 0) {
int columns = in.readInt();
- List[] batch = new List[rows];
+ List<List<Object>> batch = new ArrayList<List<Object>>(rows);
int numBytes = rows/8;
int extraRows = rows % 8;
for (int currentRow = 0; currentRow < rows; currentRow++) {
- batch[currentRow] = Arrays.asList(new Object[columns]);
+ batch.add(currentRow, Arrays.asList(new Object[columns]));
}
byte[] isNullBuffer = new byte[(extraRows > 0) ? numBytes + 1: numBytes];
for (int col = 0; col < columns; col++) {
Modified: branches/as7/client/src/main/java/org/teiid/client/ResultsMessage.java
===================================================================
--- branches/as7/client/src/main/java/org/teiid/client/ResultsMessage.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/client/src/main/java/org/teiid/client/ResultsMessage.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -26,7 +26,7 @@
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
-import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.List;
@@ -45,9 +45,9 @@
static final long serialVersionUID = 3546924172976187793L;
- private List[] results = null;
- private String[] columnNames = null;
- private String[] dataTypes = null;
+ private List<? extends List<?>> results;
+ private String[] columnNames;
+ private String[] dataTypes;
/** A description of planning that occurred as requested in the request. */
private PlanNode planDescription;
@@ -85,36 +85,29 @@
public ResultsMessage(){
}
- /**
- * Instantiate and copy relevant information from the original request message.
- * Typically, the transaction context should only be copied if this results
- * message is being returned from the connector to the query engine. Clients
- * will be unable to deserialize this object.
- * @param requestMsg
- * @param copyTransactionContext true if the transaction context should be copied; false otherwise.
- * @since 4.2
- */
- public ResultsMessage(RequestMessage requestMsg){
- this.results = new ArrayList[0];
-
- }
-
- public ResultsMessage(RequestMessage requestMsg, List[] results, String[] columnNames, String[] dataTypes){
- this (requestMsg);
- setResults( results );
+ public ResultsMessage(List<? extends List<?>> results, String[] columnNames, String[] dataTypes){
+ this.results = results;
setFirstRow( 1 );
- setLastRow( results.length );
+ setLastRow( results.size() );
this.columnNames = columnNames;
this.dataTypes = dataTypes;
}
-
- public List[] getResults() {
+
+ public List<? extends List<?>> getResultsList() {
return results;
}
+
+ /**
+ * @deprecated see {@link #getResultsList()}
+ * @return
+ */
+ public List<?>[] getResults() {
+ return results.toArray(new List[results.size()]);
+ }
- public void setResults(List[] results) {
- this.results = results;
+ public void setResults(List<?>[] results) {
+ this.results = Arrays.asList(results);
}
public String[] getColumnNames() {
@@ -341,7 +334,7 @@
*/
public String toString() {
return new StringBuffer("ResultsMessage rowCount=") //$NON-NLS-1$
- .append(results == null ? 0 : results.length)
+ .append(results == null ? 0 : results.size())
.append(" finalRow=") //$NON-NLS-1$
.append(finalRow)
.toString();
Property changes on: branches/as7/client/src/main/java/org/teiid/jdbc/ConnectionImpl.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/client/src/main/java/org/teiid/jdbc/ConnectionImpl.java:3149-3217
/trunk/client/src/main/java/org/teiid/jdbc/ConnectionImpl.java:3188-3450
+ /branches/7.4.x/client/src/main/java/org/teiid/jdbc/ConnectionImpl.java:3149-3217
/trunk/client/src/main/java/org/teiid/jdbc/ConnectionImpl.java:3188-3450,3452-3506
Property changes on: branches/as7/client/src/main/java/org/teiid/jdbc/DatabaseMetaDataImpl.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/client/src/main/java/org/teiid/jdbc/DatabaseMetaDataImpl.java:3281-3325
/trunk/client/src/main/java/org/teiid/jdbc/DatabaseMetaDataImpl.java:3188-3450
+ /branches/7.4.x/client/src/main/java/org/teiid/jdbc/DatabaseMetaDataImpl.java:3281-3325
/trunk/client/src/main/java/org/teiid/jdbc/DatabaseMetaDataImpl.java:3188-3450,3452-3506
Property changes on: branches/as7/client/src/main/java/org/teiid/jdbc/EmbeddedProfile.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/client/src/main/java/org/teiid/jdbc/EmbeddedProfile.java:3149-3217
/trunk/client/src/main/java/org/teiid/jdbc/EmbeddedProfile.java:3188-3450
+ /branches/7.4.x/client/src/main/java/org/teiid/jdbc/EmbeddedProfile.java:3149-3217
/trunk/client/src/main/java/org/teiid/jdbc/EmbeddedProfile.java:3188-3450,3452-3506
Property changes on: branches/as7/client/src/main/java/org/teiid/jdbc/JDBCURL.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/client/src/main/java/org/teiid/jdbc/JDBCURL.java:3149-3217,3281-3325
/trunk/client/src/main/java/org/teiid/jdbc/JDBCURL.java:3188-3450
+ /branches/7.4.x/client/src/main/java/org/teiid/jdbc/JDBCURL.java:3149-3217,3281-3325
/trunk/client/src/main/java/org/teiid/jdbc/JDBCURL.java:3188-3450,3452-3506
Modified: branches/as7/client/src/main/java/org/teiid/jdbc/ResultSetImpl.java
===================================================================
--- branches/as7/client/src/main/java/org/teiid/jdbc/ResultSetImpl.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/client/src/main/java/org/teiid/jdbc/ResultSetImpl.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -416,7 +416,7 @@
private Batch getCurrentBatch(ResultsMessage currentResultMsg) {
this.updatedPlanDescription = currentResultMsg.getPlanDescription();
- boolean isLast = currentResultMsg.getResults().length == 0 || currentResultMsg.getFinalRow() == currentResultMsg.getLastRow();
+ boolean isLast = currentResultMsg.getResultsList().size() == 0 || currentResultMsg.getFinalRow() == currentResultMsg.getLastRow();
Batch result = new Batch(currentResultMsg.getResults(), currentResultMsg.getFirstRow(), currentResultMsg.getLastRow(), isLast);
result.setLastRow(currentResultMsg.getFinalRow());
return result;
Property changes on: branches/as7/client/src/main/java/org/teiid/jdbc/ResultSetMetaDataImpl.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/client/src/main/java/org/teiid/jdbc/ResultSetMetaDataImpl.java:3281-3325
/trunk/client/src/main/java/org/teiid/jdbc/ResultSetMetaDataImpl.java:3188-3450
+ /branches/7.4.x/client/src/main/java/org/teiid/jdbc/ResultSetMetaDataImpl.java:3281-3325
/trunk/client/src/main/java/org/teiid/jdbc/ResultSetMetaDataImpl.java:3188-3450,3452-3506
Property changes on: branches/as7/client/src/main/java/org/teiid/jdbc/SQLStates.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/client/src/main/java/org/teiid/jdbc/SQLStates.java:3220-3275
/trunk/client/src/main/java/org/teiid/jdbc/SQLStates.java:3188-3450
+ /branches/7.4.x/client/src/main/java/org/teiid/jdbc/SQLStates.java:3220-3275
/trunk/client/src/main/java/org/teiid/jdbc/SQLStates.java:3188-3450,3452-3506
Property changes on: branches/as7/client/src/main/java/org/teiid/jdbc/StatementImpl.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/client/src/main/java/org/teiid/jdbc/StatementImpl.java:3149-3217,3220-3275,3281-3325,3355-3365
/trunk/client/src/main/java/org/teiid/jdbc/StatementImpl.java:3188-3450
+ /branches/7.4.x/client/src/main/java/org/teiid/jdbc/StatementImpl.java:3149-3217,3220-3275,3281-3325,3355-3365
/trunk/client/src/main/java/org/teiid/jdbc/StatementImpl.java:3188-3450,3452-3506
Property changes on: branches/as7/client/src/main/java/org/teiid/jdbc/TeiidSQLException.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/client/src/main/java/org/teiid/jdbc/TeiidSQLException.java:3220-3275
/trunk/client/src/main/java/org/teiid/jdbc/TeiidSQLException.java:3188-3450
+ /branches/7.4.x/client/src/main/java/org/teiid/jdbc/TeiidSQLException.java:3220-3275
/trunk/client/src/main/java/org/teiid/jdbc/TeiidSQLException.java:3188-3450,3452-3506
Property changes on: branches/as7/client/src/main/java/org/teiid/net/socket/SocketServerConnectionFactory.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/client/src/main/java/org/teiid/net/socket/SocketServerConnectionFactory.java:3149-3217
/trunk/client/src/main/java/org/teiid/net/socket/SocketServerConnectionFactory.java:3188-3450
+ /branches/7.4.x/client/src/main/java/org/teiid/net/socket/SocketServerConnectionFactory.java:3149-3217
/trunk/client/src/main/java/org/teiid/net/socket/SocketServerConnectionFactory.java:3188-3450,3452-3506
Modified: branches/as7/client/src/test/java/org/teiid/client/TestBatchSerializer.java
===================================================================
--- branches/as7/client/src/test/java/org/teiid/client/TestBatchSerializer.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/client/src/test/java/org/teiid/client/TestBatchSerializer.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -35,10 +35,10 @@
import java.util.Arrays;
import java.util.List;
-import org.teiid.client.BatchSerializer;
+import junit.framework.TestCase;
+
import org.teiid.core.types.DataTypeManager;
-import junit.framework.TestCase;
@@ -63,21 +63,22 @@
}
}
- private static void helpTestSerialization(String[] types, List[] batch) throws IOException, ClassNotFoundException {
+ private static void helpTestSerialization(String[] types, List<?>[] batch) throws IOException, ClassNotFoundException {
ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
ObjectOutputStream out = new ObjectOutputStream(byteStream);
- BatchSerializer.writeBatch(out, types, batch);
+ List<List<?>> batchList = Arrays.asList(batch);
+ BatchSerializer.writeBatch(out, types, batchList);
out.flush();
byte[] bytes = byteStream.toByteArray();
ByteArrayInputStream bytesIn = new ByteArrayInputStream(bytes);
ObjectInputStream in = new ObjectInputStream(bytesIn);
- List[] newBatch = BatchSerializer.readBatch(in, types);
+ List<List<Object>> newBatch = BatchSerializer.readBatch(in, types);
out.close();
in.close();
- assertEqual(batch, newBatch);
+ assertTrue(batchList.equals(newBatch));
}
private static final String[] sampleBatchTypes = {DataTypeManager.DefaultDataTypes.BIG_DECIMAL,
@@ -164,14 +165,6 @@
helpTestSerialization(sampleBatchTypes, sampleBatch(833)); // A bunch of rows. This should also test large strings
}
- public void testSerializeBasicTypes_NoTypeHints() throws Exception {
- helpTestSerialization(null, sampleBatch(1));
- helpTestSerialization(null, sampleBatch(8));
- helpTestSerialization(null, sampleBatch(17));
- helpTestSerialization(null, sampleBatch(120));
- helpTestSerialization(null, sampleBatch(833));
- }
-
public void testSerializeBasicTypesWithNulls() throws Exception {
helpTestSerialization(sampleBatchTypes, sampleBatchWithNulls(1));
helpTestSerialization(sampleBatchTypes, sampleBatchWithNulls(8));
@@ -186,10 +179,7 @@
}
public void testSerializeNoData() throws Exception {
- helpTestSerialization(sampleBatchTypes, null);
- helpTestSerialization(null, null);
helpTestSerialization(sampleBatchTypes, new List[0]);
- helpTestSerialization(null, new List[0]);
}
public void testSerializeDatatypeMismatch() throws Exception {
Modified: branches/as7/client/src/test/java/org/teiid/jdbc/TestAllResultsImpl.java
===================================================================
--- branches/as7/client/src/test/java/org/teiid/jdbc/TestAllResultsImpl.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/client/src/test/java/org/teiid/jdbc/TestAllResultsImpl.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -797,7 +797,7 @@
private ResultsMessage exampleMessage(List<Object>[] results, String[] columnNames, String[] datatypes) {
RequestMessage request = new RequestMessage();
request.setExecutionId(REQUEST_ID);
- ResultsMessage resultsMsg = new ResultsMessage(request);
+ ResultsMessage resultsMsg = new ResultsMessage();
resultsMsg.setResults(results);
resultsMsg.setColumnNames(columnNames);
resultsMsg.setDataTypes(datatypes);
@@ -834,7 +834,7 @@
private static ResultsMessage exampleResultsMsg4(int begin, int length, boolean lastBatch) {
RequestMessage request = new RequestMessage();
request.setExecutionId(REQUEST_ID);
- ResultsMessage resultsMsg = new ResultsMessage(request);
+ ResultsMessage resultsMsg = new ResultsMessage();
List[] results = exampleResults1(length, begin);
resultsMsg.setResults(results);
resultsMsg.setColumnNames(new String[] { "IntKey" }); //$NON-NLS-1$
@@ -862,7 +862,7 @@
@Test public void testDateType() throws SQLException {
RequestMessage request = new RequestMessage();
request.setExecutionId(REQUEST_ID);
- ResultsMessage resultsMsg = new ResultsMessage(request);
+ ResultsMessage resultsMsg = new ResultsMessage();
resultsMsg.setResults(new List[] {Arrays.asList(new Timestamp(0))});
resultsMsg.setColumnNames(new String[] { "TS" }); //$NON-NLS-1$
resultsMsg.setDataTypes(new String[] { JDBCSQLTypeInfo.TIMESTAMP });
Modified: branches/as7/client/src/test/java/org/teiid/jdbc/TestCallableStatement.java
===================================================================
--- branches/as7/client/src/test/java/org/teiid/jdbc/TestCallableStatement.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/client/src/test/java/org/teiid/jdbc/TestCallableStatement.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -38,9 +38,6 @@
import org.teiid.client.metadata.ParameterInfo;
import org.teiid.client.security.LogonResult;
import org.teiid.core.types.JDBCSQLTypeInfo;
-import org.teiid.jdbc.CallableStatementImpl;
-import org.teiid.jdbc.ConnectionImpl;
-import org.teiid.jdbc.ResultSetImpl;
import org.teiid.net.ServerConnection;
@@ -68,7 +65,7 @@
RequestMessage request = new RequestMessage();
request.setExecutionId(1);
- ResultsMessage resultsMsg = new ResultsMessage(request);
+ ResultsMessage resultsMsg = new ResultsMessage();
List[] results = new List[] {Arrays.asList(null, null, null), Arrays.asList(null, 1, 2)};
resultsMsg.setResults(results);
resultsMsg.setColumnNames(new String[] { "IntNum", "Out1", "Out2" }); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
Property changes on: branches/as7/client/src/test/java/org/teiid/jdbc/TestConnection.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/client/src/test/java/org/teiid/jdbc/TestConnection.java:3281-3325
/trunk/client/src/test/java/org/teiid/jdbc/TestConnection.java:3188-3450
+ /branches/7.4.x/client/src/test/java/org/teiid/jdbc/TestConnection.java:3281-3325
/trunk/client/src/test/java/org/teiid/jdbc/TestConnection.java:3188-3450,3452-3506
Property changes on: branches/as7/client/src/test/java/org/teiid/jdbc/TestSQLException.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/client/src/test/java/org/teiid/jdbc/TestSQLException.java:3220-3275
/trunk/client/src/test/java/org/teiid/jdbc/TestSQLException.java:3188-3450
+ /branches/7.4.x/client/src/test/java/org/teiid/jdbc/TestSQLException.java:3220-3275
/trunk/client/src/test/java/org/teiid/jdbc/TestSQLException.java:3188-3450,3452-3506
Property changes on: branches/as7/client/src/test/java/org/teiid/jdbc/TestStatement.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/client/src/test/java/org/teiid/jdbc/TestStatement.java:3149-3217,3220-3275
/trunk/client/src/test/java/org/teiid/jdbc/TestStatement.java:3188-3450
+ /branches/7.4.x/client/src/test/java/org/teiid/jdbc/TestStatement.java:3149-3217,3220-3275
/trunk/client/src/test/java/org/teiid/jdbc/TestStatement.java:3188-3450,3452-3506
Property changes on: branches/as7/client/src/test/java/org/teiid/jdbc/TestTeiidDriver.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/client/src/test/java/org/teiid/jdbc/TestTeiidDriver.java:3149-3217,3281-3325
/trunk/client/src/test/java/org/teiid/jdbc/TestTeiidDriver.java:3188-3450
+ /branches/7.4.x/client/src/test/java/org/teiid/jdbc/TestTeiidDriver.java:3149-3217,3281-3325
/trunk/client/src/test/java/org/teiid/jdbc/TestTeiidDriver.java:3188-3450,3452-3506
Property changes on: branches/as7/common-core/src/main/java/org/teiid/core/TeiidException.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/common-core/src/main/java/org/teiid/core/TeiidException.java:3220-3275
/trunk/common-core/src/main/java/org/teiid/core/TeiidException.java:3188-3450
+ /branches/7.4.x/common-core/src/main/java/org/teiid/core/TeiidException.java:3220-3275
/trunk/common-core/src/main/java/org/teiid/core/TeiidException.java:3188-3450,3452-3506
Property changes on: branches/as7/common-core/src/main/java/org/teiid/core/TeiidRuntimeException.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/common-core/src/main/java/org/teiid/core/TeiidRuntimeException.java:3220-3275
/trunk/common-core/src/main/java/org/teiid/core/TeiidRuntimeException.java:3188-3450
+ /branches/7.4.x/common-core/src/main/java/org/teiid/core/TeiidRuntimeException.java:3220-3275
/trunk/common-core/src/main/java/org/teiid/core/TeiidRuntimeException.java:3188-3450,3452-3506
Modified: branches/as7/common-core/src/main/java/org/teiid/core/types/BaseLob.java
===================================================================
--- branches/as7/common-core/src/main/java/org/teiid/core/types/BaseLob.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/common-core/src/main/java/org/teiid/core/types/BaseLob.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -22,6 +22,7 @@
package org.teiid.core.types;
+import java.io.BufferedInputStream;
import java.io.Externalizable;
import java.io.IOException;
import java.io.InputStream;
@@ -75,17 +76,8 @@
this.charset = charset;
}
- public void free() throws SQLException {
- if (this.streamFactory != null) {
- try {
- this.streamFactory.free();
- this.streamFactory = null;
- } catch (IOException e) {
- SQLException ex = new SQLException(e.getMessage());
- ex.initCause(e);
- throw ex;
- }
- }
+ public void free() {
+ this.streamFactory = null;
}
public Reader getCharacterStream() throws SQLException {
@@ -126,5 +118,35 @@
public void writeExternal(ObjectOutput out) throws IOException {
out.writeObject(streamFactory);
}
+
+ /**
+ * Returns the number of bytes.
+ */
+ public long length() throws SQLException{
+ if (getStreamFactory().getLength() == -1) {
+ getStreamFactory().setLength(length(getBinaryStream()));
+ }
+ return getStreamFactory().getLength();
+ }
+ static long length(InputStream is) throws SQLException {
+ if (!(is instanceof BufferedInputStream)) {
+ is = new BufferedInputStream(is);
+ }
+ try {
+ long length = 0;
+ while (is.read() != -1) {
+ length++;
+ }
+ return length;
+ } catch (IOException e) {
+ throw new SQLException(e);
+ } finally {
+ try {
+ is.close();
+ } catch (IOException e) {
+ }
+ }
+ }
+
}
Modified: branches/as7/common-core/src/main/java/org/teiid/core/types/BlobImpl.java
===================================================================
--- branches/as7/common-core/src/main/java/org/teiid/core/types/BlobImpl.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/common-core/src/main/java/org/teiid/core/types/BlobImpl.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -22,7 +22,6 @@
package org.teiid.core.types;
-import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -109,32 +108,6 @@
}
/**
- * Returns the number of bytes in the <code>BLOB</code> value
- * designated by this <code>Blob</code> object.
- * @return length of the <code>BLOB</code> in bytes
- */
- public long length() throws SQLException{
- if (getStreamFactory().getLength() == -1) {
- InputStream is = new BufferedInputStream(getBinaryStream());
- try {
- long length = 0;
- while (is.read() != -1) {
- length++;
- }
- getStreamFactory().setLength(length);
- } catch (IOException e) {
- throw new SQLException(e);
- } finally {
- try {
- is.close();
- } catch (IOException e) {
- }
- }
- }
- return getStreamFactory().getLength();
- }
-
- /**
* Determines the byte position in the <code>BLOB</code> value
* designated by this <code>Blob</code> object at which
* <code>pattern</code> begins. The search begins at position
Modified: branches/as7/common-core/src/main/java/org/teiid/core/types/BlobType.java
===================================================================
--- branches/as7/common-core/src/main/java/org/teiid/core/types/BlobType.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/common-core/src/main/java/org/teiid/core/types/BlobType.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -24,6 +24,7 @@
import java.io.IOException;
import java.io.InputStream;
+import java.io.ObjectInput;
import java.io.ObjectOutput;
import java.io.OutputStream;
import java.sql.Blob;
@@ -32,6 +33,7 @@
import javax.sql.rowset.serial.SerialBlob;
import org.teiid.core.TeiidRuntimeException;
+import org.teiid.core.util.ObjectConverterUtil;
/**
@@ -61,18 +63,10 @@
public byte[] getBytes(long pos, int length) throws SQLException {
return this.reference.getBytes(pos, length);
}
-
- /**
- * @see java.sql.Blob#length()
- */
- public long length() throws SQLException {
- //caching the length
- if (this.length != -1) {
- return this.length;
- }
- // if did not find before then do it again.
- this.length = this.reference.length();
- return length;
+
+ @Override
+ long computeLength() throws SQLException {
+ return this.reference.length();
}
/**
@@ -142,12 +136,38 @@
}
@Override
- public void writeExternal(ObjectOutput out) throws IOException {
+ protected void readReference(ObjectInput in) throws IOException {
+ byte[] bytes = new byte[(int)getLength()];
+ in.readFully(bytes);
try {
- length();
+ this.reference = new SerialBlob(bytes);
} catch (SQLException e) {
+ throw new IOException(e);
}
- super.writeExternal(out);
}
+ @Override
+ protected void writeReference(final ObjectOutput out) throws IOException {
+ try {
+ writeBinary(out, getBinaryStream(), (int)length);
+ } catch (SQLException e) {
+ throw new IOException();
+ }
+ }
+
+ static void writeBinary(final ObjectOutput out, InputStream is, int length) throws IOException {
+ OutputStream os = new OutputStream() {
+
+ @Override
+ public void write(int b) throws IOException {
+ out.write(b);
+ }
+ };
+ try {
+ ObjectConverterUtil.write(os, is, length, false);
+ } finally {
+ is.close();
+ }
+ }
+
}
Modified: branches/as7/common-core/src/main/java/org/teiid/core/types/ClobType.java
===================================================================
--- branches/as7/common-core/src/main/java/org/teiid/core/types/ClobType.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/common-core/src/main/java/org/teiid/core/types/ClobType.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -24,6 +24,7 @@
import java.io.IOException;
import java.io.InputStream;
+import java.io.ObjectInput;
import java.io.ObjectOutput;
import java.io.OutputStream;
import java.io.Reader;
@@ -35,6 +36,7 @@
import javax.sql.rowset.serial.SerialClob;
import org.teiid.core.TeiidRuntimeException;
+import org.teiid.core.util.ObjectConverterUtil;
/**
@@ -72,17 +74,10 @@
public String getSubString(long pos, int length) throws SQLException {
return this.reference.getSubString(pos, length);
}
-
- /**
- * @see java.sql.Clob#length()
- */
- public long length() throws SQLException {
- if (this.length != -1) {
- return this.length;
- }
-
- this.length = this.reference.length();
- return length;
+
+ @Override
+ long computeLength() throws SQLException {
+ return this.reference.length();
}
/**
@@ -218,12 +213,52 @@
}
@Override
- public void writeExternal(ObjectOutput out) throws IOException {
+ protected void readReference(ObjectInput in) throws IOException {
+ char[] chars = new char[(int)length];
+ for (int i = 0; i < chars.length; i++) {
+ chars[i] = in.readChar();
+ }
try {
- length();
+ this.reference = new SerialClob(chars);
} catch (SQLException e) {
+ throw new IOException(e);
}
- super.writeExternal(out);
}
+
+ /**
+ * Since we have the length in chars we'll just write out in double byte format.
+ * These clobs should be small, so the wasted space should be minimal.
+ */
+ @Override
+ protected void writeReference(final ObjectOutput out) throws IOException {
+ Writer w = new Writer() {
+
+ @Override
+ public void write(char[] cbuf, int off, int len) throws IOException {
+ for (int i = off; i < len; i++) {
+ out.writeShort(cbuf[i]);
+ }
+ }
+
+ @Override
+ public void flush() throws IOException {
+ }
+
+ @Override
+ public void close() throws IOException {
+ }
+ };
+ Reader r;
+ try {
+ r = getCharacterStream();
+ } catch (SQLException e) {
+ throw new IOException(e);
+ }
+ try {
+ ObjectConverterUtil.write(w, r, (int)length, false);
+ } finally {
+ r.close();
+ }
+ }
}
Modified: branches/as7/common-core/src/main/java/org/teiid/core/types/DataTypeManager.java
===================================================================
--- branches/as7/common-core/src/main/java/org/teiid/core/types/DataTypeManager.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/common-core/src/main/java/org/teiid/core/types/DataTypeManager.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -126,23 +126,40 @@
}
}
- private static Map<Class<?>, ValueCache<?>> valueMaps = new HashMap<Class<?>, ValueCache<?>>(128);
- private static HashedValueCache<String> stringCache = new HashedValueCache<String>(17) {
+ public static class WeakReferenceHashedValueCache<T> extends HashedValueCache<T> {
+ public WeakReferenceHashedValueCache(int size) {
+ super(size);
+ }
+
+ public T getByHash(Object obj) {
+ int index = hash(obj.hashCode()) & (cache.length - 1);
+ return get(index);
+ }
+
@Override
- protected Object get(int index) {
- WeakReference<?> ref = (WeakReference<?>) cache[index];
+ protected T get(int index) {
+ WeakReference<T> ref = (WeakReference<T>) cache[index];
if (ref != null) {
- return ref.get();
+ T result = ref.get();
+ if (result == null) {
+ cache[index] = null;
+ }
+ return result;
}
return null;
}
@Override
- protected void set(int index, String value) {
- cache[index] = new WeakReference<Object>(value);
- }
+ protected void set(int index, T value) {
+ cache[index] = new WeakReference<T>(value);
+ }
+ }
+
+ private static Map<Class<?>, ValueCache<?>> valueMaps = new HashMap<Class<?>, ValueCache<?>>(128);
+ private static HashedValueCache<String> stringCache = new WeakReferenceHashedValueCache<String>(17) {
+
@Override
protected int primaryHash(String value) {
if (value.length() < 14) {
@@ -536,21 +553,7 @@
valueMaps.put(DefaultDataClasses.DATE, new HashedValueCache<Date>(14));
valueMaps.put(DefaultDataClasses.TIME, new HashedValueCache<Time>(14));
valueMaps.put(DefaultDataClasses.TIMESTAMP, new HashedValueCache<Timestamp>(14));
- valueMaps.put(DefaultDataClasses.BIG_DECIMAL, new HashedValueCache<BigDecimal>(16) {
- @Override
- protected Object get(int index) {
- WeakReference<?> ref = (WeakReference<?>) cache[index];
- if (ref != null) {
- return ref.get();
- }
- return null;
- }
-
- @Override
- protected void set(int index, BigDecimal value) {
- cache[index] = new WeakReference<BigDecimal>(value);
- }
- });
+ valueMaps.put(DefaultDataClasses.BIG_DECIMAL, new WeakReferenceHashedValueCache<BigDecimal>(16));
valueMaps.put(DefaultDataClasses.STRING, stringCache);
}
}
Modified: branches/as7/common-core/src/main/java/org/teiid/core/types/InputStreamFactory.java
===================================================================
--- branches/as7/common-core/src/main/java/org/teiid/core/types/InputStreamFactory.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/common-core/src/main/java/org/teiid/core/types/InputStreamFactory.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -36,12 +36,21 @@
import java.sql.SQLXML;
import javax.activation.DataSource;
+import javax.sql.rowset.serial.SerialBlob;
+import javax.sql.rowset.serial.SerialClob;
import javax.xml.transform.Source;
import org.teiid.core.util.ReaderInputStream;
public abstract class InputStreamFactory implements Source {
+ public enum StorageMode {
+ MEMORY, //TODO: sources may return Serial values that are much too large and we should convert them to persistent.
+ PERSISTENT,
+ FREE,
+ OTHER
+ }
+
public interface StreamFactoryReference {
void setStreamFactory(InputStreamFactory inputStreamFactory);
@@ -49,7 +58,7 @@
}
private String systemId;
- private long length = -1;
+ protected long length = -1;
/**
* Get a new InputStream
@@ -67,10 +76,17 @@
this.systemId = systemId;
}
+ /**
+ * @throws IOException
+ */
public void free() throws IOException {
}
+ /**
+ * Length in bytes of the {@link InputStream}
+ * @return the length or -1 if the length is not known
+ */
public long getLength() {
return length;
}
@@ -79,12 +95,15 @@
this.length = length;
}
+ /**
+ * @throws IOException
+ */
public Reader getCharacterStream() throws IOException {
return null;
}
- public boolean isPersistent() {
- return false;
+ public StorageMode getStorageMode() {
+ return StorageMode.OTHER;
}
public static class FileInputStreamFactory extends InputStreamFactory {
@@ -107,8 +126,8 @@
}
@Override
- public boolean isPersistent() {
- return true;
+ public StorageMode getStorageMode() {
+ return StorageMode.PERSISTENT;
}
}
@@ -162,6 +181,11 @@
public OutputStream getOutputStream() throws IOException {
throw new UnsupportedOperationException();
}
+
+ @Override
+ public StorageMode getStorageMode() {
+ return getStorageMode(clob);
+ }
}
@@ -184,11 +208,13 @@
@Override
public long getLength() {
- try {
- return blob.length();
- } catch (SQLException e) {
- return -1;
- }
+ if (length == -1) {
+ try {
+ length = blob.length();
+ } catch (SQLException e) {
+ }
+ }
+ return length;
}
@Override
@@ -205,9 +231,35 @@
public OutputStream getOutputStream() throws IOException {
throw new UnsupportedOperationException();
}
+
+ @Override
+ public StorageMode getStorageMode() {
+ return getStorageMode(blob);
+ }
}
+ public static StorageMode getStorageMode(Object lob) {
+ if (lob instanceof Streamable<?>) {
+ return getStorageMode(((Streamable<?>)lob).getReference());
+ }
+ if (lob instanceof SerialClob) {
+ return StorageMode.MEMORY;
+ }
+ if (lob instanceof SerialBlob) {
+ return StorageMode.MEMORY;
+ }
+ if (lob instanceof BaseLob) {
+ BaseLob baseLob = (BaseLob)lob;
+ try {
+ return baseLob.getStreamFactory().getStorageMode();
+ } catch (SQLException e) {
+ return StorageMode.FREE;
+ }
+ }
+ return StorageMode.OTHER;
+ }
+
public static class SQLXMLInputStreamFactory extends InputStreamFactory implements DataSource {
private SQLXML sqlxml;
@@ -248,6 +300,11 @@
public OutputStream getOutputStream() throws IOException {
throw new UnsupportedOperationException();
}
+
+ @Override
+ public StorageMode getStorageMode() {
+ return getStorageMode(sqlxml);
+ }
}
Modified: branches/as7/common-core/src/main/java/org/teiid/core/types/SQLXMLImpl.java
===================================================================
--- branches/as7/common-core/src/main/java/org/teiid/core/types/SQLXMLImpl.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/common-core/src/main/java/org/teiid/core/types/SQLXMLImpl.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -71,6 +71,16 @@
public InputStream getInputStream() throws IOException {
return new ByteArrayInputStream(bytes);
}
+
+ @Override
+ public StorageMode getStorageMode() {
+ return StorageMode.MEMORY;
+ }
+
+ @Override
+ public long getLength() {
+ return bytes.length;
+ }
});
setEncoding(Streamable.ENCODING);
}
Modified: branches/as7/common-core/src/main/java/org/teiid/core/types/Streamable.java
===================================================================
--- branches/as7/common-core/src/main/java/org/teiid/core/types/Streamable.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/common-core/src/main/java/org/teiid/core/types/Streamable.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -27,7 +27,8 @@
import java.io.ObjectInput;
import java.io.ObjectOutput;
import java.nio.charset.Charset;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.sql.SQLException;
+import java.util.concurrent.atomic.AtomicLong;
import org.teiid.core.CorePlugin;
@@ -44,7 +45,7 @@
private static final long serialVersionUID = -8252488562134729374L;
- private static AtomicInteger counter = new AtomicInteger();
+ private static AtomicLong counter = new AtomicLong();
public static final String ENCODING = "UTF-8"; //$NON-NLS-1$
public static final Charset CHARSET = Charset.forName(ENCODING);
@@ -66,10 +67,23 @@
this.reference = reference;
}
+ /**
+ * Returns the cached length. May be binary or character based.
+ * @return
+ */
public long getLength() {
return length;
}
+ abstract long computeLength() throws SQLException;
+
+ public long length() throws SQLException {
+ if (length == -1) {
+ length = computeLength();
+ }
+ return length;
+ }
+
public T getReference() {
return reference;
}
@@ -82,9 +96,9 @@
return this.referenceStreamId;
}
- /*public void setReferenceStreamId(String id) {
+ public void setReferenceStreamId(String id) {
this.referenceStreamId = id;
- }*/
+ }
@Override
public String toString() {
@@ -99,12 +113,27 @@
ClassNotFoundException {
length = in.readLong();
referenceStreamId = (String)in.readObject();
+ if (referenceStreamId == null) {
+ //we expect the data inline
+ readReference(in);
+ }
}
+ protected abstract void readReference(ObjectInput in) throws IOException;
+
@Override
public void writeExternal(ObjectOutput out) throws IOException {
+ try {
+ length();
+ } catch (SQLException e) {
+ }
out.writeLong(length);
out.writeObject(referenceStreamId);
+ if (referenceStreamId == null) {
+ writeReference(out);
+ }
}
-
+
+ protected abstract void writeReference(ObjectOutput out) throws IOException;
+
}
Modified: branches/as7/common-core/src/main/java/org/teiid/core/types/XMLType.java
===================================================================
--- branches/as7/common-core/src/main/java/org/teiid/core/types/XMLType.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/common-core/src/main/java/org/teiid/core/types/XMLType.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -189,4 +189,29 @@
}
}
}
+
+ @Override
+ long computeLength() throws SQLException {
+ if (this.reference instanceof SQLXMLImpl) {
+ SQLXMLImpl impl = (SQLXMLImpl)this.reference;
+ return impl.length();
+ }
+ return BaseLob.length(getBinaryStream());
+ }
+
+ @Override
+ protected void readReference(ObjectInput in) throws IOException {
+ byte[] bytes = new byte[(int)getLength()];
+ in.readFully(bytes);
+ this.reference = new SQLXMLImpl(bytes);
+ }
+
+ @Override
+ protected void writeReference(final ObjectOutput out) throws IOException {
+ try {
+ BlobType.writeBinary(out, getBinaryStream(), (int)length);
+ } catch (SQLException e) {
+ throw new IOException();
+ }
+ }
}
Modified: branches/as7/common-core/src/main/resources/org/teiid/core/util/application.properties
===================================================================
--- branches/as7/common-core/src/main/resources/org/teiid/core/util/application.properties 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/common-core/src/main/resources/org/teiid/core/util/application.properties 2011-09-29 16:46:38 UTC (rev 3507)
@@ -1,5 +1,5 @@
-build.releaseNumber=${pom.version}
-build.number=${pom.version}
+build.releaseNumber=${project.version}
+build.number=${project.version}
build.date=@build-date@
copyright=Copyright (C) 2008-2009 Red Hat, Inc
url=${site.url}
\ No newline at end of file
Property changes on: branches/as7/common-core/src/test/java/org/teiid/core/TestMetaMatrixRuntimeException.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/common-core/src/test/java/org/teiid/core/TestMetaMatrixRuntimeException.java:3220-3275
/trunk/common-core/src/test/java/org/teiid/core/TestMetaMatrixRuntimeException.java:3188-3450
+ /branches/7.4.x/common-core/src/test/java/org/teiid/core/TestMetaMatrixRuntimeException.java:3220-3275
/trunk/common-core/src/test/java/org/teiid/core/TestMetaMatrixRuntimeException.java:3188-3450,3452-3506
Modified: branches/as7/common-core/src/test/java/org/teiid/core/types/TestBlobValue.java
===================================================================
--- branches/as7/common-core/src/test/java/org/teiid/core/types/TestBlobValue.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/common-core/src/test/java/org/teiid/core/types/TestBlobValue.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -22,21 +22,14 @@
package org.teiid.core.types;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-
import javax.sql.rowset.serial.SerialBlob;
-import org.teiid.core.types.BlobType;
-import org.teiid.core.util.UnitTestUtil;
-
-
import junit.framework.TestCase;
+import org.junit.Test;
+import org.teiid.core.util.UnitTestUtil;
+
public class TestBlobValue extends TestCase {
public void testBlobValue() throws Exception {
@@ -55,22 +48,27 @@
String key = bv.getReferenceStreamId();
// now force to serialize
- File saved = new File(UnitTestUtil.getTestScratchPath()+"/blobassaved.bin"); //$NON-NLS-1$
- ObjectOutputStream out = new ObjectOutputStream(new FileOutputStream(saved));
- out.writeObject(bv);
- out.close();
-
- // now read back the object from serilized state
- ObjectInputStream in = new ObjectInputStream(new FileInputStream(saved));
- BlobType read = (BlobType)in.readObject();
+ BlobType read = UnitTestUtil.helpSerialize(bv);
// make sure we have kept the reference stream id
assertEquals(key, read.getReferenceStreamId());
// and lost the original object
assertNull(read.getReference());
+ }
+
+ @Test public void testReferencePersistence() throws Exception {
+ String testString = "this is test clob"; //$NON-NLS-1$
+ SerialBlob blob = new SerialBlob(testString.getBytes());
- saved.delete();
+ BlobType bv = new BlobType(blob);
+ bv.setReferenceStreamId(null);
+ // now force to serialize
+ BlobType read = UnitTestUtil.helpSerialize(bv);
+
+ assertNull(read.getReferenceStreamId());
+
+ assertEquals(testString, new String(read.getBytes(1, (int)blob.length())));
}
}
Modified: branches/as7/common-core/src/test/java/org/teiid/core/types/TestClobValue.java
===================================================================
--- branches/as7/common-core/src/test/java/org/teiid/core/types/TestClobValue.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/common-core/src/test/java/org/teiid/core/types/TestClobValue.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -24,12 +24,7 @@
import static org.junit.Assert.*;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
import java.io.Reader;
import javax.sql.rowset.serial.SerialClob;
@@ -56,15 +51,8 @@
String key = cv.getReferenceStreamId();
// now force to serialize
- File saved = new File(UnitTestUtil.getTestScratchPath()+"/clobassaved.bin"); //$NON-NLS-1$
- ObjectOutputStream out = new ObjectOutputStream(new FileOutputStream(saved));
- out.writeObject(cv);
- out.close();
+ ClobType read = UnitTestUtil.helpSerialize(cv);
- // now read back the object from serilized state
- ObjectInputStream in = new ObjectInputStream(new FileInputStream(saved));
- ClobType read = (ClobType)in.readObject();
-
assertTrue(read.length() > 0);
// make sure we have kept the reference stream id
@@ -72,8 +60,21 @@
// and lost the original object
assertNull(read.getReference());
+ }
+
+ @Test public void testReferencePersistence() throws Exception {
+ String testString = "this is test clob"; //$NON-NLS-1$
+ SerialClob clob = new SerialClob(testString.toCharArray());
- saved.delete();
+ ClobType cv = new ClobType(clob);
+ cv.setReferenceStreamId(null);
+
+ // now force to serialize
+ ClobType read = UnitTestUtil.helpSerialize(cv);
+
+ assertTrue(read.length() > 0);
+
+ assertEquals(testString, read.getSubString(1, testString.length()));
}
@Test public void testClobSubstring() throws Exception {
Modified: branches/as7/common-core/src/test/java/org/teiid/core/types/TestXMLValue.java
===================================================================
--- branches/as7/common-core/src/test/java/org/teiid/core/types/TestXMLValue.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/common-core/src/test/java/org/teiid/core/types/TestXMLValue.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -22,20 +22,11 @@
package org.teiid.core.types;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
+import junit.framework.TestCase;
-import org.teiid.core.types.SQLXMLImpl;
-import org.teiid.core.types.XMLType;
import org.teiid.core.util.UnitTestUtil;
-import junit.framework.TestCase;
-
-
public class TestXMLValue extends TestCase {
public void testXMLValue() throws Exception {
@@ -55,22 +46,26 @@
String key = xv.getReferenceStreamId();
// now force to serialize
- File saved = new File(UnitTestUtil.getTestScratchPath()+"/xmlsaved.bin"); //$NON-NLS-1$
- ObjectOutputStream out = new ObjectOutputStream(new FileOutputStream(saved));
- out.writeObject(xv);
- out.close();
-
- // now read back the object from serilized state
- ObjectInputStream in = new ObjectInputStream(new FileInputStream(saved));
- XMLType read = (XMLType)in.readObject();
+ XMLType read = UnitTestUtil.helpSerialize(xv);
// make sure we have kept the reference stream id
assertEquals(key, read.getReferenceStreamId());
// and lost the original object
assertNull(read.getReference());
+ }
+
+ public void testReferencePersistence() throws Exception {
+ String testString = "<foo>this is an xml value test</foo>"; //$NON-NLS-1$
+ SQLXMLImpl xml = new SQLXMLImpl(testString);
- saved.delete();
+ XMLType xv = new XMLType(xml);
+ xv.setReferenceStreamId(null);
+
+ // now force to serialize
+ XMLType read = UnitTestUtil.helpSerialize(xv);
+
+ assertEquals(testString, read.getString());
}
}
Modified: branches/as7/common-core/src/test/java/org/teiid/core/util/UnitTestUtil.java
===================================================================
--- branches/as7/common-core/src/test/java/org/teiid/core/util/UnitTestUtil.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/common-core/src/test/java/org/teiid/core/util/UnitTestUtil.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -28,7 +28,16 @@
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
+import java.io.PrintWriter;
import java.io.Serializable;
+import java.io.Writer;
+import java.sql.Timestamp;
+import java.util.logging.ConsoleHandler;
+import java.util.logging.Formatter;
+import java.util.logging.Handler;
+import java.util.logging.Level;
+import java.util.logging.LogRecord;
+import java.util.logging.Logger;
import junit.framework.Assert;
import junit.framework.AssertionFailedError;
@@ -388,6 +397,7 @@
return filePath;
}
+ @SuppressWarnings("unchecked")
public static final <T extends Serializable> T helpSerialize(T object) throws IOException, ClassNotFoundException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
ObjectOutputStream oos = new ObjectOutputStream(baos);
@@ -398,5 +408,59 @@
return (T)ois.readObject();
}
+
+ public static void enableTraceLogging(String loggerName) {
+ Logger logger = Logger.getLogger(loggerName);
+ logger.setLevel(Level.FINEST);
+ if (logger.getHandlers().length > 0) {
+ for (Handler h : logger.getHandlers()) {
+ h.setLevel(Level.FINEST);
+ }
+ } else {
+ logger.setUseParentHandlers(false);
+ ConsoleHandler ch = new ConsoleHandler();
+ ch.setFormatter(new Formatter() {
+
+ @Override
+ public String format(LogRecord record) {
+ final StringBuilder result = new StringBuilder();
+ result.append(new Timestamp(record.getMillis()));
+ result.append(" "); //$NON-NLS-1$
+ result.append(record.getLoggerName());
+ result.append(" "); //$NON-NLS-1$
+ result.append(record.getLevel());
+ result.append(" "); //$NON-NLS-1$
+ result.append(record.getThreadID());
+ result.append(" "); //$NON-NLS-1$
+ result.append(record.getMessage());
+ result.append('\n');
+ if (record.getThrown() != null) {
+ record.getThrown().printStackTrace(new PrintWriter(new Writer() {
+ @Override
+ public void close() throws IOException {
+
+ }
+
+ @Override
+ public void flush() throws IOException {
+
+ }
+
+ @Override
+ public void write(char[] cbuf, int off, int len)
+ throws IOException {
+ result.append(new String(cbuf, off, len));
+ }
+ }));
+ result.append('\n');
+ }
+ return result.toString();
+ }
+ });
+ ch.setLevel(Level.FINEST);
+ logger.addHandler(ch);
+ }
+ }
+
}
Modified: branches/as7/documentation/caching-guide/src/main/docbook/en-US/content/hint-option.xml
===================================================================
--- branches/as7/documentation/caching-guide/src/main/docbook/en-US/content/hint-option.xml 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/documentation/caching-guide/src/main/docbook/en-US/content/hint-option.xml 2011-09-29 16:46:38 UTC (rev 3507)
@@ -19,6 +19,9 @@
<itemizedlist>
<listitem><para>The cache hint should appear at the beginning of the SQL. It will not have any affect on INSERT/UPDATE/DELETE statements or virtual update procedure definitions.</para></listitem>
<listitem><para><emphasis>pref_mem</emphasis> - if present indicates that the cached results should prefer to remain in memory. They are not however required to be memory only.
+ <note><para>Care should be taken to not over use the pref_mem option.
+ The memory preference is implemented with Java soft references. While soft references are effective at preventing out of memory conditions.
+ Too much memory held by soft references can limit the effective working memory. Consult your JVM options for clearing soft references if you need to tune their behavior.</para></note>
</para></listitem>
<listitem><para><emphasis>ttl:n</emphasis> - if present n indicates the time to live value in milliseconds.
</para></listitem>
Copied: branches/as7/engine/src/main/java/org/teiid/common/buffer/AutoCleanupUtil.java (from rev 3506, trunk/engine/src/main/java/org/teiid/common/buffer/AutoCleanupUtil.java)
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/common/buffer/AutoCleanupUtil.java (rev 0)
+++ branches/as7/engine/src/main/java/org/teiid/common/buffer/AutoCleanupUtil.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -0,0 +1,73 @@
+/*
+ * JBoss, Home of Professional Open Source.
+ * See the COPYRIGHT.txt file distributed with this work for information
+ * regarding copyright ownership. Some portions may be licensed
+ * to Red Hat, Inc. under one or more contributor license agreements.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301 USA.
+ */
+package org.teiid.common.buffer;
+
+import java.lang.ref.PhantomReference;
+import java.lang.ref.ReferenceQueue;
+import java.util.Collections;
+import java.util.IdentityHashMap;
+import java.util.Set;
+
+public class AutoCleanupUtil {
+
+ public interface Removable {
+ public void remove();
+ }
+
+ static final class PhantomCleanupReference extends PhantomReference<Object> {
+
+ private Removable removable;
+
+ public PhantomCleanupReference(Object referent, Removable removable) {
+ super(referent, QUEUE);
+ this.removable = removable;
+ }
+
+ public void cleanup() {
+ try {
+ this.removable.remove();
+ } finally {
+ this.removable = null;
+ this.clear();
+ }
+ }
+ }
+
+ private static ReferenceQueue<Object> QUEUE = new ReferenceQueue<Object>();
+ private static final Set<PhantomReference<Object>> REFERENCES = Collections.synchronizedSet(Collections.newSetFromMap(new IdentityHashMap<PhantomReference<Object>, Boolean>()));
+
+ public static void setCleanupReference(Object o, Removable r) {
+ REFERENCES.add(new PhantomCleanupReference(o, r));
+ doCleanup();
+ }
+
+ public static void doCleanup() {
+ for (int i = 0; i < 10; i++) {
+ PhantomCleanupReference ref = (PhantomCleanupReference)QUEUE.poll();
+ if (ref == null) {
+ break;
+ }
+ ref.cleanup();
+ REFERENCES.remove(ref);
+ }
+ }
+}
Modified: branches/as7/engine/src/main/java/org/teiid/common/buffer/BatchManager.java
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/common/buffer/BatchManager.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/main/java/org/teiid/common/buffer/BatchManager.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -22,36 +22,29 @@
package org.teiid.common.buffer;
+import java.lang.ref.Reference;
+import java.util.List;
+
import org.teiid.core.TeiidComponentException;
+/**
+ * Acts as a combination serializer/cachemanager
+ */
public interface BatchManager {
- public interface CleanupHook {
-
- void cleanup();
-
- }
+ List<List<?>> getBatch(Long batch, boolean retain) throws TeiidComponentException;
- public interface ManagedBatch {
-
- TupleBatch getBatch(boolean cache, String[] types) throws TeiidComponentException;
-
- void remove();
-
- void setPrefersMemory(boolean prefers);
-
- /**
- * Get an object that can cleaup the {@link ManagedBatch}, but does not hold a hard reference to
- * the {@link ManagedBatch} or the {@link BatchManager}
- * @return
- */
- CleanupHook getCleanupHook();
-
- }
+ void remove(Long batch);
- ManagedBatch createManagedBatch(TupleBatch batch, boolean softCache) throws TeiidComponentException;
+ void setPrefersMemory(boolean prefers);
+ boolean prefersMemory();
+
+ Long createManagedBatch(List<? extends List<?>> batch) throws TeiidComponentException;
+
void remove();
- FileStore createStorage(String prefix);
+ Reference<? extends BatchManager> getBatchManagerReference();
+
+ String[] getTypes();
}
Property changes on: branches/as7/engine/src/main/java/org/teiid/common/buffer/BlockedException.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/main/java/org/teiid/common/buffer/BlockedException.java:3149-3217
/trunk/engine/src/main/java/org/teiid/common/buffer/BlockedException.java:3188-3450
+ /branches/7.4.x/engine/src/main/java/org/teiid/common/buffer/BlockedException.java:3149-3217
/trunk/engine/src/main/java/org/teiid/common/buffer/BlockedException.java:3188-3450,3452-3506
Modified: branches/as7/engine/src/main/java/org/teiid/common/buffer/BufferManager.java
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/common/buffer/BufferManager.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/main/java/org/teiid/common/buffer/BufferManager.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -75,6 +75,12 @@
* Get the batch size to use during query processing.
* @return Batch size (# of rows)
*/
+ int getProcessorBatchSize(List<? extends Expression> schema);
+
+ /**
+ * Get the nominal batch size target
+ * @return
+ */
int getProcessorBatchSize();
/**
Property changes on: branches/as7/engine/src/main/java/org/teiid/common/buffer/BufferManager.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/main/java/org/teiid/common/buffer/BufferManager.java:3220-3275
/trunk/engine/src/main/java/org/teiid/common/buffer/BufferManager.java:3188-3450
+ /branches/7.4.x/engine/src/main/java/org/teiid/common/buffer/BufferManager.java:3220-3275
/trunk/engine/src/main/java/org/teiid/common/buffer/BufferManager.java:3188-3450,3452-3506
Copied: branches/as7/engine/src/main/java/org/teiid/common/buffer/Cache.java (from rev 3506, trunk/engine/src/main/java/org/teiid/common/buffer/Cache.java)
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/common/buffer/Cache.java (rev 0)
+++ branches/as7/engine/src/main/java/org/teiid/common/buffer/Cache.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -0,0 +1,39 @@
+/*
+ * JBoss, Home of Professional Open Source.
+ * See the COPYRIGHT.txt file distributed with this work for information
+ * regarding copyright ownership. Some portions may be licensed
+ * to Red Hat, Inc. under one or more contributor license agreements.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301 USA.
+ */
+
+package org.teiid.common.buffer;
+
+import java.util.Collection;
+
+import org.teiid.core.TeiidComponentException;
+
+/**
+ * Represents the storage strategy for the {@link BufferManager}
+ */
+public interface Cache extends StorageManager {
+ void createCacheGroup(Long gid); //called prior to adding an entry
+ Collection<Long> removeCacheGroup(Long gid);
+ void addToCacheGroup(Long gid, Long oid);
+ CacheEntry get(Long id, Serializer<?> serializer) throws TeiidComponentException;
+ void add(CacheEntry entry, Serializer<?> s);
+ void remove(Long gid, Long id);
+}
\ No newline at end of file
Copied: branches/as7/engine/src/main/java/org/teiid/common/buffer/CacheEntry.java (from rev 3506, trunk/engine/src/main/java/org/teiid/common/buffer/CacheEntry.java)
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/common/buffer/CacheEntry.java (rev 0)
+++ branches/as7/engine/src/main/java/org/teiid/common/buffer/CacheEntry.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -0,0 +1,101 @@
+/*
+ * JBoss, Home of Professional Open Source.
+ * See the COPYRIGHT.txt file distributed with this work for information
+ * regarding copyright ownership. Some portions may be licensed
+ * to Red Hat, Inc. under one or more contributor license agreements.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301 USA.
+ */
+
+package org.teiid.common.buffer;
+
+import java.lang.ref.WeakReference;
+
+public class CacheEntry {
+ private boolean persistent;
+ private Object object;
+ private int sizeEstimate;
+ private WeakReference<? extends Serializer<?>> serializer;
+ private Long id;
+
+ public CacheEntry(Long id) {
+ this.id = id;
+ }
+
+ public Long getId() {
+ return id;
+ }
+
+ @Override
+ public int hashCode() {
+ return getId().hashCode();
+ }
+
+ public int getSizeEstimate() {
+ return sizeEstimate;
+ }
+
+ public void setSizeEstimate(int sizeEstimate) {
+ this.sizeEstimate = sizeEstimate;
+ }
+
+ public boolean equals(Object obj) {
+ if (obj == this) {
+ return true;
+ }
+ if (!(obj instanceof CacheEntry)) {
+ return false;
+ }
+ return getId().equals(((CacheEntry)obj).getId());
+ }
+
+ @Override
+ public String toString() {
+ return getId().toString();
+ }
+
+ public Object nullOut() {
+ Object result = getObject();
+ setObject(null);
+ setSerializer(null);
+ return result;
+ }
+
+ public void setObject(Object object) {
+ this.object = object;
+ }
+
+ public Object getObject() {
+ return object;
+ }
+
+ public void setPersistent(boolean persistent) {
+ this.persistent = persistent;
+ }
+
+ public boolean isPersistent() {
+ return persistent;
+ }
+
+ public void setSerializer(WeakReference<? extends Serializer<?>> serializer) {
+ this.serializer = serializer;
+ }
+
+ public WeakReference<? extends Serializer<?>> getSerializer() {
+ return serializer;
+ }
+
+}
\ No newline at end of file
Modified: branches/as7/engine/src/main/java/org/teiid/common/buffer/FileStore.java
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/common/buffer/FileStore.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/main/java/org/teiid/common/buffer/FileStore.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -25,20 +25,14 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
-import java.lang.ref.PhantomReference;
-import java.lang.ref.ReferenceQueue;
+import java.nio.ByteBuffer;
import java.util.Arrays;
-import java.util.Collections;
-import java.util.IdentityHashMap;
-import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
-import org.teiid.core.TeiidComponentException;
+import org.teiid.common.buffer.AutoCleanupUtil.Removable;
-public abstract class FileStore {
-
- private static ReferenceQueue<Object> QUEUE = new ReferenceQueue<Object>();
- private static final Set<PhantomReference<Object>> REFERENCES = Collections.newSetFromMap(new IdentityHashMap<PhantomReference<Object>, Boolean>());
-
+public abstract class FileStore implements Removable {
+
/**
* A customized buffered stream with an exposed buffer
*/
@@ -80,12 +74,8 @@
}
private void writeDirect(byte[] b, int off, int len) throws IOException {
- try {
- FileStore.this.write(b, off, len);
- bytesWritten = true;
- } catch (TeiidComponentException e) {
- throw new IOException(e);
- }
+ FileStore.this.write(b, off, len);
+ bytesWritten = true;
}
public void flushBuffer() throws IOException {
@@ -140,84 +130,59 @@
}
- static class CleanupReference extends PhantomReference<Object> {
-
- private FileStore store;
-
- public CleanupReference(Object referent, FileStore store) {
- super(referent, QUEUE);
- this.store = store;
- }
-
- public void cleanup() {
- try {
- this.store.remove();
- } finally {
- this.clear();
- }
- }
- }
+ private AtomicBoolean removed = new AtomicBoolean();
- private boolean removed;
- protected long len;
+ public abstract long getLength();
- public void setCleanupReference(Object o) {
- REFERENCES.add(new CleanupReference(o, this));
- for (int i = 0; i < 10; i++) {
- CleanupReference ref = (CleanupReference)QUEUE.poll();
- if (ref == null) {
- break;
- }
- ref.cleanup();
- REFERENCES.remove(ref);
- }
- }
+ public abstract void setLength(long length) throws IOException;
- public synchronized long getLength() {
- return len;
- }
-
public int read(long fileOffset, byte[] b, int offSet, int length)
- throws TeiidComponentException {
- if (removed) {
- throw new TeiidComponentException("already removed"); //$NON-NLS-1$
+ throws IOException {
+ checkRemoved();
+ return readWrite(fileOffset, b, offSet, length, false);
+ }
+
+ private void checkRemoved() throws IOException {
+ if (removed.get()) {
+ throw new IOException("already removed"); //$NON-NLS-1$
}
- return readDirect(fileOffset, b, offSet, length);
}
- protected abstract int readDirect(long fileOffset, byte[] b, int offSet, int length)
- throws TeiidComponentException;
+ protected abstract int readWrite(long fileOffset, byte[] b, int offSet, int length, boolean write)
+ throws IOException;
- public void readFully(long fileOffset, byte[] b, int offSet, int length) throws TeiidComponentException {
+ public void readFully(long fileOffset, byte[] b, int offSet, int length) throws IOException {
+ if (length == 0) {
+ return;
+ }
int n = 0;
do {
int count = this.read(fileOffset + n, b, offSet + n, length - n);
- if (count < 0) {
- throw new TeiidComponentException("not enough bytes available"); //$NON-NLS-1$
+ if (count <= 0 && length > 0) {
+ throw new IOException("not enough bytes available"); //$NON-NLS-1$
}
n += count;
} while (n < length);
}
- public void write(byte[] bytes) throws TeiidComponentException {
- write(bytes, 0, bytes.length);
+ public synchronized void write(byte[] bytes, int offset, int length) throws IOException {
+ write(getLength(), bytes, offset, length);
}
-
- public synchronized long write(byte[] bytes, int offset, int length) throws TeiidComponentException {
- if (removed) {
- throw new TeiidComponentException("already removed"); //$NON-NLS-1$
- }
- writeDirect(bytes, offset, length);
- long result = len;
- len += length;
- return result;
+
+ public void write(long start, byte[] bytes, int offset, int length) throws IOException {
+ int n = 0;
+ do {
+ checkRemoved();
+ int count = this.readWrite(start + n, bytes, offset + n, length - n, true);
+ if (count <= 0 && length > 0) {
+ throw new IOException("not enough bytes available"); //$NON-NLS-1$
+ }
+ n += count;
+ } while (n < length);
}
- protected abstract void writeDirect(byte[] bytes, int offset, int length) throws TeiidComponentException;
-
public void remove() {
- if (!this.removed) {
- this.removed = true;
+ if (removed.compareAndSet(false, true)) {
this.removeDirect();
}
}
@@ -241,28 +206,46 @@
@Override
public int read(byte[] b, int off, int len) throws IOException {
- try {
- if (this.streamLength != -1 && len > this.streamLength) {
- len = (int)this.streamLength;
- }
- if (this.streamLength == -1 || this.streamLength > 0) {
- int bytes = FileStore.this.read(offset, b, off, len);
- if (bytes != -1) {
- this.offset += bytes;
- if (this.streamLength != -1) {
- this.streamLength -= bytes;
- }
+ if (this.streamLength != -1 && len > this.streamLength) {
+ len = (int)this.streamLength;
+ }
+ if (this.streamLength == -1 || this.streamLength > 0) {
+ int bytes = FileStore.this.read(offset, b, off, len);
+ if (bytes != -1) {
+ this.offset += bytes;
+ if (this.streamLength != -1) {
+ this.streamLength -= bytes;
}
- return bytes;
}
- return -1;
- } catch (TeiidComponentException e) {
- throw new IOException(e);
+ return bytes;
}
+ return -1;
}
};
}
+ public ByteBuffer getBuffer(long start, int length, boolean allocate) throws IOException {
+ byte[] b = new byte[length];
+ if (!allocate) {
+ readFully(start, b, 0, length);
+ }
+ return ByteBuffer.wrap(b);
+ }
+
+ public void updateFromBuffer(ByteBuffer bb, long start) throws IOException {
+ byte[] b = null;
+ int offset = 0;
+ bb.rewind();
+ if (bb.hasArray()) {
+ b = bb.array();
+ offset = bb.arrayOffset();
+ } else {
+ b = new byte[bb.limit()];
+ bb.get(b);
+ }
+ write(start, b, offset, bb.limit());
+ }
+
public InputStream createInputStream(final long start) {
return createInputStream(start, -1);
}
@@ -277,16 +260,12 @@
@Override
public void write(byte[] b, int off, int len) throws IOException {
- try {
- FileStore.this.write(b, off, len);
- } catch (TeiidComponentException e) {
- throw new IOException(e);
- }
+ FileStore.this.write(b, off, len);
}
};
}
- public FileStoreOutputStream createOutputStream(int maxMemorySize) {
+ public FileStoreOutputStream createOutputStream(int maxMemorySize) {
return new FileStoreOutputStream(maxMemorySize);
}
Modified: branches/as7/engine/src/main/java/org/teiid/common/buffer/FileStoreInputStreamFactory.java
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/common/buffer/FileStoreInputStreamFactory.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/main/java/org/teiid/common/buffer/FileStoreInputStreamFactory.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -42,7 +42,7 @@
public FileStoreInputStreamFactory(FileStore lobBuffer, String encoding) {
this.encoding = encoding;
this.lobBuffer = lobBuffer;
- this.lobBuffer.setCleanupReference(this);
+ AutoCleanupUtil.setCleanupReference(this, lobBuffer);
}
@Override
@@ -56,6 +56,9 @@
@Override
public long getLength() {
+ if (fsos != null && !fsos.bytesWritten()) {
+ return fsos.getCount();
+ }
return lobBuffer.getLength();
}
@@ -86,7 +89,10 @@
}
@Override
- public boolean isPersistent() {
- return fsos == null || fsos.bytesWritten();
+ public StorageMode getStorageMode() {
+ if (fsos == null) {
+ return StorageMode.PERSISTENT;
+ }
+ return StorageMode.MEMORY;
}
}
\ No newline at end of file
Copied: branches/as7/engine/src/main/java/org/teiid/common/buffer/LightWeightCopyOnWriteList.java (from rev 3506, trunk/engine/src/main/java/org/teiid/common/buffer/LightWeightCopyOnWriteList.java)
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/common/buffer/LightWeightCopyOnWriteList.java (rev 0)
+++ branches/as7/engine/src/main/java/org/teiid/common/buffer/LightWeightCopyOnWriteList.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -0,0 +1,116 @@
+/*
+ * JBoss, Home of Professional Open Source.
+ * See the COPYRIGHT.txt file distributed with this work for information
+ * regarding copyright ownership. Some portions may be licensed
+ * to Red Hat, Inc. under one or more contributor license agreements.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301 USA.
+ */
+
+package org.teiid.common.buffer;
+
+import java.util.AbstractList;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.RandomAccess;
+
+/**
+ * Creates a copy of a reference list when modified.
+ *
+ * @param <T>
+ */
+public class LightWeightCopyOnWriteList<T> extends AbstractList<T> implements RandomAccess {
+
+ private List<T> list;
+ private boolean modified;
+
+ public LightWeightCopyOnWriteList(List<T> list) {
+ this.list = list;
+ }
+
+ @Override
+ public T get(int index) {
+ return list.get(index);
+ }
+
+ public List<T> getList() {
+ return list;
+ }
+
+ public void add(int index, T element) {
+ if (!modified) {
+ List<T> next = new ArrayList<T>(list.size() + 1);
+ next.addAll(list);
+ list = next;
+ modified = true;
+ }
+ list.add(index, element);
+ }
+
+ public T set(int index, T element) {
+ checkModified();
+ return list.set(index, element);
+ }
+
+ private void checkModified() {
+ if (!modified) {
+ list = new ArrayList<T>(list);
+ modified = true;
+ }
+ }
+
+ public boolean addAll(Collection<? extends T> c) {
+ return addAll(size(), c);
+ }
+
+ @Override
+ public boolean addAll(int index, Collection<? extends T> c) {
+ checkModified();
+ return list.addAll(index, c);
+ }
+
+ @Override
+ public T remove(int index) {
+ checkModified();
+ return list.remove(index);
+ }
+
+ @Override
+ public Object[] toArray() {
+ return list.toArray();
+ }
+
+ public <U extends Object> U[] toArray(U[] a) {
+ return list.toArray(a);
+ }
+
+ @Override
+ public void clear() {
+ if (!modified) {
+ list = new ArrayList<T>();
+ modified = true;
+ } else {
+ list.clear();
+ }
+ }
+
+ @Override
+ public int size() {
+ return list.size();
+ }
+
+}
Modified: branches/as7/engine/src/main/java/org/teiid/common/buffer/LobManager.java
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/common/buffer/LobManager.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/main/java/org/teiid/common/buffer/LobManager.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -30,9 +30,10 @@
import java.sql.SQLException;
import java.sql.SQLXML;
import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
import org.teiid.core.TeiidComponentException;
import org.teiid.core.types.BaseLob;
@@ -45,6 +46,10 @@
import org.teiid.core.types.SQLXMLImpl;
import org.teiid.core.types.Streamable;
import org.teiid.core.types.XMLType;
+import org.teiid.core.types.InputStreamFactory.BlobInputStreamFactory;
+import org.teiid.core.types.InputStreamFactory.ClobInputStreamFactory;
+import org.teiid.core.types.InputStreamFactory.SQLXMLInputStreamFactory;
+import org.teiid.core.types.InputStreamFactory.StorageMode;
import org.teiid.core.util.ObjectConverterUtil;
import org.teiid.query.QueryPlugin;
import org.teiid.query.sql.symbol.Expression;
@@ -54,9 +59,57 @@
* TODO: for temp tables we may need to have a copy by value management strategy
*/
public class LobManager {
- private Map<String, Streamable<?>> lobReferences = new ConcurrentHashMap<String, Streamable<?>>();
+
+ public enum ReferenceMode {
+ ATTACH,
+ CREATE,
+ REMOVE
+ }
+
+ private static class LobHolder {
+ Streamable<?> lob;
+ int referenceCount = 1;
- public void updateReferences(int[] lobIndexes, List<?> tuple)
+ public LobHolder(Streamable<?> lob) {
+ this.lob = lob;
+ }
+ }
+
+ private Map<String, LobHolder> lobReferences = Collections.synchronizedMap(new HashMap<String, LobHolder>());
+ private boolean inlineLobs = true;
+ private int maxMemoryBytes = DataTypeManager.MAX_LOB_MEMORY_BYTES;
+ private int[] lobIndexes;
+ private FileStore lobStore;
+
+ public LobManager(int[] lobIndexes, FileStore lobStore) {
+ this.lobIndexes = lobIndexes;
+ this.lobStore = lobStore;
+ }
+
+ public LobManager clone() {
+ LobManager clone = new LobManager(lobIndexes, null);
+ clone.inlineLobs = inlineLobs;
+ clone.maxMemoryBytes = maxMemoryBytes;
+ synchronized (lobReferences) {
+ for (Map.Entry<String, LobHolder> entry : lobReferences.entrySet()) {
+ LobHolder lobHolder = new LobHolder(entry.getValue().lob);
+ lobHolder.referenceCount = entry.getValue().referenceCount;
+ clone.lobReferences.put(entry.getKey(), lobHolder);
+ }
+ }
+ return clone;
+ }
+
+ public void setInlineLobs(boolean trackMemoryLobs) {
+ this.inlineLobs = trackMemoryLobs;
+ }
+
+ public void setMaxMemoryBytes(int maxMemoryBytes) {
+ this.maxMemoryBytes = maxMemoryBytes;
+ }
+
+ @SuppressWarnings("unchecked")
+ public void updateReferences(List<?> tuple, ReferenceMode mode)
throws TeiidComponentException {
for (int i = 0; i < lobIndexes.length; i++) {
Object anObj = tuple.get(lobIndexes[i]);
@@ -64,31 +117,64 @@
continue;
}
Streamable lob = (Streamable) anObj;
- if (lob.getReference() == null) {
- lob.setReference(getLobReference(lob.getReferenceStreamId()).getReference());
- } else {
- String id = lob.getReferenceStreamId();
- this.lobReferences.put(id, lob);
+ try {
+ if (lob.getReferenceStreamId() == null || (inlineLobs
+ && (InputStreamFactory.getStorageMode(lob) == StorageMode.MEMORY
+ || lob.length()*(lob instanceof ClobType?2:1) <= maxMemoryBytes))) {
+ lob.setReferenceStreamId(null);
+ continue;
+ }
+ } catch (SQLException e) {
+ //presumably the lob is bad, but let it slide for now
}
+ String id = lob.getReferenceStreamId();
+ LobHolder lobHolder = this.lobReferences.get(id);
+ switch (mode) {
+ case REMOVE:
+ if (lobHolder != null) {
+ lobHolder.referenceCount--;
+ if (lobHolder.referenceCount < 1) {
+ this.lobReferences.remove(id);
+ }
+ }
+ break;
+ case ATTACH:
+ if (lob.getReference() == null) {
+ if (lobHolder == null) {
+ throw new TeiidComponentException(QueryPlugin.Util.getString("ProcessWorker.wrongdata")); //$NON-NLS-1$
+ }
+ lob.setReference(lobHolder.lob.getReference());
+ }
+ break;
+ case CREATE:
+ if (lob.getReference() == null) {
+ throw new TeiidComponentException(QueryPlugin.Util.getString("ProcessWorker.wrongdata")); //$NON-NLS-1$
+ }
+ if (lobHolder == null) {
+ this.lobReferences.put(id, new LobHolder(lob));
+ } else {
+ lobHolder.referenceCount++;
+ }
+ }
}
}
public Streamable<?> getLobReference(String id) throws TeiidComponentException {
- Streamable<?> lob = this.lobReferences.get(id);
+ LobHolder lob = this.lobReferences.get(id);
if (lob == null) {
throw new TeiidComponentException(QueryPlugin.Util.getString("ProcessWorker.wrongdata")); //$NON-NLS-1$
}
- return lob;
+ return lob.lob;
}
- public static int[] getLobIndexes(List expressions) {
+ public static int[] getLobIndexes(List<? extends Expression> expressions) {
if (expressions == null) {
return null;
}
int[] result = new int[expressions.size()];
int resultIndex = 0;
for (int i = 0; i < expressions.size(); i++) {
- Expression expr = (Expression) expressions.get(i);
+ Expression expr = expressions.get(i);
if (DataTypeManager.isLOB(expr.getType()) || expr.getType() == DataTypeManager.DefaultDataClasses.OBJECT) {
result[resultIndex++] = i;
}
@@ -99,12 +185,12 @@
return Arrays.copyOf(result, resultIndex);
}
- public void persist(FileStore lobStore) throws TeiidComponentException {
+ public void persist() throws TeiidComponentException {
// stream the contents of lob into file store.
- byte[] bytes = new byte[102400]; // 100k
+ byte[] bytes = new byte[1 << 14];
- for (Map.Entry<String, Streamable<?>> entry : this.lobReferences.entrySet()) {
- entry.setValue(persistLob(entry.getValue(), lobStore, bytes));
+ for (Map.Entry<String, LobHolder> entry : this.lobReferences.entrySet()) {
+ entry.getValue().lob = persistLob(entry.getValue().lob, lobStore, bytes);
}
}
@@ -115,7 +201,7 @@
try {
BaseLob baseLob = (BaseLob)lob.getReference();
InputStreamFactory isf = baseLob.getStreamFactory();
- if (isf.isPersistent()) {
+ if (isf.getStorageMode() == StorageMode.PERSISTENT) {
return lob;
}
} catch (SQLException e) {
@@ -127,19 +213,15 @@
Streamable<?> persistedLob;
try {
- InputStreamFactory isf = new InputStreamFactory() {
- @Override
- public InputStream getInputStream() throws IOException {
- if (lob instanceof BlobType) {
- return new BlobInputStreamFactory((Blob)lob).getInputStream();
- }
- else if (lob instanceof ClobType) {
- return new ClobInputStreamFactory((Clob)lob).getInputStream();
- }
- return new SQLXMLInputStreamFactory((SQLXML)lob).getInputStream();
- }
- };
- InputStream is = isf.getInputStream();
+ InputStream is = null;
+ if (lob instanceof BlobType) {
+ is = new BlobInputStreamFactory((Blob)lob).getInputStream();
+ }
+ else if (lob instanceof ClobType) {
+ is = new ClobInputStreamFactory((Clob)lob).getInputStream();
+ } else {
+ is = new SQLXMLInputStreamFactory((SQLXML)lob).getInputStream();
+ }
OutputStream fsos = store.createOutputStream();
length = ObjectConverterUtil.write(fsos, is, bytes, -1);
} catch (IOException e) {
@@ -156,8 +238,8 @@
}
@Override
- public boolean isPersistent() {
- return true;
+ public StorageMode getStorageMode() {
+ return StorageMode.PERSISTENT;
}
};
@@ -178,4 +260,13 @@
}
return persistedLob;
}
+
+ public int getLobCount() {
+ return this.lobReferences.size();
+ }
+
+ public void remove() {
+ this.lobReferences.clear();
+
+ }
}
Modified: branches/as7/engine/src/main/java/org/teiid/common/buffer/SPage.java
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/common/buffer/SPage.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/main/java/org/teiid/common/buffer/SPage.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -23,6 +23,7 @@
package org.teiid.common.buffer;
import java.lang.ref.PhantomReference;
+import java.lang.ref.Reference;
import java.lang.ref.ReferenceQueue;
import java.util.ArrayList;
import java.util.Collections;
@@ -31,17 +32,14 @@
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
-import org.teiid.common.buffer.BatchManager.CleanupHook;
-import org.teiid.common.buffer.BatchManager.ManagedBatch;
import org.teiid.core.TeiidComponentException;
import org.teiid.core.TeiidRuntimeException;
/**
* A linked list Page entry in the tree
*
- * TODO: return the tuplebatch from getvalues, since that is what we're tracking
- *
* State cloning allows a single storage reference to be shared in many trees.
* A phantom reference is used for proper cleanup once cloned.
*
@@ -56,8 +54,8 @@
static class SearchResult {
int index;
SPage page;
- TupleBatch values;
- public SearchResult(int index, SPage page, TupleBatch values) {
+ List<List<?>> values;
+ public SearchResult(int index, SPage page, List<List<?>> values) {
this.index = index;
this.page = page;
this.values = values;
@@ -68,51 +66,54 @@
private static ReferenceQueue<Object> QUEUE = new ReferenceQueue<Object>();
static class CleanupReference extends PhantomReference<Object> {
- private CleanupHook batch;
+ private Long batch;
+ private Reference<? extends BatchManager> ref;
- public CleanupReference(Object referent, CleanupHook batch) {
+ public CleanupReference(Object referent, Long batch, Reference<? extends BatchManager> ref) {
super(referent, QUEUE);
this.batch = batch;
+ this.ref = ref;
}
public void cleanup() {
try {
- this.batch.cleanup();
+ BatchManager batchManager = ref.get();
+ if (batchManager != null) {
+ batchManager.remove(batch);
+ }
} finally {
this.clear();
}
}
}
-
+
+ private static AtomicLong counter = new AtomicLong();
+
STree stree;
- private int id;
+ private long id;
protected SPage next;
protected SPage prev;
- protected ManagedBatch managedBatch;
- private Object trackingObject;
- protected TupleBatch values;
- protected ArrayList<SPage> children;
- //TODO: could track cloning more completely, which would allow for earlier batch removal
- private boolean cloned;
+ protected Long managedBatch;
+ protected Object trackingObject;
+ protected List<List<?>> values;
+ protected List<SPage> children;
SPage(STree stree, boolean leaf) {
this.stree = stree;
- this.id = stree.counter.getAndIncrement();
+ this.id = counter.getAndIncrement();
stree.pages.put(this.id, this);
- //TODO: this counter is a hack. need a better idea of a storage id
- this.values = new TupleBatch(id, new ArrayList(stree.pageSize/4));
+ this.values = new ArrayList<List<?>>();
if (!leaf) {
- children = new ArrayList<SPage>(stree.pageSize/4);
+ children = new ArrayList<SPage>();
}
}
public SPage clone(STree tree) {
try {
if (this.managedBatch != null && trackingObject == null) {
- cloned = true;
this.trackingObject = new Object();
- CleanupReference managedBatchReference = new CleanupReference(trackingObject, managedBatch.getCleanupHook());
+ CleanupReference managedBatchReference = new CleanupReference(trackingObject, managedBatch, stree.getBatchManager(children == null).getBatchManagerReference());
REFERENCES.add(managedBatchReference);
}
SPage clone = (SPage) super.clone();
@@ -121,7 +122,7 @@
clone.children = new ArrayList<SPage>(children);
}
if (values != null) {
- clone.values = new TupleBatch(stree.counter.getAndIncrement(), new ArrayList<List<?>>(values.getTuples()));
+ clone.values = new ArrayList<List<?>>(values);
}
return clone;
} catch (CloneNotSupportedException e) {
@@ -129,37 +130,37 @@
}
}
- public int getId() {
+ public long getId() {
return id;
}
static SearchResult search(SPage page, List k, LinkedList<SearchResult> parent) throws TeiidComponentException {
- TupleBatch previousValues = null;
+ List<List<?>> previousValues = null;
for (;;) {
- TupleBatch values = page.getValues();
- int index = Collections.binarySearch(values.getTuples(), k, page.stree.comparator);
+ List<List<?>> values = page.getValues();
+ int index = Collections.binarySearch(values, k, page.stree.comparator);
int flippedIndex = - index - 1;
if (previousValues != null) {
if (flippedIndex == 0) {
//systemic weakness of the algorithm
- return new SearchResult(-previousValues.getTuples().size() - 1, page.prev, previousValues);
+ return new SearchResult(-previousValues.size() - 1, page.prev, previousValues);
}
if (parent != null && index != 0) {
page.stree.updateLock.lock();
try {
- index = Collections.binarySearch(values.getTuples(), k, page.stree.comparator);
+ index = Collections.binarySearch(values, k, page.stree.comparator);
if (index != 0) {
//for non-matches move the previous pointer over to this page
SPage childPage = page;
List oldKey = null;
- List newKey = page.stree.extractKey(values.getTuples().get(0));
+ List newKey = page.stree.extractKey(values.get(0));
for (Iterator<SearchResult> desc = parent.descendingIterator(); desc.hasNext();) {
SearchResult sr = desc.next();
int parentIndex = Math.max(0, -sr.index - 2);
if (oldKey == null) {
- oldKey = sr.values.getTuples().set(parentIndex, newKey);
- } else if (page.stree.comparator.compare(oldKey, sr.values.getTuples().get(parentIndex)) == 0 ) {
- sr.values.getTuples().set(parentIndex, newKey);
+ oldKey = sr.values.set(parentIndex, newKey);
+ } else if (page.stree.comparator.compare(oldKey, sr.values.get(parentIndex)) == 0 ) {
+ sr.values.set(parentIndex, newKey);
} else {
break;
}
@@ -173,7 +174,7 @@
}
}
}
- if (flippedIndex != values.getTuples().size() || page.next == null) {
+ if (flippedIndex != values.size() || page.next == null) {
return new SearchResult(index, page, values);
}
previousValues = values;
@@ -181,36 +182,32 @@
}
}
- protected void setValues(TupleBatch values) throws TeiidComponentException {
- if (managedBatch != null && !cloned) {
- managedBatch.remove();
+ protected void setValues(List<List<?>> values) throws TeiidComponentException {
+ if (values instanceof LightWeightCopyOnWriteList<?>) {
+ values = ((LightWeightCopyOnWriteList<List<?>>)values).getList();
}
- if (values.getTuples().size() < MIN_PERSISTENT_SIZE) {
+ if (managedBatch != null && trackingObject == null) {
+ stree.getBatchManager(children == null).remove(managedBatch);
+ managedBatch = null;
+ trackingObject = null;
+ }
+ if (values.size() < MIN_PERSISTENT_SIZE) {
this.values = values;
return;
- }
- this.values = null;
- if (children != null) {
- values.setDataTypes(stree.keytypes);
- } else {
- values.setDataTypes(stree.types);
+ } else if (stree.batchInsert && children == null && values.size() < stree.leafSize) {
+ this.values = values;
+ stree.incompleteInsert = this;
+ return;
}
- if (cloned) {
- values.setRowOffset(stree.counter.getAndIncrement());
- cloned = false;
- trackingObject = null;
- }
- if (children != null) {
- managedBatch = stree.keyManager.createManagedBatch(values, true);
- } else {
- managedBatch = stree.leafManager.createManagedBatch(values, stree.preferMemory);
- }
+ this.values = null;
+ this.trackingObject = null;
+ managedBatch = stree.getBatchManager(children == null).createManagedBatch(values);
}
-
+
protected void remove(boolean force) {
if (managedBatch != null) {
- if (force || !cloned) {
- managedBatch.remove();
+ if (force || trackingObject == null) {
+ stree.getBatchManager(children == null).remove(managedBatch);
}
managedBatch = null;
trackingObject = null;
@@ -219,7 +216,7 @@
children = null;
}
- protected TupleBatch getValues() throws TeiidComponentException {
+ protected List<List<?>> getValues() throws TeiidComponentException {
if (values != null) {
return values;
}
@@ -234,19 +231,20 @@
REFERENCES.remove(ref);
ref.cleanup();
}
- if (children != null) {
- return managedBatch.getBatch(true, stree.keytypes);
+ List<List<?>> result = stree.getBatchManager(children == null).getBatch(managedBatch, true);
+ if (trackingObject != null) {
+ return new LightWeightCopyOnWriteList<List<?>>(result);
}
- return managedBatch.getBatch(true, stree.types);
+ return result;
}
- static void merge(LinkedList<SearchResult> places, TupleBatch nextValues, SPage current, TupleBatch currentValues)
+ static void merge(LinkedList<SearchResult> places, List<List<?>> nextValues, SPage current, List<List<?>> currentValues)
throws TeiidComponentException {
SearchResult parent = places.peekLast();
if (parent != null) {
- correctParents(parent.page, nextValues.getTuples().get(0), current.next, current);
+ correctParents(parent.page, nextValues.get(0), current.next, current);
}
- currentValues.getTuples().addAll(nextValues.getTuples());
+ currentValues.addAll(nextValues);
if (current.children != null) {
current.children.addAll(current.next.children);
}
@@ -292,19 +290,19 @@
public String toString() {
StringBuilder result = new StringBuilder();
try {
- TupleBatch tb = getValues();
- result.append(tb.getBeginRow());
+ List<List<?>> tb = getValues();
+ result.append(id);
if (children == null) {
- if (tb.getTuples().size() <= 1) {
- result.append(tb.getTuples());
+ if (tb.size() <= 1) {
+ result.append(tb);
} else {
- result.append("[").append(tb.getTuples().get(0)).append(" . ").append(tb.getTuples().size()). //$NON-NLS-1$ //$NON-NLS-2$
- append(" . ").append(tb.getTuples().get(tb.getTuples().size() - 1)).append("]"); //$NON-NLS-1$ //$NON-NLS-2$
+ result.append("[").append(tb.get(0)).append(" . ").append(tb.size()). //$NON-NLS-1$ //$NON-NLS-2$
+ append(" . ").append(tb.get(tb.size() - 1)).append("]"); //$NON-NLS-1$ //$NON-NLS-2$
}
} else {
result.append("["); //$NON-NLS-1$
for (int i = 0; i < children.size(); i++) {
- result.append(tb.getTuples().get(i)).append("->").append(children.get(i).getValues().getBeginRow()); //$NON-NLS-1$
+ result.append(tb.get(i)).append("->").append(children.get(i).getId()); //$NON-NLS-1$
if (i < children.size() - 1) {
result.append(", "); //$NON-NLS-1$
}
Modified: branches/as7/engine/src/main/java/org/teiid/common/buffer/STree.java
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/common/buffer/STree.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/main/java/org/teiid/common/buffer/STree.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -27,15 +27,16 @@
import java.io.ObjectOutputStream;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Random;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock;
import org.teiid.client.BatchSerializer;
+import org.teiid.common.buffer.LobManager.ReferenceMode;
import org.teiid.common.buffer.SPage.SearchResult;
import org.teiid.core.TeiidComponentException;
import org.teiid.core.TeiidRuntimeException;
@@ -51,23 +52,23 @@
public enum InsertMode {ORDERED, NEW, UPDATE}
- private static final Random seedGenerator = new Random();
+ private static final Random seedGenerator = new Random(0);
protected int randomSeed;
private int mask = 1;
private int shift = 1;
- protected AtomicInteger counter = new AtomicInteger();
- protected ConcurrentHashMap<Integer, SPage> pages = new ConcurrentHashMap<Integer, SPage>();
+ protected HashMap<Long, SPage> pages = new HashMap<Long, SPage>();
protected volatile SPage[] header = new SPage[] {new SPage(this, true)};
protected BatchManager keyManager;
protected BatchManager leafManager;
protected ListNestedSortComparator comparator;
- protected int pageSize;
+ private int pageSize;
+ protected int leafSize;
protected int keyLength;
- protected String[] types;
- protected String[] keytypes;
- protected boolean preferMemory;
+ protected boolean batchInsert;
+ protected SPage incompleteInsert;
+ protected LobManager lobManager;
protected ReentrantLock updateLock = new ReentrantLock();
@@ -77,10 +78,12 @@
BatchManager leafManager,
final ListNestedSortComparator comparator,
int pageSize,
+ int leafSize,
int keyLength,
- String[] types) {
+ LobManager lobManager) {
randomSeed = seedGenerator.nextInt() | 0x00000100; // ensure nonzero
this.keyManager = manager;
+ manager.setPrefersMemory(true);
this.leafManager = leafManager;
this.comparator = comparator;
this.pageSize = Math.max(pageSize, SPage.MIN_PERSISTENT_SIZE);
@@ -91,24 +94,27 @@
mask <<= 1;
mask++;
}
+ this.leafSize = leafSize;
this.keyLength = keyLength;
- this.types = types;
- this.keytypes = Arrays.copyOf(types, keyLength);
+ this.lobManager = lobManager;
}
public STree clone() {
updateLock.lock();
try {
STree clone = (STree) super.clone();
+ if (lobManager != null) {
+ clone.lobManager = lobManager.clone();
+ }
clone.updateLock = new ReentrantLock();
clone.rowCount = new AtomicInteger(rowCount.get());
//clone the pages
- clone.pages = new ConcurrentHashMap<Integer, SPage>(pages);
- for (Map.Entry<Integer, SPage> entry : clone.pages.entrySet()) {
+ clone.pages = new HashMap<Long, SPage>(pages);
+ for (Map.Entry<Long, SPage> entry : clone.pages.entrySet()) {
entry.setValue(entry.getValue().clone(clone));
}
//reset the pointers
- for (Map.Entry<Integer, SPage> entry : clone.pages.entrySet()) {
+ for (Map.Entry<Long, SPage> entry : clone.pages.entrySet()) {
SPage clonePage = entry.getValue();
clonePage.next = clone.getPage(clonePage.next);
clonePage.prev = clone.getPage(clonePage.prev);
@@ -141,23 +147,41 @@
SPage page = header[0];
oos.writeInt(this.rowCount.get());
while (true) {
- TupleBatch batch = page.getValues();
- BatchSerializer.writeBatch(oos, types, batch.getAllTuples());
+ List<List<?>> batch = page.getValues();
+ BatchSerializer.writeBatch(oos, leafManager.getTypes(), batch);
if (page.next == null) {
break;
}
}
}
+ public void setBatchInsert(boolean batchInsert) throws TeiidComponentException {
+ if (this.batchInsert == batchInsert) {
+ return;
+ }
+ this.batchInsert = batchInsert;
+ if (batchInsert || incompleteInsert == null) {
+ return;
+ }
+ SPage toFlush = incompleteInsert;
+ incompleteInsert = null;
+ if (toFlush.managedBatch != null) {
+ return;
+ }
+ toFlush.setValues(toFlush.getValues());
+ }
+
public void readValuesFrom(ObjectInputStream ois) throws IOException, ClassNotFoundException, TeiidComponentException {
int size = ois.readInt();
int sizeHint = this.getExpectedHeight(size);
+ batchInsert = true;
while (this.getRowCount() < size) {
- List[] batch = BatchSerializer.readBatch(ois, types);
+ List<List<Object>> batch = BatchSerializer.readBatch(ois, leafManager.getTypes());
for (List list : batch) {
this.insert(list, InsertMode.ORDERED, sizeHint);
}
}
+ batchInsert = false;
}
protected SPage findChildTail(SPage page) {
@@ -219,7 +243,7 @@
if (places != null) {
places.add(s);
}
- if ((s.index == -1 && s.page == header[i]) || s.values.getTuples().isEmpty()) {
+ if ((s.index == -1 && s.page == header[i]) || s.values.isEmpty()) {
x = null;
continue; //start at the beginning of the next level
}
@@ -234,7 +258,7 @@
if (!matched) {
return null;
}
- return s.values.getTuples().get(index);
+ return s.values.get(index);
}
x = x.children.get(index);
}
@@ -248,13 +272,16 @@
public List insert(List tuple, InsertMode mode, int sizeHint) throws TeiidComponentException {
LinkedList<SearchResult> places = new LinkedList<SearchResult>();
List match = null;
+ if (this.lobManager != null) {
+ this.lobManager.updateReferences(tuple, ReferenceMode.CREATE);
+ }
if (mode == InsertMode.ORDERED) {
SPage last = null;
while (last == null || last.children != null) {
last = findChildTail(last);
//TODO: do this lazily
- TupleBatch batch = last.getValues();
- places.add(new SearchResult(-batch.getTuples().size() -1, last, batch));
+ List<List<?>> batch = last.getValues();
+ places.add(new SearchResult(-batch.size() -1, last, batch));
}
} else {
match = find(tuple, places);
@@ -264,8 +291,11 @@
}
SearchResult last = places.getLast();
SPage page = last.page;
- last.values.getTuples().set(last.index, tuple);
+ last.values.set(last.index, tuple);
page.setValues(last.values);
+ if (this.lobManager != null) {
+ this.lobManager.updateReferences(tuple, ReferenceMode.REMOVE);
+ }
return match;
}
}
@@ -277,10 +307,10 @@
} else {
level = randomLevel();
}
- } else if (!places.isEmpty() && places.getLast().values.getTuples().size() == pageSize) {
+ } else if (!places.isEmpty() && places.getLast().values.size() == getPageSize(true)) {
int row = rowCount.get();
- while (row != 0 && row%pageSize == 0) {
- row = (row - pageSize + 1)/pageSize;
+ while (row != 0 && row%getPageSize(true) == 0) {
+ row = (row - getPageSize(true) + 1)/getPageSize(true);
level++;
}
}
@@ -293,8 +323,8 @@
for (int i = 0; i <= level; i++) {
if (places.isEmpty()) {
SPage newHead = new SPage(this, false);
- TupleBatch batch = newHead.getValues();
- batch.getTuples().add(key);
+ List<List<?>> batch = newHead.getValues();
+ batch.add(key);
newHead.setValues(batch);
newHead.children.add(page);
header[i] = newHead;
@@ -313,9 +343,9 @@
return 0;
}
int logSize = 1;
- while (sizeHint > this.pageSize) {
+ while (sizeHint > this.getPageSize(logSize==0)) {
logSize++;
- sizeHint/=this.pageSize;
+ sizeHint/=this.getPageSize(logSize==0);
}
return logSize;
}
@@ -330,10 +360,10 @@
SPage insert(List k, SearchResult result, SearchResult parent, Object value, boolean ordered) throws TeiidComponentException {
SPage page = result.page;
int index = -result.index - 1;
- if (result.values.getTuples().size() == pageSize) {
- boolean leaf = !(value instanceof SPage);
+ boolean leaf = !(value instanceof SPage);
+ if (result.values.size() == getPageSize(leaf)) {
SPage nextPage = new SPage(this, leaf);
- TupleBatch nextValues = nextPage.getValues();
+ List<List<?>> nextValues = nextPage.getValues();
nextPage.next = page.next;
nextPage.prev = page;
if (nextPage.next != null) {
@@ -343,21 +373,21 @@
boolean inNext = false;
if (!ordered) {
//split the values
- nextValues.getTuples().addAll(result.values.getTuples().subList(pageSize/2, pageSize));
- result.values.getTuples().subList(pageSize/2, pageSize).clear();
+ nextValues.addAll(result.values.subList(getPageSize(leaf)/2, getPageSize(leaf)));
+ result.values.subList(getPageSize(leaf)/2, getPageSize(leaf)).clear();
if (!leaf) {
- nextPage.children.addAll(page.children.subList(pageSize/2, pageSize));
- page.children.subList(pageSize/2, pageSize).clear();
+ nextPage.children.addAll(page.children.subList(getPageSize(leaf)/2, getPageSize(false)));
+ page.children.subList(getPageSize(false)/2, getPageSize(false)).clear();
}
- if (index <= pageSize/2) {
+ if (index <= getPageSize(leaf)/2) {
setValue(index, k, value, result.values, page);
} else {
inNext = true;
- setValue(index - pageSize/2, k, value, nextValues, nextPage);
+ setValue(index - getPageSize(leaf)/2, k, value, nextValues, nextPage);
}
page.setValues(result.values);
if (parent != null) {
- List min = nextPage.getValues().getTuples().get(0);
+ List min = nextPage.getValues().get(0);
SPage.correctParents(parent.page, min, page, nextPage);
}
} else {
@@ -375,12 +405,12 @@
return page;
}
- static void setValue(int index, List key, Object value, TupleBatch values, SPage page) {
+ static void setValue(int index, List key, Object value, List<List<?>> values, SPage page) {
if (value instanceof SPage) {
- values.getTuples().add(index, key);
+ values.add(index, key);
page.children.add(index, (SPage) value);
} else {
- values.getTuples().add(index, (List)value);
+ values.add(index, (List)value);
}
}
@@ -396,11 +426,13 @@
if (searchResult.index < 0) {
continue;
}
- searchResult.values.getTuples().remove(searchResult.index);
+ searchResult.values.remove(searchResult.index);
+ boolean leaf = true;
if (searchResult.page.children != null) {
+ leaf = false;
searchResult.page.children.remove(searchResult.index);
}
- int size = searchResult.values.getTuples().size();
+ int size = searchResult.values.size();
if (size == 0) {
if (header[i] != searchResult.page) {
searchResult.page.remove(false);
@@ -424,18 +456,18 @@
header[0] = new SPage(this, true);
}
continue;
- } else if (size < pageSize/2) {
+ } else if (size < getPageSize(leaf)/2) {
//check for merge
if (searchResult.page.next != null) {
- TupleBatch nextValues = searchResult.page.next.getValues();
- if (nextValues.getTuples().size() < pageSize/4) {
+ List<List<?>> nextValues = searchResult.page.next.getValues();
+ if (nextValues.size() < getPageSize(leaf)/4) {
SPage.merge(places, nextValues, searchResult.page, searchResult.values);
continue;
}
}
if (searchResult.page.prev != null) {
- TupleBatch prevValues = searchResult.page.prev.getValues();
- if (prevValues.getTuples().size() < pageSize/4) {
+ List<List<?>> prevValues = searchResult.page.prev.getValues();
+ if (prevValues.size() < getPageSize(leaf)/4) {
SPage.merge(places, searchResult.values, searchResult.page.prev, prevValues);
continue;
}
@@ -443,11 +475,14 @@
}
searchResult.page.setValues(searchResult.values);
}
+ if (lobManager != null) {
+ lobManager.updateReferences(tuple, ReferenceMode.REMOVE);
+ }
return tuple;
}
public void remove() {
- truncate(false);
+ truncate(true);
this.keyManager.remove();
this.leafManager.remove();
}
@@ -494,11 +529,11 @@
}
public void setPreferMemory(boolean preferMemory) {
- this.preferMemory = preferMemory;
+ this.leafManager.setPrefersMemory(preferMemory);
}
public boolean isPreferMemory() {
- return preferMemory;
+ return this.leafManager.prefersMemory();
}
public ListNestedSortComparator getComparator() {
@@ -530,4 +565,26 @@
this.comparator.setSortParameters(sortParameters);
}
+ public void clearClonedFlags() {
+ for (SPage page : pages.values()) {
+ page.trackingObject = null;
+ //we don't really care about using synchronization or a volatile here
+ //since the worst case is that we'll just use gc cleanup
+ }
+ }
+
+ public int getPageSize(boolean leaf) {
+ if (leaf) {
+ return leafSize;
+ }
+ return pageSize;
+ }
+
+ BatchManager getBatchManager(boolean leaf) {
+ if (leaf) {
+ return leafManager;
+ }
+ return keyManager;
+ }
+
}
\ No newline at end of file
Copied: branches/as7/engine/src/main/java/org/teiid/common/buffer/Serializer.java (from rev 3506, trunk/engine/src/main/java/org/teiid/common/buffer/Serializer.java)
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/common/buffer/Serializer.java (rev 0)
+++ branches/as7/engine/src/main/java/org/teiid/common/buffer/Serializer.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -0,0 +1,38 @@
+/*
+ * JBoss, Home of Professional Open Source.
+ * See the COPYRIGHT.txt file distributed with this work for information
+ * regarding copyright ownership. Some portions may be licensed
+ * to Red Hat, Inc. under one or more contributor license agreements.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301 USA.
+ */
+
+package org.teiid.common.buffer;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+
+/**
+ * Responsible for serializing {@link CacheEntry}s
+ * @param <T>
+ */
+public interface Serializer<T> {
+ void serialize(T obj, ObjectOutputStream oos) throws IOException;
+ T deserialize(ObjectInputStream ois) throws IOException, ClassNotFoundException;
+ boolean useSoftCache();
+ Long getId();
+}
\ No newline at end of file
Modified: branches/as7/engine/src/main/java/org/teiid/common/buffer/TupleBatch.java
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/common/buffer/TupleBatch.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/main/java/org/teiid/common/buffer/TupleBatch.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -22,17 +22,11 @@
package org.teiid.common.buffer;
-import java.io.Externalizable;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import org.teiid.client.BatchSerializer;
-
/**
* Represents a set of indexed tuples. The {@link #getBeginRow beginning row}
* is the first row contained in this batch; if it equals "1" then it is the
@@ -40,28 +34,17 @@
* tuples. The {@link #getEndRow ending row} is the last row contained in
* this tuple batch; it is equal to the beginning row plus the
* {@link #getRowCount number of rows} contained in this batch, minus one.
- * This object is immutable and Serializable;
*/
-public class TupleBatch implements Externalizable {
+public class TupleBatch {
private static final long serialVersionUID = 6304443387337336957L;
private int rowOffset;
- private List<List<?>> tuples;
+ protected List<List<?>> tuples;
// Optional state
private boolean terminationFlag = false;
- // for distributed cache purposes
- private String[] preservedTypes;
-
- /**
- * Contains ordered data types of each of the columns in the batch. Although it is not serialized,
- * this array is a serialization aid and must be set before serialization and deserialization using
- * the setDataTypes method.
- */
- private transient String[] types;
-
/** Required to honor Externalizable contract */
public TupleBatch() {
}
@@ -152,14 +135,6 @@
this.terminationFlag = terminationFlag;
}
- public void setDataTypes(String[] types) {
- this.types = types;
- }
-
- public String[] getDataTypes() {
- return types;
- }
-
public boolean containsRow(int row) {
return rowOffset <= row && getEndRow() >= row;
}
@@ -179,31 +154,8 @@
return s.toString();
}
- public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- rowOffset = in.readInt();
- terminationFlag = in.readBoolean();
- preservedTypes = (String[])in.readObject();
- if (types == null) {
- types = preservedTypes;
- }
- tuples = new ArrayList<List<?>>();
- for (List tuple : BatchSerializer.readBatch(in, types)) {
- tuples.add(tuple);
- }
- }
- public void writeExternal(ObjectOutput out) throws IOException {
- out.writeInt(this.rowOffset);
- out.writeBoolean(terminationFlag);
- out.writeObject(this.preservedTypes);
- BatchSerializer.writeBatch(out, types, getAllTuples());
- }
-
public void setRowOffset(int rowOffset) {
this.rowOffset = rowOffset;
}
-
- public void preserveTypes() {
- this.preservedTypes = types;
- }
}
Property changes on: branches/as7/engine/src/main/java/org/teiid/common/buffer/TupleBatch.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/main/java/org/teiid/common/buffer/TupleBatch.java:3220-3275
/trunk/engine/src/main/java/org/teiid/common/buffer/TupleBatch.java:3188-3450
+ /branches/7.4.x/engine/src/main/java/org/teiid/common/buffer/TupleBatch.java:3220-3275
/trunk/engine/src/main/java/org/teiid/common/buffer/TupleBatch.java:3188-3450,3452-3506
Modified: branches/as7/engine/src/main/java/org/teiid/common/buffer/TupleBrowser.java
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/common/buffer/TupleBrowser.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/main/java/org/teiid/common/buffer/TupleBrowser.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -48,7 +48,7 @@
private SPage bound;
private int boundIndex = -1;
- private TupleBatch values;
+ private List<List<?>> values;
private boolean updated;
private boolean direction;
@@ -106,7 +106,7 @@
if (boundIndex < 0) {
//we are guaranteed by find to not get back the -1 index, unless
//there are no tuples, in which case a bound of -1 is fine
- boundIndex = Math.min(upper.values.getTuples().size(), -boundIndex -1) - 1;
+ boundIndex = Math.min(upper.values.size(), -boundIndex -1) - 1;
}
if (!direction) {
values = upper.values;
@@ -122,7 +122,7 @@
if (page != bound || values == null) {
values = bound.getValues();
}
- boundIndex = values.getTuples().size() - 1;
+ boundIndex = values.size() - 1;
}
}
@@ -173,14 +173,14 @@
continue;
}
if (values != null) {
- int possibleIndex = Collections.binarySearch(values.getTuples(), newValue, tree.comparator);
+ int possibleIndex = Collections.binarySearch(values, newValue, tree.comparator);
if (possibleIndex >= 0) {
//value exists in the current page
index = possibleIndex;
- return values.getTuples().get(possibleIndex);
+ return values.get(possibleIndex);
}
//check for end/terminal conditions
- if (direction && possibleIndex == -values.getTuples().size() -1) {
+ if (direction && possibleIndex == -values.size() -1) {
if (page.next == null) {
resetState();
return null;
@@ -199,7 +199,7 @@
if (!setPage(newValue)) {
continue;
}
- return values.getTuples().get(index);
+ return values.get(index);
}
if (page == null) {
if (inPartial) {
@@ -213,11 +213,11 @@
if (direction) {
index = 0;
} else {
- index = values.getTuples().size() - 1;
+ index = values.size() - 1;
}
}
- if (index >= 0 && index < values.getTuples().size()) {
- List<?> result = values.getTuples().get(index);
+ if (index >= 0 && index < values.size()) {
+ List<?> result = values.get(index);
if (page == bound && index == boundIndex) {
resetState();
page = null; //terminate
@@ -257,7 +257,7 @@
* @throws TeiidComponentException
*/
public void update(List<?> tuple) throws TeiidComponentException {
- values.getTuples().set(index - getOffset(), tuple);
+ values.set(index - getOffset(), tuple);
updated = true;
}
Modified: branches/as7/engine/src/main/java/org/teiid/common/buffer/TupleBuffer.java
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/common/buffer/TupleBuffer.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/main/java/org/teiid/common/buffer/TupleBuffer.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -28,7 +28,7 @@
import java.util.Map;
import java.util.TreeMap;
-import org.teiid.common.buffer.BatchManager.ManagedBatch;
+import org.teiid.common.buffer.LobManager.ReferenceMode;
import org.teiid.core.TeiidComponentException;
import org.teiid.core.types.DataTypeManager;
import org.teiid.core.types.Streamable;
@@ -64,36 +64,32 @@
private BatchManager manager;
private String tupleSourceID;
private List<? extends Expression> schema;
- private String[] types;
private int batchSize;
private int rowCount;
private boolean isFinal;
- private TreeMap<Integer, BatchManager.ManagedBatch> batches = new TreeMap<Integer, BatchManager.ManagedBatch>();
+ private TreeMap<Integer, Long> batches = new TreeMap<Integer, Long>();
private ArrayList<List<?>> batchBuffer;
private boolean removed;
private boolean forwardOnly;
- private boolean prefersMemory;
private LobManager lobManager;
- private int[] lobIndexes;
private String uuid;
- private FileStore lobStore;
- public TupleBuffer(BatchManager manager, String id, List<? extends Expression> schema, int[] lobIndexes, int batchSize) {
+ public TupleBuffer(BatchManager manager, String id, List<? extends Expression> schema, LobManager lobManager, int batchSize) {
this.manager = manager;
this.tupleSourceID = id;
this.schema = schema;
- this.types = getTypeNames(schema);
- this.lobIndexes = lobIndexes;
- if (this.lobIndexes != null) {
- this.lobManager = new LobManager();
- this.lobStore = this.manager.createStorage("_lobs"); //$NON-NLS-1$
- this.lobStore.setCleanupReference(this);
- }
+ this.lobManager = lobManager;
this.batchSize = batchSize;
}
+ public void setInlineLobs(boolean inline) {
+ if (this.lobManager != null) {
+ this.lobManager.setInlineLobs(inline);
+ }
+ }
+
public String getId() {
if (this.uuid == null) {
this.uuid = java.util.UUID.randomUUID().toString();
@@ -106,12 +102,12 @@
}
public boolean isLobs() {
- return lobIndexes != null;
+ return lobManager != null;
}
public void addTuple(List<?> tuple) throws TeiidComponentException {
if (isLobs()) {
- lobManager.updateReferences(lobIndexes, tuple);
+ lobManager.updateReferences(tuple, ReferenceMode.CREATE);
}
this.rowCount++;
if (batchBuffer == null) {
@@ -119,7 +115,7 @@
}
batchBuffer.add(tuple);
if (batchBuffer.size() == batchSize) {
- saveBatch(false, false);
+ saveBatch(false);
}
}
@@ -138,7 +134,7 @@
//add the lob references only, since they may still be referenced later
if (isLobs()) {
for (List<?> tuple : batch.getTuples()) {
- lobManager.updateReferences(lobIndexes, tuple);
+ lobManager.updateReferences(tuple, ReferenceMode.CREATE);
}
}
}
@@ -148,7 +144,7 @@
throws TeiidComponentException {
assert this.rowCount <= rowCount;
if (this.rowCount != rowCount) {
- saveBatch(false, true);
+ saveBatch(true);
this.rowCount = rowCount;
}
}
@@ -157,15 +153,15 @@
if (this.batchBuffer != null) {
this.batchBuffer.clear();
}
- for (BatchManager.ManagedBatch batch : this.batches.values()) {
- batch.remove();
+ for (Long batch : this.batches.values()) {
+ this.manager.remove(batch);
}
this.batches.clear();
}
public void persistLobs() throws TeiidComponentException {
if (this.lobManager != null) {
- this.lobManager.persist(this.lobStore);
+ this.lobManager.persist();
}
}
@@ -174,26 +170,21 @@
* @throws TeiidComponentException
*/
public void saveBatch() throws TeiidComponentException {
- this.saveBatch(false, false);
+ this.saveBatch(false);
}
- void saveBatch(boolean finalBatch, boolean force) throws TeiidComponentException {
+ void saveBatch(boolean force) throws TeiidComponentException {
Assertion.assertTrue(!this.isRemoved());
if (batchBuffer == null || batchBuffer.isEmpty() || (!force && batchBuffer.size() < Math.max(1, batchSize / 32))) {
return;
}
- TupleBatch writeBatch = new TupleBatch(rowCount - batchBuffer.size() + 1, batchBuffer);
- if (finalBatch) {
- writeBatch.setTerminationFlag(true);
- }
- writeBatch.setDataTypes(types);
- BatchManager.ManagedBatch mbatch = manager.createManagedBatch(writeBatch, prefersMemory);
- this.batches.put(writeBatch.getBeginRow(), mbatch);
+ Long mbatch = manager.createManagedBatch(batchBuffer);
+ this.batches.put(rowCount - batchBuffer.size() + 1, mbatch);
batchBuffer = null;
}
public void close() throws TeiidComponentException {
- saveBatch(true, false);
+ saveBatch(false);
this.isFinal = true;
}
@@ -204,6 +195,8 @@
* @param row
* @return
* @throws TeiidComponentException
+ *
+ * TODO: a method to get the raw batch
*/
public TupleBatch getBatch(int row) throws TeiidComponentException {
TupleBatch result = null;
@@ -218,17 +211,20 @@
if (this.batchBuffer != null && !this.batchBuffer.isEmpty()) {
//this is just a sanity check to ensure we're not holding too many
//hard references to batches.
- saveBatch(isFinal, false);
+ saveBatch(false);
}
- Map.Entry<Integer, BatchManager.ManagedBatch> entry = batches.floorEntry(row);
+ Map.Entry<Integer, Long> entry = batches.floorEntry(row);
Assertion.isNotNull(entry);
- BatchManager.ManagedBatch batch = entry.getValue();
- result = batch.getBatch(!forwardOnly, types);
+ Long batch = entry.getValue();
+ List<List<?>> rows = manager.getBatch(batch, !forwardOnly);
+ result = new TupleBatch(entry.getKey(), rows);
+ if (isFinal && result.getEndRow() == rowCount) {
+ result.setTerminationFlag(true);
+ }
if (forwardOnly) {
batches.remove(entry.getKey());
}
}
- result.setDataTypes(types);
if (isFinal && result.getEndRow() == rowCount) {
result.setTerminationFlag(true);
}
@@ -240,8 +236,8 @@
if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.DETAIL)) {
LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, "Removing TupleBuffer:", this.tupleSourceID); //$NON-NLS-1$
}
- if (this.lobStore != null) {
- this.lobStore.remove();
+ if (this.lobManager != null) {
+ this.lobManager.remove();
}
this.batchBuffer = null;
purge();
@@ -359,18 +355,18 @@
}
public void setPrefersMemory(boolean prefersMemory) {
- this.prefersMemory = prefersMemory;
- for (ManagedBatch batch : this.batches.values()) {
- batch.setPrefersMemory(prefersMemory);
- }
+ this.manager.setPrefersMemory(prefersMemory);
}
- public boolean isPrefersMemory() {
- return prefersMemory;
+ public String[] getTypes() {
+ return manager.getTypes();
}
- public String[] getTypes() {
- return types;
+ public int getLobCount() {
+ if (this.lobManager == null) {
+ return 0;
+ }
+ return this.lobManager.getLobCount();
}
}
Property changes on: branches/as7/engine/src/main/java/org/teiid/common/buffer/TupleBuffer.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/main/java/org/teiid/common/buffer/TupleBuffer.java:3149-3217,3220-3275,3340-3349
/trunk/engine/src/main/java/org/teiid/common/buffer/TupleBuffer.java:3188-3450
+ /branches/7.4.x/engine/src/main/java/org/teiid/common/buffer/TupleBuffer.java:3149-3217,3220-3275,3340-3349
/trunk/engine/src/main/java/org/teiid/common/buffer/TupleBuffer.java:3188-3450,3452-3506
Copied: branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/BlockBitSetTree.java (from rev 3506, trunk/engine/src/main/java/org/teiid/common/buffer/impl/BlockBitSetTree.java)
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/BlockBitSetTree.java (rev 0)
+++ branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/BlockBitSetTree.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -0,0 +1,191 @@
+/*
+ * JBoss, Home of Professional Open Source.
+ * See the COPYRIGHT.txt file distributed with this work for information
+ * regarding copyright ownership. Some portions may be licensed
+ * to Red Hat, Inc. under one or more contributor license agreements.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301 USA.
+ */
+
+ package org.teiid.common.buffer.impl;
+
+import java.util.BitSet;
+
+import org.teiid.common.buffer.impl.FileStoreCache.BlockInfo;
+
+/**
+ * Extends a {@link BitSet} by adding a cumulative total and a
+ * first level index to speed queries against large bitsets.
+ */
+public class BlockBitSetTree {
+
+ private static final int LOG_BITS_PER_BLOCK = FileStoreCache.LOG_BLOCK_SIZE + 3;
+ private static final int MAX_TOP_VALUE = 1 << LOG_BITS_PER_BLOCK;
+ private int maxIndex;
+ private int bitsSet;
+ private int[] topVals;
+ private BlockManager blockManager;
+ private int blockCount = 0;
+
+ public BlockBitSetTree(int maxIndex, BlockManager blockManager) {
+ this.maxIndex = maxIndex;
+ this.blockManager = blockManager;
+ this.topVals = new int[(maxIndex >> (FileStoreCache.LOG_BLOCK_SIZE + 3)) + 1];
+ }
+
+ public int getMaxIndex() {
+ return maxIndex;
+ }
+
+ /**
+ * Set the given bit at the index.
+ * @param bitIndex
+ * @param value
+ */
+ public synchronized void set(int bitIndex, boolean value) {
+ getOrSet(bitIndex, value, true);
+ }
+
+ public synchronized boolean get(int bitIndex) {
+ return getOrSet(bitIndex, false, false);
+ }
+
+ private boolean getOrSet(int bitIndex, boolean value, boolean update) {
+ if (bitIndex > maxIndex) {
+ throw new ArrayIndexOutOfBoundsException(bitIndex);
+ }
+ int blockIndex = bitIndex>>LOG_BITS_PER_BLOCK;
+ BlockInfo bb = null;
+ if (blockIndex >= blockCount) {
+ if (!update) {
+ return false;
+ }
+ for (; blockCount < blockIndex+1; blockCount++) {
+ bb = blockManager.allocateBlock(blockCount);
+ bb.buf.position(0);
+ int longsPerBlock = FileStoreCache.BLOCK_SIZE >> 6;
+ for (int j = 0; j < longsPerBlock; j++) {
+ bb.buf.putLong(0);
+ }
+ }
+ } else {
+ bb = blockManager.getBlock(blockIndex);
+ }
+ int relativeIndex = bitIndex&(MAX_TOP_VALUE-1);
+ int longByteIndex = (relativeIndex>>6)<<3;
+ long word = bb.buf.getLong(longByteIndex);
+ long mask = 1L << bitIndex;
+ boolean currentValue = ((word & mask) != 0);
+ if (!update) {
+ return currentValue;
+ }
+ if (currentValue == value) {
+ return currentValue;
+ }
+ if (value) {
+ word |= mask;
+ } else {
+ word &= ~mask;
+ }
+ bb.buf.putLong(longByteIndex, word);
+ blockManager.updateBlock(bb);
+ int topIndex = bitIndex >> LOG_BITS_PER_BLOCK;
+ int increment = value?1:-1;
+ bitsSet+=increment;
+ topVals[topIndex]+=increment;
+ return currentValue;
+ }
+
+ public synchronized int getBitsSet() {
+ return bitsSet;
+ }
+
+ public synchronized int nextClearBit(int fromIndex) {
+ int start = fromIndex >> LOG_BITS_PER_BLOCK;
+ for (int i = start; i < topVals.length; i++) {
+ if (topVals[i] == MAX_TOP_VALUE) {
+ continue;
+ }
+ if (topVals[i] == 0) {
+ if (i == start) {
+ return fromIndex;
+ }
+ return i * MAX_TOP_VALUE;
+ }
+ int relativeIndex = 0;
+ if (i == start) {
+ relativeIndex = fromIndex&(MAX_TOP_VALUE-1);
+ }
+ BlockInfo bb = blockManager.getBlock(i);
+
+ int longByteIndex = (relativeIndex>>6)<<3;
+
+ long word = ~bb.buf.getLong(longByteIndex) & (-1l << relativeIndex);
+
+ while (true) {
+ if (word != 0) {
+ return longByteIndex*8 + (i * MAX_TOP_VALUE) + Long.numberOfTrailingZeros(word);
+ }
+ longByteIndex+=8;
+ if (longByteIndex > FileStoreCache.BLOCK_MASK) {
+ break;
+ }
+ word = ~bb.buf.getLong(longByteIndex);
+ }
+ }
+ return -1;
+ }
+
+ public synchronized int nextSetBit(int fromIndex) {
+ if (bitsSet == 0) {
+ return -1;
+ }
+ int start = fromIndex >> LOG_BITS_PER_BLOCK;
+ for (int i = start; i < topVals.length; i++) {
+ if (topVals[i] == 0) {
+ continue;
+ }
+ if (topVals[i] == MAX_TOP_VALUE) {
+ if (i == start) {
+ return fromIndex;
+ }
+ return i * MAX_TOP_VALUE;
+ }
+ int relativeIndex = 0;
+ if (i == start) {
+ relativeIndex = fromIndex&(MAX_TOP_VALUE-1);
+ }
+ BlockInfo bb = blockManager.getBlock(i);
+
+ int longByteIndex = (relativeIndex>>6)<<3;
+
+ long word = bb.buf.getLong(longByteIndex) & (-1l << relativeIndex);
+
+ while (true) {
+ if (word != 0) {
+ return longByteIndex*8 + (i * MAX_TOP_VALUE) + Long.numberOfTrailingZeros(word);
+ }
+ longByteIndex+=8;
+ if (longByteIndex > FileStoreCache.BLOCK_MASK) {
+ break;
+ }
+ word = bb.buf.getLong(longByteIndex);
+ }
+ }
+ return -1;
+ }
+
+}
Copied: branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/BlockClosedLongIntHashTable.java (from rev 3506, trunk/engine/src/main/java/org/teiid/common/buffer/impl/BlockClosedLongIntHashTable.java)
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/BlockClosedLongIntHashTable.java (rev 0)
+++ branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/BlockClosedLongIntHashTable.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -0,0 +1,238 @@
+/*
+ * JBoss, Home of Professional Open Source.
+ * See the COPYRIGHT.txt file distributed with this work for information
+ * regarding copyright ownership. Some portions may be licensed
+ * to Red Hat, Inc. under one or more contributor license agreements.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301 USA.
+ */
+
+package org.teiid.common.buffer.impl;
+
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.teiid.common.buffer.impl.FileStoreCache.BlockInfo;
+
+/**
+ * Represents the logical structure of a cache group / directory
+ *
+ * Implemented by a closed hash table using a single step linear probe with delayed removal.
+ * Uses power of 2 hashing.
+ *
+ * Provides an extremely simple hash structure that rivals {@link HashMap} performance and
+ * is directly mapped to {@link ByteBuffer}s to avoid serialization overhead.
+ *
+ * Does not expect keys or values to be negative.
+ */
+public class BlockClosedLongIntHashTable {
+
+ private enum Mode {
+ GET,
+ UPDATE,
+ REMOVE
+ }
+
+ private static final int BYTES_PER_ROW = 12; //8+4
+ private static final int BLOCK_SIZE = FileStoreCache.BLOCK_DATA_BYTES/BYTES_PER_ROW;
+ private static final float LOAD_FACTOR = .7f;
+ private static final int MIN_SIZE = 1 << (31 - Integer.numberOfLeadingZeros(BLOCK_SIZE)); //should fit in a single block
+
+ static final int EMPTY = -1;
+ private static final int REMOVED = -2;
+
+ protected int size;
+ protected int capacityMask = EMPTY;
+ protected BlockManager blockManager;
+
+ public BlockClosedLongIntHashTable(BlockManager blockManager) {
+ this.blockManager = blockManager;
+ }
+
+ private void init(int capacity) {
+ int currentBlockCount = blockCount(capacityMask + 1);
+ int desiredBlockCount = blockCount(capacity);
+ if (capacity > capacityMask) {
+ for (int i = currentBlockCount; i < desiredBlockCount; i++) {
+ BlockInfo bb = blockManager.allocateBlock(i);
+ empty(bb.buf, BLOCK_SIZE);
+ blockManager.updateBlock(bb);
+ }
+ }
+ capacityMask = capacity - 1;
+ }
+
+ private void empty(ByteBuffer bb, int toIndex) {
+ bb.position(0);
+ for (int j = 0; j < toIndex; j++) {
+ bb.putLong(j * BYTES_PER_ROW, EMPTY);
+ }
+ }
+
+ private static int blockCount(int size) {
+ int currentBlockCount = (size) / BLOCK_SIZE;
+ if (size%BLOCK_SIZE > 0) {
+ currentBlockCount++;
+ }
+ return currentBlockCount;
+ }
+
+ public synchronized int put(long key, int value) {
+ int result = getOrUpdate(key, value, Mode.UPDATE);
+ if ((result == EMPTY || result == REMOVED) && ++size > LOAD_FACTOR*capacityMask) {
+ int newCapacity = (capacityMask+1)<<1;
+ int oldLength = capacityMask + 1;
+ init(newCapacity);
+ rehash(oldLength, newCapacity);
+ }
+ return result;
+ }
+
+ public synchronized int get(long key) {
+ return getOrUpdate(key, EMPTY, Mode.GET);
+ }
+
+ public synchronized int remove(long key) {
+ int result = getOrUpdate(key, EMPTY, Mode.REMOVE);
+ if (result != EMPTY && --size*LOAD_FACTOR < capacityMask>>3 && (capacityMask+1)>>1 >= MIN_SIZE) {
+ //reduce the size of the table by half
+ int oldLength = capacityMask + 1;
+ capacityMask >>= 1;
+ rehash(oldLength, oldLength>>1);
+ int oldBlocks = blockCount(oldLength);
+ int newBlocks = blockCount(capacityMask +1);
+ for (int i = oldBlocks-1; i >= newBlocks; i--) {
+ blockManager.freeBlock(i);
+ }
+ }
+ return result;
+ }
+
+ private void rehash(int oldLength, int newLength) {
+ BlockInfo lastBlockInfo = null;
+ ByteBuffer lastBlock = null;
+ for (int i = 0; i < oldLength; i++) {
+ int relativeIndex = i%BLOCK_SIZE;
+ if (lastBlock == null || relativeIndex == 0) {
+ int lastIndex = i/BLOCK_SIZE;
+ lastBlockInfo = blockManager.getBlock(lastIndex);
+ lastBlock = lastBlockInfo.buf;
+ if (i < newLength) {
+ byte[] buf = new byte[FileStoreCache.BLOCK_DATA_BYTES];
+ lastBlock.position(0);
+ lastBlock.get(buf);
+ ByteBuffer copyBlock = ByteBuffer.wrap(buf);
+ empty(lastBlock, Math.min(BLOCK_SIZE, oldLength - lastIndex*BLOCK_SIZE));
+ blockManager.updateBlock(lastBlockInfo);
+ lastBlock = copyBlock;
+ }
+ }
+ lastBlock.position(relativeIndex*BYTES_PER_ROW);
+ long oldKey = lastBlock.getLong();
+ int oldValue = lastBlock.getInt();
+ if (oldKey != REMOVED && oldKey != EMPTY) {
+ getOrUpdate(oldKey, oldValue, Mode.UPDATE);
+ }
+ }
+ }
+
+ @SuppressWarnings("null")
+ protected int getOrUpdate(long key, int value, Mode mode) {
+ if (capacityMask == EMPTY) {
+ if (mode == Mode.GET || mode == Mode.REMOVE) {
+ return EMPTY;
+ }
+ init(MIN_SIZE);
+ }
+ int i = hashIndex(key);
+ BlockInfo lastBlockInfo = null;
+ long old = EMPTY;
+ int position = 0;
+ while (true) {
+ int relativeIndex = i%BLOCK_SIZE;
+ if (lastBlockInfo == null || relativeIndex == 0) {
+ int index = i/BLOCK_SIZE;
+ lastBlockInfo = blockManager.getBlock(index);
+ }
+ position = relativeIndex*BYTES_PER_ROW;
+ old = lastBlockInfo.buf.getLong(position);
+ if (old == EMPTY || old == key || (mode == Mode.UPDATE && old == REMOVED)) {
+ break;
+ }
+ i = (i + 1) & capacityMask;
+ }
+ int result = EMPTY;
+ if (old != EMPTY && old != REMOVED) {
+ result = lastBlockInfo.buf.getInt(position + 8);
+ }
+ switch (mode) {
+ case GET:
+ return result;
+ case UPDATE:
+ lastBlockInfo.buf.putLong(position, key);
+ lastBlockInfo.buf.putInt(position + 8, value);
+ blockManager.updateBlock(lastBlockInfo);
+ return result;
+ case REMOVE:
+ if (old == EMPTY || old == REMOVED) {
+ return EMPTY;
+ }
+ lastBlockInfo.buf.putLong(position, REMOVED);
+ blockManager.updateBlock(lastBlockInfo);
+ return result;
+ default:
+ throw new AssertionError();
+ }
+ }
+
+ private int hashIndex(long key) {
+ //start with the usual long hash
+ int primaryHash = (int)(key ^ (key >>> 32));
+ //allow the lower bits to spread the entries
+ primaryHash += primaryHash <<= 2;
+ primaryHash += primaryHash <<= 3;
+ return primaryHash & capacityMask;
+ }
+
+ public synchronized int size() {
+ return size;
+ }
+
+ public synchronized Map<Long, Integer> remove() {
+ Map<Long, Integer> result = new HashMap<Long, Integer>();
+ BlockInfo lastBlockInfo = null;
+ int blockIndex = 0;
+ for (int i = capacityMask; i >= 0; i--) {
+ int relativeIndex = i%BLOCK_SIZE;
+ if (lastBlockInfo == null || relativeIndex == BLOCK_SIZE - 1) {
+ if (lastBlockInfo != null) {
+ blockManager.freeBlock(blockIndex);
+ }
+ blockIndex = i/BLOCK_SIZE;
+ lastBlockInfo = blockManager.getBlock(blockIndex);
+ }
+ lastBlockInfo.buf.position(relativeIndex*BYTES_PER_ROW);
+ long key = lastBlockInfo.buf.getLong();
+ if (key != EMPTY && key != REMOVED) {
+ result.put(key, lastBlockInfo.buf.getInt());
+ }
+ }
+ blockManager.free();
+ return result;
+ }
+
+}
Copied: branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/BlockManager.java (from rev 3506, trunk/engine/src/main/java/org/teiid/common/buffer/impl/BlockManager.java)
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/BlockManager.java (rev 0)
+++ branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/BlockManager.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -0,0 +1,51 @@
+/*
+ * JBoss, Home of Professional Open Source.
+ * See the COPYRIGHT.txt file distributed with this work for information
+ * regarding copyright ownership. Some portions may be licensed
+ * to Red Hat, Inc. under one or more contributor license agreements.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301 USA.
+ */
+
+package org.teiid.common.buffer.impl;
+
+import org.teiid.common.buffer.impl.FileStoreCache.BlockInfo;
+
+/**
+ * Represents an INode
+ *
+ * Returned BlockInfo may be shared. If shared there and no guarantees about position and mark.
+ * in particular system/index blocks can be used by multiple threads relative methods should be
+ * avoided, but may be used for exclusive write operations.
+ * Otherwise the position will be 0.
+ *
+ * Due to buffermanager locking, non-index data blocks can be assumed to be thread-safe.
+ */
+public interface BlockManager {
+
+ int getInode();
+
+ BlockInfo allocateBlock(int index);
+
+ BlockInfo getBlock(int index);
+
+ void updateBlock(BlockInfo block);
+
+ void freeBlock(int index);
+
+ void free();
+
+}
Modified: branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/BufferManagerImpl.java
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/BufferManagerImpl.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/BufferManagerImpl.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -22,27 +22,42 @@
package org.teiid.common.buffer.impl;
-import java.io.*;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.OutputStream;
+import java.io.Serializable;
import java.lang.ref.Reference;
import java.lang.ref.ReferenceQueue;
import java.lang.ref.SoftReference;
import java.lang.ref.WeakReference;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.Condition;
-import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.teiid.client.BatchSerializer;
import org.teiid.common.buffer.*;
-import org.teiid.common.buffer.BatchManager.ManagedBatch;
+import org.teiid.common.buffer.AutoCleanupUtil.Removable;
+import org.teiid.common.buffer.LobManager.ReferenceMode;
import org.teiid.core.TeiidComponentException;
import org.teiid.core.TeiidRuntimeException;
import org.teiid.core.types.DataTypeManager;
-import org.teiid.core.util.Assertion;
+import org.teiid.core.types.DataTypeManager.WeakReferenceHashedValueCache;
import org.teiid.dqp.internal.process.DQPConfiguration;
+import org.teiid.dqp.internal.process.SerializableTupleBatch;
import org.teiid.logging.LogConstants;
import org.teiid.logging.LogManager;
import org.teiid.logging.MessageLevel;
@@ -57,341 +72,170 @@
* <p>Default implementation of BufferManager.</p>
* Responsible for creating/tracking TupleBuffers and providing access to the StorageManager.
* </p>
- * The buffering strategy attempts to purge batches from the least recently used TupleBuffer
- * from before (which wraps around circularly) the last used batch. This attempts to compensate
- * for our tendency to read buffers in a forward manner. If our processing algorithms are changed
- * to use alternating ascending/descending access, then the buffering approach could be replaced
- * with a simple LRU.
*
- * TODO: allow for cached stores to use lru - (result set/mat view)
- * TODO: account for row/content based sizing (difficult given value sharing)
- * TODO: account for memory based lobs (it would be nice if the approximate buffer size matched at 100kB)
* TODO: add detection of pinned batches to prevent unnecessary purging of non-persistent batches
* - this is not necessary for already persistent batches, since we hold a weak reference
*/
public class BufferManagerImpl implements BufferManager, StorageManager, ReplicatedObject {
- private static final int IO_BUFFER_SIZE = 1 << 14;
- private static final int COMPACTION_THRESHOLD = 1 << 25; //start checking at 32 megs
-
- private final class CleanupHook implements org.teiid.common.buffer.BatchManager.CleanupHook {
-
- private long id;
- private int beginRow;
- private WeakReference<BatchManagerImpl> ref;
-
- CleanupHook(long id, int beginRow, BatchManagerImpl batchManager) {
- this.id = id;
- this.beginRow = beginRow;
- this.ref = new WeakReference<BatchManagerImpl>(batchManager);
+ private final class BatchManagerImpl implements BatchManager, Serializer<List<? extends List<?>>> {
+ final Long id;
+ SizeUtility sizeUtility;
+ private WeakReference<BatchManagerImpl> ref = new WeakReference<BatchManagerImpl>(this);
+ AtomicBoolean prefersMemory = new AtomicBoolean();
+ String[] types;
+ private LobManager lobManager;
+
+ private BatchManagerImpl(Long newID, String[] types) {
+ this.id = newID;
+ this.sizeUtility = new SizeUtility(types);
+ this.types = types;
+ cache.createCacheGroup(newID);
}
- public void cleanup() {
- BatchManagerImpl batchManager = ref.get();
- if (batchManager == null) {
- return;
- }
- cleanupManagedBatch(batchManager, beginRow, id);
+ @Override
+ public Long getId() {
+ return id;
}
- }
-
- private final class BatchManagerImpl implements BatchManager {
- private final String id;
- private volatile FileStore store;
- private Map<Long, long[]> physicalMapping = new ConcurrentHashMap<Long, long[]>();
- private ReadWriteLock compactionLock = new ReentrantReadWriteLock();
- private AtomicLong unusedSpace = new AtomicLong();
- private int[] lobIndexes;
- private SizeUtility sizeUtility;
-
- private BatchManagerImpl(String newID, int[] lobIndexes) {
- this.id = newID;
- this.store = createFileStore(id);
- this.store.setCleanupReference(this);
- this.lobIndexes = lobIndexes;
- this.sizeUtility = new SizeUtility();
+ public void setLobManager(LobManager lobManager) {
+ this.lobManager = lobManager;
}
- public FileStore createStorage(String prefix) {
- return createFileStore(id+prefix);
- }
-
@Override
- public ManagedBatch createManagedBatch(TupleBatch batch, boolean softCache)
- throws TeiidComponentException {
- ManagedBatchImpl mbi = new ManagedBatchImpl(batch, this, softCache);
- mbi.addToCache(false);
- persistBatchReferences();
- return mbi;
+ public String[] getTypes() {
+ return types;
}
- private boolean shouldCompact(long offset) {
- return offset > COMPACTION_THRESHOLD && unusedSpace.get() * 4 > offset * 3;
+ @Override
+ public boolean prefersMemory() {
+ return prefersMemory.get();
}
- private long getOffset() throws TeiidComponentException {
- long offset = store.getLength();
- if (!shouldCompact(offset)) {
- return offset;
- }
- try {
- this.compactionLock.writeLock().lock();
- offset = store.getLength();
- //retest the condition to ensure that compaction is still needed
- if (!shouldCompact(offset)) {
- return offset;
- }
- FileStore newStore = createFileStore(id);
- newStore.setCleanupReference(this);
- byte[] buffer = new byte[IO_BUFFER_SIZE];
- List<long[]> values = new ArrayList<long[]>(physicalMapping.values());
- Collections.sort(values, new Comparator<long[]>() {
- @Override
- public int compare(long[] o1, long[] o2) {
- return Long.signum(o1[0] - o2[0]);
- }
- });
- for (long[] info : values) {
- long oldOffset = info[0];
- info[0] = newStore.getLength();
- int size = (int)info[1];
- while (size > 0) {
- int toWrite = Math.min(IO_BUFFER_SIZE, size);
- store.readFully(oldOffset, buffer, 0, toWrite);
- newStore.write(buffer, 0, toWrite);
- size -= toWrite;
- }
- }
- store.remove();
- store = newStore;
- long oldOffset = offset;
- offset = store.getLength();
- if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.TRACE)) {
- LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, "Compacted store", id, "pre-size", oldOffset, "post-size", offset); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
- }
- return offset;
- } finally {
- this.compactionLock.writeLock().unlock();
- }
- }
-
@Override
- public void remove() {
- this.store.remove();
+ public void setPrefersMemory(boolean prefers) {
+ //TODO: it's only expected to move from not preferring to prefefring
+ this.prefersMemory.set(prefers);
}
- }
-
- /**
- * Holder for active batches
- */
- private class TupleBufferInfo {
- TreeMap<Integer, ManagedBatchImpl> batches = new TreeMap<Integer, ManagedBatchImpl>();
- Integer lastUsed = null;
- ManagedBatchImpl removeBatch(int row) {
- ManagedBatchImpl result = batches.remove(row);
- if (result != null) {
- activeBatchKB -= result.sizeEstimate;
- }
- return result;
+ @Override
+ public boolean useSoftCache() {
+ return prefersMemory.get();
}
- }
-
- private final class ManagedBatchImpl implements ManagedBatch {
- private boolean persistent;
- private boolean softCache;
- private volatile TupleBatch activeBatch;
- private volatile Reference<TupleBatch> batchReference;
- private int beginRow;
- private BatchManagerImpl batchManager;
- private long id;
- private LobManager lobManager;
- private int sizeEstimate;
- public ManagedBatchImpl(TupleBatch batch, BatchManagerImpl manager, boolean softCache) {
- this.softCache = softCache;
- id = batchAdded.incrementAndGet();
- this.activeBatch = batch;
- this.beginRow = batch.getBeginRow();
- this.batchManager = manager;
- if (this.batchManager.lobIndexes != null) {
- this.lobManager = new LobManager();
- }
- sizeEstimate = (int) Math.max(1, manager.sizeUtility.getBatchSize(batch) / 1024);
- if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.TRACE)) {
- LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, "Add batch to BufferManager", id, "with size estimate", sizeEstimate); //$NON-NLS-1$ //$NON-NLS-2$
- }
+ @Override
+ public Reference<? extends BatchManager> getBatchManagerReference() {
+ return ref;
}
@Override
- public void setPrefersMemory(boolean prefers) {
- this.softCache = prefers;
- //TODO: could recreate the reference
+ public Long createManagedBatch(List<? extends List<?>> batch)
+ throws TeiidComponentException {
+ int sizeEstimate = getSizeEstimate(batch);
+ Long oid = batchAdded.getAndIncrement();
+ CacheEntry ce = new CacheEntry(oid);
+ ce.setObject(batch);
+ ce.setSizeEstimate(sizeEstimate);
+ ce.setSerializer(this.ref);
+ return addCacheEntry(ce, this);
}
- private void addToCache(boolean update) {
- synchronized (activeBatches) {
- TupleBatch batch = this.activeBatch;
- if (batch == null) {
- return; //already removed
+ @Override
+ public List<? extends List<?>> deserialize(ObjectInputStream ois)
+ throws IOException, ClassNotFoundException {
+ List<? extends List<?>> batch = BatchSerializer.readBatch(ois, types);
+ if (lobManager != null) {
+ for (List<?> list : batch) {
+ try {
+ lobManager.updateReferences(list, ReferenceMode.ATTACH);
+ } catch (TeiidComponentException e) {
+ throw new TeiidRuntimeException(e);
+ }
}
- activeBatchKB += sizeEstimate;
- TupleBufferInfo tbi = null;
- if (update) {
- tbi = activeBatches.remove(batchManager.id);
- } else {
- tbi = activeBatches.get(batchManager.id);
- }
- if (tbi == null) {
- tbi = new TupleBufferInfo();
- update = true;
- }
- if (update) {
- activeBatches.put(batchManager.id, tbi);
- }
- Assertion.isNull(tbi.batches.put(this.beginRow, this));
}
+ return batch;
}
-
+
@Override
- public TupleBatch getBatch(boolean cache, String[] types) throws TeiidComponentException {
+ public void serialize(List<? extends List<?>> obj,
+ ObjectOutputStream oos) throws IOException {
+ //it's expected that the containing structure has updated the lob manager
+ BatchSerializer.writeBatch(oos, types, obj);
+ }
+
+ public int getSizeEstimate(List<? extends List<?>> obj) {
+ return (int) Math.max(1, sizeUtility.getBatchSize(obj) / 1024);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public List<List<?>> getBatch(Long batch, boolean retain)
+ throws TeiidComponentException {
+ cleanSoftReferences();
long reads = readAttempts.incrementAndGet();
if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.TRACE)) {
- LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, batchManager.id, "getting batch", reads, "reference hits", referenceHit.get()); //$NON-NLS-1$ //$NON-NLS-2$
+ LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, id, "getting batch", batch, "total reads", reads, "reference hits", referenceHit.get()); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
}
- synchronized (activeBatches) {
- TupleBufferInfo tbi = activeBatches.remove(batchManager.id);
- if (tbi != null) {
- boolean put = true;
- if (!cache) {
- tbi.removeBatch(this.beginRow);
- if (tbi.batches.isEmpty()) {
- put = false;
- }
- }
- if (put) {
- tbi.lastUsed = this.beginRow;
- activeBatches.put(batchManager.id, tbi);
- }
- }
+ CacheEntry ce = fastGet(batch, prefersMemory.get(), retain);
+ if (ce != null) {
+ return (List<List<?>>)(!retain?ce.nullOut():ce.getObject());
}
- persistBatchReferences();
synchronized (this) {
- TupleBatch batch = this.activeBatch;
- if (batch != null){
- return batch;
+ ce = fastGet(batch, prefersMemory.get(), retain);
+ if (ce != null) {
+ return (List<List<?>>)(!retain?ce.nullOut():ce.getObject());
}
- Reference<TupleBatch> ref = this.batchReference;
- this.batchReference = null;
- if (ref != null) {
- batch = ref.get();
- if (batch != null) {
- if (cache) {
- this.activeBatch = batch;
- addToCache(true);
- }
- referenceHit.getAndIncrement();
- return batch;
- }
- }
long count = readCount.incrementAndGet();
if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.TRACE)) {
- LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, batchManager.id, id, "reading batch from disk, total reads:", count); //$NON-NLS-1$
+ LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, id, id, "reading batch", batch, "from storage, total reads:", count); //$NON-NLS-1$ //$NON-NLS-2$
}
- try {
- this.batchManager.compactionLock.readLock().lock();
- long[] info = batchManager.physicalMapping.get(this.id);
- Assertion.isNotNull(info, "Invalid batch " + id); //$NON-NLS-1$
- ObjectInputStream ois = new ObjectInputStream(new BufferedInputStream(batchManager.store.createInputStream(info[0]), IO_BUFFER_SIZE));
- batch = new TupleBatch();
- batch.setDataTypes(types);
- batch.readExternal(ois);
- batch.setRowOffset(this.beginRow);
- batch.setDataTypes(null);
- if (lobManager != null) {
- for (List<?> tuple : batch.getTuples()) {
- lobManager.updateReferences(batchManager.lobIndexes, tuple);
- }
- }
- if (cache) {
- this.activeBatch = batch;
- addToCache(true);
- }
- return batch;
- } catch(IOException e) {
- throw new TeiidComponentException(e, QueryPlugin.Util.getString("FileStoreageManager.error_reading", batchManager.id)); //$NON-NLS-1$
- } catch (ClassNotFoundException e) {
- throw new TeiidComponentException(e, QueryPlugin.Util.getString("FileStoreageManager.error_reading", batchManager.id)); //$NON-NLS-1$
- } finally {
- this.batchManager.compactionLock.readLock().unlock();
- }
- }
- }
-
- public synchronized void persist() throws TeiidComponentException {
- boolean lockheld = false;
- try {
- TupleBatch batch = activeBatch;
- if (batch != null) {
- if (!persistent) {
- long count = writeCount.incrementAndGet();
- if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.DETAIL)) {
- LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, batchManager.id, id, "writing batch to disk, total writes: ", count); //$NON-NLS-1$
- }
- long offset = 0;
- if (lobManager != null) {
- for (List<?> tuple : batch.getTuples()) {
- lobManager.updateReferences(batchManager.lobIndexes, tuple);
- }
- }
- synchronized (batchManager.store) {
- offset = batchManager.getOffset();
- OutputStream fsos = new BufferedOutputStream(batchManager.store.createOutputStream(), IO_BUFFER_SIZE);
- ObjectOutputStream oos = new ObjectOutputStream(fsos);
- batch.writeExternal(oos);
- oos.close();
- long size = batchManager.store.getLength() - offset;
- long[] info = new long[] {offset, size};
- batchManager.physicalMapping.put(this.id, info);
- }
- if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.TRACE)) {
- LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, batchManager.id, id, "batch written starting at:", offset); //$NON-NLS-1$
- }
- }
- if (softCache) {
- this.batchReference = new SoftReference<TupleBatch>(batch);
- } else if (useWeakReferences) {
- this.batchReference = new WeakReference<TupleBatch>(batch);
- }
+ ce = cache.get(batch, this);
+ if (ce == null) {
+ throw new AssertionError("Batch not found in storage " + batch); //$NON-NLS-1$
}
- } catch (IOException e) {
- throw new TeiidComponentException(e);
- } catch (Throwable e) {
- throw new TeiidComponentException(e);
- } finally {
- persistent = true;
- activeBatch = null;
- if (lockheld) {
- this.batchManager.compactionLock.writeLock().unlock();
+ if (!retain) {
+ cache.remove(this.id, batch);
}
- }
+ ce.setSerializer(this.ref);
+ if (retain) {
+ addMemoryEntry(ce);
+ }
+ }
+ return (List<List<?>>)ce.getObject();
}
+
+ @Override
+ public void remove(Long batch) {
+ cleanSoftReferences();
+ BufferManagerImpl.this.remove(id, batch, prefersMemory.get());
+ }
+ @Override
public void remove() {
- cleanupManagedBatch(batchManager, beginRow, id);
+ removeCacheGroup(id, prefersMemory.get());
}
-
+
@Override
- public CleanupHook getCleanupHook() {
- return new CleanupHook(id, beginRow, batchManager);
+ public String toString() {
+ return id.toString();
}
+ }
+
+ private static class BatchSoftReference extends SoftReference<CacheEntry> {
+
+ private int sizeEstimate;
+ private Long key;
- @Override
- public String toString() {
- return "ManagedBatch " + batchManager.id + " " + this.beginRow + " " + activeBatch; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
+ public BatchSoftReference(CacheEntry referent,
+ ReferenceQueue<? super CacheEntry> q, int sizeEstimate) {
+ super(referent, q);
+ this.sizeEstimate = sizeEstimate;
+ this.key = referent.getId();
}
}
+
+ private static final int TARGET_BYTES_PER_ROW = 1 << 11; //2k bytes per row
+ private static ReferenceQueue<CacheEntry> SOFT_QUEUE = new ReferenceQueue<CacheEntry>();
// Configuration
private int connectorBatchSize = BufferManager.DEFAULT_CONNECTOR_BATCH_SIZE;
@@ -399,21 +243,44 @@
//set to acceptable defaults for testing
private int maxProcessingKB = 1 << 11;
private Integer maxProcessingKBOrig;
- private int maxReserveKB = 1 << 25;
+ private AtomicInteger maxReserveKB = new AtomicInteger(1 << 18);
private volatile int reserveBatchKB;
private int maxActivePlans = DQPConfiguration.DEFAULT_MAX_ACTIVE_PLANS; //used as a hint to set the reserveBatchKB
private boolean useWeakReferences = true;
+ private boolean inlineLobs = true;
+ private int targetBytesPerRow = TARGET_BYTES_PER_ROW;
+ private int maxSoftReferences;
private ReentrantLock lock = new ReentrantLock(true);
private Condition batchesFreed = lock.newCondition();
- private volatile int activeBatchKB = 0;
- private Map<String, TupleBufferInfo> activeBatches = new LinkedHashMap<String, TupleBufferInfo>();
+ private AtomicInteger activeBatchKB = new AtomicInteger();
+
+ //tiered memory entries. the first tier is just a queue of adds/gets. once accessed again, the entry moves to the tenured tier.
+ private LinkedHashMap<Long, CacheEntry> memoryEntries = new LinkedHashMap<Long, CacheEntry>(16, .75f, false);
+ private LinkedHashMap<Long, CacheEntry> tenuredMemoryEntries = new LinkedHashMap<Long, CacheEntry>(16, .75f, true);
+
+ //limited size reference caches based upon the memory settings
+ private WeakReferenceHashedValueCache<CacheEntry> weakReferenceCache;
+ private Map<Long, BatchSoftReference> softCache = Collections.synchronizedMap(new LinkedHashMap<Long, BatchSoftReference>(16, .75f, false) {
+ private static final long serialVersionUID = 1L;
+
+ protected boolean removeEldestEntry(Map.Entry<Long,BatchSoftReference> eldest) {
+ if (size() > maxSoftReferences) {
+ BatchSoftReference bsr = eldest.getValue();
+ maxReserveKB.addAndGet(bsr.sizeEstimate);
+ bsr.clear();
+ return true;
+ }
+ return false;
+ };
+ });
+
+ Cache cache;
+
private Map<String, TupleReference> tupleBufferMap = new ConcurrentHashMap<String, TupleReference>();
private ReferenceQueue<TupleBuffer> tupleBufferQueue = new ReferenceQueue<TupleBuffer>();
- private StorageManager diskMgr;
-
private AtomicLong tsId = new AtomicLong();
private AtomicLong batchAdded = new AtomicLong();
private AtomicLong readCount = new AtomicLong();
@@ -463,58 +330,51 @@
public void setConnectorBatchSize(int connectorBatchSize) {
this.connectorBatchSize = connectorBatchSize;
}
+
+ public void setTargetBytesPerRow(int targetBytesPerRow) {
+ this.targetBytesPerRow = targetBytesPerRow;
+ }
public void setProcessorBatchSize(int processorBatchSize) {
this.processorBatchSize = processorBatchSize;
}
- /**
- * Add a storage manager to this buffer manager, order is unimportant
- * @param storageManager Storage manager to add
- */
- public void setStorageManager(StorageManager storageManager) {
- Assertion.isNotNull(storageManager);
- Assertion.isNull(diskMgr);
- this.diskMgr = storageManager;
- }
-
- public StorageManager getStorageManager() {
- return diskMgr;
- }
-
@Override
public TupleBuffer createTupleBuffer(final List elements, String groupName,
TupleSourceType tupleSourceType) {
- final String newID = String.valueOf(this.tsId.getAndIncrement());
+ final Long newID = this.tsId.getAndIncrement();
int[] lobIndexes = LobManager.getLobIndexes(elements);
- BatchManager batchManager = new BatchManagerImpl(newID, lobIndexes);
- TupleBuffer tupleBuffer = new TupleBuffer(batchManager, newID, elements, lobIndexes, getProcessorBatchSize());
+ String[] types = TupleBuffer.getTypeNames(elements);
+ BatchManagerImpl batchManager = createBatchManager(newID, types);
+ LobManager lobManager = null;
+ FileStore lobStore = null;
+ if (lobIndexes != null) {
+ lobStore = createFileStore(newID + "_lobs"); //$NON-NLS-1$
+ lobManager = new LobManager(lobIndexes, lobStore);
+ batchManager.setLobManager(lobManager);
+ }
+ TupleBuffer tupleBuffer = new TupleBuffer(batchManager, String.valueOf(newID), elements, lobManager, getProcessorBatchSize(elements));
+ if (lobStore != null) {
+ AutoCleanupUtil.setCleanupReference(batchManager, lobStore);
+ }
if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.DETAIL)) {
- LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, "Creating TupleBuffer:", newID, elements, Arrays.toString(tupleBuffer.getTypes()), "of type", tupleSourceType); //$NON-NLS-1$ //$NON-NLS-2$
+ LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, "Creating TupleBuffer:", newID, elements, Arrays.toString(types), "of type", tupleSourceType); //$NON-NLS-1$ //$NON-NLS-2$
}
+ tupleBuffer.setInlineLobs(inlineLobs);
return tupleBuffer;
}
- private void cleanupManagedBatch(BatchManagerImpl batchManager, int beginRow, long id) {
- synchronized (activeBatches) {
- TupleBufferInfo tbi = activeBatches.get(batchManager.id);
- if (tbi != null && tbi.removeBatch(beginRow) != null) {
- if (tbi.batches.isEmpty()) {
- activeBatches.remove(batchManager.id);
- }
- }
- }
- long[] info = batchManager.physicalMapping.remove(id);
- if (info != null) {
- batchManager.unusedSpace.addAndGet(info[1]);
- }
- }
-
public STree createSTree(final List elements, String groupName, int keyLength) {
- String newID = String.valueOf(this.tsId.getAndIncrement());
+ Long newID = this.tsId.getAndIncrement();
int[] lobIndexes = LobManager.getLobIndexes(elements);
- BatchManager bm = new BatchManagerImpl(newID, lobIndexes);
- BatchManager keyManager = new BatchManagerImpl(String.valueOf(this.tsId.getAndIncrement()), null);
+ String[] types = TupleBuffer.getTypeNames(elements);
+ BatchManagerImpl bm = createBatchManager(newID, types);
+ LobManager lobManager = null;
+ if (lobIndexes != null) {
+ lobManager = new LobManager(lobIndexes, null); //persistence is not expected yet - later we might utilize storage for out-of-line lob values
+ bm.setLobManager(lobManager);
+ }
+ BatchManager keyManager = createBatchManager(this.tsId.getAndIncrement(), Arrays.copyOf(types, keyLength));
int[] compareIndexes = new int[keyLength];
for (int i = 1; i < compareIndexes.length; i++) {
compareIndexes[i] = i;
@@ -522,16 +382,33 @@
if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.DETAIL)) {
LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, "Creating STree:", newID); //$NON-NLS-1$
}
- return new STree(keyManager, bm, new ListNestedSortComparator(compareIndexes), getProcessorBatchSize(), keyLength, TupleBuffer.getTypeNames(elements));
+ return new STree(keyManager, bm, new ListNestedSortComparator(compareIndexes), getProcessorBatchSize(elements), getProcessorBatchSize(elements.subList(0, keyLength)), keyLength, lobManager);
}
+ private BatchManagerImpl createBatchManager(final Long newID, String[] types) {
+ BatchManagerImpl bm = new BatchManagerImpl(newID, types);
+ final AtomicBoolean prefersMemory = bm.prefersMemory;
+ AutoCleanupUtil.setCleanupReference(bm, new Removable() {
+
+ @Override
+ public void remove() {
+ BufferManagerImpl.this.removeCacheGroup(newID, prefersMemory.get());
+ }
+ });
+ return bm;
+ }
+
@Override
public FileStore createFileStore(String name) {
if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.DETAIL)) {
LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, "Creating FileStore:", name); //$NON-NLS-1$
}
- return this.diskMgr.createFileStore(name);
+ return this.cache.createFileStore(name);
}
+
+ public Cache getCache() {
+ return cache;
+ }
public void setMaxActivePlans(int maxActivePlans) {
this.maxActivePlans = maxActivePlans;
@@ -542,23 +419,23 @@
}
public void setMaxReserveKB(int maxReserveBatchKB) {
- this.maxReserveKB = maxReserveBatchKB;
+ this.maxReserveKB.set(maxReserveBatchKB);
}
@Override
public void initialize() throws TeiidComponentException {
int maxMemory = (int)Math.min(Runtime.getRuntime().maxMemory() / 1024, Integer.MAX_VALUE);
maxMemory -= 300 * 1024; //assume 300 megs of overhead for the AS/system stuff
- if (maxReserveKB < 0) {
- this.maxReserveKB = 0;
+ if (getMaxReserveKB() < 0) {
+ this.setMaxReserveKB(0);
int one_gig = 1024 * 1024;
if (maxMemory > one_gig) {
//assume 75% of the memory over the first gig
- this.maxReserveKB += (int)Math.max(0, (maxMemory - one_gig) * .75);
+ this.maxReserveKB.addAndGet(((int)Math.max(0, (maxMemory - one_gig) * .75)));
}
- this.maxReserveKB += Math.max(0, Math.min(one_gig, maxMemory) * .5);
+ this.maxReserveKB.addAndGet(((int)Math.max(0, Math.min(one_gig, maxMemory) * .5)));
}
- this.reserveBatchKB = this.maxReserveKB;
+ this.reserveBatchKB = this.getMaxReserveKB();
if (this.maxProcessingKBOrig == null) {
//store the config value so that we can be reinitialized (this is not a clean approach)
this.maxProcessingKBOrig = this.maxProcessingKB;
@@ -566,6 +443,12 @@
if (this.maxProcessingKBOrig < 0) {
this.maxProcessingKB = Math.max(Math.min(8 * processorBatchSize, Integer.MAX_VALUE), (int)(.1 * maxMemory)/maxActivePlans);
}
+ int memoryBatches = (this.maxProcessingKB * maxActivePlans + this.getMaxReserveKB()) / (processorBatchSize * targetBytesPerRow / 1024);
+ int logSize = 39 - Integer.numberOfLeadingZeros(memoryBatches);
+ if (useWeakReferences) {
+ weakReferenceCache = new WeakReferenceHashedValueCache<CacheEntry>(Math.min(20, logSize));
+ }
+ this.maxSoftReferences = 1 << Math.max(28, logSize+2);
}
@Override
@@ -594,7 +477,7 @@
try {
if (mode == BufferReserveMode.WAIT) {
//don't wait for more than is available
- int waitCount = Math.min(count, this.maxReserveKB);
+ int waitCount = Math.min(count, this.getMaxReserveKB());
while (waitCount > 0 && waitCount > this.reserveBatchKB) {
try {
batchesFreed.await(100, TimeUnit.MILLISECONDS);
@@ -618,60 +501,233 @@
}
void persistBatchReferences() {
- if (activeBatchKB == 0 || activeBatchKB <= reserveBatchKB) {
- int memoryCount = activeBatchKB + maxReserveKB - reserveBatchKB;
+ if (activeBatchKB.get() == 0 || activeBatchKB.get() <= reserveBatchKB) {
+ int memoryCount = activeBatchKB.get() + getMaxReserveKB() - reserveBatchKB;
if (DataTypeManager.isValueCacheEnabled()) {
- if (memoryCount < maxReserveKB / 8) {
+ if (memoryCount < getMaxReserveKB() / 8) {
DataTypeManager.setValueCacheEnabled(false);
}
- } else if (memoryCount > maxReserveKB / 4) {
+ } else if (memoryCount > getMaxReserveKB() / 4) {
DataTypeManager.setValueCacheEnabled(true);
}
return;
}
+ boolean first = true;
while (true) {
- ManagedBatchImpl mb = null;
- synchronized (activeBatches) {
- if (activeBatchKB == 0 || activeBatchKB < reserveBatchKB * .8) {
+ CacheEntry ce = null;
+ synchronized (memoryEntries) {
+ if (activeBatchKB.get() == 0 || activeBatchKB.get() < reserveBatchKB * .8) {
break;
}
- Iterator<TupleBufferInfo> iter = activeBatches.values().iterator();
- TupleBufferInfo tbi = iter.next();
- Map.Entry<Integer, ManagedBatchImpl> entry = null;
- if (tbi.lastUsed != null) {
- entry = tbi.batches.floorEntry(tbi.lastUsed - 1);
+ if (first) { //let one entry per persist cycle loose its tenure. this helps us be more write avoident.
+ first = false;
+ if (!tenuredMemoryEntries.isEmpty()) {
+ Iterator<Map.Entry<Long, CacheEntry>> iter = tenuredMemoryEntries.entrySet().iterator();
+ Map.Entry<Long, CacheEntry> entry = iter.next();
+ iter.remove();
+ memoryEntries.put(entry.getKey(), entry.getValue());
+ }
}
- if (entry == null) {
- entry = tbi.batches.lastEntry();
+ LinkedHashMap<Long, CacheEntry> toDrain = memoryEntries;
+ if (memoryEntries.isEmpty()) {
+ toDrain = tenuredMemoryEntries;
+ if (tenuredMemoryEntries.isEmpty()) {
+ break;
+ }
+ }
+ Iterator<CacheEntry> iter = toDrain.values().iterator();
+ ce = iter.next();
+ iter.remove();
+ activeBatchKB.addAndGet(-ce.getSizeEstimate());
+ }
+ persist(ce);
+ }
+ }
+
+ void persist(CacheEntry ce) {
+ Serializer<?> s = ce.getSerializer().get();
+ if (s == null) {
+ return;
+ }
+ if (!ce.isPersistent()) {
+ long count = writeCount.incrementAndGet();
+ if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.DETAIL)) {
+ LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, ce.getId(), "writing batch to storage, total writes: ", count); //$NON-NLS-1$
+ }
+ cache.add(ce, s);
+ ce.setPersistent(true);
+ }
+ if (s.useSoftCache()) {
+ createSoftReference(ce);
+ } else if (useWeakReferences) {
+ weakReferenceCache.getValue(ce); //a get will set the value
+ }
+ }
+
+ private void createSoftReference(CacheEntry ce) {
+ BatchSoftReference ref = new BatchSoftReference(ce, SOFT_QUEUE, ce.getSizeEstimate()/2);
+ softCache.put(ce.getId(), ref);
+ maxReserveKB.addAndGet(- ce.getSizeEstimate()/2);
+ }
+
+ /**
+ * Get a CacheEntry without hitting the cache
+ */
+ CacheEntry fastGet(Long batch, boolean prefersMemory, boolean retain) {
+ CacheEntry ce = null;
+ synchronized (memoryEntries) {
+ if (retain) {
+ ce = tenuredMemoryEntries.get(batch);
+ if (ce == null) {
+ ce = memoryEntries.remove(batch);
+ if (ce != null) {
+ tenuredMemoryEntries.put(batch, ce);
+ }
}
- tbi.removeBatch(entry.getKey());
- if (tbi.batches.isEmpty()) {
- iter.remove();
+ } else {
+ ce = tenuredMemoryEntries.remove(batch);
+ if (ce == null) {
+ ce = memoryEntries.remove(batch);
}
- mb = entry.getValue();
}
- try {
- mb.persist();
- } catch (TeiidComponentException e) {
- LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, e, "Error persisting batch, attempts to read that batch later will result in an exception"); //$NON-NLS-1$
+ }
+ if (ce != null) {
+ if (!retain) {
+ BufferManagerImpl.this.remove(ce, true);
}
+ return ce;
}
+ if (prefersMemory) {
+ BatchSoftReference bsr = softCache.remove(batch);
+ if (bsr != null) {
+ ce = bsr.get();
+ if (ce != null) {
+ maxReserveKB.addAndGet(bsr.sizeEstimate);
+ }
+ }
+ } else if (useWeakReferences) {
+ ce = weakReferenceCache.getByHash(batch);
+ if (ce == null || !ce.getId().equals(batch)) {
+ return null;
+ }
+ }
+ if (ce != null && ce.getObject() != null) {
+ referenceHit.getAndIncrement();
+ if (retain) {
+ addMemoryEntry(ce);
+ } else {
+ BufferManagerImpl.this.remove(ce, false);
+ }
+ return ce;
+ }
+ return null;
}
+ void remove(Long gid, Long batch, boolean prefersMemory) {
+ if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.TRACE)) {
+ LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, "Removing batch from BufferManager", batch); //$NON-NLS-1$
+ }
+ CacheEntry ce = fastGet(batch, prefersMemory, false);
+ if (ce == null) {
+ cache.remove(gid, batch);
+ } else {
+ ce.nullOut();
+ }
+ }
+
+ private void remove(CacheEntry ce, boolean inMemory) {
+ if (inMemory) {
+ activeBatchKB.addAndGet(-ce.getSizeEstimate());
+ }
+ Serializer<?> s = ce.getSerializer().get();
+ if (s != null) {
+ cache.remove(s.getId(), ce.getId());
+ }
+ }
+
+ Long addCacheEntry(CacheEntry ce, Serializer<?> s) {
+ if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.TRACE)) {
+ LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, "Add batch to BufferManager", ce.getId(), "with size estimate", ce.getSizeEstimate()); //$NON-NLS-1$ //$NON-NLS-2$
+ }
+ cache.addToCacheGroup(s.getId(), ce.getId());
+ addMemoryEntry(ce);
+ return ce.getId();
+ }
+
+ void addMemoryEntry(CacheEntry ce) {
+ persistBatchReferences();
+ synchronized (memoryEntries) {
+ memoryEntries.put(ce.getId(), ce);
+ }
+ activeBatchKB.getAndAdd(ce.getSizeEstimate());
+ }
+
+ void removeCacheGroup(Long id, boolean prefersMemory) {
+ cleanSoftReferences();
+ Collection<Long> vals = cache.removeCacheGroup(id);
+ for (Long val : vals) {
+ fastGet(val, prefersMemory, false);
+ }
+ }
+
+ void cleanSoftReferences() {
+ for (int i = 0; i < 10; i++) {
+ BatchSoftReference ref = (BatchSoftReference)SOFT_QUEUE.poll();
+ if (ref == null) {
+ break;
+ }
+ softCache.remove(ref.key);
+ maxReserveKB.addAndGet(ref.sizeEstimate);
+ ref.clear();
+ }
+ }
+
@Override
- public int getSchemaSize(List<? extends Expression> elements) {
+ public int getProcessorBatchSize(List<? extends Expression> schema) {
+ return getSizeEstimates(schema)[0];
+ }
+
+ private int[] getSizeEstimates(List<? extends Expression> elements) {
int total = 0;
boolean isValueCacheEnabled = DataTypeManager.isValueCacheEnabled();
- //we make a assumption that the average column size under 64bits is approximately 128bytes
- //this includes alignment, row/array, and reference overhead
for (Expression element : elements) {
Class<?> type = element.getType();
total += SizeUtility.getSize(isValueCacheEnabled, type);
}
+ //assume 64-bit
total += 8*elements.size() + 36; // column list / row overhead
- total *= processorBatchSize;
- return Math.max(1, total / 1024);
+
+ //nominal targetBytesPerRow but can scale up or down
+
+ int totalCopy = total;
+ boolean less = totalCopy < targetBytesPerRow;
+ int rowCount = processorBatchSize;
+
+ for (int i = 0; i < 3; i++) {
+ if (less) {
+ totalCopy <<= 1;
+ } else {
+ totalCopy >>= 2;
+ }
+ if (less && totalCopy > targetBytesPerRow
+ || !less && totalCopy < targetBytesPerRow) {
+ break;
+ }
+ if (less) {
+ rowCount <<= 1;
+ } else {
+ rowCount >>= 1;
+ }
+ }
+ rowCount = Math.max(1, rowCount);
+ total *= rowCount;
+ return new int[]{rowCount, Math.max(1, total / 1024)};
}
+
+ @Override
+ public int getSchemaSize(List<? extends Expression> elements) {
+ return getSizeEstimates(elements)[1];
+ }
public void shutdown() {
}
@@ -758,11 +814,9 @@
out.writeInt(buffer.getRowCount());
out.writeInt(buffer.getBatchSize());
out.writeObject(buffer.getTypes());
- out.writeBoolean(buffer.isPrefersMemory());
for (int row = 1; row <= buffer.getRowCount(); row+=buffer.getBatchSize()) {
TupleBatch b = buffer.getBatch(row);
- b.preserveTypes();
- out.writeObject(b);
+ out.writeObject(new SerializableTupleBatch(b, buffer.getTypes()));
}
}
@@ -812,7 +866,6 @@
int rowCount = in.readInt();
int batchSize = in.readInt();
String[] types = (String[])in.readObject();
- boolean prefersMemory = in.readBoolean();
List<ElementSymbol> schema = new ArrayList<ElementSymbol>(types.length);
for (String type : types) {
@@ -823,7 +876,6 @@
TupleBuffer buffer = createTupleBuffer(schema, "cached", TupleSourceType.FINAL); //$NON-NLS-1$
buffer.setBatchSize(batchSize);
buffer.setId(state_id);
- buffer.setPrefersMemory(prefersMemory);
for (int row = 1; row <= rowCount; row+=batchSize) {
TupleBatch batch = (TupleBatch)in.readObject();
@@ -844,4 +896,16 @@
@Override
public void droppedMembers(Collection<Serializable> addresses) {
}
+
+ public void setInlineLobs(boolean inlineLobs) {
+ this.inlineLobs = inlineLobs;
+ }
+
+ private int getMaxReserveKB() {
+ return maxReserveKB.get();
+ }
+
+ public void setCache(Cache cache) {
+ this.cache = cache;
+ }
}
Copied: branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/ExtensibleBufferedOutputStream.java (from rev 3506, trunk/engine/src/main/java/org/teiid/common/buffer/impl/ExtensibleBufferedOutputStream.java)
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/ExtensibleBufferedOutputStream.java (rev 0)
+++ branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/ExtensibleBufferedOutputStream.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -0,0 +1,80 @@
+/*
+ * JBoss, Home of Professional Open Source.
+ * See the COPYRIGHT.txt file distributed with this work for information
+ * regarding copyright ownership. Some portions may be licensed
+ * to Red Hat, Inc. under one or more contributor license agreements.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301 USA.
+ */
+
+package org.teiid.common.buffer.impl;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+
+public abstract class ExtensibleBufferedOutputStream extends OutputStream {
+
+ protected ByteBuffer buf;
+
+ public ExtensibleBufferedOutputStream() {
+ }
+
+ public void write(int b) throws IOException {
+ ensureBuffer();
+ if (buf.remaining() == 0) {
+ flush();
+ }
+ buf.put((byte)b);
+ }
+
+ private void ensureBuffer() {
+ if (buf == null) {
+ buf = newBuffer();
+ }
+ }
+
+ public void write(byte b[], int off, int len) throws IOException {
+ while (true) {
+ ensureBuffer();
+ int toCopy = Math.min(buf.remaining(), len);
+ buf.put(b, off, toCopy);
+ len -= toCopy;
+ off += toCopy;
+ if (buf.remaining() > 0) {
+ break;
+ }
+ flush();
+ }
+ }
+
+ public void flush() throws IOException {
+ if (buf != null && buf.position() > 0) {
+ flushDirect();
+ }
+ buf = null;
+ }
+
+ protected abstract ByteBuffer newBuffer();
+
+ protected abstract void flushDirect() throws IOException;
+
+ @Override
+ public void close() throws IOException {
+ flush();
+ }
+
+}
Modified: branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/FileStorageManager.java
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/FileStorageManager.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/FileStorageManager.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -29,13 +29,11 @@
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
-import java.util.TreeMap;
import java.util.concurrent.atomic.AtomicLong;
import org.teiid.common.buffer.FileStore;
import org.teiid.common.buffer.StorageManager;
import org.teiid.core.TeiidComponentException;
-import org.teiid.core.util.Assertion;
import org.teiid.logging.LogManager;
import org.teiid.logging.MessageLevel;
import org.teiid.query.QueryPlugin;
@@ -47,7 +45,6 @@
public class FileStorageManager implements StorageManager {
public static final int DEFAULT_MAX_OPEN_FILES = 64;
- public static final long DEFAULT_MAX_FILESIZE = 2L * 1024L;
public static final long DEFAULT_MAX_BUFFERSPACE = 50L * 1024L * 1024L * 1024L;
private static final String FILE_PREFIX = "b_"; //$NON-NLS-1$
@@ -97,87 +94,85 @@
public class DiskStore extends FileStore {
private String name;
- private TreeMap<Long, FileInfo> storageFiles = new TreeMap<Long, FileInfo>();
+ private FileInfo fileInfo;
public DiskStore(String name) {
this.name = name;
}
- /**
- * Concurrent reads are possible, but only after writing is complete.
- */
- public int readDirect(long fileOffset, byte[] b, int offSet, int length) throws TeiidComponentException {
- Map.Entry<Long, FileInfo> entry = storageFiles.floorEntry(fileOffset);
- Assertion.isNotNull(entry);
- FileInfo fileInfo = entry.getValue();
- synchronized (fileInfo) {
+ @Override
+ public synchronized long getLength() {
+ if (fileInfo == null) {
+ return 0;
+ }
+ return fileInfo.file.length();
+ }
+
+ @Override
+ protected synchronized int readWrite(long fileOffset, byte[] b, int offSet,
+ int length, boolean write) throws IOException {
+ if (!write) {
try {
RandomAccessFile fileAccess = fileInfo.open();
- fileAccess.seek(fileOffset - entry.getKey());
+ fileAccess.seek(fileOffset);
return fileAccess.read(b, offSet, length);
- } catch (IOException e) {
- throw new TeiidComponentException(e, QueryPlugin.Util.getString("FileStoreageManager.error_reading", fileInfo.file.getAbsoluteFile())); //$NON-NLS-1$
} finally {
fileInfo.close();
}
- }
- }
-
- /**
- * Concurrent writes are prevented by FileStore, but in general should not happen since processing is single threaded.
- */
- public void writeDirect(byte[] bytes, int offset, int length) throws TeiidComponentException {
- long used = usedBufferSpace.addAndGet(length);
- if (used > maxBufferSpace) {
- usedBufferSpace.addAndGet(-length);
- throw new TeiidComponentException(QueryPlugin.Util.getString("FileStoreageManager.space_exhausted", maxBufferSpace)); //$NON-NLS-1$
- }
- Map.Entry<Long, FileInfo> entry = this.storageFiles.lastEntry();
- boolean createNew = false;
- FileInfo fileInfo = null;
- long fileOffset = 0;
- if (entry == null) {
- createNew = true;
- } else {
- fileInfo = entry.getValue();
- fileOffset = entry.getKey();
- createNew = entry.getValue().file.length() + length > getMaxFileSize();
- }
- if (createNew) {
- FileInfo newFileInfo = new FileInfo(createFile(name, storageFiles.size()));
- if (fileInfo != null) {
- fileOffset += fileInfo.file.length();
+ }
+ if (fileInfo == null) {
+ fileInfo = new FileInfo(createFile(name));
+ }
+ long bytesUsed = 0;
+ try {
+ RandomAccessFile fileAccess = fileInfo.open();
+ long newLength = fileOffset + length;
+ bytesUsed = newLength - fileAccess.length();
+ if (bytesUsed > 0) {
+ long used = usedBufferSpace.addAndGet(bytesUsed);
+ if (used > maxBufferSpace) {
+ usedBufferSpace.addAndGet(-bytesUsed);
+ //TODO: trigger a compaction before this is thrown
+ throw new IOException(QueryPlugin.Util.getString("FileStoreageManager.space_exhausted", maxBufferSpace)); //$NON-NLS-1$
+ }
+ fileAccess.setLength(bytesUsed);
+ bytesUsed = 0;
}
- storageFiles.put(fileOffset, newFileInfo);
- fileInfo = newFileInfo;
+ fileAccess.seek(fileOffset);
+ fileAccess.write(b, offSet, length);
+ } finally {
+ if (bytesUsed > 0) {
+ usedBufferSpace.addAndGet(-bytesUsed);
+ }
+ fileInfo.close();
+ }
+ return length;
+ }
+
+ @Override
+ public synchronized void setLength(long length) throws IOException {
+ if (fileInfo == null) {
+ fileInfo = new FileInfo(createFile(name));
}
- synchronized (fileInfo) {
- try {
- RandomAccessFile fileAccess = fileInfo.open();
- long pointer = fileAccess.length();
- fileAccess.setLength(pointer + length);
- fileAccess.seek(pointer);
- fileAccess.write(bytes, offset, length);
- } catch(IOException e) {
- throw new TeiidComponentException(e, QueryPlugin.Util.getString("FileStoreageManager.error_reading", fileInfo.file.getAbsoluteFile())); //$NON-NLS-1$
- } finally {
- fileInfo.close();
- }
- }
- }
+ try {
+ fileInfo.open().setLength(length);
+ } finally {
+ fileInfo.close();
+ }
+ }
+ @Override
public synchronized void removeDirect() {
- usedBufferSpace.addAndGet(-len);
- for (FileInfo info : storageFiles.values()) {
- info.delete();
+ usedBufferSpace.addAndGet(-getLength());
+ if (fileInfo != null){
+ fileInfo.delete();
}
}
-
+
}
// Initialization
private int maxOpenFiles = DEFAULT_MAX_OPEN_FILES;
- private long maxFileSize = DEFAULT_MAX_FILESIZE * 1024L * 1024L; // 2GB
private String directory;
private File dirFile;
@@ -216,14 +211,6 @@
}
}
- public void setMaxFileSize(long maxFileSize) {
- this.maxFileSize = maxFileSize * 1024L * 1024L;
- }
-
- void setMaxFileSizeDirect(long maxFileSize) {
- this.maxFileSize = maxFileSize;
- }
-
public void setMaxOpenFiles(int maxOpenFiles) {
this.maxOpenFiles = maxOpenFiles;
}
@@ -232,26 +219,18 @@
this.directory = directory;
}
- File createFile(String name, int fileNumber) throws TeiidComponentException {
- try {
- File storageFile = File.createTempFile(FILE_PREFIX + name + "_" + String.valueOf(fileNumber) + "_", null, this.dirFile); //$NON-NLS-1$ //$NON-NLS-2$
- if (LogManager.isMessageToBeRecorded(org.teiid.logging.LogConstants.CTX_BUFFER_MGR, MessageLevel.DETAIL)) {
- LogManager.logDetail(org.teiid.logging.LogConstants.CTX_BUFFER_MGR, "Created temporary storage area file " + storageFile.getAbsoluteFile()); //$NON-NLS-1$
- }
- return storageFile;
- } catch(IOException e) {
- throw new TeiidComponentException(e, QueryPlugin.Util.getString("FileStoreageManager.error_creating", name + "_" + fileNumber)); //$NON-NLS-1$ //$NON-NLS-2$
+ File createFile(String name) throws IOException {
+ File storageFile = File.createTempFile(FILE_PREFIX + name + "_", null, this.dirFile); //$NON-NLS-1$
+ if (LogManager.isMessageToBeRecorded(org.teiid.logging.LogConstants.CTX_BUFFER_MGR, MessageLevel.DETAIL)) {
+ LogManager.logDetail(org.teiid.logging.LogConstants.CTX_BUFFER_MGR, "Created temporary storage area file " + storageFile.getAbsoluteFile()); //$NON-NLS-1$
}
+ return storageFile;
}
public FileStore createFileStore(String name) {
return new DiskStore(name);
}
- public long getMaxFileSize() {
- return maxFileSize;
- }
-
public String getDirectory() {
return directory;
}
Copied: branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/FileStoreCache.java (from rev 3506, trunk/engine/src/main/java/org/teiid/common/buffer/impl/FileStoreCache.java)
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/FileStoreCache.java (rev 0)
+++ branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/FileStoreCache.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -0,0 +1,746 @@
+/*
+ * JBoss, Home of Professional Open Source.
+ * See the COPYRIGHT.txt file distributed with this work for information
+ * regarding copyright ownership. Some portions may be licensed
+ * to Red Hat, Inc. under one or more contributor license agreements.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301 USA.
+ */
+
+package org.teiid.common.buffer.impl;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.teiid.common.buffer.Cache;
+import org.teiid.common.buffer.CacheEntry;
+import org.teiid.common.buffer.FileStore;
+import org.teiid.common.buffer.Serializer;
+import org.teiid.common.buffer.StorageManager;
+import org.teiid.core.TeiidComponentException;
+import org.teiid.core.TeiidRuntimeException;
+import org.teiid.logging.LogConstants;
+import org.teiid.logging.LogManager;
+import org.teiid.logging.MessageLevel;
+import org.teiid.query.QueryPlugin;
+
+/**
+ * Implements storage against a {@link FileStore} abstraction using a filesystem paradigm.
+ * The filesystem uses a 31bit address space on top of 2^14 byte blocks.
+ *
+ * Therefore there is 2^31*2^14 = 2^45 or 32 terabytes max of addressable space.
+ *
+ * Some amount of the space is taken up by system information (inodes and use flags).
+ * This is held in a separate file.
+ *
+ * The 64 byte inode format is:
+ * 14 32 bit direct block pointers
+ * 1 32 bit block indirect pointer
+ * 1 32 bit block doubly indirect pointer
+ *
+ * The data block format is:
+ * data bytes | long gid | long oid | three byte int block num
+ *
+ * The gid/oid are stored with the block so that defrag/compaction can be performed
+ * with minimal blocking/lookups.
+ *
+ * This means that the maximum number of blocks available to a "file" is
+ * 14 + (2^14-18)/4 + ((2^14-18)/4)^2 ~= 2^24
+ *
+ * Thus the max serialized object size is: 2^24*(2^14-18) ~= 256GB.
+ *
+ * The root directory "physicalMapping" is held in memory for performance,
+ * but could itself be switched to using a block map. It will grow in
+ * proportion to the number of tables/tuplebuffers in use.
+ *
+ * TODO: defragment
+ * TODO: lobs could also be supported in this structure.
+ */
+public class FileStoreCache implements Cache, StorageManager {
+
+ //TODO allow the block size to be configurable
+ static final int LOG_BLOCK_SIZE = 14;
+ static final int ADDRESS_BITS = 31;
+ static final int SYSTEM_MASK = 1<<ADDRESS_BITS;
+ static final int BLOCK_SIZE = 1 << LOG_BLOCK_SIZE;
+ static final int BLOCK_MASK = BLOCK_SIZE - 1;
+ static final int BLOCK_OVERHEAD = 8+8+3;
+ static final int BLOCK_DATA_BYTES = BLOCK_SIZE - BLOCK_OVERHEAD;
+ static final int BYTES_PER_BLOCK_ADDRESS = 4;
+ static final int ADDRESSES_PER_BLOCK = BLOCK_DATA_BYTES/BYTES_PER_BLOCK_ADDRESS;
+ static final int INODE_BYTES = 16*BYTES_PER_BLOCK_ADDRESS;
+ static final int DIRECT_POINTERS = 14;
+ static final int MAX_INDIRECT = DIRECT_POINTERS + ADDRESSES_PER_BLOCK;
+ static final int MAX_DOUBLE_INDIRECT = MAX_INDIRECT + ADDRESSES_PER_BLOCK * ADDRESSES_PER_BLOCK;
+ static final int EMPTY_ADDRESS = -1;
+
+ private final class BlockOutputStream extends
+ ExtensibleBufferedOutputStream {
+ private final BlockManager blockManager;
+ int blockNum = -1;
+ BlockInfo bi;
+
+ private BlockOutputStream(BlockManager blockManager) {
+ this.blockManager = blockManager;
+ }
+
+ @Override
+ protected ByteBuffer newBuffer() {
+ bi = blockManager.allocateBlock(++blockNum);
+ return bi.buf;
+ }
+
+ @Override
+ protected void flushDirect() throws IOException {
+ blockManager.updateBlock(bi);
+ bi = null;
+ }
+ }
+
+ private final class BitSetBlockManager implements BlockManager {
+ final int offset;
+
+ BitSetBlockManager(int offset) {
+ this.offset = offset;
+ }
+
+ @Override
+ public void updateBlock(BlockInfo info) {
+ updatePhysicalBlock(info);
+ }
+
+ @Override
+ public BlockInfo getBlock(int index) {
+ return getPhysicalBlock(index + offset, true, false);
+ }
+
+ @Override
+ public BlockInfo allocateBlock(int index) {
+ return getPhysicalBlock(index + offset, true, true);
+ }
+
+ @Override
+ public int getInode() {
+ throw new AssertionError();
+ }
+
+ @Override
+ public void freeBlock(int index) {
+ throw new AssertionError();
+ }
+
+ @Override
+ public void free() {
+ throw new AssertionError();
+ }
+
+ }
+
+ private enum Mode {
+ GET,
+ UPDATE,
+ ALLOCATE
+ }
+
+ static final class BlockInfo {
+ final boolean system;
+ final int inodeOffset;
+ final ByteBuffer buf;
+ final int physicalAddress;
+ boolean dirty;
+
+ BlockInfo(boolean system, ByteBuffer ib, int index, int inodeOffset) {
+ this.system = system;
+ this.buf = ib;
+ this.physicalAddress = index;
+ this.inodeOffset = inodeOffset;
+ }
+ }
+
+ private final class InodeBlockManager implements BlockManager {
+ private final int inode;
+ private final long gid;
+ private final long oid;
+ private int lastBlock = -1;
+ boolean trackLast;
+
+ InodeBlockManager(long gid, long oid, int inode) {
+ if (inode == EMPTY_ADDRESS) {
+ synchronized (inodesInuse) {
+ inode = inodesInuse.nextClearBit(0);
+ if (inode == -1) {
+ throw new TeiidRuntimeException("no inodes available"); //$NON-NLS-1$
+ }
+ if (LogManager.isMessageToBeRecorded(LogConstants.CTX_DQP, MessageLevel.TRACE)) {
+ LogManager.logTrace(LogConstants.CTX_DQP, "Allocating inode", inode, "to", gid, oid); //$NON-NLS-1$ //$NON-NLS-2$
+ }
+ inodesInuse.set(inode, true);
+ }
+ int inodeBlock = inode/inodesPerBlock;
+ int inodePosition = INODE_BYTES*(inode%inodesPerBlock);
+ BlockInfo bb = getInodeSubBlock(inodeBlock, inodePosition);
+ bb.buf.putInt(EMPTY_ADDRESS);
+ updatePhysicalBlock(bb);
+ }
+ this.inode = inode;
+ this.gid = gid;
+ this.oid = oid;
+ }
+
+ @Override
+ public int getInode() {
+ return inode;
+ }
+
+ @Override
+ public void updateBlock(BlockInfo info) {
+ updatePhysicalBlock(info);
+ }
+
+ @Override
+ public BlockInfo getBlock(int index) {
+ int dataBlock = getOrUpdateDataBlockIndex(index, EMPTY_ADDRESS, Mode.GET);
+ BlockInfo bb = getPhysicalBlock(dataBlock, false, false);
+ bb.buf.position(0);
+ bb.buf.limit(BLOCK_DATA_BYTES);
+ return bb;
+ }
+
+ private int getOrUpdateDataBlockIndex(int index, int value, Mode mode) {
+ if (index >= MAX_DOUBLE_INDIRECT) {
+ throw new TeiidRuntimeException("Max block number exceeded"); //$NON-NLS-1$
+ }
+ int dataBlock = 0;
+ int position = 0;
+ int inodePosition = INODE_BYTES*(inode%inodesPerBlock);
+ BlockInfo info = getInodeSubBlock(inode/inodesPerBlock, inodePosition);
+ if (index >= MAX_INDIRECT) {
+ position = BYTES_PER_BLOCK_ADDRESS*(DIRECT_POINTERS+1);
+ BlockInfo next = updateIndirectBlockInfo(info, index, position, MAX_INDIRECT, MAX_DOUBLE_INDIRECT+1, value, mode);
+ if (next != info) {
+ info = next;
+ //should have traversed to the secondary
+ int indirectAddressBlock = (index - MAX_INDIRECT) / ADDRESSES_PER_BLOCK;
+ position = indirectAddressBlock * BYTES_PER_BLOCK_ADDRESS;
+ if (mode == Mode.ALLOCATE && position + BYTES_PER_BLOCK_ADDRESS < BLOCK_DATA_BYTES) {
+ info.buf.putInt(position + BYTES_PER_BLOCK_ADDRESS, EMPTY_ADDRESS);
+ }
+ next = updateIndirectBlockInfo(info, index, position, MAX_INDIRECT + indirectAddressBlock * ADDRESSES_PER_BLOCK, MAX_DOUBLE_INDIRECT + 2 + indirectAddressBlock, value, mode);
+ if (next != info) {
+ info = next;
+ position = ((index - MAX_INDIRECT)%ADDRESSES_PER_BLOCK) * BYTES_PER_BLOCK_ADDRESS;
+ if (mode == Mode.ALLOCATE && position + BYTES_PER_BLOCK_ADDRESS < BLOCK_DATA_BYTES) {
+ info.buf.putInt(position + BYTES_PER_BLOCK_ADDRESS, EMPTY_ADDRESS);
+ }
+ }
+ }
+ } else if (index >= DIRECT_POINTERS) {
+ //indirect
+ position = BYTES_PER_BLOCK_ADDRESS*DIRECT_POINTERS;
+ BlockInfo next = updateIndirectBlockInfo(info, index, position, DIRECT_POINTERS, MAX_DOUBLE_INDIRECT, value, mode);
+ if (next != info) {
+ info = next;
+ position = (index - DIRECT_POINTERS) * BYTES_PER_BLOCK_ADDRESS;
+ if (mode == Mode.ALLOCATE && position + BYTES_PER_BLOCK_ADDRESS < BLOCK_DATA_BYTES) {
+ info.buf.putInt(position + BYTES_PER_BLOCK_ADDRESS, EMPTY_ADDRESS);
+ }
+ }
+ } else {
+ position = BYTES_PER_BLOCK_ADDRESS*index;
+ if (mode == Mode.ALLOCATE) {
+ info.buf.putInt(position + BYTES_PER_BLOCK_ADDRESS, EMPTY_ADDRESS);
+ }
+ }
+ if (mode == Mode.ALLOCATE) {
+ dataBlock = nextBlock();
+ info.buf.putInt(position, dataBlock);
+ updatePhysicalBlock(info);
+ } else {
+ dataBlock = info.buf.getInt(position);
+ if (mode == Mode.UPDATE) {
+ info.buf.putInt(position, value);
+ updatePhysicalBlock(info);
+ }
+ }
+ return dataBlock;
+ }
+
+ private BlockInfo updateIndirectBlockInfo(BlockInfo info, int index, int position, int cutOff, int blockId, int value, Mode mode) {
+ int sib_index = info.buf.getInt(position);
+ boolean newBlock = false;
+ if (index == cutOff) {
+ if (mode == Mode.ALLOCATE) {
+ sib_index = nextBlock();
+ info.buf.putInt(position, sib_index);
+ updatePhysicalBlock(info);
+ newBlock = true;
+ } else if (mode == Mode.UPDATE && value == EMPTY_ADDRESS) {
+ freeDataBlock(sib_index);
+ return info;
+ }
+ }
+ info = getPhysicalBlock(sib_index, false, false);
+ if (newBlock) {
+ putBlockId(blockId, info.buf);
+ }
+ return info;
+ }
+
+ private int nextBlock() {
+ int next = -1;
+ synchronized (blocksInuse) {
+ next = blocksInuse.nextClearBit(lastBlock + 1);
+ if (next == -1) {
+ throw new TeiidRuntimeException("no freespace available"); //$NON-NLS-1$
+ }
+ blocksInuse.set(next, true);
+ }
+ if (LogManager.isMessageToBeRecorded(LogConstants.CTX_DQP, MessageLevel.TRACE)) {
+ LogManager.logTrace(LogConstants.CTX_DQP, "Allocating block", next, "to", gid, oid); //$NON-NLS-1$ //$NON-NLS-2$
+ }
+ if (trackLast) {
+ lastBlock = next;
+ }
+ return next;
+ }
+
+ @Override
+ public void freeBlock(int index) {
+ int dataBlock = getOrUpdateDataBlockIndex(index, EMPTY_ADDRESS, Mode.UPDATE);
+ freeDataBlock(dataBlock);
+ }
+
+ private void freeDataBlock(int dataBlock) {
+ if (LogManager.isMessageToBeRecorded(LogConstants.CTX_DQP, MessageLevel.TRACE)) {
+ LogManager.logTrace(LogConstants.CTX_DQP, "freeing data block", dataBlock, "for", gid, oid); //$NON-NLS-1$ //$NON-NLS-2$
+ }
+ blockCache.remove(dataBlock);
+ blocksInuse.set(dataBlock, false);
+ }
+
+ BlockInfo getInodeSubBlock(int inodeBlock, int inodePosition) {
+ BlockInfo bi = getPhysicalBlock(inodeBlock + blocksInUseBlocks + inodesInUseBlocks, true, false);
+ ByteBuffer bb = bi.buf.duplicate();
+ bb.position(inodePosition);
+ bb.limit(inodePosition + INODE_BYTES);
+ bb = bb.slice();
+ return new BlockInfo(true, bb, inodeBlock + blocksInUseBlocks + inodesInUseBlocks, inodePosition);
+ }
+
+ @Override
+ public void free() {
+ int inodeBlock = inode/inodesPerBlock;
+ int inodePosition = INODE_BYTES*(inode%inodesPerBlock);
+ BlockInfo bi = getInodeSubBlock(inodeBlock, inodePosition);
+ ByteBuffer ib = bi.buf;
+ int indirectIndexBlock = ib.getInt(ib.position() + BYTES_PER_BLOCK_ADDRESS*DIRECT_POINTERS);
+ int doublyIndirectIndexBlock = ib.getInt(ib.position() + BYTES_PER_BLOCK_ADDRESS*(DIRECT_POINTERS+1));
+ boolean freedAll = freeBlock(ib, DIRECT_POINTERS, true);
+ if (LogManager.isMessageToBeRecorded(LogConstants.CTX_DQP, MessageLevel.TRACE)) {
+ LogManager.logTrace(LogConstants.CTX_DQP, "freeing inode", inode, "for", gid, oid); //$NON-NLS-1$ //$NON-NLS-2$
+ }
+ inodesInuse.set(inode, false);
+ if (!freedAll || indirectIndexBlock == EMPTY_ADDRESS) {
+ return;
+ }
+ freedAll = freeIndirectBlock(indirectIndexBlock);
+ if (!freedAll || doublyIndirectIndexBlock == EMPTY_ADDRESS) {
+ return;
+ }
+ BlockInfo bb = getPhysicalBlock(doublyIndirectIndexBlock, false, false);
+ freeBlock(bb.buf, ADDRESSES_PER_BLOCK, false);
+ freeDataBlock(doublyIndirectIndexBlock);
+ }
+
+ private boolean freeIndirectBlock(int indirectIndexBlock) {
+ BlockInfo bb = getPhysicalBlock(indirectIndexBlock, false, false);
+ bb.buf.position(0);
+ boolean freedAll = freeBlock(bb.buf, ADDRESSES_PER_BLOCK, true);
+ freeDataBlock(indirectIndexBlock);
+ return freedAll;
+ }
+
+ private boolean freeBlock(ByteBuffer ib, int numPointers, boolean primary) {
+ for (int i = 0; i < numPointers; i++) {
+ int dataBlock = ib.getInt();
+ if (dataBlock == EMPTY_ADDRESS) {
+ return false;
+ }
+ if (primary) {
+ freeDataBlock(dataBlock);
+ } else {
+ freeIndirectBlock(dataBlock);
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public BlockInfo allocateBlock(int blockNum) {
+ int dataBlock = getOrUpdateDataBlockIndex(blockNum, EMPTY_ADDRESS, Mode.ALLOCATE);
+ BlockInfo bb = getPhysicalBlock(dataBlock, false, true);
+ putBlockId(blockNum, bb.buf);
+ bb.buf.position(0);
+ bb.buf.limit(BLOCK_DATA_BYTES);
+ return bb;
+ }
+
+ private void putBlockId(int blockNum, ByteBuffer bb) {
+ bb.position(BLOCK_DATA_BYTES);
+ bb.putLong(gid);
+ bb.putLong(oid);
+ bb.put((byte)(blockNum >> 16));
+ bb.putShort((short)blockNum);
+ }
+
+ }
+
+ private StorageManager storageManager;
+ private long maxBufferSpace = FileStorageManager.DEFAULT_MAX_BUFFERSPACE;
+ private int inodes;
+ private int blocks;
+ private int inodesPerBlock;
+ private int inodesInUseBlocks;
+ private int blocksInUseBlocks;
+ private FileStore store;
+ private FileStore systemStore;
+ private BlockBitSetTree blocksInuse;
+ private BlockBitSetTree inodesInuse;
+
+ //root directory
+ ConcurrentHashMap<Long, BlockClosedLongIntHashTable> physicalMapping = new ConcurrentHashMap<Long, BlockClosedLongIntHashTable>();
+
+ //block caching
+ int blockCacheSize = 128; //2 MB
+ ReentrantLock blockLock = new ReentrantLock();
+ LinkedHashMap<Integer, BlockInfo> blockCache = new LinkedHashMap<Integer, BlockInfo>() {
+ private static final long serialVersionUID = -4240291744435552008L;
+ BlockInfo eldestEntry = null;
+
+ protected boolean removeEldestEntry(Map.Entry<Integer,BlockInfo> eldest) {
+ if (size() > blockCacheSize) {
+ BlockInfo bi = eldest.getValue();
+ if (bi.dirty) {
+ eldestEntry = bi;
+ }
+ return true;
+ }
+ return false;
+ }
+
+ public BlockInfo put(Integer key, BlockInfo value) {
+ blockLock.lock();
+ value.dirty = true;
+ try {
+ return super.put(key, value);
+ } finally {
+ BlockInfo toUpdate = eldestEntry;
+ eldestEntry = null;
+ blockLock.unlock();
+ if (toUpdate != null) {
+ updatePhysicalBlockDirect(toUpdate);
+ }
+ }
+ }
+
+ public BlockInfo get(Object key) {
+ blockLock.lock();
+ try {
+ return super.get(key);
+ } finally {
+ blockLock.unlock();
+ }
+ }
+
+ public BlockInfo remove(Object key) {
+ blockLock.lock();
+ try {
+ return super.remove(key);
+ } finally {
+ blockLock.unlock();
+ }
+ }
+ };
+
+ BlockInfo getPhysicalBlock(int block, boolean system, boolean allocate) {
+ if (block < 0) {
+ throw new AssertionError("invalid block address " + block); //$NON-NLS-1$
+ }
+ try {
+ int key = block;
+ if (system) {
+ key |= SYSTEM_MASK;
+ }
+ BlockInfo result = blockCache.get(key);
+ assert result == null || !allocate;
+ if (result == null) {
+ ByteBuffer bb = null;
+ if (system) {
+ bb = systemStore.getBuffer(block<<LOG_BLOCK_SIZE, BLOCK_SIZE, allocate);
+ } else {
+ bb = store.getBuffer(block<<LOG_BLOCK_SIZE, BLOCK_SIZE, allocate);
+ }
+ result = new BlockInfo(system, bb, block, -1);
+ blockLock.lock();
+ try {
+ BlockInfo existing = blockCache.get(key);
+ if (existing != null) {
+ return existing;
+ }
+ blockCache.put(key, result);
+ } finally {
+ blockLock.unlock();
+ }
+ return result;
+ }
+ return result;
+ } catch (IOException e) {
+ throw new TeiidRuntimeException(e);
+ }
+ }
+
+ void updatePhysicalBlock(BlockInfo bi) {
+ int key = bi.physicalAddress;
+ if (bi.system) {
+ key |= SYSTEM_MASK;
+ }
+ if (bi.inodeOffset >= 0) {
+ blockLock.lock();
+ try {
+ BlockInfo actual = blockCache.get(key);
+ if (actual == null) {
+ //we're not in the cache, so just update storage
+ updatePhysicalBlockDirect(bi);
+ } else {
+ //TODO: check to see we're sharing the same buffer
+ for (int i = 0; i < INODE_BYTES; i++) {
+ actual.buf.put(bi.inodeOffset + i, bi.buf.get(i));
+ }
+ }
+ } finally {
+ blockLock.unlock();
+ }
+ return;
+ }
+ blockCache.put(key, bi);
+ }
+
+ private void updatePhysicalBlockDirect(BlockInfo bi) {
+ try {
+ bi.buf.rewind();
+ if (!bi.system) {
+ store.updateFromBuffer(bi.buf, bi.physicalAddress<<LOG_BLOCK_SIZE);
+ } else {
+ systemStore.updateFromBuffer(bi.buf, bi.physicalAddress<<LOG_BLOCK_SIZE);
+ }
+ } catch (IOException e) {
+ throw new TeiidRuntimeException(e);
+ }
+ }
+
+ InodeBlockManager getBlockManager(long gid, long oid, int inode) {
+ return new InodeBlockManager(gid, oid, inode);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void add(CacheEntry entry, Serializer s) {
+ boolean success = false;
+ InodeBlockManager blockManager = null;
+ try {
+ BlockClosedLongIntHashTable map = physicalMapping.get(s.getId());
+ if (map == null) {
+ return;
+ }
+ blockManager = getBlockManager(s.getId(), entry.getId(), EMPTY_ADDRESS);
+ blockManager.trackLast = true;
+ ExtensibleBufferedOutputStream fsos = new BlockOutputStream(blockManager);
+ ObjectOutputStream oos = new ObjectOutputStream(fsos);
+ oos.writeInt(entry.getSizeEstimate());
+ s.serialize(entry.getObject(), oos);
+ oos.close();
+ map.put(entry.getId(), blockManager.getInode());
+ success = true;
+ } catch (Throwable e) {
+ LogManager.logError(LogConstants.CTX_BUFFER_MGR, e, "Error persisting batch, attempts to read batch "+ entry.getId() +" later will result in an exception"); //$NON-NLS-1$ //$NON-NLS-2$
+ } finally {
+ if (!success && blockManager != null) {
+ blockManager.free();
+ }
+ }
+ }
+
+ @Override
+ public CacheEntry get(Long oid, Serializer<?> serializer) throws TeiidComponentException {
+ try {
+ BlockClosedLongIntHashTable map = physicalMapping.get(serializer.getId());
+ if (map == null) {
+ return null;
+ }
+ final int inode = map.get(oid);
+ if (inode == EMPTY_ADDRESS) {
+ return null;
+ }
+ final BlockManager manager = getBlockManager(serializer.getId(), oid, inode);
+ ObjectInputStream ois = new ObjectInputStream(new InputStream() {
+
+ int blockIndex;
+ BlockInfo buf;
+
+ @Override
+ public int read() throws IOException {
+ ensureBytes();
+ return buf.buf.get() & 0xff;
+ }
+
+ private void ensureBytes() {
+ if (buf == null || buf.buf.remaining() == 0) {
+ if (buf != null) {
+ buf = null;
+ }
+ buf = manager.getBlock(blockIndex++);
+ }
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len)
+ throws IOException {
+ ensureBytes();
+ len = Math.min(len, buf.buf.remaining());
+ buf.buf.get(b, off, len);
+ return len;
+ }
+ });
+ CacheEntry ce = new CacheEntry(oid);
+ ce.setSizeEstimate(ois.readInt());
+ ce.setObject(serializer.deserialize(ois));
+ ce.setPersistent(true);
+ return ce;
+ } catch(IOException e) {
+ throw new TeiidComponentException(e, QueryPlugin.Util.getString("FileStoreageManager.error_reading", oid)); //$NON-NLS-1$
+ } catch (ClassNotFoundException e) {
+ throw new TeiidComponentException(e, QueryPlugin.Util.getString("FileStoreageManager.error_reading", oid)); //$NON-NLS-1$
+ }
+ }
+
+ @Override
+ public FileStore createFileStore(String name) {
+ return storageManager.createFileStore(name);
+ }
+
+ @Override
+ public void initialize() throws TeiidComponentException {
+ storageManager.initialize();
+ int logSpace = Math.min(45, 63 - Long.numberOfLeadingZeros(this.maxBufferSpace));
+
+ blocks = 1 << Math.min(Math.max(12, logSpace -LOG_BLOCK_SIZE +1), ADDRESS_BITS); //blocks per segment
+
+ inodes = blocks>>1;
+ inodesPerBlock = BLOCK_SIZE/INODE_BYTES;
+
+ inodesInUseBlocks = computeInuseBlocks(inodes);
+ blocksInUseBlocks = computeInuseBlocks(blocks);
+
+ inodesInuse = new BlockBitSetTree(inodes - 1, new BitSetBlockManager(0));
+ blocksInuse = new BlockBitSetTree(blocks - 1, new BitSetBlockManager(inodesInUseBlocks));
+
+ store = storageManager.createFileStore("data_store"); //$NON-NLS-1$
+ systemStore = storageManager.createFileStore("system_store"); //$NON-NLS-1$
+ }
+
+ static int computeInuseBlocks(int number) {
+ int blockCount = (number>>LOG_BLOCK_SIZE) + ((number&BLOCK_MASK)>0?1:0);
+ return (blockCount>>3) + ((blockCount&7)>0?1:0);
+ }
+
+ @Override
+ public void addToCacheGroup(Long gid, Long oid) {
+ BlockClosedLongIntHashTable map = physicalMapping.get(gid);
+ if (map == null) {
+ return;
+ }
+ map.put(oid, EMPTY_ADDRESS);
+ }
+
+ @Override
+ public void createCacheGroup(Long gid) {
+ BlockClosedLongIntHashTable map = new BlockClosedLongIntHashTable(getBlockManager(gid, -1, EMPTY_ADDRESS));
+ physicalMapping.put(gid, map);
+ }
+
+ @Override
+ public void remove(Long gid, Long id) {
+ BlockClosedLongIntHashTable map = physicalMapping.get(gid);
+ if (map == null) {
+ return;
+ }
+ int inode = map.remove(id);
+ free(gid, id, inode);
+ }
+
+ @Override
+ public Collection<Long> removeCacheGroup(Long gid) {
+ BlockClosedLongIntHashTable map = physicalMapping.remove(gid);
+ if (map == null) {
+ return Collections.emptySet();
+ }
+ Map<Long, Integer> values = map.remove();
+ for (Map.Entry<Long, Integer> entry : values.entrySet()) {
+ if (entry.getValue() != null) {
+ free(gid, entry.getKey(), entry.getValue());
+ }
+ }
+ return values.keySet();
+ }
+
+ void free(Long gid, Long oid, int inode) {
+ if (inode == EMPTY_ADDRESS) {
+ return;
+ }
+ BlockManager bm = getBlockManager(gid, oid, inode);
+ bm.free();
+ }
+
+ public void setStorageManager(StorageManager storageManager) {
+ this.storageManager = storageManager;
+ }
+
+ public StorageManager getStorageManager() {
+ return storageManager;
+ }
+
+ public void setMaxBufferSpace(long maxBufferSpace) {
+ this.maxBufferSpace = maxBufferSpace;
+ }
+
+ public int getInodesInUse() {
+ return this.inodesInuse.getBitsSet();
+ }
+
+ public int getDataBlocksInUse() {
+ return this.blocksInuse.getBitsSet();
+ }
+
+}
Modified: branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/MemoryStorageManager.java
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/MemoryStorageManager.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/MemoryStorageManager.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -22,16 +22,89 @@
package org.teiid.common.buffer.impl;
+import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
+import org.teiid.common.buffer.Cache;
+import org.teiid.common.buffer.CacheEntry;
import org.teiid.common.buffer.FileStore;
-import org.teiid.common.buffer.StorageManager;
+import org.teiid.common.buffer.Serializer;
import org.teiid.core.TeiidComponentException;
-public class MemoryStorageManager implements StorageManager {
+public class MemoryStorageManager implements Cache {
+
+ public static final int MAX_FILE_SIZE = 1 << 17;
+ private final class MemoryFileStore extends FileStore {
+ private ByteBuffer buffer = ByteBuffer.allocate(MAX_FILE_SIZE);
+
+ public MemoryFileStore() {
+ buffer.limit(0);
+ }
+
+ @Override
+ public synchronized void removeDirect() {
+ removed.incrementAndGet();
+ buffer = ByteBuffer.allocate(0);
+ }
+
+ @Override
+ protected synchronized int readWrite(long fileOffset, byte[] b, int offSet,
+ int length, boolean write) {
+ if (!write) {
+ if (fileOffset >= getLength()) {
+ return -1;
+ }
+ int position = (int)fileOffset;
+ buffer.position(position);
+ length = Math.min(length, (int)getLength() - position);
+ buffer.get(b, offSet, length);
+ return length;
+ }
+ int requiredLength = (int)(fileOffset + length);
+ if (requiredLength > buffer.limit()) {
+ buffer.limit(requiredLength);
+ }
+ buffer.position((int)fileOffset);
+ buffer.put(b, offSet, length);
+ return length;
+ }
+
+ @Override
+ public synchronized void setLength(long length) {
+ buffer.limit((int)length);
+ }
+
+ @Override
+ public synchronized long getLength() {
+ return buffer.limit();
+ }
+
+ @Override
+ public synchronized ByteBuffer getBuffer(long start, int length, boolean allocate) {
+ int position = (int)start;
+ buffer.limit(position + length);
+ buffer.position(position);
+ return buffer.slice();
+ }
+
+ @Override
+ public void updateFromBuffer(ByteBuffer bb, long start)
+ throws IOException {
+ //do nothing we are sharing the bytes
+ }
+
+ }
+
+ private Map<Long, Map<Long, CacheEntry>> groups = new ConcurrentHashMap<Long, Map<Long, CacheEntry>>();
private AtomicInteger created = new AtomicInteger();
private AtomicInteger removed = new AtomicInteger();
@@ -41,40 +114,7 @@
@Override
public FileStore createFileStore(String name) {
created.incrementAndGet();
- return new FileStore() {
- private ByteBuffer buffer = ByteBuffer.allocate(1 << 16);
-
- @Override
- public void writeDirect(byte[] bytes, int offset, int length) throws TeiidComponentException {
- if (getLength() + length > buffer.capacity()) {
- ByteBuffer newBuffer = ByteBuffer.allocate(buffer.capacity() * 2 + length);
- buffer.position(0);
- newBuffer.put(buffer);
- buffer = newBuffer;
- }
- buffer.position((int)getLength());
- buffer.put(bytes, offset, length);
- }
-
- @Override
- public synchronized void removeDirect() {
- removed.incrementAndGet();
- buffer = ByteBuffer.allocate(0);
- }
-
- @Override
- public synchronized int readDirect(long fileOffset, byte[] b, int offset, int length)
- throws TeiidComponentException {
- if (fileOffset >= getLength()) {
- return -1;
- }
- int position = (int)fileOffset;
- buffer.position(position);
- length = Math.min(length, (int)getLength() - position);
- buffer.get(b, offset, length);
- return length;
- }
- };
+ return new MemoryFileStore();
}
public int getCreated() {
@@ -84,4 +124,55 @@
public int getRemoved() {
return removed.get();
}
+
+ @Override
+ public void add(CacheEntry entry, Serializer<?> s) {
+ Map<Long, CacheEntry> group = groups.get(s.getId());
+ if (group != null) {
+ group.put(entry.getId(), entry);
+ }
+ }
+
+ @Override
+ public void addToCacheGroup(Long gid, Long oid) {
+ Map<Long, CacheEntry> group = groups.get(gid);
+ if (group != null) {
+ group.put(oid, null);
+ }
+ }
+
+ @Override
+ public void createCacheGroup(Long gid) {
+ groups.put(gid, Collections.synchronizedMap(new HashMap<Long, CacheEntry>()));
+ }
+
+ @Override
+ public CacheEntry get(Long id, Serializer<?> serializer)
+ throws TeiidComponentException {
+ Map<Long, CacheEntry> group = groups.get(serializer.getId());
+ if (group != null) {
+ return group.get(id);
+ }
+ return null;
+ }
+
+ @Override
+ public void remove(Long gid, Long id) {
+ Map<Long, CacheEntry> group = groups.get(gid);
+ if (group != null) {
+ group.remove(id);
+ }
+ }
+
+ @Override
+ public Collection<Long> removeCacheGroup(Long gid) {
+ Map<Long, CacheEntry> group = groups.remove(gid);
+ if (group == null) {
+ return Collections.emptySet();
+ }
+ synchronized (group) {
+ return new ArrayList<Long>(group.keySet());
+ }
+ }
+
}
\ No newline at end of file
Modified: branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/SizeUtility.java
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/SizeUtility.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/SizeUtility.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -25,8 +25,10 @@
import java.lang.reflect.Array;
import java.math.BigDecimal;
import java.math.BigInteger;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
-import org.teiid.common.buffer.TupleBatch;
import org.teiid.core.types.DataTypeManager;
@@ -40,29 +42,52 @@
public final class SizeUtility {
public static final int REFERENCE_SIZE = 8;
+ private static Map<Class<?>, int[]> SIZE_ESTIMATES = new HashMap<Class<?>, int[]>(128);
+
+ static {
+ SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.STRING, new int[] {100, 256});
+ SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.DATE, new int[] {20, 28});
+ SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.TIME, new int[] {20, 28});
+ SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.TIMESTAMP, new int[] {20, 28});
+ SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.LONG, new int[] {12, 16});
+ SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.DOUBLE, new int[] {12, 16});
+ SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.INTEGER, new int[] {6, 12});
+ SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.FLOAT, new int[] {6, 12});
+ SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.CHAR, new int[] {4, 10});
+ SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.SHORT, new int[] {4, 10});
+ SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.OBJECT, new int[] {1024, 1024});
+ SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.NULL, new int[] {0, 0});
+ SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.BYTE, new int[] {1, 1});
+ SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.BOOLEAN, new int[] {1, 1});
+ SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.BIG_INTEGER, new int[] {75, 100});
+ SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.BIG_DECIMAL, new int[] {150, 200});
+ }
+
private long bigIntegerEstimate;
private long bigDecimalEstimate;
+ private String[] types;
- public SizeUtility() {
+ public SizeUtility(String[] types) {
boolean isValueCacheEnabled = DataTypeManager.isValueCacheEnabled();
bigIntegerEstimate = getSize(isValueCacheEnabled, DataTypeManager.DefaultDataClasses.BIG_INTEGER);
bigDecimalEstimate = getSize(isValueCacheEnabled, DataTypeManager.DefaultDataClasses.BIG_DECIMAL);
+ this.types = types;
}
- public long getBatchSize(TupleBatch data) {
+ public long getBatchSize(List<? extends List<?>> data) {
return getBatchSize(DataTypeManager.isValueCacheEnabled(), data);
}
- private long getBatchSize(boolean accountForValueCache, TupleBatch data) {
- int colLength = data.getDataTypes().length;
- int rowLength = data.getRowCount();
+ private long getBatchSize(boolean accountForValueCache, List<? extends List<?>> data) {
+ int colLength = types.length;
+ int rowLength = data.size();
// Array overhead for row array
long size = 16 + alignMemory(rowLength * REFERENCE_SIZE);
// array overhead for all the columns ( 8 object overhead + 4 ref + 4 int)
size += (rowLength * (48 + alignMemory(colLength * REFERENCE_SIZE)));
for (int col = 0; col < colLength; col++) {
- Class<?> type = DataTypeManager.getDataTypeClass(data.getDataTypes()[col]);
+ Class<?> type = DataTypeManager.getDataTypeClass(types[col]);
if (type == DataTypeManager.DefaultDataClasses.STRING
|| type == DataTypeManager.DefaultDataClasses.OBJECT
@@ -71,7 +96,7 @@
int estRow = 0;
for (int row = 0; row < rowLength; row++) {
boolean updateEst = row == estRow;
- size += getSize(data.getTuples().get(row).get(col), updateEst, accountForValueCache);
+ size += getSize(data.get(row).get(col), updateEst, accountForValueCache);
if (updateEst) {
estRow = estRow * 2 + 1;
}
@@ -85,36 +110,12 @@
static int getSize(boolean isValueCacheEnabled,
Class<?> type) {
- if (type == DataTypeManager.DefaultDataClasses.STRING) {
- return isValueCacheEnabled?100:256; //assumes an "average" string length of approximately 100 chars
- } else if (type == DataTypeManager.DefaultDataClasses.DATE
- || type == DataTypeManager.DefaultDataClasses.TIME
- || type == DataTypeManager.DefaultDataClasses.TIMESTAMP) {
- return isValueCacheEnabled?20:28;
- } else if (type == DataTypeManager.DefaultDataClasses.LONG
- || type == DataTypeManager.DefaultDataClasses.DOUBLE) {
- return isValueCacheEnabled?12:16;
- } else if (type == DataTypeManager.DefaultDataClasses.INTEGER
- || type == DataTypeManager.DefaultDataClasses.FLOAT) {
- return isValueCacheEnabled?6:12;
- } else if (type == DataTypeManager.DefaultDataClasses.CHAR
- || type == DataTypeManager.DefaultDataClasses.SHORT) {
- return isValueCacheEnabled?4:10;
- } else if (type == DataTypeManager.DefaultDataClasses.OBJECT) {
- return 1024;
- } else if (type == DataTypeManager.DefaultDataClasses.NULL) {
- return 0; //it's free
- } else if (type == DataTypeManager.DefaultDataClasses.BYTE
- || type == DataTypeManager.DefaultDataClasses.BOOLEAN) {
- return 1; //should always be value cached, but there's a small chance it's not
- } else if (type == DataTypeManager.DefaultDataClasses.BIG_INTEGER){
- return isValueCacheEnabled?75:100;
- } else if (type == DataTypeManager.DefaultDataClasses.BIG_DECIMAL) {
- return isValueCacheEnabled?150:200;
- }
- return 512; //assumes buffer overhead in the case of lobs
- //however the account for lobs is misleading as the lob
- //references are not actually removed from memory
+ int[] vals = SIZE_ESTIMATES.get(type);
+ if (vals == null) {
+ return 512; //this is is misleading for lobs
+ //most references are not actually removed from memory
+ }
+ return vals[isValueCacheEnabled?0:1];
}
/**
Copied: branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/SplittableStorageManager.java (from rev 3506, trunk/engine/src/main/java/org/teiid/common/buffer/impl/SplittableStorageManager.java)
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/SplittableStorageManager.java (rev 0)
+++ branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/SplittableStorageManager.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -0,0 +1,190 @@
+/*
+ * JBoss, Home of Professional Open Source.
+ * See the COPYRIGHT.txt file distributed with this work for information
+ * regarding copyright ownership. Some portions may be licensed
+ * to Red Hat, Inc. under one or more contributor license agreements.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301 USA.
+ */
+
+package org.teiid.common.buffer.impl;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.teiid.common.buffer.FileStore;
+import org.teiid.common.buffer.StorageManager;
+import org.teiid.core.TeiidComponentException;
+
+/**
+ * A storage manager that combines smaller files into a larger
+ * logical file.
+ *
+ * The buffer methods assume that buffers cannot go beyond single
+ * file boundaries.
+ */
+public class SplittableStorageManager implements StorageManager {
+
+ public static final long DEFAULT_MAX_FILESIZE = 2 * 1024l;
+ private long maxFileSize = DEFAULT_MAX_FILESIZE * 1024l * 1024l; // 2GB
+ private StorageManager storageManager;
+
+ public SplittableStorageManager(StorageManager storageManager) {
+ this.storageManager = storageManager;
+ }
+
+ @Override
+ public FileStore createFileStore(String name) {
+ return new SplittableFileStore(name);
+ }
+
+ @Override
+ public void initialize() throws TeiidComponentException {
+ storageManager.initialize();
+ }
+
+ public class SplittableFileStore extends FileStore {
+ private String name;
+ private List<FileStore> storageFiles = new ArrayList<FileStore>();
+
+ private volatile long len;
+
+ public SplittableFileStore(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public long getLength() {
+ return len;
+ }
+
+ @Override
+ protected int readWrite(long fileOffset, byte[] b, int offSet,
+ int length, boolean write) throws IOException {
+ FileStore store = null;
+ if (!write) {
+ synchronized (this) {
+ if (fileOffset + length > len) {
+ throw new IOException("Invalid file position " + fileOffset + " length " + length); //$NON-NLS-1$ //$NON-NLS-2$
+ }
+ store = storageFiles.get((int)(fileOffset/maxFileSize));
+ }
+ return store.read(fileOffset%maxFileSize, b, offSet, length);
+ }
+ synchronized (this) {
+ ensureLength(fileOffset + length);
+ store = storageFiles.get((int)(fileOffset/maxFileSize));
+ }
+ long fileBegin = fileOffset%maxFileSize;
+ length = Math.min(length, (int)Math.min(Integer.MAX_VALUE, maxFileSize - fileBegin));
+ store.write(fileBegin, b, offSet, length);
+ return length;
+ }
+
+ private void ensureLength(long length) throws IOException {
+ if (length <= len) {
+ return;
+ }
+ int numFiles = (int)(length/maxFileSize);
+ long lastFileSize = length%maxFileSize;
+ if (lastFileSize > 0) {
+ numFiles++;
+ }
+ for (int i = storageFiles.size(); i < numFiles; i++) {
+ FileStore newFileInfo = storageManager.createFileStore(name + "_" + storageFiles.size()); //$NON-NLS-1$
+ storageFiles.add(newFileInfo);
+ if (lastFileSize == 0 || i != numFiles - 1) {
+ newFileInfo.setLength(maxFileSize);
+ }
+ }
+ if (lastFileSize > 0) {
+ storageFiles.get(storageFiles.size() - 1).setLength(lastFileSize);
+ }
+ len = length;
+ }
+
+ @Override
+ public ByteBuffer getBuffer(long start, int length, boolean allocate) throws IOException {
+ FileStore store = null;
+ synchronized (this) {
+ ensureLength(start + length);
+ store = storageFiles.get((int)(start/maxFileSize));
+ }
+ long fileBegin = start%maxFileSize;
+ return store.getBuffer(fileBegin, length, allocate);
+ }
+
+ @Override
+ public void updateFromBuffer(ByteBuffer bb, long start)
+ throws IOException {
+ FileStore store = null;
+ synchronized (this) {
+ store = storageFiles.get((int)(start/maxFileSize));
+ }
+ long fileBegin = start%maxFileSize;
+ store.updateFromBuffer(bb, fileBegin);
+ }
+
+ @Override
+ public synchronized void setLength(long length) throws IOException {
+ if (length > len) {
+ ensureLength(length);
+ } else {
+ int numFiles = (int)(length/maxFileSize);
+ long lastFileSize = length%maxFileSize;
+ if (lastFileSize > 0) {
+ numFiles++;
+ }
+ int toRemove = storageFiles.size() - numFiles;
+ for (int i = 0; i < toRemove; i++) {
+ FileStore store = storageFiles.remove(storageFiles.size() -1);
+ store.remove();
+ }
+ if (lastFileSize > 0) {
+ storageFiles.get(storageFiles.size() - 1).setLength(lastFileSize);
+ }
+ }
+ len = length;
+ }
+
+ public synchronized void removeDirect() {
+ for (FileStore info : storageFiles) {
+ info.remove();
+ }
+ storageFiles.clear();
+ }
+
+ }
+
+ public long getMaxFileSize() {
+ return maxFileSize;
+ }
+
+ public void setMaxFileSize(long maxFileSize) {
+ this.maxFileSize = maxFileSize * 1024l * 1024l;
+ }
+
+ public void setMaxFileSizeDirect(long maxFileSize) {
+ this.maxFileSize = maxFileSize;
+ }
+
+ public StorageManager getStorageManager() {
+ return storageManager;
+ }
+
+}
Modified: branches/as7/engine/src/main/java/org/teiid/dqp/internal/datamgr/ConnectorWorkItem.java
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/dqp/internal/datamgr/ConnectorWorkItem.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/main/java/org/teiid/dqp/internal/datamgr/ConnectorWorkItem.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -31,7 +31,6 @@
import org.teiid.adminapi.impl.VDBMetaData;
import org.teiid.common.buffer.BlockedException;
-import org.teiid.common.buffer.TupleBuffer;
import org.teiid.core.TeiidProcessingException;
import org.teiid.core.util.Assertion;
import org.teiid.dqp.message.AtomicRequestID;
@@ -332,7 +331,7 @@
LogManager.logWarning(LogConstants.CTX_CONNECTOR, QueryPlugin.Util.getString("ConnectorWorker.zero_size_non_last_batch", requestMsg.getConnectorName())); //$NON-NLS-1$
}
- AtomicResultsMessage response = createResultsMessage(rows.toArray(new List[currentRowCount]), requestMsg.getCommand().getProjectedSymbols());
+ AtomicResultsMessage response = createResultsMessage(rows.toArray(new List[currentRowCount]));
// if we need to keep the execution alive, then we can not support implicit close.
response.setSupportsImplicitClose(!this.securityContext.keepExecutionAlive());
@@ -346,9 +345,8 @@
return response;
}
- public static AtomicResultsMessage createResultsMessage(List[] batch, List columnSymbols) {
- String[] dataTypes = TupleBuffer.getTypeNames(columnSymbols);
- return new AtomicResultsMessage(batch, dataTypes);
+ public static AtomicResultsMessage createResultsMessage(List[] batch) {
+ return new AtomicResultsMessage(batch);
}
boolean isCancelled() {
Modified: branches/as7/engine/src/main/java/org/teiid/dqp/internal/process/DataTierTupleSource.java
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/dqp/internal/process/DataTierTupleSource.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/main/java/org/teiid/dqp/internal/process/DataTierTupleSource.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -398,7 +398,7 @@
fullyCloseSource();
}
if(workItem.requestMsg.supportsPartialResults()) {
- AtomicResultsMessage emptyResults = new AtomicResultsMessage(new List[0], null);
+ AtomicResultsMessage emptyResults = new AtomicResultsMessage(new List[0]);
emptyResults.setWarnings(Arrays.asList((Exception)exception));
emptyResults.setFinalRow(this.rowsProcessed);
return emptyResults;
Modified: branches/as7/engine/src/main/java/org/teiid/dqp/internal/process/Request.java
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/dqp/internal/process/Request.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/main/java/org/teiid/dqp/internal/process/Request.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -233,7 +233,6 @@
props,
this.requestMsg.getShowPlan() != ShowPlan.OFF);
this.context.setProcessorBatchSize(bufferManager.getProcessorBatchSize());
- this.context.setConnectorBatchSize(bufferManager.getConnectorBatchSize());
this.context.setGlobalTableStore(this.globalTables);
if (multiSourceModels != null) {
MultiSourcePlanToProcessConverter modifier = new MultiSourcePlanToProcessConverter(
Modified: branches/as7/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -25,12 +25,13 @@
import java.sql.ResultSet;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
import org.teiid.client.RequestMessage;
import org.teiid.client.ResultsMessage;
@@ -144,7 +145,7 @@
* maintained during processing
*/
private Throwable processingException;
- private Map<AtomicRequestID, DataTierTupleSource> connectorInfo = new ConcurrentHashMap<AtomicRequestID, DataTierTupleSource>(4);
+ private Map<AtomicRequestID, DataTierTupleSource> connectorInfo = Collections.synchronizedMap(new HashMap<AtomicRequestID, DataTierTupleSource>(4));
// This exception contains details of all the atomic requests that failed when query is run in partial results mode.
private List<TeiidException> warnings = new LinkedList<TeiidException>();
private volatile boolean doneProducingBatches;
@@ -157,7 +158,7 @@
private int begin;
private int end;
private TupleBatch savedBatch;
- private Map<Integer, LobWorkItem> lobStreams = new ConcurrentHashMap<Integer, LobWorkItem>(4);
+ private Map<Integer, LobWorkItem> lobStreams = Collections.synchronizedMap(new HashMap<Integer, LobWorkItem>(4));
/**The time when command begins processing on the server.*/
private long processingTimestamp = System.currentTimeMillis();
@@ -362,7 +363,7 @@
* break the read of a lob from a transactional source under a transaction
* if the source does not support holding the clob open after commit
*/
- for (DataTierTupleSource connectorRequest : this.connectorInfo.values()) {
+ for (DataTierTupleSource connectorRequest : getConnectorRequests()) {
if (connectorRequest.isTransactional()) {
connectorRequest.fullyCloseSource();
}
@@ -401,15 +402,21 @@
}
}
- for (DataTierTupleSource connectorRequest : this.connectorInfo.values()) {
+ for (DataTierTupleSource connectorRequest : getConnectorRequests()) {
connectorRequest.fullyCloseSource();
}
}
this.resultsBuffer = null;
- for (LobWorkItem lobWorkItem : this.lobStreams.values()) {
- lobWorkItem.close();
+ if (!this.lobStreams.isEmpty()) {
+ List<LobWorkItem> lobs = null;
+ synchronized (lobStreams) {
+ lobs = new ArrayList<LobWorkItem>(this.lobStreams.values());
+ }
+ for (LobWorkItem lobWorkItem : lobs) {
+ lobWorkItem.close();
+ }
}
}
@@ -592,7 +599,7 @@
}
int finalRowCount = this.resultsBuffer.isFinal()?this.resultsBuffer.getRowCount():(batch.getTerminationFlag()?batch.getEndRow():-1);
- response = createResultsMessage(batch.getAllTuples(), this.originalCommand.getProjectedSymbols());
+ response = createResultsMessage(batch.getTuples(), this.originalCommand.getProjectedSymbols());
response.setFirstRow(batch.getBeginRow());
response.setLastRow(batch.getEndRow());
response.setUpdateResult(this.returnsUpdateCount);
@@ -631,7 +638,7 @@
return result;
}
- public ResultsMessage createResultsMessage(List[] batch, List columnSymbols) {
+ public ResultsMessage createResultsMessage(List<? extends List<?>> batch, List columnSymbols) {
String[] columnNames = new String[columnSymbols.size()];
String[] dataTypes = new String[columnSymbols.size()];
@@ -640,7 +647,7 @@
columnNames[i] = SingleElementSymbol.getShortName(symbol.getOutputName());
dataTypes[i] = DataTypeManager.getDataTypeName(symbol.getType());
}
- ResultsMessage result = new ResultsMessage(requestMsg, batch, columnNames, dataTypes);
+ ResultsMessage result = new ResultsMessage(batch, columnNames, dataTypes);
setAnalysisRecords(result);
return result;
}
@@ -672,7 +679,7 @@
}
}
LogManager.logDetail(LogConstants.CTX_DQP, processingException, "Sending error to client", requestID); //$NON-NLS-1$
- ResultsMessage response = new ResultsMessage(requestMsg);
+ ResultsMessage response = new ResultsMessage();
Throwable exception = this.processingException;
if (isCanceled) {
exception = addCancelCode(exception);
@@ -737,7 +744,7 @@
// Cancel Connector atomic requests
try {
- for (DataTierTupleSource connectorRequest : this.connectorInfo.values()) {
+ for (DataTierTupleSource connectorRequest : getConnectorRequests()) {
connectorRequest.cancelRequest();
}
} finally {
@@ -837,7 +844,9 @@
Collection<DataTierTupleSource> getConnectorRequests() {
- return new LinkedList<DataTierTupleSource>(this.connectorInfo.values());
+ synchronized (this.connectorInfo) {
+ return new ArrayList<DataTierTupleSource>(this.connectorInfo.values());
+ }
}
DataTierTupleSource getConnectorRequest(AtomicRequestID id) {
@@ -872,6 +881,12 @@
private void doneProducingBatches() {
this.doneProducingBatches = true;
+ //TODO: we could perform more tracking to know what source lobs are in use
+ if (this.resultsBuffer.getLobCount() == 0) {
+ for (DataTierTupleSource connectorRequest : getConnectorRequests()) {
+ connectorRequest.fullyCloseSource();
+ }
+ }
dqpCore.finishProcessing(this);
}
Copied: branches/as7/engine/src/main/java/org/teiid/dqp/internal/process/SerializableTupleBatch.java (from rev 3506, trunk/engine/src/main/java/org/teiid/dqp/internal/process/SerializableTupleBatch.java)
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/dqp/internal/process/SerializableTupleBatch.java (rev 0)
+++ branches/as7/engine/src/main/java/org/teiid/dqp/internal/process/SerializableTupleBatch.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -0,0 +1,65 @@
+/*
+ * JBoss, Home of Professional Open Source.
+ * See the COPYRIGHT.txt file distributed with this work for information
+ * regarding copyright ownership. Some portions may be licensed
+ * to Red Hat, Inc. under one or more contributor license agreements.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301 USA.
+ */
+
+package org.teiid.dqp.internal.process;
+
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.List;
+
+import org.teiid.client.BatchSerializer;
+import org.teiid.common.buffer.TupleBatch;
+import org.teiid.core.util.ExternalizeUtil;
+
+public class SerializableTupleBatch extends TupleBatch implements Externalizable {
+
+ private String[] types;
+
+ public SerializableTupleBatch() {
+ //for Externalizable
+ }
+
+ public SerializableTupleBatch(TupleBatch batch, String[] types) {
+ super(batch.getBeginRow(), batch.getTuples());
+ this.types = types;
+ }
+
+ @Override
+ public void readExternal(ObjectInput in) throws IOException,
+ ClassNotFoundException {
+ String[] types = ExternalizeUtil.readStringArray(in);
+ this.setRowOffset(in.readInt());
+ this.setTerminationFlag(in.readBoolean());
+ this.tuples = (List)BatchSerializer.readBatch(in, types);
+ }
+
+ @Override
+ public void writeExternal(ObjectOutput out) throws IOException {
+ ExternalizeUtil.writeArray(out, types);
+ out.writeInt(this.getBeginRow());
+ out.writeBoolean(this.getTerminationFlag());
+ BatchSerializer.writeBatch(out, types, this.getTuples());
+ }
+
+}
Modified: branches/as7/engine/src/main/java/org/teiid/dqp/message/AtomicResultsMessage.java
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/dqp/message/AtomicResultsMessage.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/main/java/org/teiid/dqp/message/AtomicResultsMessage.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -22,20 +22,12 @@
package org.teiid.dqp.message;
-import java.io.Externalizable;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
import java.util.List;
-import org.teiid.client.BatchSerializer;
-import org.teiid.core.util.ExternalizeUtil;
+public class AtomicResultsMessage {
-public class AtomicResultsMessage implements Externalizable {
-
private List[] results;
- private String[] dataTypes;
// Final row index in complete result set, if known
private int finalRow = -1;
@@ -53,8 +45,7 @@
public AtomicResultsMessage() {
}
- public AtomicResultsMessage(List[] results, String[] dataTypes) {
- this.dataTypes = dataTypes;
+ public AtomicResultsMessage(List[] results) {
this.results = results;
}
@@ -86,26 +77,6 @@
return results;
}
- public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- dataTypes = ExternalizeUtil.readStringArray(in);
- results = BatchSerializer.readBatch(in, dataTypes);
- finalRow = in.readInt();
- supportsImplicitClose = in.readBoolean();
- warnings = (List<Exception>)in.readObject();
- isTransactional = in.readBoolean();
- supportsCloseWithLobs = in.readBoolean();
- }
-
- public void writeExternal(ObjectOutput out) throws IOException {
- ExternalizeUtil.writeArray(out, dataTypes);
- BatchSerializer.writeBatch(out, dataTypes, results);
- out.writeInt(finalRow);
- out.writeBoolean(supportsImplicitClose);
- out.writeObject(warnings);
- out.writeBoolean(isTransactional);
- out.writeBoolean(supportsCloseWithLobs);
- }
-
public boolean isTransactional() {
return isTransactional;
}
Property changes on: branches/as7/engine/src/main/java/org/teiid/query/function/FunctionDescriptor.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/main/java/org/teiid/query/function/FunctionDescriptor.java:3149-3171
/trunk/engine/src/main/java/org/teiid/query/function/FunctionDescriptor.java:3188-3450
+ /branches/7.4.x/engine/src/main/java/org/teiid/query/function/FunctionDescriptor.java:3149-3171
/trunk/engine/src/main/java/org/teiid/query/function/FunctionDescriptor.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/main/java/org/teiid/query/function/FunctionTree.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/main/java/org/teiid/query/function/FunctionTree.java:3149-3171
/trunk/engine/src/main/java/org/teiid/query/function/FunctionTree.java:3188-3450
+ /branches/7.4.x/engine/src/main/java/org/teiid/query/function/FunctionTree.java:3149-3171
/trunk/engine/src/main/java/org/teiid/query/function/FunctionTree.java:3188-3450,3452-3506
Modified: branches/as7/engine/src/main/java/org/teiid/query/function/source/SecuritySystemFunctions.java
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/query/function/source/SecuritySystemFunctions.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/main/java/org/teiid/query/function/source/SecuritySystemFunctions.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -31,7 +31,7 @@
public class SecuritySystemFunctions {
- public static boolean hasRole(CommandContext context, Object roleName) throws FunctionExecutionException {
+ public static boolean hasRole(CommandContext context, String roleName) throws FunctionExecutionException {
SecurityFunctionEvaluator eval = context.getSecurityFunctionEvaluator();
if (eval == null) {
@@ -39,13 +39,13 @@
}
try {
- return eval.hasRole(SecurityFunctionEvaluator.DATA_ROLE, (String)roleName);
+ return eval.hasRole(SecurityFunctionEvaluator.DATA_ROLE, roleName);
} catch (TeiidComponentException err) {
throw new FunctionExecutionException(err, err.getMessage());
}
}
- public static boolean hasRole(CommandContext context, Object roleType, Object roleName) throws FunctionExecutionException {
+ public static boolean hasRole(CommandContext context, String roleType, String roleName) throws FunctionExecutionException {
SecurityFunctionEvaluator eval = context.getSecurityFunctionEvaluator();
@@ -54,7 +54,7 @@
}
try {
- return eval.hasRole((String)roleType, (String)roleName);
+ return eval.hasRole(roleType, roleName);
} catch (TeiidComponentException err) {
throw new FunctionExecutionException(err, err.getMessage());
}
Modified: branches/as7/engine/src/main/java/org/teiid/query/metadata/TransformationMetadata.java
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/query/metadata/TransformationMetadata.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/main/java/org/teiid/query/metadata/TransformationMetadata.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -105,6 +105,11 @@
} catch (IOException e) {
}
}
+
+ @Override
+ public StorageMode getStorageMode() {
+ return StorageMode.PERSISTENT;
+ }
}
public static class Resource {
Property changes on: branches/as7/engine/src/main/java/org/teiid/query/optimizer/relational/PlanToProcessConverter.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/main/java/org/teiid/query/optimizer/relational/PlanToProcessConverter.java:3149-3217
/trunk/engine/src/main/java/org/teiid/query/optimizer/relational/PlanToProcessConverter.java:3188-3450
+ /branches/7.4.x/engine/src/main/java/org/teiid/query/optimizer/relational/PlanToProcessConverter.java:3149-3217
/trunk/engine/src/main/java/org/teiid/query/optimizer/relational/PlanToProcessConverter.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java:3149-3217
/trunk/engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java:3188-3450
+ /branches/7.4.x/engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java:3149-3217
/trunk/engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/main/java/org/teiid/query/processor/QueryProcessor.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/main/java/org/teiid/query/processor/QueryProcessor.java:3220-3275,3340-3349
/trunk/engine/src/main/java/org/teiid/query/processor/QueryProcessor.java:3188-3450
+ /branches/7.4.x/engine/src/main/java/org/teiid/query/processor/QueryProcessor.java:3220-3275,3340-3349
/trunk/engine/src/main/java/org/teiid/query/processor/QueryProcessor.java:3188-3450,3452-3506
Modified: branches/as7/engine/src/main/java/org/teiid/query/processor/proc/ProcedurePlan.java
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/query/processor/proc/ProcedurePlan.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/main/java/org/teiid/query/processor/proc/ProcedurePlan.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -163,7 +163,7 @@
*/
public void initialize(CommandContext context, ProcessorDataManager dataMgr, BufferManager bufferMgr) {
this.bufferMgr = bufferMgr;
- this.batchSize = bufferMgr.getProcessorBatchSize();
+ this.batchSize = bufferMgr.getProcessorBatchSize(getOutputElements());
setContext(context.clone());
this.dataMgr = new ProcessorDataManager() {
Modified: branches/as7/engine/src/main/java/org/teiid/query/processor/relational/EnhancedSortMergeJoinStrategy.java
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/query/processor/relational/EnhancedSortMergeJoinStrategy.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/main/java/org/teiid/query/processor/relational/EnhancedSortMergeJoinStrategy.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -141,6 +141,7 @@
List<?> lastTuple = null;
boolean sortedDistinct = sorted && !state.isDistinct();
int sizeHint = index.getExpectedHeight(state.getTupleBuffer().getRowCount());
+ index.setBatchInsert(sorted);
outer: while (its.hasNext()) {
//detect if sorted and distinct
List<?> originalTuple = its.nextTuple();
@@ -162,6 +163,8 @@
}
if (!sorted) {
index.compact();
+ } else {
+ index.setBatchInsert(false);
}
its.closeSource();
this.reverseIndexes = new int[elements.size()];
Modified: branches/as7/engine/src/main/java/org/teiid/query/processor/relational/NullNode.java
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/query/processor/relational/NullNode.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/main/java/org/teiid/query/processor/relational/NullNode.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -22,6 +22,9 @@
package org.teiid.query.processor.relational;
+import java.util.Collections;
+import java.util.List;
+
import org.teiid.common.buffer.TupleBatch;
import org.teiid.core.TeiidComponentException;
@@ -38,11 +41,12 @@
this.terminateBatches();
return pullBatch();
}
+
+ @Override
+ public List getOutputElements() {
+ return Collections.emptyList();
+ }
- protected void getNodeString(StringBuffer str) {
- super.getNodeString(str);
- }
-
public Object clone(){
NullNode clonedNode = new NullNode(super.getID());
super.copy(this, clonedNode);
Modified: branches/as7/engine/src/main/java/org/teiid/query/processor/relational/RelationalNode.java
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/query/processor/relational/RelationalNode.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/main/java/org/teiid/query/processor/relational/RelationalNode.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -114,7 +114,11 @@
this.getProcessingState().nodeStatistics = new RelationalNodeStatistics();
}
- this.getProcessingState().batchSize = bufferManager.getProcessorBatchSize();
+ if (getOutputElements() != null) {
+ this.getProcessingState().batchSize = bufferManager.getProcessorBatchSize(getOutputElements());
+ } else {
+ this.getProcessingState().batchSize = bufferManager.getProcessorBatchSize();
+ }
}
public CommandContext getContext() {
Modified: branches/as7/engine/src/main/java/org/teiid/query/processor/relational/SortUtility.java
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/query/processor/relational/SortUtility.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/main/java/org/teiid/query/processor/relational/SortUtility.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -94,6 +94,7 @@
private String groupName;
private List<? extends Expression> schema;
private int schemaSize;
+ private int batchSize;
private ListNestedSortComparator comparator;
private TupleBuffer output;
@@ -164,6 +165,7 @@
this.groupName = groupName;
this.schema = schema;
this.schemaSize = bufferManager.getSchemaSize(this.schema);
+ this.batchSize = bufferManager.getProcessorBatchSize(this.schema);
this.comparator = new ListNestedSortComparator(cols, sortTypes);
int distinctIndex = cols.length - 1;
this.comparator.setDistinctIndex(distinctIndex);
@@ -226,7 +228,7 @@
int totalReservedBuffers = 0;
try {
- int maxRows = this.bufferManager.getProcessorBatchSize();
+ int maxRows = this.batchSize;
while(!doneReading) {
//attempt to reserve more working memory if there are additional rows available before blocking
if (workingTuples.size() >= maxRows) {
@@ -236,7 +238,7 @@
break;
}
totalReservedBuffers += reserved;
- maxRows += bufferManager.getProcessorBatchSize();
+ maxRows += this.batchSize;
}
try {
List<?> tuple = source.nextTuple();
@@ -249,7 +251,7 @@
this.collected++;
}
} catch(BlockedException e) {
- if (workingTuples.size() >= bufferManager.getProcessorBatchSize()) {
+ if (workingTuples.size() >= this.batchSize) {
break;
}
if (mode != Mode.DUP_REMOVE
Modified: branches/as7/engine/src/main/java/org/teiid/query/processor/xml/RelationalPlanExecutor.java
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/query/processor/xml/RelationalPlanExecutor.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/main/java/org/teiid/query/processor/xml/RelationalPlanExecutor.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -144,11 +144,8 @@
if (this.resultInfo.isAutoStaged() && tempTable != null) {
AlterTempTable att = new AlterTempTable(tempTable);
- int size = (Integer)this.currentRow.get(0);
- if (size > this.bufferMgr.getProcessorBatchSize() * 2) {
- //TODO: if the parent is small, then this is not necessary
- att.setIndexColumns(this.resultInfo.getFkColumns());
- }
+ //TODO: if the parent is small, then this is not necessary
+ att.setIndexColumns(this.resultInfo.getFkColumns());
this.dataManager.registerRequest(this.internalProcessor.getContext(), att, TempMetadataAdapter.TEMP_MODEL.getName(), null, 0, -1);
}
Property changes on: branches/as7/engine/src/main/java/org/teiid/query/sql/LanguageObject.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/main/java/org/teiid/query/sql/LanguageObject.java:3149-3217
/trunk/engine/src/main/java/org/teiid/query/sql/LanguageObject.java:3188-3450
+ /branches/7.4.x/engine/src/main/java/org/teiid/query/sql/LanguageObject.java:3149-3217
/trunk/engine/src/main/java/org/teiid/query/sql/LanguageObject.java:3188-3450,3452-3506
Modified: branches/as7/engine/src/main/java/org/teiid/query/tempdata/GlobalTableStoreImpl.java
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/query/tempdata/GlobalTableStoreImpl.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/main/java/org/teiid/query/tempdata/GlobalTableStoreImpl.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -290,7 +290,7 @@
if (tempTable != null) {
TempMetadataID id = tableStore.getMetadataStore().getTempGroupID(matTableName);
synchronized (id) {
- boolean clone = tempTable.getActiveReaders().get() != 0;
+ boolean clone = tempTable.getActive().get() != 0;
if (clone) {
tempTable = tempTable.clone();
}
Modified: branches/as7/engine/src/main/java/org/teiid/query/tempdata/TempTable.java
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/query/tempdata/TempTable.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/main/java/org/teiid/query/tempdata/TempTable.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -36,6 +36,7 @@
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.teiid.api.exception.query.ExpressionEvaluationException;
@@ -89,6 +90,19 @@
this.addRowId = addRowId;
this.indexes = indexes;
}
+
+ @Override
+ int process() throws ExpressionEvaluationException,
+ TeiidComponentException, TeiidProcessingException {
+ tree.setBatchInsert(addRowId);
+ return super.process();
+ }
+
+ @Override
+ protected void afterCompletion() throws TeiidComponentException {
+ tree.setBatchInsert(false);
+ }
+
@Override
protected void tuplePassed(List tuple) throws BlockedException,
@@ -236,6 +250,7 @@
success = true;
} finally {
try {
+ afterCompletion();
if (!success && undoLog != null) {
undoLog.setFinal(true);
TupleSource undoTs = undoLog.createIndexedTupleSource();
@@ -259,6 +274,14 @@
return updateCount;
}
+ /**
+ *
+ * @throws TeiidComponentException
+ */
+ protected void afterCompletion() throws TeiidComponentException {
+
+ }
+
@SuppressWarnings("unused")
void success() throws TeiidComponentException, ExpressionEvaluationException, TeiidProcessingException {}
@@ -275,9 +298,9 @@
}
}
- private static AtomicInteger ID_GENERATOR = new AtomicInteger();
+ private static AtomicLong ID_GENERATOR = new AtomicLong();
- private int id = ID_GENERATOR.getAndIncrement();
+ private Long id = ID_GENERATOR.getAndIncrement();
private STree tree;
private AtomicInteger rowId;
private List<ElementSymbol> columns;
@@ -361,7 +384,7 @@
}
}
- public AtomicInteger getActiveReaders() {
+ public AtomicInteger getActive() {
return activeReaders;
}
@@ -803,13 +826,13 @@
return tid;
}
- public int getId() {
+ public Long getId() {
return id;
}
@Override
public int hashCode() {
- return id;
+ return id.hashCode();
}
@Override
@@ -821,7 +844,7 @@
return false;
}
TempTable other = (TempTable)obj;
- return id == other.id;
+ return id.equals(other.id);
}
}
\ No newline at end of file
Modified: branches/as7/engine/src/main/java/org/teiid/query/tempdata/TempTableDataManager.java
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/query/tempdata/TempTableDataManager.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/main/java/org/teiid/query/tempdata/TempTableDataManager.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -213,7 +213,7 @@
TempTable tt = contextStore.getTempTable(att.getTempTable().toUpperCase());
Assertion.isNotNull(tt, "Table doesn't exist"); //$NON-NLS-1$
tt.setUpdatable(false);
- if (att.getIndexColumns() != null) {
+ if (att.getIndexColumns() != null && tt.getRowCount() > 2*tt.getTree().getPageSize(true)) {
tt.addIndex(att.getIndexColumns(), false);
}
return CollectionTupleSource.createUpdateCountTupleSource(0);
Modified: branches/as7/engine/src/main/java/org/teiid/query/tempdata/TempTableStore.java
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/query/tempdata/TempTableStore.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/main/java/org/teiid/query/tempdata/TempTableStore.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -82,7 +82,7 @@
public class TempTableSynchronization implements Synchronization {
private String id;
- Set<Integer> existingTables = new HashSet<Integer>();
+ Set<Long> existingTables = new HashSet<Long>();
ConcurrentHashMap<String, TempTable> tables = new ConcurrentHashMap<String, TempTable>();
private List<TransactionCallback> callbacks = new LinkedList<TransactionCallback>();
@@ -131,10 +131,21 @@
@Override
public synchronized void afterCompletion(int status) {
- //TODO: cleanup tables
completed = true;
synchronizations.remove(id);
- for (TransactionCallback callback : callbacks) {
+ if (transactionMode == TransactionMode.ISOLATE_READS) {
+ for (TempTable table : tables.values()) {
+ table.getActive().decrementAndGet();
+ }
+ } else {
+ HashSet<TempTable> current = new HashSet<TempTable>(tempTables.values());
+ current.retainAll(tables.values());
+ for (TempTable table : current) {
+ table.getActive().set(0);
+ table.getTree().clearClonedFlags();
+ }
+ }
+ for (TransactionCallback callback : callbacks) {
if (status == Status.STATUS_COMMITTED) {
callback.commit();
} else {
@@ -144,6 +155,10 @@
callbacks.clear();
}
+ public boolean isCompleted() {
+ return completed;
+ }
+
@Override
public void beforeCompletion() {
@@ -307,11 +322,21 @@
if (synch != null && synch.existingTables.contains(tempTable.getId())) {
TempTable result = synch.tables.get(tempTableID);
if (result == null) {
- synch.tables.put(tempTableID, tempTable.clone());
+ synchronized (synch) {
+ if (synch.isCompleted()) {
+ throw new AssertionError("Expected active transaction"); //$NON-NLS-1$
+ }
+ if (!tempTable.getActive().compareAndSet(0, 1)) {
+ throw new TeiidProcessingException(QueryPlugin.Util.getString("TempTableStore.pending_update", tempTableID)); //$NON-NLS-1$
+ }
+ synch.tables.put(tempTableID, tempTable.clone());
+ }
}
return tempTable;
}
- }
+ } else if (tempTable.getActive().get() != 0) {
+ throw new TeiidProcessingException(QueryPlugin.Util.getString("TempTableStore.pending_update", tempTableID)); //$NON-NLS-1$
+ }
}
} else if (transactionMode == TransactionMode.ISOLATE_READS) {
TransactionContext tc = context.getTransactionContext();
@@ -320,24 +345,13 @@
if (synch != null) {
TempTable result = synch.tables.get(tempTableID);
if (result == null) {
- synch.tables.put(tempTableID, tempTable);
result = tempTable;
- result.getActiveReaders().getAndIncrement();
- TransactionCallback callback = new TransactionCallback() {
-
- @Override
- public void rollback() {
- tempTable.getActiveReaders().getAndDecrement();
- }
-
- @Override
- public void commit() {
- tempTable.getActiveReaders().getAndDecrement();
- }
- };
- if (!synch.addCallback(callback)) {
- callback.rollback();
- }
+ synchronized (synch) {
+ if (!synch.isCompleted()) {
+ synch.tables.put(tempTableID, tempTable);
+ result.getActive().getAndIncrement();
+ }
+ }
}
return result;
}
Modified: branches/as7/engine/src/main/java/org/teiid/query/util/CommandContext.java
===================================================================
--- branches/as7/engine/src/main/java/org/teiid/query/util/CommandContext.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/main/java/org/teiid/query/util/CommandContext.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -26,6 +26,7 @@
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedList;
+import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Random;
@@ -78,8 +79,6 @@
private int processorBatchSize = BufferManager.DEFAULT_PROCESSOR_BATCH_SIZE;
- private int connectorBatchSize = BufferManager.DEFAULT_CONNECTOR_BATCH_SIZE;
-
private String userName;
private Serializable commandPayload;
@@ -312,20 +311,15 @@
return this.globalState.collectNodeStatistics;
}
- public int getConnectorBatchSize() {
- return this.globalState.connectorBatchSize;
- }
-
- public void setConnectorBatchSize(int connectorBatchSize) {
- this.globalState.connectorBatchSize = connectorBatchSize;
- }
-
-
+ @Override
public int getProcessorBatchSize() {
return this.globalState.processorBatchSize;
}
-
+ public int getProcessorBatchSize(List<Expression> schema) {
+ return this.globalState.bufferManager.getProcessorBatchSize(schema);
+ }
+
public void setProcessorBatchSize(int processorBatchSize) {
this.globalState.processorBatchSize = processorBatchSize;
}
Modified: branches/as7/engine/src/main/resources/org/teiid/query/i18n.properties
===================================================================
--- branches/as7/engine/src/main/resources/org/teiid/query/i18n.properties 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/main/resources/org/teiid/query/i18n.properties 2011-09-29 16:46:38 UTC (rev 3507)
@@ -791,6 +791,7 @@
RulePlanJoins.cantSatisfy=Join region with unsatisfied access patterns cannot be satisfied by the join criteria, Access patterns: {0}
TempTableStore.table_exist_error=Temporary table "{0}" already exists.
TempTableStore.table_doesnt_exist_error=Temporary table "{0}" does not exist.
+TempTableStore.pending_update=Table {0} is locked by pending transaction update.
XMLQueryPlanner.cannot_plan=Cannot create a query for MappingClass with user criteria {0}
XMLQueryPlanner.invalid_relationship=Conjunct "{0}" has no relationship with target context {1}.
Modified: branches/as7/engine/src/test/java/org/teiid/common/buffer/BufferManagerFactory.java
===================================================================
--- branches/as7/engine/src/test/java/org/teiid/common/buffer/BufferManagerFactory.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/test/java/org/teiid/common/buffer/BufferManagerFactory.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -22,9 +22,10 @@
package org.teiid.common.buffer;
-import org.teiid.common.buffer.BufferManager;
import org.teiid.common.buffer.impl.BufferManagerImpl;
+import org.teiid.common.buffer.impl.FileStoreCache;
import org.teiid.common.buffer.impl.MemoryStorageManager;
+import org.teiid.common.buffer.impl.SplittableStorageManager;
import org.teiid.core.TeiidComponentException;
@@ -81,12 +82,18 @@
public static BufferManagerImpl initBufferManager(BufferManagerImpl bufferManager) {
try {
bufferManager.initialize();
- } catch (TeiidComponentException e) {
+ MemoryStorageManager storageManager = new MemoryStorageManager();
+ SplittableStorageManager ssm = new SplittableStorageManager(storageManager);
+ ssm.setMaxFileSizeDirect(MemoryStorageManager.MAX_FILE_SIZE);
+ FileStoreCache fsc = new FileStoreCache();
+ fsc.setMaxBufferSpace(Runtime.getRuntime().maxMemory()/4);
+ fsc.setStorageManager(ssm);
+ fsc.initialize();
+ bufferManager.setCache(fsc);
+ return bufferManager;
+ } catch (TeiidComponentException e) {
throw new RuntimeException(e);
}
-
- bufferManager.setStorageManager(new MemoryStorageManager());
- return bufferManager;
}
}
Modified: branches/as7/engine/src/test/java/org/teiid/common/buffer/TestLobManager.java
===================================================================
--- branches/as7/engine/src/test/java/org/teiid/common/buffer/TestLobManager.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/test/java/org/teiid/common/buffer/TestLobManager.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -28,8 +28,10 @@
import java.io.StringReader;
import java.nio.charset.Charset;
import java.util.Arrays;
+import java.util.List;
import org.junit.Test;
+import org.teiid.common.buffer.LobManager.ReferenceMode;
import org.teiid.common.buffer.impl.BufferManagerImpl;
import org.teiid.core.types.BlobImpl;
import org.teiid.core.types.BlobType;
@@ -65,9 +67,11 @@
}));
- LobManager lobManager = new LobManager();
- lobManager.updateReferences(new int[] {0,1}, Arrays.asList(clob, blob));
- lobManager.persist(fs);
+ LobManager lobManager = new LobManager(new int[] {0, 1}, fs);
+ lobManager.setMaxMemoryBytes(4);
+ List<Streamable<? extends Object>> tuple = Arrays.asList(clob, blob);
+ lobManager.updateReferences(tuple, ReferenceMode.CREATE);
+ lobManager.persist();
Streamable<?>lob = lobManager.getLobReference(clob.getReferenceStreamId());
assertTrue(lob.getClass().isAssignableFrom(ClobType.class));
@@ -80,6 +84,10 @@
BlobType blobRead = (BlobType)lob;
assertTrue(Arrays.equals(ObjectConverterUtil.convertToByteArray(blob.getBinaryStream()), ObjectConverterUtil.convertToByteArray(blobRead.getBinaryStream())));
+ lobManager.updateReferences(tuple, ReferenceMode.REMOVE);
+
+ assertEquals(0, lobManager.getLobCount());
+
}
}
Modified: branches/as7/engine/src/test/java/org/teiid/common/buffer/TestSTree.java
===================================================================
--- branches/as7/engine/src/test/java/org/teiid/common/buffer/TestSTree.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/test/java/org/teiid/common/buffer/TestSTree.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -79,7 +79,7 @@
@Test public void testOrderedInsert() throws TeiidComponentException {
BufferManagerImpl bm = BufferManagerFactory.createBufferManager();
- bm.setProcessorBatchSize(16);
+ bm.setProcessorBatchSize(4);
ElementSymbol e1 = new ElementSymbol("x");
e1.setType(Integer.class);
@@ -100,5 +100,33 @@
}
}
-
+
+ /**
+ * Forces the logic through several compaction cycles by using large strings
+ * @throws TeiidComponentException
+ */
+ @Test public void testCompaction() throws TeiidComponentException {
+ BufferManagerImpl bm = BufferManagerFactory.createBufferManager();
+ bm.setProcessorBatchSize(32);
+ bm.setMaxReserveKB(0);//force all to disk
+ bm.initialize();
+
+ ElementSymbol e1 = new ElementSymbol("x");
+ e1.setType(String.class);
+ List<ElementSymbol> elements = Arrays.asList(e1);
+ STree map = bm.createSTree(elements, "1", 1);
+
+ int size = 1000;
+
+ for (int i = 0; i < size; i++) {
+ assertNull(map.insert(Arrays.asList(new String(new byte[1000])), InsertMode.ORDERED, size));
+ assertEquals(i + 1, map.getRowCount());
+ }
+
+ for (int i = 0; i < size; i++) {
+ assertNotNull(map.remove(Arrays.asList(new String(new byte[1000]))));
+ }
+
+ }
+
}
Modified: branches/as7/engine/src/test/java/org/teiid/common/buffer/TestTupleBuffer.java
===================================================================
--- branches/as7/engine/src/test/java/org/teiid/common/buffer/TestTupleBuffer.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/test/java/org/teiid/common/buffer/TestTupleBuffer.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -30,60 +30,18 @@
import javax.sql.rowset.serial.SerialClob;
import org.junit.Test;
-import org.mockito.Mockito;
-import org.teiid.core.TeiidComponentException;
+import org.teiid.common.buffer.BufferManager.TupleSourceType;
import org.teiid.core.types.ClobType;
import org.teiid.core.types.DataTypeManager;
import org.teiid.query.sql.symbol.ElementSymbol;
-
public class TestTupleBuffer {
- public static final class FakeBatchManager implements BatchManager {
- @Override
- public void remove() {
-
- }
-
- @Override
- public ManagedBatch createManagedBatch(final TupleBatch batch, boolean softCache)
- throws TeiidComponentException {
- return new ManagedBatch() {
-
- @Override
- public void remove() {
-
- }
-
- @Override
- public TupleBatch getBatch(boolean cache, String[] types)
- throws TeiidComponentException {
- return batch;
- }
-
- @Override
- public void setPrefersMemory(boolean prefers) {
-
- }
-
- @Override
- public CleanupHook getCleanupHook() {
- return null;
- }
- };
- }
-
- @Override
- public FileStore createStorage(String prefix) {
- return Mockito.mock(FileStore.class);
- }
- }
-
@Test public void testForwardOnly() throws Exception {
ElementSymbol x = new ElementSymbol("x"); //$NON-NLS-1$
x.setType(DataTypeManager.DefaultDataClasses.INTEGER);
List<ElementSymbol> schema = Arrays.asList(x);
- TupleBuffer tb = new TupleBuffer(new FakeBatchManager(), "x", schema, null, 32); //$NON-NLS-1$
+ TupleBuffer tb = BufferManagerFactory.getStandaloneBufferManager().createTupleBuffer(schema, "x", TupleSourceType.PROCESSOR); //$NON-NLS-1$
tb.setForwardOnly(true);
tb.addTuple(Arrays.asList(1));
TupleBatch batch = tb.getBatch(1);
@@ -106,7 +64,8 @@
ElementSymbol x = new ElementSymbol("x"); //$NON-NLS-1$
x.setType(DataTypeManager.DefaultDataClasses.CLOB);
List<ElementSymbol> schema = Arrays.asList(x);
- TupleBuffer tb = new TupleBuffer(new FakeBatchManager(), "x", schema, LobManager.getLobIndexes(schema), 32); //$NON-NLS-1$
+ TupleBuffer tb = BufferManagerFactory.getStandaloneBufferManager().createTupleBuffer(schema, "x", TupleSourceType.PROCESSOR); //$NON-NLS-1$
+ tb.setInlineLobs(false);
ClobType c = new ClobType(new SerialClob(new char[0]));
TupleBatch batch = new TupleBatch(1, new List[] {Arrays.asList(c)});
tb.addTupleBatch(batch, false);
Copied: branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestBlockBitSetTree.java (from rev 3506, trunk/engine/src/test/java/org/teiid/common/buffer/impl/TestBlockBitSetTree.java)
===================================================================
--- branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestBlockBitSetTree.java (rev 0)
+++ branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestBlockBitSetTree.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -0,0 +1,66 @@
+/*
+ * JBoss, Home of Professional Open Source.
+ * See the COPYRIGHT.txt file distributed with this work for information
+ * regarding copyright ownership. Some portions may be licensed
+ * to Red Hat, Inc. under one or more contributor license agreements.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301 USA.
+ */
+
+ package org.teiid.common.buffer.impl;
+
+import static org.junit.Assert.*;
+
+import java.util.BitSet;
+import java.util.Random;
+
+import org.junit.Test;
+
+public class TestBlockBitSetTree {
+
+ @Test public void testBitsSet() {
+ BlockBitSetTree bst = new BlockBitSetTree((1 << 20) -1, new TestBlockClosedLongIntHashTable.DummyBlockManager());
+ bst.set(1, true);
+ bst.set(100, true);
+ bst.set(10000, true);
+ bst.set(1000000, true);
+ assertEquals(4, bst.getBitsSet());
+ bst.set(1, false);
+ assertEquals(3, bst.getBitsSet());
+ assertFalse(bst.get(1));
+ }
+
+ @Test public void testNextClearSet() {
+ BlockBitSetTree bst = new BlockBitSetTree((1 << 20) -1, new TestBlockClosedLongIntHashTable.DummyBlockManager());
+ BitSet bst1 = new BitSet();
+ Random r = new Random(1);
+ for (int i = 0; i < 1000; i++) {
+ int rand = r.nextInt() & bst.getMaxIndex();
+ bst.set(rand, true);
+ bst1.set(rand, true);
+ assertTrue(bst.get(rand));
+ assertEquals(bst1.nextSetBit(rand), bst.nextSetBit(rand));
+ assertEquals(String.valueOf(i), bst1.nextSetBit(rand), bst.nextSetBit(rand));
+ }
+
+ for (int i = 0; i < 10000; i++) {
+ int rand = r.nextInt() & bst.getMaxIndex();
+ assertEquals(bst1.nextClearBit(rand), bst.nextClearBit(rand));
+ assertEquals(bst1.nextSetBit(rand), bst.nextSetBit(rand));
+ }
+ }
+
+}
Copied: branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestBlockClosedLongIntHashTable.java (from rev 3506, trunk/engine/src/test/java/org/teiid/common/buffer/impl/TestBlockClosedLongIntHashTable.java)
===================================================================
--- branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestBlockClosedLongIntHashTable.java (rev 0)
+++ branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestBlockClosedLongIntHashTable.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -0,0 +1,92 @@
+/*
+ * JBoss, Home of Professional Open Source.
+ * See the COPYRIGHT.txt file distributed with this work for information
+ * regarding copyright ownership. Some portions may be licensed
+ * to Red Hat, Inc. under one or more contributor license agreements.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301 USA.
+ */
+
+package org.teiid.common.buffer.impl;
+
+import static org.junit.Assert.*;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Random;
+
+import org.junit.Test;
+import org.teiid.common.buffer.impl.FileStoreCache.BlockInfo;
+
+public class TestBlockClosedLongIntHashTable {
+
+ public static final class DummyBlockManager implements BlockManager {
+ List<BlockInfo> blocks = new ArrayList<BlockInfo>();
+
+ @Override
+ public int getInode() {
+ return 0;
+ }
+
+ @Override
+ public void updateBlock(BlockInfo block) {
+
+ }
+
+ @Override
+ public void free() {
+ blocks.clear();
+ }
+
+ @Override
+ public BlockInfo getBlock(int index) {
+ BlockInfo block = blocks.get(index);
+ block.buf.rewind();
+ return block;
+ }
+
+ @Override
+ public void freeBlock(int index) {
+ blocks.remove(index);
+ }
+
+ @Override
+ public BlockInfo allocateBlock(int index) {
+ assertEquals(index, blocks.size());
+ ByteBuffer result = ByteBuffer.wrap(new byte[FileStoreCache.BLOCK_SIZE]);
+ blocks.add(new BlockInfo(false, result, index, -1));
+ return blocks.get(blocks.size() - 1);
+ }
+ }
+
+ @Test public void testAgainstHashMap() {
+ BlockClosedLongIntHashTable table = new BlockClosedLongIntHashTable(new DummyBlockManager());
+ HashMap<Long, Integer> table1 = new HashMap<Long, Integer>(16);
+ for (long i = 1; i < 200000; i++) {
+ table.put(i, (int)i);
+ table1.put(i, (int)i);
+ }
+ Random r = new Random(0);
+ for (int i = 1; i < 2000000; i++) {
+ long toRemove = r.nextInt(i);
+ boolean removed = table.remove(toRemove) != BlockClosedLongIntHashTable.EMPTY;
+ assertEquals(table1.remove(toRemove) != null, removed);
+ }
+ }
+
+}
Modified: branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestFileStorageManager.java
===================================================================
--- branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestFileStorageManager.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestFileStorageManager.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -25,11 +25,10 @@
import static org.junit.Assert.*;
import java.io.File;
+import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
-import java.io.RandomAccessFile;
import java.util.Arrays;
-import java.util.Map;
import java.util.Random;
import org.junit.Test;
@@ -41,12 +40,9 @@
@SuppressWarnings("nls")
public class TestFileStorageManager {
- public FileStorageManager getStorageManager(Integer maxFileSize, Integer openFiles, String dir) throws TeiidComponentException {
+ public FileStorageManager getStorageManager(Integer openFiles, String dir) throws TeiidComponentException {
FileStorageManager sm = new FileStorageManager();
sm.setStorageDirectory(UnitTestUtil.getTestScratchPath() + (dir != null ? File.separator + dir : "")); //$NON-NLS-1$
- if (maxFileSize != null) {
- sm.setMaxFileSizeDirect(maxFileSize);
- }
if (openFiles != null) {
sm.setMaxOpenFiles(openFiles);
}
@@ -55,7 +51,7 @@
}
@Test public void testWrite() throws Exception {
- FileStorageManager sm = getStorageManager(null, null, null);
+ FileStorageManager sm = getStorageManager(null, null);
String tsID = "0"; //$NON-NLS-1$
FileStore store = sm.createFileStore(tsID);
writeBytes(store);
@@ -64,27 +60,8 @@
assertEquals(0, sm.getUsedBufferSpace());
}
- @Test public void testCreatesSpillFiles() throws Exception {
- FileStorageManager sm = getStorageManager(1024, null, null); // 1KB
- String tsID = "0"; //$NON-NLS-1$
- // Add one batch
- FileStore store = sm.createFileStore(tsID);
- writeBytes(store);
-
- Map<File, RandomAccessFile> cache = sm.getFileCache();
- assertEquals(1, cache.size());
-
- writeBytes(store);
-
- assertEquals(2, cache.size());
-
- store.remove();
-
- assertEquals(0, cache.size());
- }
-
- @Test(expected=TeiidComponentException.class) public void testMaxSpace() throws Exception {
- FileStorageManager sm = getStorageManager(null, null, null);
+ @Test(expected=IOException.class) public void testMaxSpace() throws Exception {
+ FileStorageManager sm = getStorageManager(null, null);
sm.setMaxBufferSpace(1);
String tsID = "0"; //$NON-NLS-1$
// Add one batch
@@ -93,7 +70,7 @@
}
@Test public void testFlush() throws Exception {
- FileStorageManager sm = getStorageManager(null, null, null);
+ FileStorageManager sm = getStorageManager(null, null);
FileStore store = sm.createFileStore("0");
FileStoreOutputStream fsos = store.createOutputStream(2);
fsos.write(new byte[3]);
@@ -104,19 +81,19 @@
static Random r = new Random();
- private void writeBytes(FileStore store)
- throws TeiidComponentException {
+ static void writeBytes(FileStore store)
+ throws IOException {
byte[] bytes = new byte[2048];
r.nextBytes(bytes);
- long start = store.write(bytes, 0, bytes.length);
+ long start = store.getLength();
+ store.write(bytes, 0, bytes.length);
byte[] bytesRead = new byte[2048];
store.readFully(start, bytesRead, 0, bytesRead.length);
assertTrue(Arrays.equals(bytes, bytesRead));
}
-
@Test public void testWritingMultipleFiles() throws Exception {
- FileStorageManager sm = getStorageManager(1024, null, null);
+ FileStorageManager sm = getStorageManager(null, null);
String tsID = "0"; //$NON-NLS-1$
// Add one batch
FileStore store = sm.createFileStore(tsID);
Copied: branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestFileStoreCache.java (from rev 3506, trunk/engine/src/test/java/org/teiid/common/buffer/impl/TestFileStoreCache.java)
===================================================================
--- branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestFileStoreCache.java (rev 0)
+++ branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestFileStoreCache.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -0,0 +1,119 @@
+/*
+ * JBoss, Home of Professional Open Source.
+ * See the COPYRIGHT.txt file distributed with this work for information
+ * regarding copyright ownership. Some portions may be licensed
+ * to Red Hat, Inc. under one or more contributor license agreements.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301 USA.
+ */
+
+package org.teiid.common.buffer.impl;
+
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+
+import org.junit.Test;
+import org.teiid.common.buffer.CacheEntry;
+import org.teiid.common.buffer.Serializer;
+import org.teiid.core.util.UnitTestUtil;
+
+public class TestFileStoreCache {
+
+ private final class SimpleSerializer implements Serializer<Integer> {
+ @Override
+ public Integer deserialize(ObjectInputStream ois)
+ throws IOException, ClassNotFoundException {
+ Integer result = ois.readInt();
+ for (int i = 0; i < result; i++) {
+ assertEquals(i, ois.readInt());
+ }
+ return result;
+ }
+
+ @Override
+ public Long getId() {
+ return 1l;
+ }
+
+ @Override
+ public void serialize(Integer obj, ObjectOutputStream oos)
+ throws IOException {
+ oos.writeInt(obj);
+ for (int i = 0; i < obj; i++) {
+ oos.writeInt(i);
+ }
+ }
+
+ @Override
+ public boolean useSoftCache() {
+ return false;
+ }
+ }
+
+ @Test public void testAddGetMultiBlock() throws Exception {
+ FileStoreCache fsc = new FileStoreCache();
+ fsc.setMaxBufferSpace(1 << 28);
+ SplittableStorageManager ssm = new SplittableStorageManager(new MemoryStorageManager());
+ ssm.setMaxFileSizeDirect(MemoryStorageManager.MAX_FILE_SIZE);
+ fsc.setStorageManager(ssm);
+ fsc.initialize();
+
+ UnitTestUtil.enableTraceLogging("org.teiid"); //$NON-NLS-1$
+
+ CacheEntry ce = new CacheEntry(2l);
+ Serializer<Integer> s = new SimpleSerializer();
+ fsc.createCacheGroup(s.getId());
+ Integer cacheObject = Integer.valueOf(2);
+ ce.setObject(cacheObject);
+ fsc.add(ce, s);
+
+ ce = fsc.get(2l, s);
+ assertEquals(cacheObject, ce.getObject());
+
+ //test something that exceeds the direct inode data blocks
+ ce = new CacheEntry(3l);
+ cacheObject = Integer.valueOf(80000);
+ ce.setObject(cacheObject);
+ fsc.add(ce, s);
+
+ ce = fsc.get(3l, s);
+ assertEquals(cacheObject, ce.getObject());
+
+ fsc.removeCacheGroup(1l);
+
+ assertEquals(0, fsc.getDataBlocksInUse());
+ assertEquals(0, fsc.getInodesInUse());
+
+ //test something that exceeds the indirect inode data blocks
+ ce = new CacheEntry(3l);
+ fsc.createCacheGroup(s.getId());
+ cacheObject = Integer.valueOf(5000000);
+ ce.setObject(cacheObject);
+ fsc.add(ce, s);
+
+ ce = fsc.get(3l, s);
+ assertEquals(cacheObject, ce.getObject());
+
+ fsc.removeCacheGroup(1l);
+
+ assertEquals(0, fsc.getDataBlocksInUse());
+ assertEquals(0, fsc.getInodesInUse());
+ }
+
+}
Modified: branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestSizeUtility.java
===================================================================
--- branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestSizeUtility.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestSizeUtility.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -34,7 +34,6 @@
import java.util.List;
import org.junit.Test;
-import org.teiid.common.buffer.TupleBatch;
public class TestSizeUtility {
@@ -43,7 +42,7 @@
}
public void helpTestGetSize(Object obj, long expectedSize) {
- long actualSize = new SizeUtility().getSize(obj, true, false);
+ long actualSize = new SizeUtility(null).getSize(obj, true, false);
assertEquals("Got unexpected size: ", expectedSize, actualSize); //$NON-NLS-1$
}
@@ -99,18 +98,6 @@
helpTestGetSize("abcdefghij", 64); //$NON-NLS-1$
}
- public void XtestGetSizeLongString() {
- // There is no clear way of figuring out the actual size of a string that is created
- // from a StringBuffer because the buffer can sometimes be twice as big as the actual length of the string
- // Since the data comin from the connector is not created this way, this test is an inaccurate setup
- int size = 10000;
- StringBuffer str = new StringBuffer();
- for(int i=0; i<size; i++) {
- str.append("a"); //$NON-NLS-1$
- }
- helpTestGetSize(str.toString(), size+3);
- }
-
@Test public void testGetSizeRow1() {
List<Object> row = new ArrayList<Object>(1);
row.add(new Integer(0));
@@ -176,9 +163,7 @@
String[] types = {"string", "integer", "boolean", "double", "string", "integer"}; //$NON-NLS-1$//$NON-NLS-2$//$NON-NLS-3$//$NON-NLS-4$ //$NON-NLS-5$//$NON-NLS-6$
- TupleBatch tb = new TupleBatch(1, expected);
- tb.setDataTypes(types);
- long actualSize = new SizeUtility().getBatchSize(tb);
+ long actualSize = new SizeUtility(types).getBatchSize(Arrays.asList(expected));
assertEquals("Got unexpected size: ", 2667, actualSize); //$NON-NLS-1$
}
Copied: branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestSplittableStorageManager.java (from rev 3506, trunk/engine/src/test/java/org/teiid/common/buffer/impl/TestSplittableStorageManager.java)
===================================================================
--- branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestSplittableStorageManager.java (rev 0)
+++ branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestSplittableStorageManager.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -0,0 +1,81 @@
+/*
+ * JBoss, Home of Professional Open Source.
+ * See the COPYRIGHT.txt file distributed with this work for information
+ * regarding copyright ownership. Some portions may be licensed
+ * to Red Hat, Inc. under one or more contributor license agreements.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301 USA.
+ */
+
+package org.teiid.common.buffer.impl;
+
+import static org.junit.Assert.*;
+
+import org.junit.Test;
+import org.teiid.common.buffer.FileStore;
+
+public class TestSplittableStorageManager {
+
+ @Test public void testCreatesSpillFiles() throws Exception {
+ MemoryStorageManager msm = new MemoryStorageManager();
+ SplittableStorageManager ssm = new SplittableStorageManager(msm);
+ ssm.setMaxFileSizeDirect(2048);
+ String tsID = "0"; //$NON-NLS-1$
+ // Add one batch
+ FileStore store = ssm.createFileStore(tsID);
+ TestFileStorageManager.writeBytes(store);
+
+ assertEquals(1, msm.getCreated());
+
+ TestFileStorageManager.writeBytes(store);
+
+ assertEquals(2, msm.getCreated());
+
+ store.setLength(10000);
+
+ assertEquals(5, msm.getCreated());
+
+ store.setLength(100);
+
+ assertEquals(4, msm.getRemoved());
+
+ store.remove();
+
+ assertEquals(5, msm.getRemoved());
+ }
+
+ @Test public void testTruncate() throws Exception {
+ MemoryStorageManager msm = new MemoryStorageManager();
+ SplittableStorageManager ssm = new SplittableStorageManager(msm);
+ ssm.setMaxFileSizeDirect(2048);
+ String tsID = "0"; //$NON-NLS-1$
+ // Add one batch
+ FileStore store = ssm.createFileStore(tsID);
+ TestFileStorageManager.writeBytes(store);
+
+ assertEquals(1, msm.getCreated());
+
+ TestFileStorageManager.writeBytes(store);
+
+ assertEquals(2, msm.getCreated());
+
+ store.setLength(100);
+
+ assertEquals(1, msm.getRemoved());
+
+ }
+
+}
Modified: branches/as7/engine/src/test/java/org/teiid/dqp/internal/process/TestCachedResults.java
===================================================================
--- branches/as7/engine/src/test/java/org/teiid/dqp/internal/process/TestCachedResults.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/test/java/org/teiid/dqp/internal/process/TestCachedResults.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -23,7 +23,6 @@
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import java.util.Arrays;
@@ -33,8 +32,8 @@
import org.teiid.cache.Cache;
import org.teiid.cache.DefaultCache;
import org.teiid.common.buffer.BufferManager;
+import org.teiid.common.buffer.BufferManager.TupleSourceType;
import org.teiid.common.buffer.BufferManagerFactory;
-import org.teiid.common.buffer.TestTupleBuffer.FakeBatchManager;
import org.teiid.common.buffer.TupleBuffer;
import org.teiid.core.types.DataTypeManager;
import org.teiid.core.util.UnitTestUtil;
@@ -62,7 +61,7 @@
ElementSymbol x = new ElementSymbol("x"); //$NON-NLS-1$
x.setType(DataTypeManager.DefaultDataClasses.INTEGER);
List<ElementSymbol> schema = Arrays.asList(x);
- TupleBuffer tb = new TupleBuffer(new FakeBatchManager(), "x", schema, null, 4); //$NON-NLS-1$
+ TupleBuffer tb = BufferManagerFactory.getStandaloneBufferManager().createTupleBuffer(schema, "x", TupleSourceType.PROCESSOR); //$NON-NLS-1$
tb.setForwardOnly(false);
tb.addTuple(Arrays.asList(1));
Modified: branches/as7/engine/src/test/java/org/teiid/dqp/internal/process/TestDQPCore.java
===================================================================
--- branches/as7/engine/src/test/java/org/teiid/dqp/internal/process/TestDQPCore.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/test/java/org/teiid/dqp/internal/process/TestDQPCore.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -25,6 +25,7 @@
import static org.junit.Assert.*;
import java.sql.ResultSet;
+import java.util.HashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
@@ -35,6 +36,8 @@
import org.junit.Ignore;
import org.junit.Test;
import org.mockito.Mockito;
+import org.teiid.adminapi.DataPolicy;
+import org.teiid.adminapi.impl.DataPolicyMetadata;
import org.teiid.api.exception.query.QueryResolverException;
import org.teiid.cache.CacheConfiguration;
import org.teiid.cache.DefaultCacheFactory;
@@ -104,6 +107,9 @@
DQPWorkContext context = RealMetadataFactory.buildWorkContext(RealMetadataFactory.createTransformationMetadata(RealMetadataFactory.exampleBQTCached().getMetadataStore(), "bqt"));
context.getVDB().getModel("BQT3").setVisible(false); //$NON-NLS-1$
context.getVDB().getModel("VQT").setVisible(false); //$NON-NLS-1$
+ HashMap<String, DataPolicy> policies = new HashMap<String, DataPolicy>();
+ policies.put("foo", new DataPolicyMetadata());
+ context.setPolicies(policies);
ConnectorManagerRepository repo = Mockito.mock(ConnectorManagerRepository.class);
context.getVDB().addAttchment(ConnectorManagerRepository.class, repo);
@@ -112,7 +118,9 @@
@Override
public BufferManager getBufferManager() {
- return BufferManagerFactory.createBufferManager();
+ BufferManagerImpl bm = BufferManagerFactory.createBufferManager();
+ bm.setInlineLobs(false);
+ return bm;
}
};
core = new DQPCore();
@@ -124,6 +132,9 @@
config = new DQPConfiguration();
config.setMaxActivePlans(1);
config.setUserRequestSourceConcurrency(2);
+ DefaultAuthorizationValidator daa = new DefaultAuthorizationValidator();
+ daa.setPolicyDecider(new DataRolePolicyDecider());
+ config.setAuthorizationValidator(daa);
core.start(config);
core.getPrepPlanCache().setModTime(1);
core.getRsCache().setBufferManager(bs.getBufferManager());
@@ -151,6 +162,20 @@
@Test public void testRequest1() throws Exception {
helpExecute("SELECT IntKey FROM BQT1.SmallA", "a"); //$NON-NLS-1$ //$NON-NLS-2$
}
+
+ @Test public void testHasRole() throws Exception {
+ String sql = "SELECT hasRole('foo')"; //$NON-NLS-1$
+ String userName = "logon"; //$NON-NLS-1$
+ ResultsMessage rm = helpExecute(sql, userName);
+ assertTrue((Boolean)rm.getResults()[0].get(0));
+ }
+
+ @Test public void testNotHasRole() throws Exception {
+ String sql = "SELECT hasRole('bar')"; //$NON-NLS-1$
+ String userName = "logon"; //$NON-NLS-1$
+ ResultsMessage rm = helpExecute(sql, userName);
+ assertFalse((Boolean)rm.getResults()[0].get(0));
+ }
@Test public void testUser1() throws Exception {
String sql = "SELECT IntKey FROM BQT1.SmallA WHERE user() = 'logon'"; //$NON-NLS-1$
@@ -277,8 +302,8 @@
}
@Test public void testBufferLimit() throws Exception {
- //the sql should return 100 rows
- String sql = "SELECT A.IntKey FROM BQT1.SmallA as A, BQT1.SmallA as B"; //$NON-NLS-1$
+ //the sql should return 400 rows
+ String sql = "SELECT A.IntKey FROM BQT1.SmallA as A, BQT1.SmallA as B, (select intkey from BQT1.SmallA limit 4) as C"; //$NON-NLS-1$
String userName = "1"; //$NON-NLS-1$
String sessionid = "1"; //$NON-NLS-1$
@@ -286,31 +311,33 @@
reqMsg.setCursorType(ResultSet.TYPE_FORWARD_ONLY);
DQPWorkContext.getWorkContext().getSession().setSessionId(sessionid);
DQPWorkContext.getWorkContext().getSession().setUserName(userName);
- ((BufferManagerImpl)core.getBufferManager()).setProcessorBatchSize(2);
+ ((BufferManagerImpl)core.getBufferManager()).setProcessorBatchSize(1);
Future<ResultsMessage> message = core.executeRequest(reqMsg.getExecutionId(), reqMsg);
ResultsMessage rm = message.get(500000, TimeUnit.MILLISECONDS);
assertNull(rm.getException());
- assertEquals(2, rm.getResults().length);
+
+ int rowsPerBatch = 8;
+ assertEquals(rowsPerBatch, rm.getResults().length);
RequestWorkItem item = core.getRequestWorkItem(DQPWorkContext.getWorkContext().getRequestID(reqMsg.getExecutionId()));
- message = core.processCursorRequest(reqMsg.getExecutionId(), 3, 2);
+ message = core.processCursorRequest(reqMsg.getExecutionId(), 9, rowsPerBatch);
rm = message.get(500000, TimeUnit.MILLISECONDS);
assertNull(rm.getException());
- assertEquals(2, rm.getResults().length);
+ assertEquals(rowsPerBatch, rm.getResults().length);
//ensure that we are idle
for (int i = 0; i < 10 && item.getThreadState() != ThreadState.IDLE; i++) {
Thread.sleep(100);
}
assertEquals(ThreadState.IDLE, item.getThreadState());
- assertTrue(item.resultsBuffer.getManagedRowCount() <= 46);
+ assertTrue(item.resultsBuffer.getManagedRowCount() <= rowsPerBatch*23);
//pull the rest of the results
for (int j = 0; j < 48; j++) {
item = core.getRequestWorkItem(DQPWorkContext.getWorkContext().getRequestID(reqMsg.getExecutionId()));
- message = core.processCursorRequest(reqMsg.getExecutionId(), j * 2 + 5, 2);
+ message = core.processCursorRequest(reqMsg.getExecutionId(), (j + 2) * rowsPerBatch + 1, rowsPerBatch);
rm = message.get(5000, TimeUnit.MILLISECONDS);
assertNull(rm.getException());
- assertEquals(2, rm.getResults().length);
+ assertEquals(rowsPerBatch, rm.getResultsList().size());
}
}
@@ -324,11 +351,11 @@
reqMsg.setCursorType(ResultSet.TYPE_FORWARD_ONLY);
DQPWorkContext.getWorkContext().getSession().setSessionId(sessionid);
DQPWorkContext.getWorkContext().getSession().setUserName(userName);
- ((BufferManagerImpl)core.getBufferManager()).setProcessorBatchSize(2);
+ ((BufferManagerImpl)core.getBufferManager()).setProcessorBatchSize(1);
Future<ResultsMessage> message = core.executeRequest(reqMsg.getExecutionId(), reqMsg);
ResultsMessage rm = message.get(500000, TimeUnit.MILLISECONDS);
assertNull(rm.getException());
- assertEquals(2, rm.getResults().length);
+ assertEquals(8, rm.getResultsList().size());
RequestWorkItem item = core.getRequestWorkItem(DQPWorkContext.getWorkContext().getRequestID(reqMsg.getExecutionId()));
assertEquals(100, item.resultsBuffer.getRowCount());
}
Modified: branches/as7/engine/src/test/java/org/teiid/dqp/service/AutoGenDataService.java
===================================================================
--- branches/as7/engine/src/test/java/org/teiid/dqp/service/AutoGenDataService.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/test/java/org/teiid/dqp/service/AutoGenDataService.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -95,7 +95,7 @@
results = new List[] {Arrays.asList(1)};
}
- final AtomicResultsMessage msg = ConnectorWorkItem.createResultsMessage(results, projectedSymbols);
+ final AtomicResultsMessage msg = ConnectorWorkItem.createResultsMessage(results);
msg.setFinalRow(rows);
return new ConnectorWork() {
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/function/TestFunction.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/function/TestFunction.java:3281-3325
/trunk/engine/src/test/java/org/teiid/query/function/TestFunction.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/function/TestFunction.java:3281-3325
/trunk/engine/src/test/java/org/teiid/query/function/TestFunction.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/function/TestFunctionLibrary.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/function/TestFunctionLibrary.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/function/TestFunctionLibrary.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/function/TestFunctionLibrary.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/function/TestFunctionLibrary.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/function/TestFunctionTree.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/function/TestFunctionTree.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/function/TestFunctionTree.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/function/TestFunctionTree.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/function/TestFunctionTree.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/function/TestResolvedFunctions.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/function/TestResolvedFunctions.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/function/TestResolvedFunctions.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/function/TestResolvedFunctions.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/function/TestResolvedFunctions.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestAccessPatterns.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestAccessPatterns.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestAccessPatterns.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestAccessPatterns.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestAccessPatterns.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestAggregatePushdown.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestAggregatePushdown.java:3149-3217,3220-3275
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestAggregatePushdown.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestAggregatePushdown.java:3149-3217,3220-3275
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestAggregatePushdown.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestAnsiJoinPushdown.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestAnsiJoinPushdown.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestAnsiJoinPushdown.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestAnsiJoinPushdown.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestAnsiJoinPushdown.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestBatchedUpdatePlanner.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestBatchedUpdatePlanner.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestBatchedUpdatePlanner.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestBatchedUpdatePlanner.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestBatchedUpdatePlanner.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestDependentJoins.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestDependentJoins.java:3149-3217,3220-3275
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestDependentJoins.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestDependentJoins.java:3149-3217,3220-3275
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestDependentJoins.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestExpressionsInGroupBy.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestExpressionsInGroupBy.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestExpressionsInGroupBy.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestExpressionsInGroupBy.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestExpressionsInGroupBy.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestInlineView.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestInlineView.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestInlineView.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestInlineView.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestInlineView.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestJoinOptimization.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestJoinOptimization.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestJoinOptimization.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestJoinOptimization.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestJoinOptimization.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestJoinPushdownRestrictions.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestJoinPushdownRestrictions.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestJoinPushdownRestrictions.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestJoinPushdownRestrictions.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestJoinPushdownRestrictions.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestLimit.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestLimit.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestLimit.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestLimit.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestLimit.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestOptimizer.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestOptimizer.java:3149-3217,3220-3275
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestOptimizer.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestOptimizer.java:3149-3217,3220-3275
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestOptimizer.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestOptionalJoins.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestOptionalJoins.java:3149-3217,3220-3275
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestOptionalJoins.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestOptionalJoins.java:3149-3217,3220-3275
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestOptionalJoins.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestPartitionedJoinPlanning.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestPartitionedJoinPlanning.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestPartitionedJoinPlanning.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestPartitionedJoinPlanning.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestPartitionedJoinPlanning.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestRuleMergeVirtual.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestRuleMergeVirtual.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestRuleMergeVirtual.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestRuleMergeVirtual.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestRuleMergeVirtual.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestRuleRaiseNull.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestRuleRaiseNull.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestRuleRaiseNull.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestRuleRaiseNull.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestRuleRaiseNull.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestRuleRemoveSorts.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestRuleRemoveSorts.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestRuleRemoveSorts.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestRuleRemoveSorts.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestRuleRemoveSorts.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestSortOptimization.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestSortOptimization.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestSortOptimization.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestSortOptimization.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestSortOptimization.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestStoredProcedurePlanning.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestStoredProcedurePlanning.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestStoredProcedurePlanning.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestStoredProcedurePlanning.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestStoredProcedurePlanning.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestSubqueryPushdown.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestSubqueryPushdown.java:3149-3217,3281-3325
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestSubqueryPushdown.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestSubqueryPushdown.java:3149-3217,3281-3325
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestSubqueryPushdown.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestUnionPlanning.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestUnionPlanning.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestUnionPlanning.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestUnionPlanning.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/optimizer/TestUnionPlanning.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/relational/TestAliasGenerator.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/relational/TestAliasGenerator.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/optimizer/relational/TestAliasGenerator.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/relational/TestAliasGenerator.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/optimizer/relational/TestAliasGenerator.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/processor/FakeDataManager.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/FakeDataManager.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/processor/FakeDataManager.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/FakeDataManager.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/processor/FakeDataManager.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/processor/FakeDataStore.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/FakeDataStore.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/processor/FakeDataStore.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/FakeDataStore.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/processor/FakeDataStore.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/processor/TestAggregateProcessing.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestAggregateProcessing.java:3149-3217,3220-3275
/trunk/engine/src/test/java/org/teiid/query/processor/TestAggregateProcessing.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestAggregateProcessing.java:3149-3217,3220-3275
/trunk/engine/src/test/java/org/teiid/query/processor/TestAggregateProcessing.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/processor/TestDependentJoins.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestDependentJoins.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/processor/TestDependentJoins.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestDependentJoins.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/processor/TestDependentJoins.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/processor/TestInsertProcessing.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestInsertProcessing.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/processor/TestInsertProcessing.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestInsertProcessing.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/processor/TestInsertProcessing.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/processor/TestJoinWithFunction.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestJoinWithFunction.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/processor/TestJoinWithFunction.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestJoinWithFunction.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/processor/TestJoinWithFunction.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/processor/TestOptionalJoins.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestOptionalJoins.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/processor/TestOptionalJoins.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestOptionalJoins.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/processor/TestOptionalJoins.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/processor/TestOrderByProcessing.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestOrderByProcessing.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/processor/TestOrderByProcessing.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestOrderByProcessing.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/processor/TestOrderByProcessing.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/processor/TestProcedureRelational.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestProcedureRelational.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/processor/TestProcedureRelational.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestProcedureRelational.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/processor/TestProcedureRelational.java:3188-3450,3452-3506
Modified: branches/as7/engine/src/test/java/org/teiid/query/processor/TestProcessor.java
===================================================================
--- branches/as7/engine/src/test/java/org/teiid/query/processor/TestProcessor.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/test/java/org/teiid/query/processor/TestProcessor.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -365,7 +365,6 @@
props.setProperty("soap_port", "12345"); //$NON-NLS-1$ //$NON-NLS-2$
CommandContext context = new CommandContext("0", "test", "user", null, "myvdb", 1, props, DEBUG); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$
context.setProcessorBatchSize(BufferManager.DEFAULT_PROCESSOR_BATCH_SIZE);
- context.setConnectorBatchSize(BufferManager.DEFAULT_CONNECTOR_BATCH_SIZE);
context.setBufferManager(BufferManagerFactory.getStandaloneBufferManager());
context.setPreparedPlanCache(new SessionAwareCache<PreparedPlan>());
return context;
@@ -1080,7 +1079,6 @@
CommandContext context = createCommandContext();
context.setProcessorBatchSize(2);
- context.setConnectorBatchSize(2);
context.setMetadata(RealMetadataFactory.example1Cached());
// Plan query
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/processor/TestProcessor.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestProcessor.java:3149-3217,3220-3275
/trunk/engine/src/test/java/org/teiid/query/processor/TestProcessor.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestProcessor.java:3149-3217,3220-3275
/trunk/engine/src/test/java/org/teiid/query/processor/TestProcessor.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/processor/TestSQLXMLProcessing.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestSQLXMLProcessing.java:3149-3217,3220-3275,3281-3325,3340-3349
/trunk/engine/src/test/java/org/teiid/query/processor/TestSQLXMLProcessing.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestSQLXMLProcessing.java:3149-3217,3220-3275,3281-3325,3340-3349
/trunk/engine/src/test/java/org/teiid/query/processor/TestSQLXMLProcessing.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/processor/TestSecurityFunctions.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestSecurityFunctions.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/processor/TestSecurityFunctions.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestSecurityFunctions.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/processor/TestSecurityFunctions.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/processor/TestSetProcessing.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestSetProcessing.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/processor/TestSetProcessing.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestSetProcessing.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/processor/TestSetProcessing.java:3188-3450,3452-3506
Modified: branches/as7/engine/src/test/java/org/teiid/query/processor/TestTempTables.java
===================================================================
--- branches/as7/engine/src/test/java/org/teiid/query/processor/TestTempTables.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/test/java/org/teiid/query/processor/TestTempTables.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -135,6 +135,34 @@
execute("select * from x", new List[] {});
}
+ @Test public void testCommitExistingRemoved() throws Exception {
+ execute("create local temporary table x (e1 string, e2 integer)", new List[] {Arrays.asList(0)}); //$NON-NLS-1$
+ setupTransaction(Connection.TRANSACTION_SERIALIZABLE);
+ execute("drop table x", new List[] {Arrays.asList(0)}); //$NON-NLS-1$
+ synch.afterCompletion(Status.STATUS_COMMITTED);
+ try {
+ execute("select * from x", new List[] {});
+ fail();
+ } catch (Exception e) {
+
+ }
+ }
+
+ @Test public void testUpdateLock() throws Exception {
+ execute("create local temporary table x (e1 string, e2 integer)", new List[] {Arrays.asList(0)}); //$NON-NLS-1$
+ setupTransaction(Connection.TRANSACTION_SERIALIZABLE);
+ execute("insert into x (e2, e1) select e2, e1 from pm1.g1", new List[] {Arrays.asList(6)}); //$NON-NLS-1$
+ tc = null;
+ try {
+ execute("insert into x (e2, e1) select e2, e1 from pm1.g1", new List[] {Arrays.asList(6)}); //$NON-NLS-1$
+ fail();
+ } catch (Exception e) {
+
+ }
+ synch.afterCompletion(Status.STATUS_COMMITTED);
+ execute("insert into x (e2, e1) select e2, e1 from pm1.g1", new List[] {Arrays.asList(6)}); //$NON-NLS-1$
+ }
+
@Test public void testRollbackExisting1() throws Exception {
execute("create local temporary table x (e1 string, e2 integer)", new List[] {Arrays.asList(0)}); //$NON-NLS-1$
for (int i = 0; i < 86; i++) {
@@ -160,19 +188,19 @@
tempStore = gtsi.getTempTableStore();
metadata = new TempMetadataAdapter(RealMetadataFactory.example1Cached(), tempStore.getMetadataStore());
execute("create local temporary table x (e1 string, e2 integer)", new List[] {Arrays.asList(0)}); //$NON-NLS-1$
- for (int i = 0; i < 86; i++) {
+ for (int i = 0; i < 300; i++) {
execute("insert into x (e2, e1) select e2, e1 from pm1.g1", new List[] {Arrays.asList(6)}); //$NON-NLS-1$
}
setupTransaction(Connection.TRANSACTION_SERIALIZABLE);
- execute("select count(*) from x", new List[] {Arrays.asList(516)});
- gtsi.updateMatViewRow("X", Arrays.asList(1), true);
+ execute("select count(e1) from x", new List[] {Arrays.asList(1500)});
+ gtsi.updateMatViewRow("X", Arrays.asList(2), true);
tc=null;
//outside of the transaction we can see the row removed
- execute("select count(*) from x", new List[] {Arrays.asList(515)});
+ execute("select count(e1) from x", new List[] {Arrays.asList(1499)});
//back in the transaction we see the original state
setupTransaction(Connection.TRANSACTION_SERIALIZABLE);
- execute("select count(*) from x", new List[] {Arrays.asList(516)});
+ execute("select count(e1) from x", new List[] {Arrays.asList(1500)});
synch.afterCompletion(Status.STATUS_COMMITTED);
}
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/processor/TestTempTables.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestTempTables.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/processor/TestTempTables.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestTempTables.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/processor/TestTempTables.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/processor/TestTextTable.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestTextTable.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/processor/TestTextTable.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestTextTable.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/processor/TestTextTable.java:3188-3450,3452-3506
Modified: branches/as7/engine/src/test/java/org/teiid/query/processor/TestVirtualDepJoin.java
===================================================================
--- branches/as7/engine/src/test/java/org/teiid/query/processor/TestVirtualDepJoin.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/test/java/org/teiid/query/processor/TestVirtualDepJoin.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -33,9 +33,8 @@
import org.junit.Test;
import org.teiid.common.buffer.BufferManager;
+import org.teiid.common.buffer.BufferManagerFactory;
import org.teiid.common.buffer.TupleBuffer;
-import org.teiid.common.buffer.impl.BufferManagerImpl;
-import org.teiid.common.buffer.impl.MemoryStorageManager;
import org.teiid.core.TeiidComponentException;
import org.teiid.core.TeiidException;
import org.teiid.core.types.DataTypeManager;
@@ -461,16 +460,8 @@
TestProcessor.examineResults((List[])expected.toArray(new List[expected.size()]), bufferMgr, id);
}
- private BufferManager createCustomBufferMgr(int batchSize) throws TeiidComponentException {
- BufferManagerImpl bufferMgr = new BufferManagerImpl();
- bufferMgr.setConnectorBatchSize(batchSize);
- bufferMgr.setProcessorBatchSize(batchSize);
- bufferMgr.initialize();
-
- // Add unmanaged memory storage manager
- bufferMgr.setStorageManager(new MemoryStorageManager());
-
- return bufferMgr;
+ private BufferManager createCustomBufferMgr(int batchSize) {
+ return BufferManagerFactory.getTestBufferManager(200000, batchSize);
}
public void helpTestVirtualDepJoin(boolean pushCriteria) throws Exception {
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/processor/TestVirtualDepJoin.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestVirtualDepJoin.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/processor/TestVirtualDepJoin.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestVirtualDepJoin.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/processor/TestVirtualDepJoin.java:3188-3450,3452-3506
Property changes on: branches/as7/engine/src/test/java/org/teiid/query/processor/TestWithClauseProcessing.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestWithClauseProcessing.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/processor/TestWithClauseProcessing.java:3188-3450
+ /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestWithClauseProcessing.java:3149-3217
/trunk/engine/src/test/java/org/teiid/query/processor/TestWithClauseProcessing.java:3188-3450,3452-3506
Modified: branches/as7/engine/src/test/java/org/teiid/query/processor/relational/TestJoinNode.java
===================================================================
--- branches/as7/engine/src/test/java/org/teiid/query/processor/relational/TestJoinNode.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/test/java/org/teiid/query/processor/relational/TestJoinNode.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -33,9 +33,9 @@
import org.junit.Before;
import org.junit.Test;
import org.teiid.common.buffer.BlockedException;
-import org.teiid.common.buffer.BufferManager;
import org.teiid.common.buffer.BufferManagerFactory;
import org.teiid.common.buffer.TupleBatch;
+import org.teiid.common.buffer.impl.BufferManagerImpl;
import org.teiid.core.TeiidComponentException;
import org.teiid.core.TeiidProcessingException;
import org.teiid.core.types.DataTypeManager;
@@ -232,7 +232,8 @@
}
public void helpTestJoinDirect(List[] expectedResults, int batchSize, int processingBytes) throws TeiidComponentException, TeiidProcessingException {
- BufferManager mgr = BufferManagerFactory.getTestBufferManager(processingBytes, batchSize);
+ BufferManagerImpl mgr = BufferManagerFactory.getTestBufferManager(processingBytes, batchSize);
+ mgr.setTargetBytesPerRow(100);
CommandContext context = new CommandContext("pid", "test", null, null, 1); //$NON-NLS-1$ //$NON-NLS-2$
join.addChild(leftNode);
Modified: branches/as7/engine/src/test/java/org/teiid/query/processor/relational/TestProjectIntoNode.java
===================================================================
--- branches/as7/engine/src/test/java/org/teiid/query/processor/relational/TestProjectIntoNode.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/test/java/org/teiid/query/processor/relational/TestProjectIntoNode.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -72,10 +72,9 @@
ElementSymbol elementSymbol_2 = new ElementSymbol("myGroup.myElement2"); //$NON-NLS-1$
elementSymbol_1.setType(Integer.class);
elementSymbol_2.setType(String.class);
- ArrayList elements = new ArrayList();
- elements.add(elementSymbol_1);
- elements.add(elementSymbol_2);
+ List<ElementSymbol> elements = Arrays.asList(elementSymbol_1, elementSymbol_2);
node.setIntoElements(elements);
+ child.setElements(elements);
node.setMode(mode);
node.setModelName("myModel"); //$NON-NLS-1$
Modified: branches/as7/engine/src/test/java/org/teiid/query/sql/symbol/TestElementSymbol.java
===================================================================
--- branches/as7/engine/src/test/java/org/teiid/query/sql/symbol/TestElementSymbol.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/engine/src/test/java/org/teiid/query/sql/symbol/TestElementSymbol.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -127,22 +127,22 @@
@Test public void testEquals5() {
ElementSymbol es = sampleElement1();
- helpEquals(es, (ElementSymbol)es.clone(), true);
+ helpEquals(es, es.clone(), true);
}
@Test public void testEquals6() {
ElementSymbol es = sampleElement2();
- helpEquals(es, (ElementSymbol)es.clone(), true);
+ helpEquals(es, es.clone(), true);
}
@Test public void testEquals7() {
ElementSymbol es = sampleElement3();
- helpEquals(es, (ElementSymbol)es.clone(), true);
+ helpEquals(es, es.clone(), true);
}
@Test public void testEquals8() {
ElementSymbol es = sampleElement4();
- helpEquals(es, (ElementSymbol)es.clone(), true);
+ helpEquals(es, es.clone(), true);
}
// Compare fully-qualified to short versions
@@ -190,7 +190,7 @@
@Test public void testCloneEquivalence(){
ElementSymbol s1 = sampleElement1();
- ElementSymbol s2 = (ElementSymbol)s1.clone();
+ ElementSymbol s2 = s1.clone();
int equals = 0;
UnitTestUtil.helpTestEquivalence(equals, s1, s2);
}
@@ -225,31 +225,31 @@
@Test public void testClone1() {
ElementSymbol e1 = sampleElement1();
- ElementSymbol copy = (ElementSymbol) e1.clone();
+ ElementSymbol copy = e1.clone();
helpEquals(e1, copy, true);
}
@Test public void testClone2() {
ElementSymbol e1 = sampleElement2();
- ElementSymbol copy = (ElementSymbol) e1.clone();
+ ElementSymbol copy = e1.clone();
helpEquals(e1, copy, true);
}
@Test public void testClone3() {
ElementSymbol e1 = sampleElement3();
- ElementSymbol copy = (ElementSymbol) e1.clone();
+ ElementSymbol copy = e1.clone();
helpEquals(e1, copy, true);
}
@Test public void testClone4() {
ElementSymbol e1 = sampleElement4();
- ElementSymbol copy = (ElementSymbol) e1.clone();
+ ElementSymbol copy = e1.clone();
helpEquals(e1, copy, true);
}
@Test public void testClone5() {
ElementSymbol e1 = sampleElement1();
- ElementSymbol copy = (ElementSymbol) e1.clone();
+ ElementSymbol copy = e1.clone();
helpEquals(e1, copy, true);
// Change original, clone shouldn't change
Property changes on: branches/as7/engine/src/test/resources/text/cdm_dos_win.txt
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/engine/src/test/resources/text/cdm_dos_win.txt:3149-3164,3172-3217
/trunk/engine/src/test/resources/text/cdm_dos_win.txt:3188-3450
+ /branches/7.4.x/engine/src/test/resources/text/cdm_dos_win.txt:3149-3164,3172-3217
/trunk/engine/src/test/resources/text/cdm_dos_win.txt:3188-3450,3452-3506
Modified: branches/as7/jboss-integration/pom.xml
===================================================================
--- branches/as7/jboss-integration/pom.xml 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/jboss-integration/pom.xml 2011-09-29 16:46:38 UTC (rev 3507)
@@ -203,7 +203,12 @@
<scope>test</scope>
</dependency>
-->
-
+ <dependency>
+ <groupId>org.jboss.teiid</groupId>
+ <artifactId>teiid-common-core</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
Modified: branches/as7/metadata/src/main/java/org/teiid/internal/core/index/BlocksIndexInput.java
===================================================================
--- branches/as7/metadata/src/main/java/org/teiid/internal/core/index/BlocksIndexInput.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/metadata/src/main/java/org/teiid/internal/core/index/BlocksIndexInput.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -60,9 +60,12 @@
//System.out.println(" Closing Index File: " + this.indexFile.getName());
summary= null;
setOpen(false);
- if (raf != null)
+ if (raf != null) {
raf.close();
- vraf.close();
+ }
+ if (vraf != null) {
+ vraf.close();
+ }
}
}
Modified: branches/as7/runtime/src/main/java/org/teiid/services/BufferServiceImpl.java
===================================================================
--- branches/as7/runtime/src/main/java/org/teiid/services/BufferServiceImpl.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/runtime/src/main/java/org/teiid/services/BufferServiceImpl.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -29,7 +29,9 @@
import org.teiid.common.buffer.BufferManager;
import org.teiid.common.buffer.impl.BufferManagerImpl;
import org.teiid.common.buffer.impl.FileStorageManager;
+import org.teiid.common.buffer.impl.FileStoreCache;
import org.teiid.common.buffer.impl.MemoryStorageManager;
+import org.teiid.common.buffer.impl.SplittableStorageManager;
import org.teiid.core.TeiidComponentException;
import org.teiid.core.TeiidRuntimeException;
import org.teiid.core.util.FileUtils;
@@ -56,10 +58,11 @@
private int processorBatchSize = BufferManager.DEFAULT_PROCESSOR_BATCH_SIZE;
private int connectorBatchSize = BufferManager.DEFAULT_CONNECTOR_BATCH_SIZE;
private int maxOpenFiles = FileStorageManager.DEFAULT_MAX_OPEN_FILES;
- private long maxFileSize = FileStorageManager.DEFAULT_MAX_FILESIZE; // 2GB
+ private long maxFileSize = SplittableStorageManager.DEFAULT_MAX_FILESIZE; // 2GB
private int maxProcessingKb = BufferManager.DEFAULT_MAX_PROCESSING_KB;
private int maxReserveKb = BufferManager.DEFAULT_RESERVE_BUFFER_KB;
- private long maxBufferSpace = FileStorageManager.DEFAULT_MAX_BUFFERSPACE;
+ private long maxBufferSpace = FileStorageManager.DEFAULT_MAX_BUFFERSPACE>>20;
+ private boolean inlineLobs = true;
private FileStorageManager fsm;
/**
@@ -94,13 +97,17 @@
// Get the properties for FileStorageManager and create.
fsm = new FileStorageManager();
fsm.setStorageDirectory(bufferDir.getCanonicalPath());
- fsm.setMaxFileSize(maxFileSize);
fsm.setMaxOpenFiles(maxOpenFiles);
fsm.setMaxBufferSpace(maxBufferSpace*MB);
- fsm.initialize();
- this.bufferMgr.setStorageManager(fsm);
+ SplittableStorageManager ssm = new SplittableStorageManager(fsm);
+ ssm.setMaxFileSize(maxFileSize);
+ FileStoreCache fsc = new FileStoreCache();
+ fsc.setStorageManager(ssm);
+ fsc.setMaxBufferSpace(maxBufferSpace*MB);
+ fsc.initialize();
+ this.bufferMgr.setCache(fsc);
} else {
- this.bufferMgr.setStorageManager(new MemoryStorageManager());
+ this.bufferMgr.setCache(new MemoryStorageManager());
}
} catch(TeiidComponentException e) {
@@ -121,7 +128,7 @@
}
}
- public BufferManager getBufferManager() {
+ public BufferManagerImpl getBufferManager() {
return this.bufferMgr;
}
@@ -139,6 +146,10 @@
public void setConnectorBatchSize(int size) {
this.connectorBatchSize = size;
}
+
+ public void setInlineLobs(boolean inlineLobs) {
+ this.inlineLobs = inlineLobs;
+ }
public File getBufferDirectory() {
return bufferDir;
@@ -147,6 +158,10 @@
public boolean isUseDisk() {
return this.useDisk;
}
+
+ public boolean isInlineLobs() {
+ return inlineLobs;
+ }
public int getProcessorBatchSize() {
return this.processorBatchSize;
Modified: branches/as7/runtime/src/test/java/org/teiid/dqp/service/buffer/TestLocalBufferService.java
===================================================================
--- branches/as7/runtime/src/test/java/org/teiid/dqp/service/buffer/TestLocalBufferService.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/runtime/src/test/java/org/teiid/dqp/service/buffer/TestLocalBufferService.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -37,6 +37,8 @@
import org.teiid.common.buffer.BufferManager.TupleSourceType;
import org.teiid.common.buffer.impl.BufferManagerImpl;
import org.teiid.common.buffer.impl.FileStorageManager;
+import org.teiid.common.buffer.impl.FileStoreCache;
+import org.teiid.common.buffer.impl.SplittableStorageManager;
import org.teiid.core.types.DataTypeManager;
import org.teiid.core.types.DataTypeManager.DefaultDataTypes;
import org.teiid.core.util.UnitTestUtil;
@@ -60,8 +62,9 @@
assertTrue("does not end with one", svc.getBufferDirectory().getParent().endsWith("1")); //$NON-NLS-1$ //$NON-NLS-2$
assertTrue(svc.isUseDisk());
- BufferManagerImpl mgr = (BufferManagerImpl) svc.getBufferManager();
- assertTrue(((FileStorageManager)mgr.getStorageManager()).getDirectory().endsWith(svc.getBufferDirectory().getName()));
+ BufferManagerImpl mgr = svc.getBufferManager();
+ SplittableStorageManager ssm = (SplittableStorageManager)((FileStoreCache)mgr.getCache()).getStorageManager();
+ assertTrue(((FileStorageManager)ssm.getStorageManager()).getDirectory().endsWith(svc.getBufferDirectory().getName()));
}
@Test public void testCheckMemPropertyGotSet2() throws Exception {
@@ -99,7 +102,8 @@
svc.start();
BufferManager mgr = svc.getBufferManager();
- assertEquals(13141, mgr.getSchemaSize(schema));
+ assertEquals(6570, mgr.getSchemaSize(schema));
+ assertEquals(256, mgr.getProcessorBatchSize(schema));
}
@Test
Modified: branches/as7/test-integration/common/src/test/java/org/teiid/systemmodel/TestMatViewReplication.java
===================================================================
--- branches/as7/test-integration/common/src/test/java/org/teiid/systemmodel/TestMatViewReplication.java 2011-09-29 01:08:46 UTC (rev 3506)
+++ branches/as7/test-integration/common/src/test/java/org/teiid/systemmodel/TestMatViewReplication.java 2011-09-29 16:46:38 UTC (rev 3507)
@@ -30,9 +30,6 @@
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
-import java.util.logging.Handler;
-import java.util.logging.Level;
-import java.util.logging.Logger;
import org.jboss.as.clustering.jgroups.ChannelFactory;
import org.jgroups.Channel;
@@ -60,16 +57,7 @@
@Test public void testReplication() throws Exception {
if (DEBUG) {
- Logger logger = Logger.getLogger("org.teiid");
- logger.setLevel(Level.FINEST);
- for (Handler h : logger.getHandlers()) {
- h.setLevel(Level.FINEST);
- }
- /*org.apache.log4j.Logger l = LogManager.getLogger("org.jgroups");
- l.setLevel(org.apache.log4j.Level.TRACE);
- ConsoleAppender ca = new ConsoleAppender(new PatternLayout());
- ca.setName("x");
- l.addAppender(ca);*/
+ UnitTestUtil.enableTraceLogging("org.teiid");
}
FakeServer server1 = createServer();
Property changes on: branches/as7/test-integration/common/src/test/java/org/teiid/transport/TestODBCSocketTransport.java
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/test-integration/common/src/test/java/org/teiid/transport/TestODBCSocketTransport.java:3149-3217,3281-3325
/trunk/test-integration/common/src/test/java/org/teiid/transport/TestODBCSocketTransport.java:3188-3450
+ /branches/7.4.x/test-integration/common/src/test/java/org/teiid/transport/TestODBCSocketTransport.java:3149-3217,3281-3325
/trunk/test-integration/common/src/test/java/org/teiid/transport/TestODBCSocketTransport.java:3188-3450,3452-3506
Property changes on: branches/as7/test-integration/common/src/test/resources/TestODBCSocketTransport/testPk.expected
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/test-integration/common/src/test/resources/TestODBCSocketTransport/testPk.expected:3281-3325
/trunk/test-integration/common/src/test/resources/TestODBCSocketTransport/testPk.expected:3188-3450
+ /branches/7.4.x/test-integration/common/src/test/resources/TestODBCSocketTransport/testPk.expected:3281-3325
/trunk/test-integration/common/src/test/resources/TestODBCSocketTransport/testPk.expected:3188-3450,3452-3506
Property changes on: branches/as7/test-integration/common/src/test/resources/TestODBCSocketTransport/testPkPrepared.expected
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/test-integration/common/src/test/resources/TestODBCSocketTransport/testPkPrepared.expected:3281-3325
/trunk/test-integration/common/src/test/resources/TestODBCSocketTransport/testPkPrepared.expected:3188-3450
+ /branches/7.4.x/test-integration/common/src/test/resources/TestODBCSocketTransport/testPkPrepared.expected:3281-3325
/trunk/test-integration/common/src/test/resources/TestODBCSocketTransport/testPkPrepared.expected:3188-3450,3452-3506
Property changes on: branches/as7/test-integration/common/src/test/resources/TestSystemVirtualModel/testProcedureParams.expected
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/test-integration/common/src/test/resources/TestSystemVirtualModel/testProcedureParams.expected:3220-3275
/trunk/test-integration/common/src/test/resources/TestSystemVirtualModel/testProcedureParams.expected:3188-3450
+ /branches/7.4.x/test-integration/common/src/test/resources/TestSystemVirtualModel/testProcedureParams.expected:3220-3275
/trunk/test-integration/common/src/test/resources/TestSystemVirtualModel/testProcedureParams.expected:3188-3450,3452-3506
Property changes on: branches/as7/test-integration/common/src/test/resources/TestSystemVirtualModel/testProcedures.expected
___________________________________________________________________
Modified: svn:mergeinfo
- /branches/7.4.x/test-integration/common/src/test/resources/TestSystemVirtualModel/testProcedures.expected:3220-3275
/trunk/test-integration/common/src/test/resources/TestSystemVirtualModel/testProcedures.expected:3188-3450
+ /branches/7.4.x/test-integration/common/src/test/resources/TestSystemVirtualModel/testProcedures.expected:3220-3275
/trunk/test-integration/common/src/test/resources/TestSystemVirtualModel/testProcedures.expected:3188-3450,3452-3506
More information about the teiid-commits
mailing list