[teiid-commits] teiid SVN: r3507 - in branches/as7: adminshell/src/main/java/org/teiid/adminshell and 53 other directories.

Author: rareddy Date: 2011-09-29 12:46:38 -0400 (Thu, 29 Sep 2011) New Revision: 3507 Added: branches/as7/engine/src/main/java/org/teiid/common/buffer/AutoCleanupUtil.java branches/as7/engine/src/main/java/org/teiid/common/buffer/Cache.java branches/as7/engine/src/main/java/org/teiid/common/buffer/CacheEntry.java branches/as7/engine/src/main/java/org/teiid/common/buffer/LightWeightCopyOnWriteList.java branches/as7/engine/src/main/java/org/teiid/common/buffer/Serializer.java branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/BlockBitSetTree.java branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/BlockClosedLongIntHashTable.java branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/BlockManager.java branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/ExtensibleBufferedOutputStream.java branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/FileStoreCache.java branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/SplittableStorageManager.java branches/as7/engine/src/main/java/org/teiid/dqp/internal/process/SerializableTupleBatch.java branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestBlockBitSetTree.java branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestBlockClosedLongIntHashTable.java branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestFileStoreCache.java branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestSplittableStorageManager.java Removed: branches/as7/engine/src/test/java/org/teiid/core/ Modified: branches/as7/ branches/as7/adminshell/src/main/java/org/teiid/adminshell/AdminShell.java branches/as7/api/src/main/java/org/teiid/CommandContext.java branches/as7/api/src/main/java/org/teiid/events/EventDistributor.java branches/as7/build/kits/jboss-container/deploy/teiid/teiid-jboss-beans.xml branches/as7/build/kits/jboss-container/teiid-releasenotes.html branches/as7/client/src/main/java/org/teiid/client/BatchSerializer.java branches/as7/client/src/main/java/org/teiid/client/ResultsMessage.java branches/as7/client/src/main/java/org/teiid/jdbc/ConnectionImpl.java branches/as7/client/src/main/java/org/teiid/jdbc/DatabaseMetaDataImpl.java branches/as7/client/src/main/java/org/teiid/jdbc/EmbeddedProfile.java branches/as7/client/src/main/java/org/teiid/jdbc/JDBCURL.java branches/as7/client/src/main/java/org/teiid/jdbc/ResultSetImpl.java branches/as7/client/src/main/java/org/teiid/jdbc/ResultSetMetaDataImpl.java branches/as7/client/src/main/java/org/teiid/jdbc/SQLStates.java branches/as7/client/src/main/java/org/teiid/jdbc/StatementImpl.java branches/as7/client/src/main/java/org/teiid/jdbc/TeiidSQLException.java branches/as7/client/src/main/java/org/teiid/net/socket/SocketServerConnectionFactory.java branches/as7/client/src/test/java/org/teiid/client/TestBatchSerializer.java branches/as7/client/src/test/java/org/teiid/jdbc/TestAllResultsImpl.java branches/as7/client/src/test/java/org/teiid/jdbc/TestCallableStatement.java branches/as7/client/src/test/java/org/teiid/jdbc/TestConnection.java branches/as7/client/src/test/java/org/teiid/jdbc/TestSQLException.java branches/as7/client/src/test/java/org/teiid/jdbc/TestStatement.java branches/as7/client/src/test/java/org/teiid/jdbc/TestTeiidDriver.java branches/as7/common-core/src/main/java/org/teiid/core/TeiidException.java branches/as7/common-core/src/main/java/org/teiid/core/TeiidRuntimeException.java branches/as7/common-core/src/main/java/org/teiid/core/types/BaseLob.java branches/as7/common-core/src/main/java/org/teiid/core/types/BlobImpl.java branches/as7/common-core/src/main/java/org/teiid/core/types/BlobType.java branches/as7/common-core/src/main/java/org/teiid/core/types/ClobType.java branches/as7/common-core/src/main/java/org/teiid/core/types/DataTypeManager.java branches/as7/common-core/src/main/java/org/teiid/core/types/InputStreamFactory.java branches/as7/common-core/src/main/java/org/teiid/core/types/SQLXMLImpl.java branches/as7/common-core/src/main/java/org/teiid/core/types/Streamable.java branches/as7/common-core/src/main/java/org/teiid/core/types/XMLType.java branches/as7/common-core/src/main/resources/org/teiid/core/util/application.properties branches/as7/common-core/src/test/java/org/teiid/core/TestMetaMatrixRuntimeException.java branches/as7/common-core/src/test/java/org/teiid/core/types/TestBlobValue.java branches/as7/common-core/src/test/java/org/teiid/core/types/TestClobValue.java branches/as7/common-core/src/test/java/org/teiid/core/types/TestXMLValue.java branches/as7/common-core/src/test/java/org/teiid/core/util/UnitTestUtil.java branches/as7/documentation/caching-guide/src/main/docbook/en-US/content/hint-option.xml branches/as7/engine/src/main/java/org/teiid/common/buffer/BatchManager.java branches/as7/engine/src/main/java/org/teiid/common/buffer/BlockedException.java branches/as7/engine/src/main/java/org/teiid/common/buffer/BufferManager.java branches/as7/engine/src/main/java/org/teiid/common/buffer/FileStore.java branches/as7/engine/src/main/java/org/teiid/common/buffer/FileStoreInputStreamFactory.java branches/as7/engine/src/main/java/org/teiid/common/buffer/LobManager.java branches/as7/engine/src/main/java/org/teiid/common/buffer/SPage.java branches/as7/engine/src/main/java/org/teiid/common/buffer/STree.java branches/as7/engine/src/main/java/org/teiid/common/buffer/TupleBatch.java branches/as7/engine/src/main/java/org/teiid/common/buffer/TupleBrowser.java branches/as7/engine/src/main/java/org/teiid/common/buffer/TupleBuffer.java branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/BufferManagerImpl.java branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/FileStorageManager.java branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/MemoryStorageManager.java branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/SizeUtility.java branches/as7/engine/src/main/java/org/teiid/dqp/internal/datamgr/ConnectorWorkItem.java branches/as7/engine/src/main/java/org/teiid/dqp/internal/process/DataTierTupleSource.java branches/as7/engine/src/main/java/org/teiid/dqp/internal/process/Request.java branches/as7/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java branches/as7/engine/src/main/java/org/teiid/dqp/message/AtomicResultsMessage.java branches/as7/engine/src/main/java/org/teiid/query/function/FunctionDescriptor.java branches/as7/engine/src/main/java/org/teiid/query/function/FunctionTree.java branches/as7/engine/src/main/java/org/teiid/query/function/source/SecuritySystemFunctions.java branches/as7/engine/src/main/java/org/teiid/query/metadata/TransformationMetadata.java branches/as7/engine/src/main/java/org/teiid/query/optimizer/relational/PlanToProcessConverter.java branches/as7/engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java branches/as7/engine/src/main/java/org/teiid/query/processor/QueryProcessor.java branches/as7/engine/src/main/java/org/teiid/query/processor/proc/ProcedurePlan.java branches/as7/engine/src/main/java/org/teiid/query/processor/relational/EnhancedSortMergeJoinStrategy.java branches/as7/engine/src/main/java/org/teiid/query/processor/relational/NullNode.java branches/as7/engine/src/main/java/org/teiid/query/processor/relational/RelationalNode.java branches/as7/engine/src/main/java/org/teiid/query/processor/relational/SortUtility.java branches/as7/engine/src/main/java/org/teiid/query/processor/xml/RelationalPlanExecutor.java branches/as7/engine/src/main/java/org/teiid/query/sql/LanguageObject.java branches/as7/engine/src/main/java/org/teiid/query/tempdata/GlobalTableStoreImpl.java branches/as7/engine/src/main/java/org/teiid/query/tempdata/TempTable.java branches/as7/engine/src/main/java/org/teiid/query/tempdata/TempTableDataManager.java branches/as7/engine/src/main/java/org/teiid/query/tempdata/TempTableStore.java branches/as7/engine/src/main/java/org/teiid/query/util/CommandContext.java branches/as7/engine/src/main/resources/org/teiid/query/i18n.properties branches/as7/engine/src/test/java/org/teiid/common/buffer/BufferManagerFactory.java branches/as7/engine/src/test/java/org/teiid/common/buffer/TestLobManager.java branches/as7/engine/src/test/java/org/teiid/common/buffer/TestSTree.java branches/as7/engine/src/test/java/org/teiid/common/buffer/TestTupleBuffer.java branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestFileStorageManager.java branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestSizeUtility.java branches/as7/engine/src/test/java/org/teiid/dqp/internal/process/TestCachedResults.java branches/as7/engine/src/test/java/org/teiid/dqp/internal/process/TestDQPCore.java branches/as7/engine/src/test/java/org/teiid/dqp/service/AutoGenDataService.java branches/as7/engine/src/test/java/org/teiid/query/function/TestFunction.java branches/as7/engine/src/test/java/org/teiid/query/function/TestFunctionLibrary.java branches/as7/engine/src/test/java/org/teiid/query/function/TestFunctionTree.java branches/as7/engine/src/test/java/org/teiid/query/function/TestResolvedFunctions.java branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestAccessPatterns.java branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestAggregatePushdown.java branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestAnsiJoinPushdown.java branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestBatchedUpdatePlanner.java branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestDependentJoins.java branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestExpressionsInGroupBy.java branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestInlineView.java branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestJoinOptimization.java branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestJoinPushdownRestrictions.java branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestLimit.java branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestOptimizer.java branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestOptionalJoins.java branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestPartitionedJoinPlanning.java branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestRuleMergeVirtual.java branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestRuleRaiseNull.java branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestRuleRemoveSorts.java branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestSortOptimization.java branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestStoredProcedurePlanning.java branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestSubqueryPushdown.java branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestUnionPlanning.java branches/as7/engine/src/test/java/org/teiid/query/optimizer/relational/TestAliasGenerator.java branches/as7/engine/src/test/java/org/teiid/query/processor/FakeDataManager.java branches/as7/engine/src/test/java/org/teiid/query/processor/FakeDataStore.java branches/as7/engine/src/test/java/org/teiid/query/processor/TestAggregateProcessing.java branches/as7/engine/src/test/java/org/teiid/query/processor/TestDependentJoins.java branches/as7/engine/src/test/java/org/teiid/query/processor/TestInsertProcessing.java branches/as7/engine/src/test/java/org/teiid/query/processor/TestJoinWithFunction.java branches/as7/engine/src/test/java/org/teiid/query/processor/TestOptionalJoins.java branches/as7/engine/src/test/java/org/teiid/query/processor/TestOrderByProcessing.java branches/as7/engine/src/test/java/org/teiid/query/processor/TestProcedureRelational.java branches/as7/engine/src/test/java/org/teiid/query/processor/TestProcessor.java branches/as7/engine/src/test/java/org/teiid/query/processor/TestSQLXMLProcessing.java branches/as7/engine/src/test/java/org/teiid/query/processor/TestSecurityFunctions.java branches/as7/engine/src/test/java/org/teiid/query/processor/TestSetProcessing.java branches/as7/engine/src/test/java/org/teiid/query/processor/TestTempTables.java branches/as7/engine/src/test/java/org/teiid/query/processor/TestTextTable.java branches/as7/engine/src/test/java/org/teiid/query/processor/TestVirtualDepJoin.java branches/as7/engine/src/test/java/org/teiid/query/processor/TestWithClauseProcessing.java branches/as7/engine/src/test/java/org/teiid/query/processor/relational/TestJoinNode.java branches/as7/engine/src/test/java/org/teiid/query/processor/relational/TestProjectIntoNode.java branches/as7/engine/src/test/java/org/teiid/query/sql/symbol/TestElementSymbol.java branches/as7/engine/src/test/resources/text/cdm_dos_win.txt branches/as7/jboss-integration/pom.xml branches/as7/metadata/src/main/java/org/teiid/internal/core/index/BlocksIndexInput.java branches/as7/runtime/src/main/java/org/teiid/services/BufferServiceImpl.java branches/as7/runtime/src/test/java/org/teiid/dqp/service/buffer/TestLocalBufferService.java branches/as7/test-integration/common/src/test/java/org/teiid/systemmodel/TestMatViewReplication.java branches/as7/test-integration/common/src/test/java/org/teiid/transport/TestODBCSocketTransport.java branches/as7/test-integration/common/src/test/resources/TestODBCSocketTransport/testPk.expected branches/as7/test-integration/common/src/test/resources/TestODBCSocketTransport/testPkPrepared.expected branches/as7/test-integration/common/src/test/resources/TestSystemVirtualModel/testProcedureParams.expected branches/as7/test-integration/common/src/test/resources/TestSystemVirtualModel/testProcedures.expected Log: merge -r 3451:3506 from trunk to as7 Property changes on: branches/as7 ___________________________________________________________________ Modified: svn:mergeinfo - /trunk:3188-3450 + /trunk:3188-3450,3452-3506 Property changes on: branches/as7/adminshell/src/main/java/org/teiid/adminshell/AdminShell.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.1.x/adminshell/src/main/java/org/teiid/adminshell/AdminShell.java:2901-2940 /trunk/adminshell/src/main/java/org/teiid/adminshell/AdminShell.java:3188-3450 + /branches/7.1.x/adminshell/src/main/java/org/teiid/adminshell/AdminShell.java:2901-2940 /trunk/adminshell/src/main/java/org/teiid/adminshell/AdminShell.java:3188-3450,3452-3506 Modified: branches/as7/api/src/main/java/org/teiid/CommandContext.java =================================================================== --- branches/as7/api/src/main/java/org/teiid/CommandContext.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/api/src/main/java/org/teiid/CommandContext.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -87,7 +87,8 @@ /** * Get the processor batch size set on the BufferManager - * @return + * @return - the nominal batch size target. actual batch sizes will vary based + * upon the column types */ int getProcessorBatchSize(); Property changes on: branches/as7/api/src/main/java/org/teiid/events/EventDistributor.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/api/src/main/java/org/teiid/events/EventDistributor.java:3340-3349 /trunk/api/src/main/java/org/teiid/events/EventDistributor.java:3188-3450 + /branches/7.4.x/api/src/main/java/org/teiid/events/EventDistributor.java:3340-3349 /trunk/api/src/main/java/org/teiid/events/EventDistributor.java:3188-3450,3452-3506 Modified: branches/as7/build/kits/jboss-container/deploy/teiid/teiid-jboss-beans.xml =================================================================== --- branches/as7/build/kits/jboss-container/deploy/teiid/teiid-jboss-beans.xml 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/build/kits/jboss-container/deploy/teiid/teiid-jboss-beans.xml 2011-09-29 16:46:38 UTC (rev 3507) @@ -43,7 +43,11 @@ <!-- Max storage space, in MB, to be used for buffer files (default 50G) --> <property name="maxBufferSpace">51200</property> <!-- Max open buffer files (default 64) --> - <property name="maxOpenFiles">64</property> + <property name="maxOpenFiles">64</property> + <!-- Set to true to allow inlining of memory based and small lobs into results. + However inline lob values are not supported by pre-7.6 clients, so disable this + property if using older clients utilizing lobs. (default true) --> + <property name="inlineLobs">true</property> </bean> <bean name="CacheFactory" class="org.teiid.cache.jboss.ClusterableCacheFactory"> Modified: branches/as7/build/kits/jboss-container/teiid-releasenotes.html =================================================================== --- branches/as7/build/kits/jboss-container/teiid-releasenotes.html 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/build/kits/jboss-container/teiid-releasenotes.html 2011-09-29 16:46:38 UTC (rev 3507) @@ -30,6 +30,7 @@ <LI><B>File Enhancements</B> - the file translator can now optionally (via the ExceptionIfFileNotFound property) throw an exception if the path refers to a file that doesn't exist. The file resource adapter can be configured to map file names and can prevent parent path .. references. See the Admin Guide or the file-ds.xml template for more. <LI><B>TEXTTABLE Enhancements</B> - TEXTTABLE can now parse fixed width files that do not use a row delimiter and can optionally produce fixed values that haven't been trimmed. <LI><B>Temp table transactions</B> - Internal materialized views and temp table usage from a session and within procedures can take advantage of greater transaction support. + <LI><B>Buffering Improvements</B> - Added the ability to inline memory based or small lobs and added tracking of the memory held by soft references. </UL> <h2><a name="Compatibility">Compatibility Issues</a></h2> @@ -39,9 +40,10 @@ <li>Support for using the FROM clause post item hints MAKEDEP/MAKENOTDEP has been deprecated. Use the pre item comment hint syntax instead, e.g. /*+ MAKEDEP */ tbl </ul> -<h4>from 7.4</h4> +<h4>from 7.5</h4> <ul> - <li>Leave was added as a reserved word. + <li>Leave was added as a reserved word. + <li>Lob inlining is incompatible with clients older than 7.6. If a 7.6 server will have older clients that use lobs connect to it, then the BufferService property inlineLobs should be set to false in the teiid-jboss-beans.xml file. </ul> <h4>from 7.4</h4> Modified: branches/as7/client/src/main/java/org/teiid/client/BatchSerializer.java =================================================================== --- branches/as7/client/src/main/java/org/teiid/client/BatchSerializer.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/client/src/main/java/org/teiid/client/BatchSerializer.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -29,6 +29,7 @@ import java.math.BigInteger; import java.sql.Time; import java.sql.Timestamp; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -75,23 +76,23 @@ * @throws IOException * @since 4.2 */ - static void writeIsNullData(ObjectOutput out, int col, List[] batch) throws IOException { - int numBytes = batch.length / 8, row = 0, currentByte = 0; + static void writeIsNullData(ObjectOutput out, int col, List<? extends List<?>> batch) throws IOException { + int numBytes = batch.size() / 8, row = 0, currentByte = 0; for (int byteNum = 0; byteNum < numBytes; byteNum++, row+=8) { - currentByte = (batch[row].get(col) == null) ? 0x80 : 0; - if (batch[row+1].get(col) == null) currentByte |= 0x40; - if (batch[row+2].get(col) == null) currentByte |= 0x20; - if (batch[row+3].get(col) == null) currentByte |= 0x10; - if (batch[row+4].get(col) == null) currentByte |= 0x08; - if (batch[row+5].get(col) == null) currentByte |= 0x04; - if (batch[row+6].get(col) == null) currentByte |= 0x02; - if (batch[row+7].get(col) == null) currentByte |= 0x01; + currentByte = (batch.get(row).get(col) == null) ? 0x80 : 0; + if (batch.get(row+1).get(col) == null) currentByte |= 0x40; + if (batch.get(row+2).get(col) == null) currentByte |= 0x20; + if (batch.get(row+3).get(col) == null) currentByte |= 0x10; + if (batch.get(row+4).get(col) == null) currentByte |= 0x08; + if (batch.get(row+5).get(col) == null) currentByte |= 0x04; + if (batch.get(row+6).get(col) == null) currentByte |= 0x02; + if (batch.get(row+7).get(col) == null) currentByte |= 0x01; out.write(currentByte); } - if (batch.length % 8 > 0) { + if (batch.size() % 8 > 0) { currentByte = 0; - for (int mask = 0x80; row < batch.length; row++, mask >>= 1) { - if (batch[row].get(col) == null) currentByte |= mask; + for (int mask = 0x80; row < batch.size(); row++, mask >>= 1) { + if (batch.get(row).get(col) == null) currentByte |= mask; } out.write(currentByte); } @@ -126,22 +127,22 @@ * @since 4.2 */ private static class ColumnSerializer { - public void writeColumn(ObjectOutput out, int col, List[] batch) throws IOException { + public void writeColumn(ObjectOutput out, int col, List<? extends List<?>> batch) throws IOException { writeIsNullData(out, col, batch); Object obj = null; - for (int i = 0; i < batch.length; i++) { - obj = batch[i].get(col); + for (int i = 0; i < batch.size(); i++) { + obj = batch.get(i).get(col); if (obj != null) { writeObject(out, obj); } } } - public void readColumn(ObjectInput in, int col, List[] batch, byte[] isNull) throws IOException, ClassNotFoundException { + public void readColumn(ObjectInput in, int col, List<List<Object>> batch, byte[] isNull) throws IOException, ClassNotFoundException { readIsNullData(in, isNull); - for (int i = 0; i < batch.length; i++) { + for (int i = 0; i < batch.size(); i++) { if (!isNullObject(isNull, i)) { - batch[i].set(col, DataTypeManager.getCanonicalValue(readObject(in))); + batch.get(i).set(col, DataTypeManager.getCanonicalValue(readObject(in))); } } } @@ -203,13 +204,14 @@ /* This implementation compacts the isNull and boolean data for non-null values into a byte[] * by using a 8 bit mask that is bit-shifted to mask each value. */ - public void writeColumn(ObjectOutput out, int col, List[] batch) throws IOException { + @Override + public void writeColumn(ObjectOutput out, int col, List<? extends List<?>> batch) throws IOException { int currentByte = 0; int mask = 0x80; Object obj; - for (int row = 0; row < batch.length; row++) { + for (int row = 0; row < batch.size(); row++) { // Write the isNull value - obj = batch[row].get(col); + obj = batch.get(row).get(col); if (obj == null ) { currentByte |= mask; } @@ -241,10 +243,13 @@ } } - public void readColumn(ObjectInput in, int col, List[] batch, byte[] isNull) throws IOException, ClassNotFoundException { + @Override + public void readColumn(ObjectInput in, int col, + List<List<Object>> batch, byte[] isNull) throws IOException, + ClassNotFoundException { int currentByte = 0, mask = 0; // Initialize the mask so that it is reset in the loop boolean isNullVal; - for (int row = 0; row < batch.length; row++) { + for (int row = 0; row < batch.size(); row++) { if (mask == 0) { // If we used up the byte, read the next one, and reset the mask currentByte = in.read(); @@ -257,7 +262,7 @@ currentByte = in.read(); mask = 0x80; } - batch[row].set(col, ((currentByte & mask) == 0) ? Boolean.FALSE : Boolean.TRUE); + batch.get(row).set(col, ((currentByte & mask) == 0) ? Boolean.FALSE : Boolean.TRUE); mask >>= 1; } } @@ -354,17 +359,12 @@ return cs; } - public static void writeBatch(ObjectOutput out, String[] types, List[] batch) throws IOException { - // If there are no type hints, simply use the default mechanism to serialize - if (types == null || types.length == 0) { - out.writeObject(batch); - return; - } + public static void writeBatch(ObjectOutput out, String[] types, List<? extends List<?>> batch) throws IOException { if (batch == null) { out.writeInt(-1); } else { - out.writeInt(batch.length); - if (batch.length > 0) { + out.writeInt(batch.size()); + if (batch.size() > 0) { int columns = types.length; out.writeInt(columns); for(int i = 0; i < columns; i++) { @@ -374,8 +374,8 @@ } catch (ClassCastException e) { Object obj = null; String objectClass = null; - objectSearch: for (int row = 0; row < batch.length; row++) { - obj = batch[row].get(i); + objectSearch: for (int row = 0; row < batch.size(); row++) { + obj = batch.get(row).get(i); if (obj != null) { objectClass = obj.getClass().getName(); break objectSearch; @@ -388,21 +388,17 @@ } } - public static List[] readBatch(ObjectInput in, String[] types) throws IOException, ClassNotFoundException { - // If there are no type hints, use the default mechanism to deserialize - if (types == null || types.length == 0) { - return (List[])in.readObject(); - } + public static List<List<Object>> readBatch(ObjectInput in, String[] types) throws IOException, ClassNotFoundException { int rows = in.readInt(); if (rows == 0) { - return new List[0]; + return new ArrayList<List<Object>>(0); } else if (rows > 0) { int columns = in.readInt(); - List[] batch = new List[rows]; + List<List<Object>> batch = new ArrayList<List<Object>>(rows); int numBytes = rows/8; int extraRows = rows % 8; for (int currentRow = 0; currentRow < rows; currentRow++) { - batch[currentRow] = Arrays.asList(new Object[columns]); + batch.add(currentRow, Arrays.asList(new Object[columns])); } byte[] isNullBuffer = new byte[(extraRows > 0) ? numBytes + 1: numBytes]; for (int col = 0; col < columns; col++) { Modified: branches/as7/client/src/main/java/org/teiid/client/ResultsMessage.java =================================================================== --- branches/as7/client/src/main/java/org/teiid/client/ResultsMessage.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/client/src/main/java/org/teiid/client/ResultsMessage.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -26,7 +26,7 @@ import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; -import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.List; @@ -45,9 +45,9 @@ static final long serialVersionUID = 3546924172976187793L; - private List[] results = null; - private String[] columnNames = null; - private String[] dataTypes = null; + private List<? extends List<?>> results; + private String[] columnNames; + private String[] dataTypes; /** A description of planning that occurred as requested in the request. */ private PlanNode planDescription; @@ -85,36 +85,29 @@ public ResultsMessage(){ } - /** - * Instantiate and copy relevant information from the original request message. - * Typically, the transaction context should only be copied if this results - * message is being returned from the connector to the query engine. Clients - * will be unable to deserialize this object. - * @param requestMsg - * @param copyTransactionContext true if the transaction context should be copied; false otherwise. - * @since 4.2 - */ - public ResultsMessage(RequestMessage requestMsg){ - this.results = new ArrayList[0]; - - } - - public ResultsMessage(RequestMessage requestMsg, List[] results, String[] columnNames, String[] dataTypes){ - this (requestMsg); - setResults( results ); + public ResultsMessage(List<? extends List<?>> results, String[] columnNames, String[] dataTypes){ + this.results = results; setFirstRow( 1 ); - setLastRow( results.length ); + setLastRow( results.size() ); this.columnNames = columnNames; this.dataTypes = dataTypes; } - - public List[] getResults() { + + public List<? extends List<?>> getResultsList() { return results; } + + /** + * @deprecated see {@link #getResultsList()} + * @return + */ + public List<?>[] getResults() { + return results.toArray(new List[results.size()]); + } - public void setResults(List[] results) { - this.results = results; + public void setResults(List<?>[] results) { + this.results = Arrays.asList(results); } public String[] getColumnNames() { @@ -341,7 +334,7 @@ */ public String toString() { return new StringBuffer("ResultsMessage rowCount=") //$NON-NLS-1$ - .append(results == null ? 0 : results.length) + .append(results == null ? 0 : results.size()) .append(" finalRow=") //$NON-NLS-1$ .append(finalRow) .toString(); Property changes on: branches/as7/client/src/main/java/org/teiid/jdbc/ConnectionImpl.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/client/src/main/java/org/teiid/jdbc/ConnectionImpl.java:3149-3217 /trunk/client/src/main/java/org/teiid/jdbc/ConnectionImpl.java:3188-3450 + /branches/7.4.x/client/src/main/java/org/teiid/jdbc/ConnectionImpl.java:3149-3217 /trunk/client/src/main/java/org/teiid/jdbc/ConnectionImpl.java:3188-3450,3452-3506 Property changes on: branches/as7/client/src/main/java/org/teiid/jdbc/DatabaseMetaDataImpl.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/client/src/main/java/org/teiid/jdbc/DatabaseMetaDataImpl.java:3281-3325 /trunk/client/src/main/java/org/teiid/jdbc/DatabaseMetaDataImpl.java:3188-3450 + /branches/7.4.x/client/src/main/java/org/teiid/jdbc/DatabaseMetaDataImpl.java:3281-3325 /trunk/client/src/main/java/org/teiid/jdbc/DatabaseMetaDataImpl.java:3188-3450,3452-3506 Property changes on: branches/as7/client/src/main/java/org/teiid/jdbc/EmbeddedProfile.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/client/src/main/java/org/teiid/jdbc/EmbeddedProfile.java:3149-3217 /trunk/client/src/main/java/org/teiid/jdbc/EmbeddedProfile.java:3188-3450 + /branches/7.4.x/client/src/main/java/org/teiid/jdbc/EmbeddedProfile.java:3149-3217 /trunk/client/src/main/java/org/teiid/jdbc/EmbeddedProfile.java:3188-3450,3452-3506 Property changes on: branches/as7/client/src/main/java/org/teiid/jdbc/JDBCURL.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/client/src/main/java/org/teiid/jdbc/JDBCURL.java:3149-3217,3281-3325 /trunk/client/src/main/java/org/teiid/jdbc/JDBCURL.java:3188-3450 + /branches/7.4.x/client/src/main/java/org/teiid/jdbc/JDBCURL.java:3149-3217,3281-3325 /trunk/client/src/main/java/org/teiid/jdbc/JDBCURL.java:3188-3450,3452-3506 Modified: branches/as7/client/src/main/java/org/teiid/jdbc/ResultSetImpl.java =================================================================== --- branches/as7/client/src/main/java/org/teiid/jdbc/ResultSetImpl.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/client/src/main/java/org/teiid/jdbc/ResultSetImpl.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -416,7 +416,7 @@ private Batch getCurrentBatch(ResultsMessage currentResultMsg) { this.updatedPlanDescription = currentResultMsg.getPlanDescription(); - boolean isLast = currentResultMsg.getResults().length == 0 || currentResultMsg.getFinalRow() == currentResultMsg.getLastRow(); + boolean isLast = currentResultMsg.getResultsList().size() == 0 || currentResultMsg.getFinalRow() == currentResultMsg.getLastRow(); Batch result = new Batch(currentResultMsg.getResults(), currentResultMsg.getFirstRow(), currentResultMsg.getLastRow(), isLast); result.setLastRow(currentResultMsg.getFinalRow()); return result; Property changes on: branches/as7/client/src/main/java/org/teiid/jdbc/ResultSetMetaDataImpl.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/client/src/main/java/org/teiid/jdbc/ResultSetMetaDataImpl.java:3281-3325 /trunk/client/src/main/java/org/teiid/jdbc/ResultSetMetaDataImpl.java:3188-3450 + /branches/7.4.x/client/src/main/java/org/teiid/jdbc/ResultSetMetaDataImpl.java:3281-3325 /trunk/client/src/main/java/org/teiid/jdbc/ResultSetMetaDataImpl.java:3188-3450,3452-3506 Property changes on: branches/as7/client/src/main/java/org/teiid/jdbc/SQLStates.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/client/src/main/java/org/teiid/jdbc/SQLStates.java:3220-3275 /trunk/client/src/main/java/org/teiid/jdbc/SQLStates.java:3188-3450 + /branches/7.4.x/client/src/main/java/org/teiid/jdbc/SQLStates.java:3220-3275 /trunk/client/src/main/java/org/teiid/jdbc/SQLStates.java:3188-3450,3452-3506 Property changes on: branches/as7/client/src/main/java/org/teiid/jdbc/StatementImpl.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/client/src/main/java/org/teiid/jdbc/StatementImpl.java:3149-3217,3220-3275,3281-3325,3355-3365 /trunk/client/src/main/java/org/teiid/jdbc/StatementImpl.java:3188-3450 + /branches/7.4.x/client/src/main/java/org/teiid/jdbc/StatementImpl.java:3149-3217,3220-3275,3281-3325,3355-3365 /trunk/client/src/main/java/org/teiid/jdbc/StatementImpl.java:3188-3450,3452-3506 Property changes on: branches/as7/client/src/main/java/org/teiid/jdbc/TeiidSQLException.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/client/src/main/java/org/teiid/jdbc/TeiidSQLException.java:3220-3275 /trunk/client/src/main/java/org/teiid/jdbc/TeiidSQLException.java:3188-3450 + /branches/7.4.x/client/src/main/java/org/teiid/jdbc/TeiidSQLException.java:3220-3275 /trunk/client/src/main/java/org/teiid/jdbc/TeiidSQLException.java:3188-3450,3452-3506 Property changes on: branches/as7/client/src/main/java/org/teiid/net/socket/SocketServerConnectionFactory.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/client/src/main/java/org/teiid/net/socket/SocketServerConnectionFactory.java:3149-3217 /trunk/client/src/main/java/org/teiid/net/socket/SocketServerConnectionFactory.java:3188-3450 + /branches/7.4.x/client/src/main/java/org/teiid/net/socket/SocketServerConnectionFactory.java:3149-3217 /trunk/client/src/main/java/org/teiid/net/socket/SocketServerConnectionFactory.java:3188-3450,3452-3506 Modified: branches/as7/client/src/test/java/org/teiid/client/TestBatchSerializer.java =================================================================== --- branches/as7/client/src/test/java/org/teiid/client/TestBatchSerializer.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/client/src/test/java/org/teiid/client/TestBatchSerializer.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -35,10 +35,10 @@ import java.util.Arrays; import java.util.List; -import org.teiid.client.BatchSerializer; +import junit.framework.TestCase; + import org.teiid.core.types.DataTypeManager; -import junit.framework.TestCase; @@ -63,21 +63,22 @@ } } - private static void helpTestSerialization(String[] types, List[] batch) throws IOException, ClassNotFoundException { + private static void helpTestSerialization(String[] types, List<?>[] batch) throws IOException, ClassNotFoundException { ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); ObjectOutputStream out = new ObjectOutputStream(byteStream); - BatchSerializer.writeBatch(out, types, batch); + List<List<?>> batchList = Arrays.asList(batch); + BatchSerializer.writeBatch(out, types, batchList); out.flush(); byte[] bytes = byteStream.toByteArray(); ByteArrayInputStream bytesIn = new ByteArrayInputStream(bytes); ObjectInputStream in = new ObjectInputStream(bytesIn); - List[] newBatch = BatchSerializer.readBatch(in, types); + List<List<Object>> newBatch = BatchSerializer.readBatch(in, types); out.close(); in.close(); - assertEqual(batch, newBatch); + assertTrue(batchList.equals(newBatch)); } private static final String[] sampleBatchTypes = {DataTypeManager.DefaultDataTypes.BIG_DECIMAL, @@ -164,14 +165,6 @@ helpTestSerialization(sampleBatchTypes, sampleBatch(833)); // A bunch of rows. This should also test large strings } - public void testSerializeBasicTypes_NoTypeHints() throws Exception { - helpTestSerialization(null, sampleBatch(1)); - helpTestSerialization(null, sampleBatch(8)); - helpTestSerialization(null, sampleBatch(17)); - helpTestSerialization(null, sampleBatch(120)); - helpTestSerialization(null, sampleBatch(833)); - } - public void testSerializeBasicTypesWithNulls() throws Exception { helpTestSerialization(sampleBatchTypes, sampleBatchWithNulls(1)); helpTestSerialization(sampleBatchTypes, sampleBatchWithNulls(8)); @@ -186,10 +179,7 @@ } public void testSerializeNoData() throws Exception { - helpTestSerialization(sampleBatchTypes, null); - helpTestSerialization(null, null); helpTestSerialization(sampleBatchTypes, new List[0]); - helpTestSerialization(null, new List[0]); } public void testSerializeDatatypeMismatch() throws Exception { Modified: branches/as7/client/src/test/java/org/teiid/jdbc/TestAllResultsImpl.java =================================================================== --- branches/as7/client/src/test/java/org/teiid/jdbc/TestAllResultsImpl.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/client/src/test/java/org/teiid/jdbc/TestAllResultsImpl.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -797,7 +797,7 @@ private ResultsMessage exampleMessage(List<Object>[] results, String[] columnNames, String[] datatypes) { RequestMessage request = new RequestMessage(); request.setExecutionId(REQUEST_ID); - ResultsMessage resultsMsg = new ResultsMessage(request); + ResultsMessage resultsMsg = new ResultsMessage(); resultsMsg.setResults(results); resultsMsg.setColumnNames(columnNames); resultsMsg.setDataTypes(datatypes); @@ -834,7 +834,7 @@ private static ResultsMessage exampleResultsMsg4(int begin, int length, boolean lastBatch) { RequestMessage request = new RequestMessage(); request.setExecutionId(REQUEST_ID); - ResultsMessage resultsMsg = new ResultsMessage(request); + ResultsMessage resultsMsg = new ResultsMessage(); List[] results = exampleResults1(length, begin); resultsMsg.setResults(results); resultsMsg.setColumnNames(new String[] { "IntKey" }); //$NON-NLS-1$ @@ -862,7 +862,7 @@ @Test public void testDateType() throws SQLException { RequestMessage request = new RequestMessage(); request.setExecutionId(REQUEST_ID); - ResultsMessage resultsMsg = new ResultsMessage(request); + ResultsMessage resultsMsg = new ResultsMessage(); resultsMsg.setResults(new List[] {Arrays.asList(new Timestamp(0))}); resultsMsg.setColumnNames(new String[] { "TS" }); //$NON-NLS-1$ resultsMsg.setDataTypes(new String[] { JDBCSQLTypeInfo.TIMESTAMP }); Modified: branches/as7/client/src/test/java/org/teiid/jdbc/TestCallableStatement.java =================================================================== --- branches/as7/client/src/test/java/org/teiid/jdbc/TestCallableStatement.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/client/src/test/java/org/teiid/jdbc/TestCallableStatement.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -38,9 +38,6 @@ import org.teiid.client.metadata.ParameterInfo; import org.teiid.client.security.LogonResult; import org.teiid.core.types.JDBCSQLTypeInfo; -import org.teiid.jdbc.CallableStatementImpl; -import org.teiid.jdbc.ConnectionImpl; -import org.teiid.jdbc.ResultSetImpl; import org.teiid.net.ServerConnection; @@ -68,7 +65,7 @@ RequestMessage request = new RequestMessage(); request.setExecutionId(1); - ResultsMessage resultsMsg = new ResultsMessage(request); + ResultsMessage resultsMsg = new ResultsMessage(); List[] results = new List[] {Arrays.asList(null, null, null), Arrays.asList(null, 1, 2)}; resultsMsg.setResults(results); resultsMsg.setColumnNames(new String[] { "IntNum", "Out1", "Out2" }); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ Property changes on: branches/as7/client/src/test/java/org/teiid/jdbc/TestConnection.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/client/src/test/java/org/teiid/jdbc/TestConnection.java:3281-3325 /trunk/client/src/test/java/org/teiid/jdbc/TestConnection.java:3188-3450 + /branches/7.4.x/client/src/test/java/org/teiid/jdbc/TestConnection.java:3281-3325 /trunk/client/src/test/java/org/teiid/jdbc/TestConnection.java:3188-3450,3452-3506 Property changes on: branches/as7/client/src/test/java/org/teiid/jdbc/TestSQLException.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/client/src/test/java/org/teiid/jdbc/TestSQLException.java:3220-3275 /trunk/client/src/test/java/org/teiid/jdbc/TestSQLException.java:3188-3450 + /branches/7.4.x/client/src/test/java/org/teiid/jdbc/TestSQLException.java:3220-3275 /trunk/client/src/test/java/org/teiid/jdbc/TestSQLException.java:3188-3450,3452-3506 Property changes on: branches/as7/client/src/test/java/org/teiid/jdbc/TestStatement.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/client/src/test/java/org/teiid/jdbc/TestStatement.java:3149-3217,3220-3275 /trunk/client/src/test/java/org/teiid/jdbc/TestStatement.java:3188-3450 + /branches/7.4.x/client/src/test/java/org/teiid/jdbc/TestStatement.java:3149-3217,3220-3275 /trunk/client/src/test/java/org/teiid/jdbc/TestStatement.java:3188-3450,3452-3506 Property changes on: branches/as7/client/src/test/java/org/teiid/jdbc/TestTeiidDriver.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/client/src/test/java/org/teiid/jdbc/TestTeiidDriver.java:3149-3217,3281-3325 /trunk/client/src/test/java/org/teiid/jdbc/TestTeiidDriver.java:3188-3450 + /branches/7.4.x/client/src/test/java/org/teiid/jdbc/TestTeiidDriver.java:3149-3217,3281-3325 /trunk/client/src/test/java/org/teiid/jdbc/TestTeiidDriver.java:3188-3450,3452-3506 Property changes on: branches/as7/common-core/src/main/java/org/teiid/core/TeiidException.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/common-core/src/main/java/org/teiid/core/TeiidException.java:3220-3275 /trunk/common-core/src/main/java/org/teiid/core/TeiidException.java:3188-3450 + /branches/7.4.x/common-core/src/main/java/org/teiid/core/TeiidException.java:3220-3275 /trunk/common-core/src/main/java/org/teiid/core/TeiidException.java:3188-3450,3452-3506 Property changes on: branches/as7/common-core/src/main/java/org/teiid/core/TeiidRuntimeException.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/common-core/src/main/java/org/teiid/core/TeiidRuntimeException.java:3220-3275 /trunk/common-core/src/main/java/org/teiid/core/TeiidRuntimeException.java:3188-3450 + /branches/7.4.x/common-core/src/main/java/org/teiid/core/TeiidRuntimeException.java:3220-3275 /trunk/common-core/src/main/java/org/teiid/core/TeiidRuntimeException.java:3188-3450,3452-3506 Modified: branches/as7/common-core/src/main/java/org/teiid/core/types/BaseLob.java =================================================================== --- branches/as7/common-core/src/main/java/org/teiid/core/types/BaseLob.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/common-core/src/main/java/org/teiid/core/types/BaseLob.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -22,6 +22,7 @@ package org.teiid.core.types; +import java.io.BufferedInputStream; import java.io.Externalizable; import java.io.IOException; import java.io.InputStream; @@ -75,17 +76,8 @@ this.charset = charset; } - public void free() throws SQLException { - if (this.streamFactory != null) { - try { - this.streamFactory.free(); - this.streamFactory = null; - } catch (IOException e) { - SQLException ex = new SQLException(e.getMessage()); - ex.initCause(e); - throw ex; - } - } + public void free() { + this.streamFactory = null; } public Reader getCharacterStream() throws SQLException { @@ -126,5 +118,35 @@ public void writeExternal(ObjectOutput out) throws IOException { out.writeObject(streamFactory); } + + /** + * Returns the number of bytes. + */ + public long length() throws SQLException{ + if (getStreamFactory().getLength() == -1) { + getStreamFactory().setLength(length(getBinaryStream())); + } + return getStreamFactory().getLength(); + } + static long length(InputStream is) throws SQLException { + if (!(is instanceof BufferedInputStream)) { + is = new BufferedInputStream(is); + } + try { + long length = 0; + while (is.read() != -1) { + length++; + } + return length; + } catch (IOException e) { + throw new SQLException(e); + } finally { + try { + is.close(); + } catch (IOException e) { + } + } + } + } Modified: branches/as7/common-core/src/main/java/org/teiid/core/types/BlobImpl.java =================================================================== --- branches/as7/common-core/src/main/java/org/teiid/core/types/BlobImpl.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/common-core/src/main/java/org/teiid/core/types/BlobImpl.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -22,7 +22,6 @@ package org.teiid.core.types; -import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -109,32 +108,6 @@ } /** - * Returns the number of bytes in the <code>BLOB</code> value - * designated by this <code>Blob</code> object. - * @return length of the <code>BLOB</code> in bytes - */ - public long length() throws SQLException{ - if (getStreamFactory().getLength() == -1) { - InputStream is = new BufferedInputStream(getBinaryStream()); - try { - long length = 0; - while (is.read() != -1) { - length++; - } - getStreamFactory().setLength(length); - } catch (IOException e) { - throw new SQLException(e); - } finally { - try { - is.close(); - } catch (IOException e) { - } - } - } - return getStreamFactory().getLength(); - } - - /** * Determines the byte position in the <code>BLOB</code> value * designated by this <code>Blob</code> object at which * <code>pattern</code> begins. The search begins at position Modified: branches/as7/common-core/src/main/java/org/teiid/core/types/BlobType.java =================================================================== --- branches/as7/common-core/src/main/java/org/teiid/core/types/BlobType.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/common-core/src/main/java/org/teiid/core/types/BlobType.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -24,6 +24,7 @@ import java.io.IOException; import java.io.InputStream; +import java.io.ObjectInput; import java.io.ObjectOutput; import java.io.OutputStream; import java.sql.Blob; @@ -32,6 +33,7 @@ import javax.sql.rowset.serial.SerialBlob; import org.teiid.core.TeiidRuntimeException; +import org.teiid.core.util.ObjectConverterUtil; /** @@ -61,18 +63,10 @@ public byte[] getBytes(long pos, int length) throws SQLException { return this.reference.getBytes(pos, length); } - - /** - * @see java.sql.Blob#length() - */ - public long length() throws SQLException { - //caching the length - if (this.length != -1) { - return this.length; - } - // if did not find before then do it again. - this.length = this.reference.length(); - return length; + + @Override + long computeLength() throws SQLException { + return this.reference.length(); } /** @@ -142,12 +136,38 @@ } @Override - public void writeExternal(ObjectOutput out) throws IOException { + protected void readReference(ObjectInput in) throws IOException { + byte[] bytes = new byte[(int)getLength()]; + in.readFully(bytes); try { - length(); + this.reference = new SerialBlob(bytes); } catch (SQLException e) { + throw new IOException(e); } - super.writeExternal(out); } + @Override + protected void writeReference(final ObjectOutput out) throws IOException { + try { + writeBinary(out, getBinaryStream(), (int)length); + } catch (SQLException e) { + throw new IOException(); + } + } + + static void writeBinary(final ObjectOutput out, InputStream is, int length) throws IOException { + OutputStream os = new OutputStream() { + + @Override + public void write(int b) throws IOException { + out.write(b); + } + }; + try { + ObjectConverterUtil.write(os, is, length, false); + } finally { + is.close(); + } + } + } Modified: branches/as7/common-core/src/main/java/org/teiid/core/types/ClobType.java =================================================================== --- branches/as7/common-core/src/main/java/org/teiid/core/types/ClobType.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/common-core/src/main/java/org/teiid/core/types/ClobType.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -24,6 +24,7 @@ import java.io.IOException; import java.io.InputStream; +import java.io.ObjectInput; import java.io.ObjectOutput; import java.io.OutputStream; import java.io.Reader; @@ -35,6 +36,7 @@ import javax.sql.rowset.serial.SerialClob; import org.teiid.core.TeiidRuntimeException; +import org.teiid.core.util.ObjectConverterUtil; /** @@ -72,17 +74,10 @@ public String getSubString(long pos, int length) throws SQLException { return this.reference.getSubString(pos, length); } - - /** - * @see java.sql.Clob#length() - */ - public long length() throws SQLException { - if (this.length != -1) { - return this.length; - } - - this.length = this.reference.length(); - return length; + + @Override + long computeLength() throws SQLException { + return this.reference.length(); } /** @@ -218,12 +213,52 @@ } @Override - public void writeExternal(ObjectOutput out) throws IOException { + protected void readReference(ObjectInput in) throws IOException { + char[] chars = new char[(int)length]; + for (int i = 0; i < chars.length; i++) { + chars[i] = in.readChar(); + } try { - length(); + this.reference = new SerialClob(chars); } catch (SQLException e) { + throw new IOException(e); } - super.writeExternal(out); } + + /** + * Since we have the length in chars we'll just write out in double byte format. + * These clobs should be small, so the wasted space should be minimal. + */ + @Override + protected void writeReference(final ObjectOutput out) throws IOException { + Writer w = new Writer() { + + @Override + public void write(char[] cbuf, int off, int len) throws IOException { + for (int i = off; i < len; i++) { + out.writeShort(cbuf[i]); + } + } + + @Override + public void flush() throws IOException { + } + + @Override + public void close() throws IOException { + } + }; + Reader r; + try { + r = getCharacterStream(); + } catch (SQLException e) { + throw new IOException(e); + } + try { + ObjectConverterUtil.write(w, r, (int)length, false); + } finally { + r.close(); + } + } } Modified: branches/as7/common-core/src/main/java/org/teiid/core/types/DataTypeManager.java =================================================================== --- branches/as7/common-core/src/main/java/org/teiid/core/types/DataTypeManager.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/common-core/src/main/java/org/teiid/core/types/DataTypeManager.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -126,23 +126,40 @@ } } - private static Map<Class<?>, ValueCache<?>> valueMaps = new HashMap<Class<?>, ValueCache<?>>(128); - private static HashedValueCache<String> stringCache = new HashedValueCache<String>(17) { + public static class WeakReferenceHashedValueCache<T> extends HashedValueCache<T> { + public WeakReferenceHashedValueCache(int size) { + super(size); + } + + public T getByHash(Object obj) { + int index = hash(obj.hashCode()) & (cache.length - 1); + return get(index); + } + @Override - protected Object get(int index) { - WeakReference<?> ref = (WeakReference<?>) cache[index]; + protected T get(int index) { + WeakReference<T> ref = (WeakReference<T>) cache[index]; if (ref != null) { - return ref.get(); + T result = ref.get(); + if (result == null) { + cache[index] = null; + } + return result; } return null; } @Override - protected void set(int index, String value) { - cache[index] = new WeakReference<Object>(value); - } + protected void set(int index, T value) { + cache[index] = new WeakReference<T>(value); + } + } + + private static Map<Class<?>, ValueCache<?>> valueMaps = new HashMap<Class<?>, ValueCache<?>>(128); + private static HashedValueCache<String> stringCache = new WeakReferenceHashedValueCache<String>(17) { + @Override protected int primaryHash(String value) { if (value.length() < 14) { @@ -536,21 +553,7 @@ valueMaps.put(DefaultDataClasses.DATE, new HashedValueCache<Date>(14)); valueMaps.put(DefaultDataClasses.TIME, new HashedValueCache<Time>(14)); valueMaps.put(DefaultDataClasses.TIMESTAMP, new HashedValueCache<Timestamp>(14)); - valueMaps.put(DefaultDataClasses.BIG_DECIMAL, new HashedValueCache<BigDecimal>(16) { - @Override - protected Object get(int index) { - WeakReference<?> ref = (WeakReference<?>) cache[index]; - if (ref != null) { - return ref.get(); - } - return null; - } - - @Override - protected void set(int index, BigDecimal value) { - cache[index] = new WeakReference<BigDecimal>(value); - } - }); + valueMaps.put(DefaultDataClasses.BIG_DECIMAL, new WeakReferenceHashedValueCache<BigDecimal>(16)); valueMaps.put(DefaultDataClasses.STRING, stringCache); } } Modified: branches/as7/common-core/src/main/java/org/teiid/core/types/InputStreamFactory.java =================================================================== --- branches/as7/common-core/src/main/java/org/teiid/core/types/InputStreamFactory.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/common-core/src/main/java/org/teiid/core/types/InputStreamFactory.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -36,12 +36,21 @@ import java.sql.SQLXML; import javax.activation.DataSource; +import javax.sql.rowset.serial.SerialBlob; +import javax.sql.rowset.serial.SerialClob; import javax.xml.transform.Source; import org.teiid.core.util.ReaderInputStream; public abstract class InputStreamFactory implements Source { + public enum StorageMode { + MEMORY, //TODO: sources may return Serial values that are much too large and we should convert them to persistent. + PERSISTENT, + FREE, + OTHER + } + public interface StreamFactoryReference { void setStreamFactory(InputStreamFactory inputStreamFactory); @@ -49,7 +58,7 @@ } private String systemId; - private long length = -1; + protected long length = -1; /** * Get a new InputStream @@ -67,10 +76,17 @@ this.systemId = systemId; } + /** + * @throws IOException + */ public void free() throws IOException { } + /** + * Length in bytes of the {@link InputStream} + * @return the length or -1 if the length is not known + */ public long getLength() { return length; } @@ -79,12 +95,15 @@ this.length = length; } + /** + * @throws IOException + */ public Reader getCharacterStream() throws IOException { return null; } - public boolean isPersistent() { - return false; + public StorageMode getStorageMode() { + return StorageMode.OTHER; } public static class FileInputStreamFactory extends InputStreamFactory { @@ -107,8 +126,8 @@ } @Override - public boolean isPersistent() { - return true; + public StorageMode getStorageMode() { + return StorageMode.PERSISTENT; } } @@ -162,6 +181,11 @@ public OutputStream getOutputStream() throws IOException { throw new UnsupportedOperationException(); } + + @Override + public StorageMode getStorageMode() { + return getStorageMode(clob); + } } @@ -184,11 +208,13 @@ @Override public long getLength() { - try { - return blob.length(); - } catch (SQLException e) { - return -1; - } + if (length == -1) { + try { + length = blob.length(); + } catch (SQLException e) { + } + } + return length; } @Override @@ -205,9 +231,35 @@ public OutputStream getOutputStream() throws IOException { throw new UnsupportedOperationException(); } + + @Override + public StorageMode getStorageMode() { + return getStorageMode(blob); + } } + public static StorageMode getStorageMode(Object lob) { + if (lob instanceof Streamable<?>) { + return getStorageMode(((Streamable<?>)lob).getReference()); + } + if (lob instanceof SerialClob) { + return StorageMode.MEMORY; + } + if (lob instanceof SerialBlob) { + return StorageMode.MEMORY; + } + if (lob instanceof BaseLob) { + BaseLob baseLob = (BaseLob)lob; + try { + return baseLob.getStreamFactory().getStorageMode(); + } catch (SQLException e) { + return StorageMode.FREE; + } + } + return StorageMode.OTHER; + } + public static class SQLXMLInputStreamFactory extends InputStreamFactory implements DataSource { private SQLXML sqlxml; @@ -248,6 +300,11 @@ public OutputStream getOutputStream() throws IOException { throw new UnsupportedOperationException(); } + + @Override + public StorageMode getStorageMode() { + return getStorageMode(sqlxml); + } } Modified: branches/as7/common-core/src/main/java/org/teiid/core/types/SQLXMLImpl.java =================================================================== --- branches/as7/common-core/src/main/java/org/teiid/core/types/SQLXMLImpl.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/common-core/src/main/java/org/teiid/core/types/SQLXMLImpl.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -71,6 +71,16 @@ public InputStream getInputStream() throws IOException { return new ByteArrayInputStream(bytes); } + + @Override + public StorageMode getStorageMode() { + return StorageMode.MEMORY; + } + + @Override + public long getLength() { + return bytes.length; + } }); setEncoding(Streamable.ENCODING); } Modified: branches/as7/common-core/src/main/java/org/teiid/core/types/Streamable.java =================================================================== --- branches/as7/common-core/src/main/java/org/teiid/core/types/Streamable.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/common-core/src/main/java/org/teiid/core/types/Streamable.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -27,7 +27,8 @@ import java.io.ObjectInput; import java.io.ObjectOutput; import java.nio.charset.Charset; -import java.util.concurrent.atomic.AtomicInteger; +import java.sql.SQLException; +import java.util.concurrent.atomic.AtomicLong; import org.teiid.core.CorePlugin; @@ -44,7 +45,7 @@ private static final long serialVersionUID = -8252488562134729374L; - private static AtomicInteger counter = new AtomicInteger(); + private static AtomicLong counter = new AtomicLong(); public static final String ENCODING = "UTF-8"; //$NON-NLS-1$ public static final Charset CHARSET = Charset.forName(ENCODING); @@ -66,10 +67,23 @@ this.reference = reference; } + /** + * Returns the cached length. May be binary or character based. + * @return + */ public long getLength() { return length; } + abstract long computeLength() throws SQLException; + + public long length() throws SQLException { + if (length == -1) { + length = computeLength(); + } + return length; + } + public T getReference() { return reference; } @@ -82,9 +96,9 @@ return this.referenceStreamId; } - /*public void setReferenceStreamId(String id) { + public void setReferenceStreamId(String id) { this.referenceStreamId = id; - }*/ + } @Override public String toString() { @@ -99,12 +113,27 @@ ClassNotFoundException { length = in.readLong(); referenceStreamId = (String)in.readObject(); + if (referenceStreamId == null) { + //we expect the data inline + readReference(in); + } } + protected abstract void readReference(ObjectInput in) throws IOException; + @Override public void writeExternal(ObjectOutput out) throws IOException { + try { + length(); + } catch (SQLException e) { + } out.writeLong(length); out.writeObject(referenceStreamId); + if (referenceStreamId == null) { + writeReference(out); + } } - + + protected abstract void writeReference(ObjectOutput out) throws IOException; + } Modified: branches/as7/common-core/src/main/java/org/teiid/core/types/XMLType.java =================================================================== --- branches/as7/common-core/src/main/java/org/teiid/core/types/XMLType.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/common-core/src/main/java/org/teiid/core/types/XMLType.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -189,4 +189,29 @@ } } } + + @Override + long computeLength() throws SQLException { + if (this.reference instanceof SQLXMLImpl) { + SQLXMLImpl impl = (SQLXMLImpl)this.reference; + return impl.length(); + } + return BaseLob.length(getBinaryStream()); + } + + @Override + protected void readReference(ObjectInput in) throws IOException { + byte[] bytes = new byte[(int)getLength()]; + in.readFully(bytes); + this.reference = new SQLXMLImpl(bytes); + } + + @Override + protected void writeReference(final ObjectOutput out) throws IOException { + try { + BlobType.writeBinary(out, getBinaryStream(), (int)length); + } catch (SQLException e) { + throw new IOException(); + } + } } Modified: branches/as7/common-core/src/main/resources/org/teiid/core/util/application.properties =================================================================== --- branches/as7/common-core/src/main/resources/org/teiid/core/util/application.properties 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/common-core/src/main/resources/org/teiid/core/util/application.properties 2011-09-29 16:46:38 UTC (rev 3507) @@ -1,5 +1,5 @@ -build.releaseNumber=${pom.version} -build.number=${pom.version} +build.releaseNumber=${project.version} +build.number=${project.version} build.date=@build-date@ copyright=Copyright (C) 2008-2009 Red Hat, Inc url=${site.url} \ No newline at end of file Property changes on: branches/as7/common-core/src/test/java/org/teiid/core/TestMetaMatrixRuntimeException.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/common-core/src/test/java/org/teiid/core/TestMetaMatrixRuntimeException.java:3220-3275 /trunk/common-core/src/test/java/org/teiid/core/TestMetaMatrixRuntimeException.java:3188-3450 + /branches/7.4.x/common-core/src/test/java/org/teiid/core/TestMetaMatrixRuntimeException.java:3220-3275 /trunk/common-core/src/test/java/org/teiid/core/TestMetaMatrixRuntimeException.java:3188-3450,3452-3506 Modified: branches/as7/common-core/src/test/java/org/teiid/core/types/TestBlobValue.java =================================================================== --- branches/as7/common-core/src/test/java/org/teiid/core/types/TestBlobValue.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/common-core/src/test/java/org/teiid/core/types/TestBlobValue.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -22,21 +22,14 @@ package org.teiid.core.types; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; - import javax.sql.rowset.serial.SerialBlob; -import org.teiid.core.types.BlobType; -import org.teiid.core.util.UnitTestUtil; - - import junit.framework.TestCase; +import org.junit.Test; +import org.teiid.core.util.UnitTestUtil; + public class TestBlobValue extends TestCase { public void testBlobValue() throws Exception { @@ -55,22 +48,27 @@ String key = bv.getReferenceStreamId(); // now force to serialize - File saved = new File(UnitTestUtil.getTestScratchPath()+"/blobassaved.bin"); //$NON-NLS-1$ - ObjectOutputStream out = new ObjectOutputStream(new FileOutputStream(saved)); - out.writeObject(bv); - out.close(); - - // now read back the object from serilized state - ObjectInputStream in = new ObjectInputStream(new FileInputStream(saved)); - BlobType read = (BlobType)in.readObject(); + BlobType read = UnitTestUtil.helpSerialize(bv); // make sure we have kept the reference stream id assertEquals(key, read.getReferenceStreamId()); // and lost the original object assertNull(read.getReference()); + } + + @Test public void testReferencePersistence() throws Exception { + String testString = "this is test clob"; //$NON-NLS-1$ + SerialBlob blob = new SerialBlob(testString.getBytes()); - saved.delete(); + BlobType bv = new BlobType(blob); + bv.setReferenceStreamId(null); + // now force to serialize + BlobType read = UnitTestUtil.helpSerialize(bv); + + assertNull(read.getReferenceStreamId()); + + assertEquals(testString, new String(read.getBytes(1, (int)blob.length()))); } } Modified: branches/as7/common-core/src/test/java/org/teiid/core/types/TestClobValue.java =================================================================== --- branches/as7/common-core/src/test/java/org/teiid/core/types/TestClobValue.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/common-core/src/test/java/org/teiid/core/types/TestClobValue.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -24,12 +24,7 @@ import static org.junit.Assert.*; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; import java.io.Reader; import javax.sql.rowset.serial.SerialClob; @@ -56,15 +51,8 @@ String key = cv.getReferenceStreamId(); // now force to serialize - File saved = new File(UnitTestUtil.getTestScratchPath()+"/clobassaved.bin"); //$NON-NLS-1$ - ObjectOutputStream out = new ObjectOutputStream(new FileOutputStream(saved)); - out.writeObject(cv); - out.close(); + ClobType read = UnitTestUtil.helpSerialize(cv); - // now read back the object from serilized state - ObjectInputStream in = new ObjectInputStream(new FileInputStream(saved)); - ClobType read = (ClobType)in.readObject(); - assertTrue(read.length() > 0); // make sure we have kept the reference stream id @@ -72,8 +60,21 @@ // and lost the original object assertNull(read.getReference()); + } + + @Test public void testReferencePersistence() throws Exception { + String testString = "this is test clob"; //$NON-NLS-1$ + SerialClob clob = new SerialClob(testString.toCharArray()); - saved.delete(); + ClobType cv = new ClobType(clob); + cv.setReferenceStreamId(null); + + // now force to serialize + ClobType read = UnitTestUtil.helpSerialize(cv); + + assertTrue(read.length() > 0); + + assertEquals(testString, read.getSubString(1, testString.length())); } @Test public void testClobSubstring() throws Exception { Modified: branches/as7/common-core/src/test/java/org/teiid/core/types/TestXMLValue.java =================================================================== --- branches/as7/common-core/src/test/java/org/teiid/core/types/TestXMLValue.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/common-core/src/test/java/org/teiid/core/types/TestXMLValue.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -22,20 +22,11 @@ package org.teiid.core.types; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; +import junit.framework.TestCase; -import org.teiid.core.types.SQLXMLImpl; -import org.teiid.core.types.XMLType; import org.teiid.core.util.UnitTestUtil; -import junit.framework.TestCase; - - public class TestXMLValue extends TestCase { public void testXMLValue() throws Exception { @@ -55,22 +46,26 @@ String key = xv.getReferenceStreamId(); // now force to serialize - File saved = new File(UnitTestUtil.getTestScratchPath()+"/xmlsaved.bin"); //$NON-NLS-1$ - ObjectOutputStream out = new ObjectOutputStream(new FileOutputStream(saved)); - out.writeObject(xv); - out.close(); - - // now read back the object from serilized state - ObjectInputStream in = new ObjectInputStream(new FileInputStream(saved)); - XMLType read = (XMLType)in.readObject(); + XMLType read = UnitTestUtil.helpSerialize(xv); // make sure we have kept the reference stream id assertEquals(key, read.getReferenceStreamId()); // and lost the original object assertNull(read.getReference()); + } + + public void testReferencePersistence() throws Exception { + String testString = "<foo>this is an xml value test</foo>"; //$NON-NLS-1$ + SQLXMLImpl xml = new SQLXMLImpl(testString); - saved.delete(); + XMLType xv = new XMLType(xml); + xv.setReferenceStreamId(null); + + // now force to serialize + XMLType read = UnitTestUtil.helpSerialize(xv); + + assertEquals(testString, read.getString()); } } Modified: branches/as7/common-core/src/test/java/org/teiid/core/util/UnitTestUtil.java =================================================================== --- branches/as7/common-core/src/test/java/org/teiid/core/util/UnitTestUtil.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/common-core/src/test/java/org/teiid/core/util/UnitTestUtil.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -28,7 +28,16 @@ import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; +import java.io.PrintWriter; import java.io.Serializable; +import java.io.Writer; +import java.sql.Timestamp; +import java.util.logging.ConsoleHandler; +import java.util.logging.Formatter; +import java.util.logging.Handler; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.logging.Logger; import junit.framework.Assert; import junit.framework.AssertionFailedError; @@ -388,6 +397,7 @@ return filePath; } + @SuppressWarnings("unchecked") public static final <T extends Serializable> T helpSerialize(T object) throws IOException, ClassNotFoundException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); ObjectOutputStream oos = new ObjectOutputStream(baos); @@ -398,5 +408,59 @@ return (T)ois.readObject(); } + + public static void enableTraceLogging(String loggerName) { + Logger logger = Logger.getLogger(loggerName); + logger.setLevel(Level.FINEST); + if (logger.getHandlers().length > 0) { + for (Handler h : logger.getHandlers()) { + h.setLevel(Level.FINEST); + } + } else { + logger.setUseParentHandlers(false); + ConsoleHandler ch = new ConsoleHandler(); + ch.setFormatter(new Formatter() { + + @Override + public String format(LogRecord record) { + final StringBuilder result = new StringBuilder(); + result.append(new Timestamp(record.getMillis())); + result.append(" "); //$NON-NLS-1$ + result.append(record.getLoggerName()); + result.append(" "); //$NON-NLS-1$ + result.append(record.getLevel()); + result.append(" "); //$NON-NLS-1$ + result.append(record.getThreadID()); + result.append(" "); //$NON-NLS-1$ + result.append(record.getMessage()); + result.append('\n'); + if (record.getThrown() != null) { + record.getThrown().printStackTrace(new PrintWriter(new Writer() { + @Override + public void close() throws IOException { + + } + + @Override + public void flush() throws IOException { + + } + + @Override + public void write(char[] cbuf, int off, int len) + throws IOException { + result.append(new String(cbuf, off, len)); + } + })); + result.append('\n'); + } + return result.toString(); + } + }); + ch.setLevel(Level.FINEST); + logger.addHandler(ch); + } + } + } Modified: branches/as7/documentation/caching-guide/src/main/docbook/en-US/content/hint-option.xml =================================================================== --- branches/as7/documentation/caching-guide/src/main/docbook/en-US/content/hint-option.xml 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/documentation/caching-guide/src/main/docbook/en-US/content/hint-option.xml 2011-09-29 16:46:38 UTC (rev 3507) @@ -19,6 +19,9 @@ <itemizedlist> <listitem><para>The cache hint should appear at the beginning of the SQL. It will not have any affect on INSERT/UPDATE/DELETE statements or virtual update procedure definitions.</para></listitem> <listitem><para><emphasis>pref_mem</emphasis> - if present indicates that the cached results should prefer to remain in memory. They are not however required to be memory only. + <note><para>Care should be taken to not over use the pref_mem option. + The memory preference is implemented with Java soft references. While soft references are effective at preventing out of memory conditions. + Too much memory held by soft references can limit the effective working memory. Consult your JVM options for clearing soft references if you need to tune their behavior.</para></note> </para></listitem> <listitem><para><emphasis>ttl:n</emphasis> - if present n indicates the time to live value in milliseconds. </para></listitem> Copied: branches/as7/engine/src/main/java/org/teiid/common/buffer/AutoCleanupUtil.java (from rev 3506, trunk/engine/src/main/java/org/teiid/common/buffer/AutoCleanupUtil.java) =================================================================== --- branches/as7/engine/src/main/java/org/teiid/common/buffer/AutoCleanupUtil.java (rev 0) +++ branches/as7/engine/src/main/java/org/teiid/common/buffer/AutoCleanupUtil.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -0,0 +1,73 @@ +/* + * JBoss, Home of Professional Open Source. + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA. + */ +package org.teiid.common.buffer; + +import java.lang.ref.PhantomReference; +import java.lang.ref.ReferenceQueue; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.Set; + +public class AutoCleanupUtil { + + public interface Removable { + public void remove(); + } + + static final class PhantomCleanupReference extends PhantomReference<Object> { + + private Removable removable; + + public PhantomCleanupReference(Object referent, Removable removable) { + super(referent, QUEUE); + this.removable = removable; + } + + public void cleanup() { + try { + this.removable.remove(); + } finally { + this.removable = null; + this.clear(); + } + } + } + + private static ReferenceQueue<Object> QUEUE = new ReferenceQueue<Object>(); + private static final Set<PhantomReference<Object>> REFERENCES = Collections.synchronizedSet(Collections.newSetFromMap(new IdentityHashMap<PhantomReference<Object>, Boolean>())); + + public static void setCleanupReference(Object o, Removable r) { + REFERENCES.add(new PhantomCleanupReference(o, r)); + doCleanup(); + } + + public static void doCleanup() { + for (int i = 0; i < 10; i++) { + PhantomCleanupReference ref = (PhantomCleanupReference)QUEUE.poll(); + if (ref == null) { + break; + } + ref.cleanup(); + REFERENCES.remove(ref); + } + } +} Modified: branches/as7/engine/src/main/java/org/teiid/common/buffer/BatchManager.java =================================================================== --- branches/as7/engine/src/main/java/org/teiid/common/buffer/BatchManager.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/main/java/org/teiid/common/buffer/BatchManager.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -22,36 +22,29 @@ package org.teiid.common.buffer; +import java.lang.ref.Reference; +import java.util.List; + import org.teiid.core.TeiidComponentException; +/** + * Acts as a combination serializer/cachemanager + */ public interface BatchManager { - public interface CleanupHook { - - void cleanup(); - - } + List<List<?>> getBatch(Long batch, boolean retain) throws TeiidComponentException; - public interface ManagedBatch { - - TupleBatch getBatch(boolean cache, String[] types) throws TeiidComponentException; - - void remove(); - - void setPrefersMemory(boolean prefers); - - /** - * Get an object that can cleaup the {@link ManagedBatch}, but does not hold a hard reference to - * the {@link ManagedBatch} or the {@link BatchManager} - * @return - */ - CleanupHook getCleanupHook(); - - } + void remove(Long batch); - ManagedBatch createManagedBatch(TupleBatch batch, boolean softCache) throws TeiidComponentException; + void setPrefersMemory(boolean prefers); + boolean prefersMemory(); + + Long createManagedBatch(List<? extends List<?>> batch) throws TeiidComponentException; + void remove(); - FileStore createStorage(String prefix); + Reference<? extends BatchManager> getBatchManagerReference(); + + String[] getTypes(); } Property changes on: branches/as7/engine/src/main/java/org/teiid/common/buffer/BlockedException.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/main/java/org/teiid/common/buffer/BlockedException.java:3149-3217 /trunk/engine/src/main/java/org/teiid/common/buffer/BlockedException.java:3188-3450 + /branches/7.4.x/engine/src/main/java/org/teiid/common/buffer/BlockedException.java:3149-3217 /trunk/engine/src/main/java/org/teiid/common/buffer/BlockedException.java:3188-3450,3452-3506 Modified: branches/as7/engine/src/main/java/org/teiid/common/buffer/BufferManager.java =================================================================== --- branches/as7/engine/src/main/java/org/teiid/common/buffer/BufferManager.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/main/java/org/teiid/common/buffer/BufferManager.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -75,6 +75,12 @@ * Get the batch size to use during query processing. * @return Batch size (# of rows) */ + int getProcessorBatchSize(List<? extends Expression> schema); + + /** + * Get the nominal batch size target + * @return + */ int getProcessorBatchSize(); /** Property changes on: branches/as7/engine/src/main/java/org/teiid/common/buffer/BufferManager.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/main/java/org/teiid/common/buffer/BufferManager.java:3220-3275 /trunk/engine/src/main/java/org/teiid/common/buffer/BufferManager.java:3188-3450 + /branches/7.4.x/engine/src/main/java/org/teiid/common/buffer/BufferManager.java:3220-3275 /trunk/engine/src/main/java/org/teiid/common/buffer/BufferManager.java:3188-3450,3452-3506 Copied: branches/as7/engine/src/main/java/org/teiid/common/buffer/Cache.java (from rev 3506, trunk/engine/src/main/java/org/teiid/common/buffer/Cache.java) =================================================================== --- branches/as7/engine/src/main/java/org/teiid/common/buffer/Cache.java (rev 0) +++ branches/as7/engine/src/main/java/org/teiid/common/buffer/Cache.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -0,0 +1,39 @@ +/* + * JBoss, Home of Professional Open Source. + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA. + */ + +package org.teiid.common.buffer; + +import java.util.Collection; + +import org.teiid.core.TeiidComponentException; + +/** + * Represents the storage strategy for the {@link BufferManager} + */ +public interface Cache extends StorageManager { + void createCacheGroup(Long gid); //called prior to adding an entry + Collection<Long> removeCacheGroup(Long gid); + void addToCacheGroup(Long gid, Long oid); + CacheEntry get(Long id, Serializer<?> serializer) throws TeiidComponentException; + void add(CacheEntry entry, Serializer<?> s); + void remove(Long gid, Long id); +} \ No newline at end of file Copied: branches/as7/engine/src/main/java/org/teiid/common/buffer/CacheEntry.java (from rev 3506, trunk/engine/src/main/java/org/teiid/common/buffer/CacheEntry.java) =================================================================== --- branches/as7/engine/src/main/java/org/teiid/common/buffer/CacheEntry.java (rev 0) +++ branches/as7/engine/src/main/java/org/teiid/common/buffer/CacheEntry.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -0,0 +1,101 @@ +/* + * JBoss, Home of Professional Open Source. + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA. + */ + +package org.teiid.common.buffer; + +import java.lang.ref.WeakReference; + +public class CacheEntry { + private boolean persistent; + private Object object; + private int sizeEstimate; + private WeakReference<? extends Serializer<?>> serializer; + private Long id; + + public CacheEntry(Long id) { + this.id = id; + } + + public Long getId() { + return id; + } + + @Override + public int hashCode() { + return getId().hashCode(); + } + + public int getSizeEstimate() { + return sizeEstimate; + } + + public void setSizeEstimate(int sizeEstimate) { + this.sizeEstimate = sizeEstimate; + } + + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof CacheEntry)) { + return false; + } + return getId().equals(((CacheEntry)obj).getId()); + } + + @Override + public String toString() { + return getId().toString(); + } + + public Object nullOut() { + Object result = getObject(); + setObject(null); + setSerializer(null); + return result; + } + + public void setObject(Object object) { + this.object = object; + } + + public Object getObject() { + return object; + } + + public void setPersistent(boolean persistent) { + this.persistent = persistent; + } + + public boolean isPersistent() { + return persistent; + } + + public void setSerializer(WeakReference<? extends Serializer<?>> serializer) { + this.serializer = serializer; + } + + public WeakReference<? extends Serializer<?>> getSerializer() { + return serializer; + } + +} \ No newline at end of file Modified: branches/as7/engine/src/main/java/org/teiid/common/buffer/FileStore.java =================================================================== --- branches/as7/engine/src/main/java/org/teiid/common/buffer/FileStore.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/main/java/org/teiid/common/buffer/FileStore.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -25,20 +25,14 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.lang.ref.PhantomReference; -import java.lang.ref.ReferenceQueue; +import java.nio.ByteBuffer; import java.util.Arrays; -import java.util.Collections; -import java.util.IdentityHashMap; -import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; -import org.teiid.core.TeiidComponentException; +import org.teiid.common.buffer.AutoCleanupUtil.Removable; -public abstract class FileStore { - - private static ReferenceQueue<Object> QUEUE = new ReferenceQueue<Object>(); - private static final Set<PhantomReference<Object>> REFERENCES = Collections.newSetFromMap(new IdentityHashMap<PhantomReference<Object>, Boolean>()); - +public abstract class FileStore implements Removable { + /** * A customized buffered stream with an exposed buffer */ @@ -80,12 +74,8 @@ } private void writeDirect(byte[] b, int off, int len) throws IOException { - try { - FileStore.this.write(b, off, len); - bytesWritten = true; - } catch (TeiidComponentException e) { - throw new IOException(e); - } + FileStore.this.write(b, off, len); + bytesWritten = true; } public void flushBuffer() throws IOException { @@ -140,84 +130,59 @@ } - static class CleanupReference extends PhantomReference<Object> { - - private FileStore store; - - public CleanupReference(Object referent, FileStore store) { - super(referent, QUEUE); - this.store = store; - } - - public void cleanup() { - try { - this.store.remove(); - } finally { - this.clear(); - } - } - } + private AtomicBoolean removed = new AtomicBoolean(); - private boolean removed; - protected long len; + public abstract long getLength(); - public void setCleanupReference(Object o) { - REFERENCES.add(new CleanupReference(o, this)); - for (int i = 0; i < 10; i++) { - CleanupReference ref = (CleanupReference)QUEUE.poll(); - if (ref == null) { - break; - } - ref.cleanup(); - REFERENCES.remove(ref); - } - } + public abstract void setLength(long length) throws IOException; - public synchronized long getLength() { - return len; - } - public int read(long fileOffset, byte[] b, int offSet, int length) - throws TeiidComponentException { - if (removed) { - throw new TeiidComponentException("already removed"); //$NON-NLS-1$ + throws IOException { + checkRemoved(); + return readWrite(fileOffset, b, offSet, length, false); + } + + private void checkRemoved() throws IOException { + if (removed.get()) { + throw new IOException("already removed"); //$NON-NLS-1$ } - return readDirect(fileOffset, b, offSet, length); } - protected abstract int readDirect(long fileOffset, byte[] b, int offSet, int length) - throws TeiidComponentException; + protected abstract int readWrite(long fileOffset, byte[] b, int offSet, int length, boolean write) + throws IOException; - public void readFully(long fileOffset, byte[] b, int offSet, int length) throws TeiidComponentException { + public void readFully(long fileOffset, byte[] b, int offSet, int length) throws IOException { + if (length == 0) { + return; + } int n = 0; do { int count = this.read(fileOffset + n, b, offSet + n, length - n); - if (count < 0) { - throw new TeiidComponentException("not enough bytes available"); //$NON-NLS-1$ + if (count <= 0 && length > 0) { + throw new IOException("not enough bytes available"); //$NON-NLS-1$ } n += count; } while (n < length); } - public void write(byte[] bytes) throws TeiidComponentException { - write(bytes, 0, bytes.length); + public synchronized void write(byte[] bytes, int offset, int length) throws IOException { + write(getLength(), bytes, offset, length); } - - public synchronized long write(byte[] bytes, int offset, int length) throws TeiidComponentException { - if (removed) { - throw new TeiidComponentException("already removed"); //$NON-NLS-1$ - } - writeDirect(bytes, offset, length); - long result = len; - len += length; - return result; + + public void write(long start, byte[] bytes, int offset, int length) throws IOException { + int n = 0; + do { + checkRemoved(); + int count = this.readWrite(start + n, bytes, offset + n, length - n, true); + if (count <= 0 && length > 0) { + throw new IOException("not enough bytes available"); //$NON-NLS-1$ + } + n += count; + } while (n < length); } - protected abstract void writeDirect(byte[] bytes, int offset, int length) throws TeiidComponentException; - public void remove() { - if (!this.removed) { - this.removed = true; + if (removed.compareAndSet(false, true)) { this.removeDirect(); } } @@ -241,28 +206,46 @@ @Override public int read(byte[] b, int off, int len) throws IOException { - try { - if (this.streamLength != -1 && len > this.streamLength) { - len = (int)this.streamLength; - } - if (this.streamLength == -1 || this.streamLength > 0) { - int bytes = FileStore.this.read(offset, b, off, len); - if (bytes != -1) { - this.offset += bytes; - if (this.streamLength != -1) { - this.streamLength -= bytes; - } + if (this.streamLength != -1 && len > this.streamLength) { + len = (int)this.streamLength; + } + if (this.streamLength == -1 || this.streamLength > 0) { + int bytes = FileStore.this.read(offset, b, off, len); + if (bytes != -1) { + this.offset += bytes; + if (this.streamLength != -1) { + this.streamLength -= bytes; } - return bytes; } - return -1; - } catch (TeiidComponentException e) { - throw new IOException(e); + return bytes; } + return -1; } }; } + public ByteBuffer getBuffer(long start, int length, boolean allocate) throws IOException { + byte[] b = new byte[length]; + if (!allocate) { + readFully(start, b, 0, length); + } + return ByteBuffer.wrap(b); + } + + public void updateFromBuffer(ByteBuffer bb, long start) throws IOException { + byte[] b = null; + int offset = 0; + bb.rewind(); + if (bb.hasArray()) { + b = bb.array(); + offset = bb.arrayOffset(); + } else { + b = new byte[bb.limit()]; + bb.get(b); + } + write(start, b, offset, bb.limit()); + } + public InputStream createInputStream(final long start) { return createInputStream(start, -1); } @@ -277,16 +260,12 @@ @Override public void write(byte[] b, int off, int len) throws IOException { - try { - FileStore.this.write(b, off, len); - } catch (TeiidComponentException e) { - throw new IOException(e); - } + FileStore.this.write(b, off, len); } }; } - public FileStoreOutputStream createOutputStream(int maxMemorySize) { + public FileStoreOutputStream createOutputStream(int maxMemorySize) { return new FileStoreOutputStream(maxMemorySize); } Modified: branches/as7/engine/src/main/java/org/teiid/common/buffer/FileStoreInputStreamFactory.java =================================================================== --- branches/as7/engine/src/main/java/org/teiid/common/buffer/FileStoreInputStreamFactory.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/main/java/org/teiid/common/buffer/FileStoreInputStreamFactory.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -42,7 +42,7 @@ public FileStoreInputStreamFactory(FileStore lobBuffer, String encoding) { this.encoding = encoding; this.lobBuffer = lobBuffer; - this.lobBuffer.setCleanupReference(this); + AutoCleanupUtil.setCleanupReference(this, lobBuffer); } @Override @@ -56,6 +56,9 @@ @Override public long getLength() { + if (fsos != null && !fsos.bytesWritten()) { + return fsos.getCount(); + } return lobBuffer.getLength(); } @@ -86,7 +89,10 @@ } @Override - public boolean isPersistent() { - return fsos == null || fsos.bytesWritten(); + public StorageMode getStorageMode() { + if (fsos == null) { + return StorageMode.PERSISTENT; + } + return StorageMode.MEMORY; } } \ No newline at end of file Copied: branches/as7/engine/src/main/java/org/teiid/common/buffer/LightWeightCopyOnWriteList.java (from rev 3506, trunk/engine/src/main/java/org/teiid/common/buffer/LightWeightCopyOnWriteList.java) =================================================================== --- branches/as7/engine/src/main/java/org/teiid/common/buffer/LightWeightCopyOnWriteList.java (rev 0) +++ branches/as7/engine/src/main/java/org/teiid/common/buffer/LightWeightCopyOnWriteList.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -0,0 +1,116 @@ +/* + * JBoss, Home of Professional Open Source. + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA. + */ + +package org.teiid.common.buffer; + +import java.util.AbstractList; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.RandomAccess; + +/** + * Creates a copy of a reference list when modified. + * + * @param <T> + */ +public class LightWeightCopyOnWriteList<T> extends AbstractList<T> implements RandomAccess { + + private List<T> list; + private boolean modified; + + public LightWeightCopyOnWriteList(List<T> list) { + this.list = list; + } + + @Override + public T get(int index) { + return list.get(index); + } + + public List<T> getList() { + return list; + } + + public void add(int index, T element) { + if (!modified) { + List<T> next = new ArrayList<T>(list.size() + 1); + next.addAll(list); + list = next; + modified = true; + } + list.add(index, element); + } + + public T set(int index, T element) { + checkModified(); + return list.set(index, element); + } + + private void checkModified() { + if (!modified) { + list = new ArrayList<T>(list); + modified = true; + } + } + + public boolean addAll(Collection<? extends T> c) { + return addAll(size(), c); + } + + @Override + public boolean addAll(int index, Collection<? extends T> c) { + checkModified(); + return list.addAll(index, c); + } + + @Override + public T remove(int index) { + checkModified(); + return list.remove(index); + } + + @Override + public Object[] toArray() { + return list.toArray(); + } + + public <U extends Object> U[] toArray(U[] a) { + return list.toArray(a); + } + + @Override + public void clear() { + if (!modified) { + list = new ArrayList<T>(); + modified = true; + } else { + list.clear(); + } + } + + @Override + public int size() { + return list.size(); + } + +} Modified: branches/as7/engine/src/main/java/org/teiid/common/buffer/LobManager.java =================================================================== --- branches/as7/engine/src/main/java/org/teiid/common/buffer/LobManager.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/main/java/org/teiid/common/buffer/LobManager.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -30,9 +30,10 @@ import java.sql.SQLException; import java.sql.SQLXML; import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; import org.teiid.core.TeiidComponentException; import org.teiid.core.types.BaseLob; @@ -45,6 +46,10 @@ import org.teiid.core.types.SQLXMLImpl; import org.teiid.core.types.Streamable; import org.teiid.core.types.XMLType; +import org.teiid.core.types.InputStreamFactory.BlobInputStreamFactory; +import org.teiid.core.types.InputStreamFactory.ClobInputStreamFactory; +import org.teiid.core.types.InputStreamFactory.SQLXMLInputStreamFactory; +import org.teiid.core.types.InputStreamFactory.StorageMode; import org.teiid.core.util.ObjectConverterUtil; import org.teiid.query.QueryPlugin; import org.teiid.query.sql.symbol.Expression; @@ -54,9 +59,57 @@ * TODO: for temp tables we may need to have a copy by value management strategy */ public class LobManager { - private Map<String, Streamable<?>> lobReferences = new ConcurrentHashMap<String, Streamable<?>>(); + + public enum ReferenceMode { + ATTACH, + CREATE, + REMOVE + } + + private static class LobHolder { + Streamable<?> lob; + int referenceCount = 1; - public void updateReferences(int[] lobIndexes, List<?> tuple) + public LobHolder(Streamable<?> lob) { + this.lob = lob; + } + } + + private Map<String, LobHolder> lobReferences = Collections.synchronizedMap(new HashMap<String, LobHolder>()); + private boolean inlineLobs = true; + private int maxMemoryBytes = DataTypeManager.MAX_LOB_MEMORY_BYTES; + private int[] lobIndexes; + private FileStore lobStore; + + public LobManager(int[] lobIndexes, FileStore lobStore) { + this.lobIndexes = lobIndexes; + this.lobStore = lobStore; + } + + public LobManager clone() { + LobManager clone = new LobManager(lobIndexes, null); + clone.inlineLobs = inlineLobs; + clone.maxMemoryBytes = maxMemoryBytes; + synchronized (lobReferences) { + for (Map.Entry<String, LobHolder> entry : lobReferences.entrySet()) { + LobHolder lobHolder = new LobHolder(entry.getValue().lob); + lobHolder.referenceCount = entry.getValue().referenceCount; + clone.lobReferences.put(entry.getKey(), lobHolder); + } + } + return clone; + } + + public void setInlineLobs(boolean trackMemoryLobs) { + this.inlineLobs = trackMemoryLobs; + } + + public void setMaxMemoryBytes(int maxMemoryBytes) { + this.maxMemoryBytes = maxMemoryBytes; + } + + @SuppressWarnings("unchecked") + public void updateReferences(List<?> tuple, ReferenceMode mode) throws TeiidComponentException { for (int i = 0; i < lobIndexes.length; i++) { Object anObj = tuple.get(lobIndexes[i]); @@ -64,31 +117,64 @@ continue; } Streamable lob = (Streamable) anObj; - if (lob.getReference() == null) { - lob.setReference(getLobReference(lob.getReferenceStreamId()).getReference()); - } else { - String id = lob.getReferenceStreamId(); - this.lobReferences.put(id, lob); + try { + if (lob.getReferenceStreamId() == null || (inlineLobs + && (InputStreamFactory.getStorageMode(lob) == StorageMode.MEMORY + || lob.length()*(lob instanceof ClobType?2:1) <= maxMemoryBytes))) { + lob.setReferenceStreamId(null); + continue; + } + } catch (SQLException e) { + //presumably the lob is bad, but let it slide for now } + String id = lob.getReferenceStreamId(); + LobHolder lobHolder = this.lobReferences.get(id); + switch (mode) { + case REMOVE: + if (lobHolder != null) { + lobHolder.referenceCount--; + if (lobHolder.referenceCount < 1) { + this.lobReferences.remove(id); + } + } + break; + case ATTACH: + if (lob.getReference() == null) { + if (lobHolder == null) { + throw new TeiidComponentException(QueryPlugin.Util.getString("ProcessWorker.wrongdata")); //$NON-NLS-1$ + } + lob.setReference(lobHolder.lob.getReference()); + } + break; + case CREATE: + if (lob.getReference() == null) { + throw new TeiidComponentException(QueryPlugin.Util.getString("ProcessWorker.wrongdata")); //$NON-NLS-1$ + } + if (lobHolder == null) { + this.lobReferences.put(id, new LobHolder(lob)); + } else { + lobHolder.referenceCount++; + } + } } } public Streamable<?> getLobReference(String id) throws TeiidComponentException { - Streamable<?> lob = this.lobReferences.get(id); + LobHolder lob = this.lobReferences.get(id); if (lob == null) { throw new TeiidComponentException(QueryPlugin.Util.getString("ProcessWorker.wrongdata")); //$NON-NLS-1$ } - return lob; + return lob.lob; } - public static int[] getLobIndexes(List expressions) { + public static int[] getLobIndexes(List<? extends Expression> expressions) { if (expressions == null) { return null; } int[] result = new int[expressions.size()]; int resultIndex = 0; for (int i = 0; i < expressions.size(); i++) { - Expression expr = (Expression) expressions.get(i); + Expression expr = expressions.get(i); if (DataTypeManager.isLOB(expr.getType()) || expr.getType() == DataTypeManager.DefaultDataClasses.OBJECT) { result[resultIndex++] = i; } @@ -99,12 +185,12 @@ return Arrays.copyOf(result, resultIndex); } - public void persist(FileStore lobStore) throws TeiidComponentException { + public void persist() throws TeiidComponentException { // stream the contents of lob into file store. - byte[] bytes = new byte[102400]; // 100k + byte[] bytes = new byte[1 << 14]; - for (Map.Entry<String, Streamable<?>> entry : this.lobReferences.entrySet()) { - entry.setValue(persistLob(entry.getValue(), lobStore, bytes)); + for (Map.Entry<String, LobHolder> entry : this.lobReferences.entrySet()) { + entry.getValue().lob = persistLob(entry.getValue().lob, lobStore, bytes); } } @@ -115,7 +201,7 @@ try { BaseLob baseLob = (BaseLob)lob.getReference(); InputStreamFactory isf = baseLob.getStreamFactory(); - if (isf.isPersistent()) { + if (isf.getStorageMode() == StorageMode.PERSISTENT) { return lob; } } catch (SQLException e) { @@ -127,19 +213,15 @@ Streamable<?> persistedLob; try { - InputStreamFactory isf = new InputStreamFactory() { - @Override - public InputStream getInputStream() throws IOException { - if (lob instanceof BlobType) { - return new BlobInputStreamFactory((Blob)lob).getInputStream(); - } - else if (lob instanceof ClobType) { - return new ClobInputStreamFactory((Clob)lob).getInputStream(); - } - return new SQLXMLInputStreamFactory((SQLXML)lob).getInputStream(); - } - }; - InputStream is = isf.getInputStream(); + InputStream is = null; + if (lob instanceof BlobType) { + is = new BlobInputStreamFactory((Blob)lob).getInputStream(); + } + else if (lob instanceof ClobType) { + is = new ClobInputStreamFactory((Clob)lob).getInputStream(); + } else { + is = new SQLXMLInputStreamFactory((SQLXML)lob).getInputStream(); + } OutputStream fsos = store.createOutputStream(); length = ObjectConverterUtil.write(fsos, is, bytes, -1); } catch (IOException e) { @@ -156,8 +238,8 @@ } @Override - public boolean isPersistent() { - return true; + public StorageMode getStorageMode() { + return StorageMode.PERSISTENT; } }; @@ -178,4 +260,13 @@ } return persistedLob; } + + public int getLobCount() { + return this.lobReferences.size(); + } + + public void remove() { + this.lobReferences.clear(); + + } } Modified: branches/as7/engine/src/main/java/org/teiid/common/buffer/SPage.java =================================================================== --- branches/as7/engine/src/main/java/org/teiid/common/buffer/SPage.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/main/java/org/teiid/common/buffer/SPage.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -23,6 +23,7 @@ package org.teiid.common.buffer; import java.lang.ref.PhantomReference; +import java.lang.ref.Reference; import java.lang.ref.ReferenceQueue; import java.util.ArrayList; import java.util.Collections; @@ -31,17 +32,14 @@ import java.util.LinkedList; import java.util.List; import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; -import org.teiid.common.buffer.BatchManager.CleanupHook; -import org.teiid.common.buffer.BatchManager.ManagedBatch; import org.teiid.core.TeiidComponentException; import org.teiid.core.TeiidRuntimeException; /** * A linked list Page entry in the tree * - * TODO: return the tuplebatch from getvalues, since that is what we're tracking - * * State cloning allows a single storage reference to be shared in many trees. * A phantom reference is used for proper cleanup once cloned. * @@ -56,8 +54,8 @@ static class SearchResult { int index; SPage page; - TupleBatch values; - public SearchResult(int index, SPage page, TupleBatch values) { + List<List<?>> values; + public SearchResult(int index, SPage page, List<List<?>> values) { this.index = index; this.page = page; this.values = values; @@ -68,51 +66,54 @@ private static ReferenceQueue<Object> QUEUE = new ReferenceQueue<Object>(); static class CleanupReference extends PhantomReference<Object> { - private CleanupHook batch; + private Long batch; + private Reference<? extends BatchManager> ref; - public CleanupReference(Object referent, CleanupHook batch) { + public CleanupReference(Object referent, Long batch, Reference<? extends BatchManager> ref) { super(referent, QUEUE); this.batch = batch; + this.ref = ref; } public void cleanup() { try { - this.batch.cleanup(); + BatchManager batchManager = ref.get(); + if (batchManager != null) { + batchManager.remove(batch); + } } finally { this.clear(); } } } - + + private static AtomicLong counter = new AtomicLong(); + STree stree; - private int id; + private long id; protected SPage next; protected SPage prev; - protected ManagedBatch managedBatch; - private Object trackingObject; - protected TupleBatch values; - protected ArrayList<SPage> children; - //TODO: could track cloning more completely, which would allow for earlier batch removal - private boolean cloned; + protected Long managedBatch; + protected Object trackingObject; + protected List<List<?>> values; + protected List<SPage> children; SPage(STree stree, boolean leaf) { this.stree = stree; - this.id = stree.counter.getAndIncrement(); + this.id = counter.getAndIncrement(); stree.pages.put(this.id, this); - //TODO: this counter is a hack. need a better idea of a storage id - this.values = new TupleBatch(id, new ArrayList(stree.pageSize/4)); + this.values = new ArrayList<List<?>>(); if (!leaf) { - children = new ArrayList<SPage>(stree.pageSize/4); + children = new ArrayList<SPage>(); } } public SPage clone(STree tree) { try { if (this.managedBatch != null && trackingObject == null) { - cloned = true; this.trackingObject = new Object(); - CleanupReference managedBatchReference = new CleanupReference(trackingObject, managedBatch.getCleanupHook()); + CleanupReference managedBatchReference = new CleanupReference(trackingObject, managedBatch, stree.getBatchManager(children == null).getBatchManagerReference()); REFERENCES.add(managedBatchReference); } SPage clone = (SPage) super.clone(); @@ -121,7 +122,7 @@ clone.children = new ArrayList<SPage>(children); } if (values != null) { - clone.values = new TupleBatch(stree.counter.getAndIncrement(), new ArrayList<List<?>>(values.getTuples())); + clone.values = new ArrayList<List<?>>(values); } return clone; } catch (CloneNotSupportedException e) { @@ -129,37 +130,37 @@ } } - public int getId() { + public long getId() { return id; } static SearchResult search(SPage page, List k, LinkedList<SearchResult> parent) throws TeiidComponentException { - TupleBatch previousValues = null; + List<List<?>> previousValues = null; for (;;) { - TupleBatch values = page.getValues(); - int index = Collections.binarySearch(values.getTuples(), k, page.stree.comparator); + List<List<?>> values = page.getValues(); + int index = Collections.binarySearch(values, k, page.stree.comparator); int flippedIndex = - index - 1; if (previousValues != null) { if (flippedIndex == 0) { //systemic weakness of the algorithm - return new SearchResult(-previousValues.getTuples().size() - 1, page.prev, previousValues); + return new SearchResult(-previousValues.size() - 1, page.prev, previousValues); } if (parent != null && index != 0) { page.stree.updateLock.lock(); try { - index = Collections.binarySearch(values.getTuples(), k, page.stree.comparator); + index = Collections.binarySearch(values, k, page.stree.comparator); if (index != 0) { //for non-matches move the previous pointer over to this page SPage childPage = page; List oldKey = null; - List newKey = page.stree.extractKey(values.getTuples().get(0)); + List newKey = page.stree.extractKey(values.get(0)); for (Iterator<SearchResult> desc = parent.descendingIterator(); desc.hasNext();) { SearchResult sr = desc.next(); int parentIndex = Math.max(0, -sr.index - 2); if (oldKey == null) { - oldKey = sr.values.getTuples().set(parentIndex, newKey); - } else if (page.stree.comparator.compare(oldKey, sr.values.getTuples().get(parentIndex)) == 0 ) { - sr.values.getTuples().set(parentIndex, newKey); + oldKey = sr.values.set(parentIndex, newKey); + } else if (page.stree.comparator.compare(oldKey, sr.values.get(parentIndex)) == 0 ) { + sr.values.set(parentIndex, newKey); } else { break; } @@ -173,7 +174,7 @@ } } } - if (flippedIndex != values.getTuples().size() || page.next == null) { + if (flippedIndex != values.size() || page.next == null) { return new SearchResult(index, page, values); } previousValues = values; @@ -181,36 +182,32 @@ } } - protected void setValues(TupleBatch values) throws TeiidComponentException { - if (managedBatch != null && !cloned) { - managedBatch.remove(); + protected void setValues(List<List<?>> values) throws TeiidComponentException { + if (values instanceof LightWeightCopyOnWriteList<?>) { + values = ((LightWeightCopyOnWriteList<List<?>>)values).getList(); } - if (values.getTuples().size() < MIN_PERSISTENT_SIZE) { + if (managedBatch != null && trackingObject == null) { + stree.getBatchManager(children == null).remove(managedBatch); + managedBatch = null; + trackingObject = null; + } + if (values.size() < MIN_PERSISTENT_SIZE) { this.values = values; return; - } - this.values = null; - if (children != null) { - values.setDataTypes(stree.keytypes); - } else { - values.setDataTypes(stree.types); + } else if (stree.batchInsert && children == null && values.size() < stree.leafSize) { + this.values = values; + stree.incompleteInsert = this; + return; } - if (cloned) { - values.setRowOffset(stree.counter.getAndIncrement()); - cloned = false; - trackingObject = null; - } - if (children != null) { - managedBatch = stree.keyManager.createManagedBatch(values, true); - } else { - managedBatch = stree.leafManager.createManagedBatch(values, stree.preferMemory); - } + this.values = null; + this.trackingObject = null; + managedBatch = stree.getBatchManager(children == null).createManagedBatch(values); } - + protected void remove(boolean force) { if (managedBatch != null) { - if (force || !cloned) { - managedBatch.remove(); + if (force || trackingObject == null) { + stree.getBatchManager(children == null).remove(managedBatch); } managedBatch = null; trackingObject = null; @@ -219,7 +216,7 @@ children = null; } - protected TupleBatch getValues() throws TeiidComponentException { + protected List<List<?>> getValues() throws TeiidComponentException { if (values != null) { return values; } @@ -234,19 +231,20 @@ REFERENCES.remove(ref); ref.cleanup(); } - if (children != null) { - return managedBatch.getBatch(true, stree.keytypes); + List<List<?>> result = stree.getBatchManager(children == null).getBatch(managedBatch, true); + if (trackingObject != null) { + return new LightWeightCopyOnWriteList<List<?>>(result); } - return managedBatch.getBatch(true, stree.types); + return result; } - static void merge(LinkedList<SearchResult> places, TupleBatch nextValues, SPage current, TupleBatch currentValues) + static void merge(LinkedList<SearchResult> places, List<List<?>> nextValues, SPage current, List<List<?>> currentValues) throws TeiidComponentException { SearchResult parent = places.peekLast(); if (parent != null) { - correctParents(parent.page, nextValues.getTuples().get(0), current.next, current); + correctParents(parent.page, nextValues.get(0), current.next, current); } - currentValues.getTuples().addAll(nextValues.getTuples()); + currentValues.addAll(nextValues); if (current.children != null) { current.children.addAll(current.next.children); } @@ -292,19 +290,19 @@ public String toString() { StringBuilder result = new StringBuilder(); try { - TupleBatch tb = getValues(); - result.append(tb.getBeginRow()); + List<List<?>> tb = getValues(); + result.append(id); if (children == null) { - if (tb.getTuples().size() <= 1) { - result.append(tb.getTuples()); + if (tb.size() <= 1) { + result.append(tb); } else { - result.append("[").append(tb.getTuples().get(0)).append(" . ").append(tb.getTuples().size()). //$NON-NLS-1$ //$NON-NLS-2$ - append(" . ").append(tb.getTuples().get(tb.getTuples().size() - 1)).append("]"); //$NON-NLS-1$ //$NON-NLS-2$ + result.append("[").append(tb.get(0)).append(" . ").append(tb.size()). //$NON-NLS-1$ //$NON-NLS-2$ + append(" . ").append(tb.get(tb.size() - 1)).append("]"); //$NON-NLS-1$ //$NON-NLS-2$ } } else { result.append("["); //$NON-NLS-1$ for (int i = 0; i < children.size(); i++) { - result.append(tb.getTuples().get(i)).append("->").append(children.get(i).getValues().getBeginRow()); //$NON-NLS-1$ + result.append(tb.get(i)).append("->").append(children.get(i).getId()); //$NON-NLS-1$ if (i < children.size() - 1) { result.append(", "); //$NON-NLS-1$ } Modified: branches/as7/engine/src/main/java/org/teiid/common/buffer/STree.java =================================================================== --- branches/as7/engine/src/main/java/org/teiid/common/buffer/STree.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/main/java/org/teiid/common/buffer/STree.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -27,15 +27,16 @@ import java.io.ObjectOutputStream; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Random; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReentrantLock; import org.teiid.client.BatchSerializer; +import org.teiid.common.buffer.LobManager.ReferenceMode; import org.teiid.common.buffer.SPage.SearchResult; import org.teiid.core.TeiidComponentException; import org.teiid.core.TeiidRuntimeException; @@ -51,23 +52,23 @@ public enum InsertMode {ORDERED, NEW, UPDATE} - private static final Random seedGenerator = new Random(); + private static final Random seedGenerator = new Random(0); protected int randomSeed; private int mask = 1; private int shift = 1; - protected AtomicInteger counter = new AtomicInteger(); - protected ConcurrentHashMap<Integer, SPage> pages = new ConcurrentHashMap<Integer, SPage>(); + protected HashMap<Long, SPage> pages = new HashMap<Long, SPage>(); protected volatile SPage[] header = new SPage[] {new SPage(this, true)}; protected BatchManager keyManager; protected BatchManager leafManager; protected ListNestedSortComparator comparator; - protected int pageSize; + private int pageSize; + protected int leafSize; protected int keyLength; - protected String[] types; - protected String[] keytypes; - protected boolean preferMemory; + protected boolean batchInsert; + protected SPage incompleteInsert; + protected LobManager lobManager; protected ReentrantLock updateLock = new ReentrantLock(); @@ -77,10 +78,12 @@ BatchManager leafManager, final ListNestedSortComparator comparator, int pageSize, + int leafSize, int keyLength, - String[] types) { + LobManager lobManager) { randomSeed = seedGenerator.nextInt() | 0x00000100; // ensure nonzero this.keyManager = manager; + manager.setPrefersMemory(true); this.leafManager = leafManager; this.comparator = comparator; this.pageSize = Math.max(pageSize, SPage.MIN_PERSISTENT_SIZE); @@ -91,24 +94,27 @@ mask <<= 1; mask++; } + this.leafSize = leafSize; this.keyLength = keyLength; - this.types = types; - this.keytypes = Arrays.copyOf(types, keyLength); + this.lobManager = lobManager; } public STree clone() { updateLock.lock(); try { STree clone = (STree) super.clone(); + if (lobManager != null) { + clone.lobManager = lobManager.clone(); + } clone.updateLock = new ReentrantLock(); clone.rowCount = new AtomicInteger(rowCount.get()); //clone the pages - clone.pages = new ConcurrentHashMap<Integer, SPage>(pages); - for (Map.Entry<Integer, SPage> entry : clone.pages.entrySet()) { + clone.pages = new HashMap<Long, SPage>(pages); + for (Map.Entry<Long, SPage> entry : clone.pages.entrySet()) { entry.setValue(entry.getValue().clone(clone)); } //reset the pointers - for (Map.Entry<Integer, SPage> entry : clone.pages.entrySet()) { + for (Map.Entry<Long, SPage> entry : clone.pages.entrySet()) { SPage clonePage = entry.getValue(); clonePage.next = clone.getPage(clonePage.next); clonePage.prev = clone.getPage(clonePage.prev); @@ -141,23 +147,41 @@ SPage page = header[0]; oos.writeInt(this.rowCount.get()); while (true) { - TupleBatch batch = page.getValues(); - BatchSerializer.writeBatch(oos, types, batch.getAllTuples()); + List<List<?>> batch = page.getValues(); + BatchSerializer.writeBatch(oos, leafManager.getTypes(), batch); if (page.next == null) { break; } } } + public void setBatchInsert(boolean batchInsert) throws TeiidComponentException { + if (this.batchInsert == batchInsert) { + return; + } + this.batchInsert = batchInsert; + if (batchInsert || incompleteInsert == null) { + return; + } + SPage toFlush = incompleteInsert; + incompleteInsert = null; + if (toFlush.managedBatch != null) { + return; + } + toFlush.setValues(toFlush.getValues()); + } + public void readValuesFrom(ObjectInputStream ois) throws IOException, ClassNotFoundException, TeiidComponentException { int size = ois.readInt(); int sizeHint = this.getExpectedHeight(size); + batchInsert = true; while (this.getRowCount() < size) { - List[] batch = BatchSerializer.readBatch(ois, types); + List<List<Object>> batch = BatchSerializer.readBatch(ois, leafManager.getTypes()); for (List list : batch) { this.insert(list, InsertMode.ORDERED, sizeHint); } } + batchInsert = false; } protected SPage findChildTail(SPage page) { @@ -219,7 +243,7 @@ if (places != null) { places.add(s); } - if ((s.index == -1 && s.page == header[i]) || s.values.getTuples().isEmpty()) { + if ((s.index == -1 && s.page == header[i]) || s.values.isEmpty()) { x = null; continue; //start at the beginning of the next level } @@ -234,7 +258,7 @@ if (!matched) { return null; } - return s.values.getTuples().get(index); + return s.values.get(index); } x = x.children.get(index); } @@ -248,13 +272,16 @@ public List insert(List tuple, InsertMode mode, int sizeHint) throws TeiidComponentException { LinkedList<SearchResult> places = new LinkedList<SearchResult>(); List match = null; + if (this.lobManager != null) { + this.lobManager.updateReferences(tuple, ReferenceMode.CREATE); + } if (mode == InsertMode.ORDERED) { SPage last = null; while (last == null || last.children != null) { last = findChildTail(last); //TODO: do this lazily - TupleBatch batch = last.getValues(); - places.add(new SearchResult(-batch.getTuples().size() -1, last, batch)); + List<List<?>> batch = last.getValues(); + places.add(new SearchResult(-batch.size() -1, last, batch)); } } else { match = find(tuple, places); @@ -264,8 +291,11 @@ } SearchResult last = places.getLast(); SPage page = last.page; - last.values.getTuples().set(last.index, tuple); + last.values.set(last.index, tuple); page.setValues(last.values); + if (this.lobManager != null) { + this.lobManager.updateReferences(tuple, ReferenceMode.REMOVE); + } return match; } } @@ -277,10 +307,10 @@ } else { level = randomLevel(); } - } else if (!places.isEmpty() && places.getLast().values.getTuples().size() == pageSize) { + } else if (!places.isEmpty() && places.getLast().values.size() == getPageSize(true)) { int row = rowCount.get(); - while (row != 0 && row%pageSize == 0) { - row = (row - pageSize + 1)/pageSize; + while (row != 0 && row%getPageSize(true) == 0) { + row = (row - getPageSize(true) + 1)/getPageSize(true); level++; } } @@ -293,8 +323,8 @@ for (int i = 0; i <= level; i++) { if (places.isEmpty()) { SPage newHead = new SPage(this, false); - TupleBatch batch = newHead.getValues(); - batch.getTuples().add(key); + List<List<?>> batch = newHead.getValues(); + batch.add(key); newHead.setValues(batch); newHead.children.add(page); header[i] = newHead; @@ -313,9 +343,9 @@ return 0; } int logSize = 1; - while (sizeHint > this.pageSize) { + while (sizeHint > this.getPageSize(logSize==0)) { logSize++; - sizeHint/=this.pageSize; + sizeHint/=this.getPageSize(logSize==0); } return logSize; } @@ -330,10 +360,10 @@ SPage insert(List k, SearchResult result, SearchResult parent, Object value, boolean ordered) throws TeiidComponentException { SPage page = result.page; int index = -result.index - 1; - if (result.values.getTuples().size() == pageSize) { - boolean leaf = !(value instanceof SPage); + boolean leaf = !(value instanceof SPage); + if (result.values.size() == getPageSize(leaf)) { SPage nextPage = new SPage(this, leaf); - TupleBatch nextValues = nextPage.getValues(); + List<List<?>> nextValues = nextPage.getValues(); nextPage.next = page.next; nextPage.prev = page; if (nextPage.next != null) { @@ -343,21 +373,21 @@ boolean inNext = false; if (!ordered) { //split the values - nextValues.getTuples().addAll(result.values.getTuples().subList(pageSize/2, pageSize)); - result.values.getTuples().subList(pageSize/2, pageSize).clear(); + nextValues.addAll(result.values.subList(getPageSize(leaf)/2, getPageSize(leaf))); + result.values.subList(getPageSize(leaf)/2, getPageSize(leaf)).clear(); if (!leaf) { - nextPage.children.addAll(page.children.subList(pageSize/2, pageSize)); - page.children.subList(pageSize/2, pageSize).clear(); + nextPage.children.addAll(page.children.subList(getPageSize(leaf)/2, getPageSize(false))); + page.children.subList(getPageSize(false)/2, getPageSize(false)).clear(); } - if (index <= pageSize/2) { + if (index <= getPageSize(leaf)/2) { setValue(index, k, value, result.values, page); } else { inNext = true; - setValue(index - pageSize/2, k, value, nextValues, nextPage); + setValue(index - getPageSize(leaf)/2, k, value, nextValues, nextPage); } page.setValues(result.values); if (parent != null) { - List min = nextPage.getValues().getTuples().get(0); + List min = nextPage.getValues().get(0); SPage.correctParents(parent.page, min, page, nextPage); } } else { @@ -375,12 +405,12 @@ return page; } - static void setValue(int index, List key, Object value, TupleBatch values, SPage page) { + static void setValue(int index, List key, Object value, List<List<?>> values, SPage page) { if (value instanceof SPage) { - values.getTuples().add(index, key); + values.add(index, key); page.children.add(index, (SPage) value); } else { - values.getTuples().add(index, (List)value); + values.add(index, (List)value); } } @@ -396,11 +426,13 @@ if (searchResult.index < 0) { continue; } - searchResult.values.getTuples().remove(searchResult.index); + searchResult.values.remove(searchResult.index); + boolean leaf = true; if (searchResult.page.children != null) { + leaf = false; searchResult.page.children.remove(searchResult.index); } - int size = searchResult.values.getTuples().size(); + int size = searchResult.values.size(); if (size == 0) { if (header[i] != searchResult.page) { searchResult.page.remove(false); @@ -424,18 +456,18 @@ header[0] = new SPage(this, true); } continue; - } else if (size < pageSize/2) { + } else if (size < getPageSize(leaf)/2) { //check for merge if (searchResult.page.next != null) { - TupleBatch nextValues = searchResult.page.next.getValues(); - if (nextValues.getTuples().size() < pageSize/4) { + List<List<?>> nextValues = searchResult.page.next.getValues(); + if (nextValues.size() < getPageSize(leaf)/4) { SPage.merge(places, nextValues, searchResult.page, searchResult.values); continue; } } if (searchResult.page.prev != null) { - TupleBatch prevValues = searchResult.page.prev.getValues(); - if (prevValues.getTuples().size() < pageSize/4) { + List<List<?>> prevValues = searchResult.page.prev.getValues(); + if (prevValues.size() < getPageSize(leaf)/4) { SPage.merge(places, searchResult.values, searchResult.page.prev, prevValues); continue; } @@ -443,11 +475,14 @@ } searchResult.page.setValues(searchResult.values); } + if (lobManager != null) { + lobManager.updateReferences(tuple, ReferenceMode.REMOVE); + } return tuple; } public void remove() { - truncate(false); + truncate(true); this.keyManager.remove(); this.leafManager.remove(); } @@ -494,11 +529,11 @@ } public void setPreferMemory(boolean preferMemory) { - this.preferMemory = preferMemory; + this.leafManager.setPrefersMemory(preferMemory); } public boolean isPreferMemory() { - return preferMemory; + return this.leafManager.prefersMemory(); } public ListNestedSortComparator getComparator() { @@ -530,4 +565,26 @@ this.comparator.setSortParameters(sortParameters); } + public void clearClonedFlags() { + for (SPage page : pages.values()) { + page.trackingObject = null; + //we don't really care about using synchronization or a volatile here + //since the worst case is that we'll just use gc cleanup + } + } + + public int getPageSize(boolean leaf) { + if (leaf) { + return leafSize; + } + return pageSize; + } + + BatchManager getBatchManager(boolean leaf) { + if (leaf) { + return leafManager; + } + return keyManager; + } + } \ No newline at end of file Copied: branches/as7/engine/src/main/java/org/teiid/common/buffer/Serializer.java (from rev 3506, trunk/engine/src/main/java/org/teiid/common/buffer/Serializer.java) =================================================================== --- branches/as7/engine/src/main/java/org/teiid/common/buffer/Serializer.java (rev 0) +++ branches/as7/engine/src/main/java/org/teiid/common/buffer/Serializer.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -0,0 +1,38 @@ +/* + * JBoss, Home of Professional Open Source. + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA. + */ + +package org.teiid.common.buffer; + +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; + +/** + * Responsible for serializing {@link CacheEntry}s + * @param <T> + */ +public interface Serializer<T> { + void serialize(T obj, ObjectOutputStream oos) throws IOException; + T deserialize(ObjectInputStream ois) throws IOException, ClassNotFoundException; + boolean useSoftCache(); + Long getId(); +} \ No newline at end of file Modified: branches/as7/engine/src/main/java/org/teiid/common/buffer/TupleBatch.java =================================================================== --- branches/as7/engine/src/main/java/org/teiid/common/buffer/TupleBatch.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/main/java/org/teiid/common/buffer/TupleBatch.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -22,17 +22,11 @@ package org.teiid.common.buffer; -import java.io.Externalizable; -import java.io.IOException; -import java.io.ObjectInput; -import java.io.ObjectOutput; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import org.teiid.client.BatchSerializer; - /** * Represents a set of indexed tuples. The {@link #getBeginRow beginning row} * is the first row contained in this batch; if it equals "1" then it is the @@ -40,28 +34,17 @@ * tuples. The {@link #getEndRow ending row} is the last row contained in * this tuple batch; it is equal to the beginning row plus the * {@link #getRowCount number of rows} contained in this batch, minus one. - * This object is immutable and Serializable; */ -public class TupleBatch implements Externalizable { +public class TupleBatch { private static final long serialVersionUID = 6304443387337336957L; private int rowOffset; - private List<List<?>> tuples; + protected List<List<?>> tuples; // Optional state private boolean terminationFlag = false; - // for distributed cache purposes - private String[] preservedTypes; - - /** - * Contains ordered data types of each of the columns in the batch. Although it is not serialized, - * this array is a serialization aid and must be set before serialization and deserialization using - * the setDataTypes method. - */ - private transient String[] types; - /** Required to honor Externalizable contract */ public TupleBatch() { } @@ -152,14 +135,6 @@ this.terminationFlag = terminationFlag; } - public void setDataTypes(String[] types) { - this.types = types; - } - - public String[] getDataTypes() { - return types; - } - public boolean containsRow(int row) { return rowOffset <= row && getEndRow() >= row; } @@ -179,31 +154,8 @@ return s.toString(); } - public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { - rowOffset = in.readInt(); - terminationFlag = in.readBoolean(); - preservedTypes = (String[])in.readObject(); - if (types == null) { - types = preservedTypes; - } - tuples = new ArrayList<List<?>>(); - for (List tuple : BatchSerializer.readBatch(in, types)) { - tuples.add(tuple); - } - } - public void writeExternal(ObjectOutput out) throws IOException { - out.writeInt(this.rowOffset); - out.writeBoolean(terminationFlag); - out.writeObject(this.preservedTypes); - BatchSerializer.writeBatch(out, types, getAllTuples()); - } - public void setRowOffset(int rowOffset) { this.rowOffset = rowOffset; } - - public void preserveTypes() { - this.preservedTypes = types; - } } Property changes on: branches/as7/engine/src/main/java/org/teiid/common/buffer/TupleBatch.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/main/java/org/teiid/common/buffer/TupleBatch.java:3220-3275 /trunk/engine/src/main/java/org/teiid/common/buffer/TupleBatch.java:3188-3450 + /branches/7.4.x/engine/src/main/java/org/teiid/common/buffer/TupleBatch.java:3220-3275 /trunk/engine/src/main/java/org/teiid/common/buffer/TupleBatch.java:3188-3450,3452-3506 Modified: branches/as7/engine/src/main/java/org/teiid/common/buffer/TupleBrowser.java =================================================================== --- branches/as7/engine/src/main/java/org/teiid/common/buffer/TupleBrowser.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/main/java/org/teiid/common/buffer/TupleBrowser.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -48,7 +48,7 @@ private SPage bound; private int boundIndex = -1; - private TupleBatch values; + private List<List<?>> values; private boolean updated; private boolean direction; @@ -106,7 +106,7 @@ if (boundIndex < 0) { //we are guaranteed by find to not get back the -1 index, unless //there are no tuples, in which case a bound of -1 is fine - boundIndex = Math.min(upper.values.getTuples().size(), -boundIndex -1) - 1; + boundIndex = Math.min(upper.values.size(), -boundIndex -1) - 1; } if (!direction) { values = upper.values; @@ -122,7 +122,7 @@ if (page != bound || values == null) { values = bound.getValues(); } - boundIndex = values.getTuples().size() - 1; + boundIndex = values.size() - 1; } } @@ -173,14 +173,14 @@ continue; } if (values != null) { - int possibleIndex = Collections.binarySearch(values.getTuples(), newValue, tree.comparator); + int possibleIndex = Collections.binarySearch(values, newValue, tree.comparator); if (possibleIndex >= 0) { //value exists in the current page index = possibleIndex; - return values.getTuples().get(possibleIndex); + return values.get(possibleIndex); } //check for end/terminal conditions - if (direction && possibleIndex == -values.getTuples().size() -1) { + if (direction && possibleIndex == -values.size() -1) { if (page.next == null) { resetState(); return null; @@ -199,7 +199,7 @@ if (!setPage(newValue)) { continue; } - return values.getTuples().get(index); + return values.get(index); } if (page == null) { if (inPartial) { @@ -213,11 +213,11 @@ if (direction) { index = 0; } else { - index = values.getTuples().size() - 1; + index = values.size() - 1; } } - if (index >= 0 && index < values.getTuples().size()) { - List<?> result = values.getTuples().get(index); + if (index >= 0 && index < values.size()) { + List<?> result = values.get(index); if (page == bound && index == boundIndex) { resetState(); page = null; //terminate @@ -257,7 +257,7 @@ * @throws TeiidComponentException */ public void update(List<?> tuple) throws TeiidComponentException { - values.getTuples().set(index - getOffset(), tuple); + values.set(index - getOffset(), tuple); updated = true; } Modified: branches/as7/engine/src/main/java/org/teiid/common/buffer/TupleBuffer.java =================================================================== --- branches/as7/engine/src/main/java/org/teiid/common/buffer/TupleBuffer.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/main/java/org/teiid/common/buffer/TupleBuffer.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -28,7 +28,7 @@ import java.util.Map; import java.util.TreeMap; -import org.teiid.common.buffer.BatchManager.ManagedBatch; +import org.teiid.common.buffer.LobManager.ReferenceMode; import org.teiid.core.TeiidComponentException; import org.teiid.core.types.DataTypeManager; import org.teiid.core.types.Streamable; @@ -64,36 +64,32 @@ private BatchManager manager; private String tupleSourceID; private List<? extends Expression> schema; - private String[] types; private int batchSize; private int rowCount; private boolean isFinal; - private TreeMap<Integer, BatchManager.ManagedBatch> batches = new TreeMap<Integer, BatchManager.ManagedBatch>(); + private TreeMap<Integer, Long> batches = new TreeMap<Integer, Long>(); private ArrayList<List<?>> batchBuffer; private boolean removed; private boolean forwardOnly; - private boolean prefersMemory; private LobManager lobManager; - private int[] lobIndexes; private String uuid; - private FileStore lobStore; - public TupleBuffer(BatchManager manager, String id, List<? extends Expression> schema, int[] lobIndexes, int batchSize) { + public TupleBuffer(BatchManager manager, String id, List<? extends Expression> schema, LobManager lobManager, int batchSize) { this.manager = manager; this.tupleSourceID = id; this.schema = schema; - this.types = getTypeNames(schema); - this.lobIndexes = lobIndexes; - if (this.lobIndexes != null) { - this.lobManager = new LobManager(); - this.lobStore = this.manager.createStorage("_lobs"); //$NON-NLS-1$ - this.lobStore.setCleanupReference(this); - } + this.lobManager = lobManager; this.batchSize = batchSize; } + public void setInlineLobs(boolean inline) { + if (this.lobManager != null) { + this.lobManager.setInlineLobs(inline); + } + } + public String getId() { if (this.uuid == null) { this.uuid = java.util.UUID.randomUUID().toString(); @@ -106,12 +102,12 @@ } public boolean isLobs() { - return lobIndexes != null; + return lobManager != null; } public void addTuple(List<?> tuple) throws TeiidComponentException { if (isLobs()) { - lobManager.updateReferences(lobIndexes, tuple); + lobManager.updateReferences(tuple, ReferenceMode.CREATE); } this.rowCount++; if (batchBuffer == null) { @@ -119,7 +115,7 @@ } batchBuffer.add(tuple); if (batchBuffer.size() == batchSize) { - saveBatch(false, false); + saveBatch(false); } } @@ -138,7 +134,7 @@ //add the lob references only, since they may still be referenced later if (isLobs()) { for (List<?> tuple : batch.getTuples()) { - lobManager.updateReferences(lobIndexes, tuple); + lobManager.updateReferences(tuple, ReferenceMode.CREATE); } } } @@ -148,7 +144,7 @@ throws TeiidComponentException { assert this.rowCount <= rowCount; if (this.rowCount != rowCount) { - saveBatch(false, true); + saveBatch(true); this.rowCount = rowCount; } } @@ -157,15 +153,15 @@ if (this.batchBuffer != null) { this.batchBuffer.clear(); } - for (BatchManager.ManagedBatch batch : this.batches.values()) { - batch.remove(); + for (Long batch : this.batches.values()) { + this.manager.remove(batch); } this.batches.clear(); } public void persistLobs() throws TeiidComponentException { if (this.lobManager != null) { - this.lobManager.persist(this.lobStore); + this.lobManager.persist(); } } @@ -174,26 +170,21 @@ * @throws TeiidComponentException */ public void saveBatch() throws TeiidComponentException { - this.saveBatch(false, false); + this.saveBatch(false); } - void saveBatch(boolean finalBatch, boolean force) throws TeiidComponentException { + void saveBatch(boolean force) throws TeiidComponentException { Assertion.assertTrue(!this.isRemoved()); if (batchBuffer == null || batchBuffer.isEmpty() || (!force && batchBuffer.size() < Math.max(1, batchSize / 32))) { return; } - TupleBatch writeBatch = new TupleBatch(rowCount - batchBuffer.size() + 1, batchBuffer); - if (finalBatch) { - writeBatch.setTerminationFlag(true); - } - writeBatch.setDataTypes(types); - BatchManager.ManagedBatch mbatch = manager.createManagedBatch(writeBatch, prefersMemory); - this.batches.put(writeBatch.getBeginRow(), mbatch); + Long mbatch = manager.createManagedBatch(batchBuffer); + this.batches.put(rowCount - batchBuffer.size() + 1, mbatch); batchBuffer = null; } public void close() throws TeiidComponentException { - saveBatch(true, false); + saveBatch(false); this.isFinal = true; } @@ -204,6 +195,8 @@ * @param row * @return * @throws TeiidComponentException + * + * TODO: a method to get the raw batch */ public TupleBatch getBatch(int row) throws TeiidComponentException { TupleBatch result = null; @@ -218,17 +211,20 @@ if (this.batchBuffer != null && !this.batchBuffer.isEmpty()) { //this is just a sanity check to ensure we're not holding too many //hard references to batches. - saveBatch(isFinal, false); + saveBatch(false); } - Map.Entry<Integer, BatchManager.ManagedBatch> entry = batches.floorEntry(row); + Map.Entry<Integer, Long> entry = batches.floorEntry(row); Assertion.isNotNull(entry); - BatchManager.ManagedBatch batch = entry.getValue(); - result = batch.getBatch(!forwardOnly, types); + Long batch = entry.getValue(); + List<List<?>> rows = manager.getBatch(batch, !forwardOnly); + result = new TupleBatch(entry.getKey(), rows); + if (isFinal && result.getEndRow() == rowCount) { + result.setTerminationFlag(true); + } if (forwardOnly) { batches.remove(entry.getKey()); } } - result.setDataTypes(types); if (isFinal && result.getEndRow() == rowCount) { result.setTerminationFlag(true); } @@ -240,8 +236,8 @@ if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.DETAIL)) { LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, "Removing TupleBuffer:", this.tupleSourceID); //$NON-NLS-1$ } - if (this.lobStore != null) { - this.lobStore.remove(); + if (this.lobManager != null) { + this.lobManager.remove(); } this.batchBuffer = null; purge(); @@ -359,18 +355,18 @@ } public void setPrefersMemory(boolean prefersMemory) { - this.prefersMemory = prefersMemory; - for (ManagedBatch batch : this.batches.values()) { - batch.setPrefersMemory(prefersMemory); - } + this.manager.setPrefersMemory(prefersMemory); } - public boolean isPrefersMemory() { - return prefersMemory; + public String[] getTypes() { + return manager.getTypes(); } - public String[] getTypes() { - return types; + public int getLobCount() { + if (this.lobManager == null) { + return 0; + } + return this.lobManager.getLobCount(); } } Property changes on: branches/as7/engine/src/main/java/org/teiid/common/buffer/TupleBuffer.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/main/java/org/teiid/common/buffer/TupleBuffer.java:3149-3217,3220-3275,3340-3349 /trunk/engine/src/main/java/org/teiid/common/buffer/TupleBuffer.java:3188-3450 + /branches/7.4.x/engine/src/main/java/org/teiid/common/buffer/TupleBuffer.java:3149-3217,3220-3275,3340-3349 /trunk/engine/src/main/java/org/teiid/common/buffer/TupleBuffer.java:3188-3450,3452-3506 Copied: branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/BlockBitSetTree.java (from rev 3506, trunk/engine/src/main/java/org/teiid/common/buffer/impl/BlockBitSetTree.java) =================================================================== --- branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/BlockBitSetTree.java (rev 0) +++ branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/BlockBitSetTree.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -0,0 +1,191 @@ +/* + * JBoss, Home of Professional Open Source. + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA. + */ + + package org.teiid.common.buffer.impl; + +import java.util.BitSet; + +import org.teiid.common.buffer.impl.FileStoreCache.BlockInfo; + +/** + * Extends a {@link BitSet} by adding a cumulative total and a + * first level index to speed queries against large bitsets. + */ +public class BlockBitSetTree { + + private static final int LOG_BITS_PER_BLOCK = FileStoreCache.LOG_BLOCK_SIZE + 3; + private static final int MAX_TOP_VALUE = 1 << LOG_BITS_PER_BLOCK; + private int maxIndex; + private int bitsSet; + private int[] topVals; + private BlockManager blockManager; + private int blockCount = 0; + + public BlockBitSetTree(int maxIndex, BlockManager blockManager) { + this.maxIndex = maxIndex; + this.blockManager = blockManager; + this.topVals = new int[(maxIndex >> (FileStoreCache.LOG_BLOCK_SIZE + 3)) + 1]; + } + + public int getMaxIndex() { + return maxIndex; + } + + /** + * Set the given bit at the index. + * @param bitIndex + * @param value + */ + public synchronized void set(int bitIndex, boolean value) { + getOrSet(bitIndex, value, true); + } + + public synchronized boolean get(int bitIndex) { + return getOrSet(bitIndex, false, false); + } + + private boolean getOrSet(int bitIndex, boolean value, boolean update) { + if (bitIndex > maxIndex) { + throw new ArrayIndexOutOfBoundsException(bitIndex); + } + int blockIndex = bitIndex>>LOG_BITS_PER_BLOCK; + BlockInfo bb = null; + if (blockIndex >= blockCount) { + if (!update) { + return false; + } + for (; blockCount < blockIndex+1; blockCount++) { + bb = blockManager.allocateBlock(blockCount); + bb.buf.position(0); + int longsPerBlock = FileStoreCache.BLOCK_SIZE >> 6; + for (int j = 0; j < longsPerBlock; j++) { + bb.buf.putLong(0); + } + } + } else { + bb = blockManager.getBlock(blockIndex); + } + int relativeIndex = bitIndex&(MAX_TOP_VALUE-1); + int longByteIndex = (relativeIndex>>6)<<3; + long word = bb.buf.getLong(longByteIndex); + long mask = 1L << bitIndex; + boolean currentValue = ((word & mask) != 0); + if (!update) { + return currentValue; + } + if (currentValue == value) { + return currentValue; + } + if (value) { + word |= mask; + } else { + word &= ~mask; + } + bb.buf.putLong(longByteIndex, word); + blockManager.updateBlock(bb); + int topIndex = bitIndex >> LOG_BITS_PER_BLOCK; + int increment = value?1:-1; + bitsSet+=increment; + topVals[topIndex]+=increment; + return currentValue; + } + + public synchronized int getBitsSet() { + return bitsSet; + } + + public synchronized int nextClearBit(int fromIndex) { + int start = fromIndex >> LOG_BITS_PER_BLOCK; + for (int i = start; i < topVals.length; i++) { + if (topVals[i] == MAX_TOP_VALUE) { + continue; + } + if (topVals[i] == 0) { + if (i == start) { + return fromIndex; + } + return i * MAX_TOP_VALUE; + } + int relativeIndex = 0; + if (i == start) { + relativeIndex = fromIndex&(MAX_TOP_VALUE-1); + } + BlockInfo bb = blockManager.getBlock(i); + + int longByteIndex = (relativeIndex>>6)<<3; + + long word = ~bb.buf.getLong(longByteIndex) & (-1l << relativeIndex); + + while (true) { + if (word != 0) { + return longByteIndex*8 + (i * MAX_TOP_VALUE) + Long.numberOfTrailingZeros(word); + } + longByteIndex+=8; + if (longByteIndex > FileStoreCache.BLOCK_MASK) { + break; + } + word = ~bb.buf.getLong(longByteIndex); + } + } + return -1; + } + + public synchronized int nextSetBit(int fromIndex) { + if (bitsSet == 0) { + return -1; + } + int start = fromIndex >> LOG_BITS_PER_BLOCK; + for (int i = start; i < topVals.length; i++) { + if (topVals[i] == 0) { + continue; + } + if (topVals[i] == MAX_TOP_VALUE) { + if (i == start) { + return fromIndex; + } + return i * MAX_TOP_VALUE; + } + int relativeIndex = 0; + if (i == start) { + relativeIndex = fromIndex&(MAX_TOP_VALUE-1); + } + BlockInfo bb = blockManager.getBlock(i); + + int longByteIndex = (relativeIndex>>6)<<3; + + long word = bb.buf.getLong(longByteIndex) & (-1l << relativeIndex); + + while (true) { + if (word != 0) { + return longByteIndex*8 + (i * MAX_TOP_VALUE) + Long.numberOfTrailingZeros(word); + } + longByteIndex+=8; + if (longByteIndex > FileStoreCache.BLOCK_MASK) { + break; + } + word = bb.buf.getLong(longByteIndex); + } + } + return -1; + } + +} Copied: branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/BlockClosedLongIntHashTable.java (from rev 3506, trunk/engine/src/main/java/org/teiid/common/buffer/impl/BlockClosedLongIntHashTable.java) =================================================================== --- branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/BlockClosedLongIntHashTable.java (rev 0) +++ branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/BlockClosedLongIntHashTable.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -0,0 +1,238 @@ +/* + * JBoss, Home of Professional Open Source. + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA. + */ + +package org.teiid.common.buffer.impl; + +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; + +import org.teiid.common.buffer.impl.FileStoreCache.BlockInfo; + +/** + * Represents the logical structure of a cache group / directory + * + * Implemented by a closed hash table using a single step linear probe with delayed removal. + * Uses power of 2 hashing. + * + * Provides an extremely simple hash structure that rivals {@link HashMap} performance and + * is directly mapped to {@link ByteBuffer}s to avoid serialization overhead. + * + * Does not expect keys or values to be negative. + */ +public class BlockClosedLongIntHashTable { + + private enum Mode { + GET, + UPDATE, + REMOVE + } + + private static final int BYTES_PER_ROW = 12; //8+4 + private static final int BLOCK_SIZE = FileStoreCache.BLOCK_DATA_BYTES/BYTES_PER_ROW; + private static final float LOAD_FACTOR = .7f; + private static final int MIN_SIZE = 1 << (31 - Integer.numberOfLeadingZeros(BLOCK_SIZE)); //should fit in a single block + + static final int EMPTY = -1; + private static final int REMOVED = -2; + + protected int size; + protected int capacityMask = EMPTY; + protected BlockManager blockManager; + + public BlockClosedLongIntHashTable(BlockManager blockManager) { + this.blockManager = blockManager; + } + + private void init(int capacity) { + int currentBlockCount = blockCount(capacityMask + 1); + int desiredBlockCount = blockCount(capacity); + if (capacity > capacityMask) { + for (int i = currentBlockCount; i < desiredBlockCount; i++) { + BlockInfo bb = blockManager.allocateBlock(i); + empty(bb.buf, BLOCK_SIZE); + blockManager.updateBlock(bb); + } + } + capacityMask = capacity - 1; + } + + private void empty(ByteBuffer bb, int toIndex) { + bb.position(0); + for (int j = 0; j < toIndex; j++) { + bb.putLong(j * BYTES_PER_ROW, EMPTY); + } + } + + private static int blockCount(int size) { + int currentBlockCount = (size) / BLOCK_SIZE; + if (size%BLOCK_SIZE > 0) { + currentBlockCount++; + } + return currentBlockCount; + } + + public synchronized int put(long key, int value) { + int result = getOrUpdate(key, value, Mode.UPDATE); + if ((result == EMPTY || result == REMOVED) && ++size > LOAD_FACTOR*capacityMask) { + int newCapacity = (capacityMask+1)<<1; + int oldLength = capacityMask + 1; + init(newCapacity); + rehash(oldLength, newCapacity); + } + return result; + } + + public synchronized int get(long key) { + return getOrUpdate(key, EMPTY, Mode.GET); + } + + public synchronized int remove(long key) { + int result = getOrUpdate(key, EMPTY, Mode.REMOVE); + if (result != EMPTY && --size*LOAD_FACTOR < capacityMask>>3 && (capacityMask+1)>>1 >= MIN_SIZE) { + //reduce the size of the table by half + int oldLength = capacityMask + 1; + capacityMask >>= 1; + rehash(oldLength, oldLength>>1); + int oldBlocks = blockCount(oldLength); + int newBlocks = blockCount(capacityMask +1); + for (int i = oldBlocks-1; i >= newBlocks; i--) { + blockManager.freeBlock(i); + } + } + return result; + } + + private void rehash(int oldLength, int newLength) { + BlockInfo lastBlockInfo = null; + ByteBuffer lastBlock = null; + for (int i = 0; i < oldLength; i++) { + int relativeIndex = i%BLOCK_SIZE; + if (lastBlock == null || relativeIndex == 0) { + int lastIndex = i/BLOCK_SIZE; + lastBlockInfo = blockManager.getBlock(lastIndex); + lastBlock = lastBlockInfo.buf; + if (i < newLength) { + byte[] buf = new byte[FileStoreCache.BLOCK_DATA_BYTES]; + lastBlock.position(0); + lastBlock.get(buf); + ByteBuffer copyBlock = ByteBuffer.wrap(buf); + empty(lastBlock, Math.min(BLOCK_SIZE, oldLength - lastIndex*BLOCK_SIZE)); + blockManager.updateBlock(lastBlockInfo); + lastBlock = copyBlock; + } + } + lastBlock.position(relativeIndex*BYTES_PER_ROW); + long oldKey = lastBlock.getLong(); + int oldValue = lastBlock.getInt(); + if (oldKey != REMOVED && oldKey != EMPTY) { + getOrUpdate(oldKey, oldValue, Mode.UPDATE); + } + } + } + + @SuppressWarnings("null") + protected int getOrUpdate(long key, int value, Mode mode) { + if (capacityMask == EMPTY) { + if (mode == Mode.GET || mode == Mode.REMOVE) { + return EMPTY; + } + init(MIN_SIZE); + } + int i = hashIndex(key); + BlockInfo lastBlockInfo = null; + long old = EMPTY; + int position = 0; + while (true) { + int relativeIndex = i%BLOCK_SIZE; + if (lastBlockInfo == null || relativeIndex == 0) { + int index = i/BLOCK_SIZE; + lastBlockInfo = blockManager.getBlock(index); + } + position = relativeIndex*BYTES_PER_ROW; + old = lastBlockInfo.buf.getLong(position); + if (old == EMPTY || old == key || (mode == Mode.UPDATE && old == REMOVED)) { + break; + } + i = (i + 1) & capacityMask; + } + int result = EMPTY; + if (old != EMPTY && old != REMOVED) { + result = lastBlockInfo.buf.getInt(position + 8); + } + switch (mode) { + case GET: + return result; + case UPDATE: + lastBlockInfo.buf.putLong(position, key); + lastBlockInfo.buf.putInt(position + 8, value); + blockManager.updateBlock(lastBlockInfo); + return result; + case REMOVE: + if (old == EMPTY || old == REMOVED) { + return EMPTY; + } + lastBlockInfo.buf.putLong(position, REMOVED); + blockManager.updateBlock(lastBlockInfo); + return result; + default: + throw new AssertionError(); + } + } + + private int hashIndex(long key) { + //start with the usual long hash + int primaryHash = (int)(key ^ (key >>> 32)); + //allow the lower bits to spread the entries + primaryHash += primaryHash <<= 2; + primaryHash += primaryHash <<= 3; + return primaryHash & capacityMask; + } + + public synchronized int size() { + return size; + } + + public synchronized Map<Long, Integer> remove() { + Map<Long, Integer> result = new HashMap<Long, Integer>(); + BlockInfo lastBlockInfo = null; + int blockIndex = 0; + for (int i = capacityMask; i >= 0; i--) { + int relativeIndex = i%BLOCK_SIZE; + if (lastBlockInfo == null || relativeIndex == BLOCK_SIZE - 1) { + if (lastBlockInfo != null) { + blockManager.freeBlock(blockIndex); + } + blockIndex = i/BLOCK_SIZE; + lastBlockInfo = blockManager.getBlock(blockIndex); + } + lastBlockInfo.buf.position(relativeIndex*BYTES_PER_ROW); + long key = lastBlockInfo.buf.getLong(); + if (key != EMPTY && key != REMOVED) { + result.put(key, lastBlockInfo.buf.getInt()); + } + } + blockManager.free(); + return result; + } + +} Copied: branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/BlockManager.java (from rev 3506, trunk/engine/src/main/java/org/teiid/common/buffer/impl/BlockManager.java) =================================================================== --- branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/BlockManager.java (rev 0) +++ branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/BlockManager.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -0,0 +1,51 @@ +/* + * JBoss, Home of Professional Open Source. + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA. + */ + +package org.teiid.common.buffer.impl; + +import org.teiid.common.buffer.impl.FileStoreCache.BlockInfo; + +/** + * Represents an INode + * + * Returned BlockInfo may be shared. If shared there and no guarantees about position and mark. + * in particular system/index blocks can be used by multiple threads relative methods should be + * avoided, but may be used for exclusive write operations. + * Otherwise the position will be 0. + * + * Due to buffermanager locking, non-index data blocks can be assumed to be thread-safe. + */ +public interface BlockManager { + + int getInode(); + + BlockInfo allocateBlock(int index); + + BlockInfo getBlock(int index); + + void updateBlock(BlockInfo block); + + void freeBlock(int index); + + void free(); + +} Modified: branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/BufferManagerImpl.java =================================================================== --- branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/BufferManagerImpl.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/BufferManagerImpl.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -22,27 +22,42 @@ package org.teiid.common.buffer.impl; -import java.io.*; +import java.io.IOException; +import java.io.InputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.OutputStream; +import java.io.Serializable; import java.lang.ref.Reference; import java.lang.ref.ReferenceQueue; import java.lang.ref.SoftReference; import java.lang.ref.WeakReference; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.Condition; -import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; +import org.teiid.client.BatchSerializer; import org.teiid.common.buffer.*; -import org.teiid.common.buffer.BatchManager.ManagedBatch; +import org.teiid.common.buffer.AutoCleanupUtil.Removable; +import org.teiid.common.buffer.LobManager.ReferenceMode; import org.teiid.core.TeiidComponentException; import org.teiid.core.TeiidRuntimeException; import org.teiid.core.types.DataTypeManager; -import org.teiid.core.util.Assertion; +import org.teiid.core.types.DataTypeManager.WeakReferenceHashedValueCache; import org.teiid.dqp.internal.process.DQPConfiguration; +import org.teiid.dqp.internal.process.SerializableTupleBatch; import org.teiid.logging.LogConstants; import org.teiid.logging.LogManager; import org.teiid.logging.MessageLevel; @@ -57,341 +72,170 @@ * <p>Default implementation of BufferManager.</p> * Responsible for creating/tracking TupleBuffers and providing access to the StorageManager. * </p> - * The buffering strategy attempts to purge batches from the least recently used TupleBuffer - * from before (which wraps around circularly) the last used batch. This attempts to compensate - * for our tendency to read buffers in a forward manner. If our processing algorithms are changed - * to use alternating ascending/descending access, then the buffering approach could be replaced - * with a simple LRU. * - * TODO: allow for cached stores to use lru - (result set/mat view) - * TODO: account for row/content based sizing (difficult given value sharing) - * TODO: account for memory based lobs (it would be nice if the approximate buffer size matched at 100kB) * TODO: add detection of pinned batches to prevent unnecessary purging of non-persistent batches * - this is not necessary for already persistent batches, since we hold a weak reference */ public class BufferManagerImpl implements BufferManager, StorageManager, ReplicatedObject { - private static final int IO_BUFFER_SIZE = 1 << 14; - private static final int COMPACTION_THRESHOLD = 1 << 25; //start checking at 32 megs - - private final class CleanupHook implements org.teiid.common.buffer.BatchManager.CleanupHook { - - private long id; - private int beginRow; - private WeakReference<BatchManagerImpl> ref; - - CleanupHook(long id, int beginRow, BatchManagerImpl batchManager) { - this.id = id; - this.beginRow = beginRow; - this.ref = new WeakReference<BatchManagerImpl>(batchManager); + private final class BatchManagerImpl implements BatchManager, Serializer<List<? extends List<?>>> { + final Long id; + SizeUtility sizeUtility; + private WeakReference<BatchManagerImpl> ref = new WeakReference<BatchManagerImpl>(this); + AtomicBoolean prefersMemory = new AtomicBoolean(); + String[] types; + private LobManager lobManager; + + private BatchManagerImpl(Long newID, String[] types) { + this.id = newID; + this.sizeUtility = new SizeUtility(types); + this.types = types; + cache.createCacheGroup(newID); } - public void cleanup() { - BatchManagerImpl batchManager = ref.get(); - if (batchManager == null) { - return; - } - cleanupManagedBatch(batchManager, beginRow, id); + @Override + public Long getId() { + return id; } - } - - private final class BatchManagerImpl implements BatchManager { - private final String id; - private volatile FileStore store; - private Map<Long, long[]> physicalMapping = new ConcurrentHashMap<Long, long[]>(); - private ReadWriteLock compactionLock = new ReentrantReadWriteLock(); - private AtomicLong unusedSpace = new AtomicLong(); - private int[] lobIndexes; - private SizeUtility sizeUtility; - - private BatchManagerImpl(String newID, int[] lobIndexes) { - this.id = newID; - this.store = createFileStore(id); - this.store.setCleanupReference(this); - this.lobIndexes = lobIndexes; - this.sizeUtility = new SizeUtility(); + public void setLobManager(LobManager lobManager) { + this.lobManager = lobManager; } - public FileStore createStorage(String prefix) { - return createFileStore(id+prefix); - } - @Override - public ManagedBatch createManagedBatch(TupleBatch batch, boolean softCache) - throws TeiidComponentException { - ManagedBatchImpl mbi = new ManagedBatchImpl(batch, this, softCache); - mbi.addToCache(false); - persistBatchReferences(); - return mbi; + public String[] getTypes() { + return types; } - private boolean shouldCompact(long offset) { - return offset > COMPACTION_THRESHOLD && unusedSpace.get() * 4 > offset * 3; + @Override + public boolean prefersMemory() { + return prefersMemory.get(); } - private long getOffset() throws TeiidComponentException { - long offset = store.getLength(); - if (!shouldCompact(offset)) { - return offset; - } - try { - this.compactionLock.writeLock().lock(); - offset = store.getLength(); - //retest the condition to ensure that compaction is still needed - if (!shouldCompact(offset)) { - return offset; - } - FileStore newStore = createFileStore(id); - newStore.setCleanupReference(this); - byte[] buffer = new byte[IO_BUFFER_SIZE]; - List<long[]> values = new ArrayList<long[]>(physicalMapping.values()); - Collections.sort(values, new Comparator<long[]>() { - @Override - public int compare(long[] o1, long[] o2) { - return Long.signum(o1[0] - o2[0]); - } - }); - for (long[] info : values) { - long oldOffset = info[0]; - info[0] = newStore.getLength(); - int size = (int)info[1]; - while (size > 0) { - int toWrite = Math.min(IO_BUFFER_SIZE, size); - store.readFully(oldOffset, buffer, 0, toWrite); - newStore.write(buffer, 0, toWrite); - size -= toWrite; - } - } - store.remove(); - store = newStore; - long oldOffset = offset; - offset = store.getLength(); - if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.TRACE)) { - LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, "Compacted store", id, "pre-size", oldOffset, "post-size", offset); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ - } - return offset; - } finally { - this.compactionLock.writeLock().unlock(); - } - } - @Override - public void remove() { - this.store.remove(); + public void setPrefersMemory(boolean prefers) { + //TODO: it's only expected to move from not preferring to prefefring + this.prefersMemory.set(prefers); } - } - - /** - * Holder for active batches - */ - private class TupleBufferInfo { - TreeMap<Integer, ManagedBatchImpl> batches = new TreeMap<Integer, ManagedBatchImpl>(); - Integer lastUsed = null; - ManagedBatchImpl removeBatch(int row) { - ManagedBatchImpl result = batches.remove(row); - if (result != null) { - activeBatchKB -= result.sizeEstimate; - } - return result; + @Override + public boolean useSoftCache() { + return prefersMemory.get(); } - } - - private final class ManagedBatchImpl implements ManagedBatch { - private boolean persistent; - private boolean softCache; - private volatile TupleBatch activeBatch; - private volatile Reference<TupleBatch> batchReference; - private int beginRow; - private BatchManagerImpl batchManager; - private long id; - private LobManager lobManager; - private int sizeEstimate; - public ManagedBatchImpl(TupleBatch batch, BatchManagerImpl manager, boolean softCache) { - this.softCache = softCache; - id = batchAdded.incrementAndGet(); - this.activeBatch = batch; - this.beginRow = batch.getBeginRow(); - this.batchManager = manager; - if (this.batchManager.lobIndexes != null) { - this.lobManager = new LobManager(); - } - sizeEstimate = (int) Math.max(1, manager.sizeUtility.getBatchSize(batch) / 1024); - if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.TRACE)) { - LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, "Add batch to BufferManager", id, "with size estimate", sizeEstimate); //$NON-NLS-1$ //$NON-NLS-2$ - } + @Override + public Reference<? extends BatchManager> getBatchManagerReference() { + return ref; } @Override - public void setPrefersMemory(boolean prefers) { - this.softCache = prefers; - //TODO: could recreate the reference + public Long createManagedBatch(List<? extends List<?>> batch) + throws TeiidComponentException { + int sizeEstimate = getSizeEstimate(batch); + Long oid = batchAdded.getAndIncrement(); + CacheEntry ce = new CacheEntry(oid); + ce.setObject(batch); + ce.setSizeEstimate(sizeEstimate); + ce.setSerializer(this.ref); + return addCacheEntry(ce, this); } - private void addToCache(boolean update) { - synchronized (activeBatches) { - TupleBatch batch = this.activeBatch; - if (batch == null) { - return; //already removed + @Override + public List<? extends List<?>> deserialize(ObjectInputStream ois) + throws IOException, ClassNotFoundException { + List<? extends List<?>> batch = BatchSerializer.readBatch(ois, types); + if (lobManager != null) { + for (List<?> list : batch) { + try { + lobManager.updateReferences(list, ReferenceMode.ATTACH); + } catch (TeiidComponentException e) { + throw new TeiidRuntimeException(e); + } } - activeBatchKB += sizeEstimate; - TupleBufferInfo tbi = null; - if (update) { - tbi = activeBatches.remove(batchManager.id); - } else { - tbi = activeBatches.get(batchManager.id); - } - if (tbi == null) { - tbi = new TupleBufferInfo(); - update = true; - } - if (update) { - activeBatches.put(batchManager.id, tbi); - } - Assertion.isNull(tbi.batches.put(this.beginRow, this)); } + return batch; } - + @Override - public TupleBatch getBatch(boolean cache, String[] types) throws TeiidComponentException { + public void serialize(List<? extends List<?>> obj, + ObjectOutputStream oos) throws IOException { + //it's expected that the containing structure has updated the lob manager + BatchSerializer.writeBatch(oos, types, obj); + } + + public int getSizeEstimate(List<? extends List<?>> obj) { + return (int) Math.max(1, sizeUtility.getBatchSize(obj) / 1024); + } + + @SuppressWarnings("unchecked") + @Override + public List<List<?>> getBatch(Long batch, boolean retain) + throws TeiidComponentException { + cleanSoftReferences(); long reads = readAttempts.incrementAndGet(); if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.TRACE)) { - LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, batchManager.id, "getting batch", reads, "reference hits", referenceHit.get()); //$NON-NLS-1$ //$NON-NLS-2$ + LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, id, "getting batch", batch, "total reads", reads, "reference hits", referenceHit.get()); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ } - synchronized (activeBatches) { - TupleBufferInfo tbi = activeBatches.remove(batchManager.id); - if (tbi != null) { - boolean put = true; - if (!cache) { - tbi.removeBatch(this.beginRow); - if (tbi.batches.isEmpty()) { - put = false; - } - } - if (put) { - tbi.lastUsed = this.beginRow; - activeBatches.put(batchManager.id, tbi); - } - } + CacheEntry ce = fastGet(batch, prefersMemory.get(), retain); + if (ce != null) { + return (List<List<?>>)(!retain?ce.nullOut():ce.getObject()); } - persistBatchReferences(); synchronized (this) { - TupleBatch batch = this.activeBatch; - if (batch != null){ - return batch; + ce = fastGet(batch, prefersMemory.get(), retain); + if (ce != null) { + return (List<List<?>>)(!retain?ce.nullOut():ce.getObject()); } - Reference<TupleBatch> ref = this.batchReference; - this.batchReference = null; - if (ref != null) { - batch = ref.get(); - if (batch != null) { - if (cache) { - this.activeBatch = batch; - addToCache(true); - } - referenceHit.getAndIncrement(); - return batch; - } - } long count = readCount.incrementAndGet(); if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.TRACE)) { - LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, batchManager.id, id, "reading batch from disk, total reads:", count); //$NON-NLS-1$ + LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, id, id, "reading batch", batch, "from storage, total reads:", count); //$NON-NLS-1$ //$NON-NLS-2$ } - try { - this.batchManager.compactionLock.readLock().lock(); - long[] info = batchManager.physicalMapping.get(this.id); - Assertion.isNotNull(info, "Invalid batch " + id); //$NON-NLS-1$ - ObjectInputStream ois = new ObjectInputStream(new BufferedInputStream(batchManager.store.createInputStream(info[0]), IO_BUFFER_SIZE)); - batch = new TupleBatch(); - batch.setDataTypes(types); - batch.readExternal(ois); - batch.setRowOffset(this.beginRow); - batch.setDataTypes(null); - if (lobManager != null) { - for (List<?> tuple : batch.getTuples()) { - lobManager.updateReferences(batchManager.lobIndexes, tuple); - } - } - if (cache) { - this.activeBatch = batch; - addToCache(true); - } - return batch; - } catch(IOException e) { - throw new TeiidComponentException(e, QueryPlugin.Util.getString("FileStoreageManager.error_reading", batchManager.id)); //$NON-NLS-1$ - } catch (ClassNotFoundException e) { - throw new TeiidComponentException(e, QueryPlugin.Util.getString("FileStoreageManager.error_reading", batchManager.id)); //$NON-NLS-1$ - } finally { - this.batchManager.compactionLock.readLock().unlock(); - } - } - } - - public synchronized void persist() throws TeiidComponentException { - boolean lockheld = false; - try { - TupleBatch batch = activeBatch; - if (batch != null) { - if (!persistent) { - long count = writeCount.incrementAndGet(); - if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.DETAIL)) { - LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, batchManager.id, id, "writing batch to disk, total writes: ", count); //$NON-NLS-1$ - } - long offset = 0; - if (lobManager != null) { - for (List<?> tuple : batch.getTuples()) { - lobManager.updateReferences(batchManager.lobIndexes, tuple); - } - } - synchronized (batchManager.store) { - offset = batchManager.getOffset(); - OutputStream fsos = new BufferedOutputStream(batchManager.store.createOutputStream(), IO_BUFFER_SIZE); - ObjectOutputStream oos = new ObjectOutputStream(fsos); - batch.writeExternal(oos); - oos.close(); - long size = batchManager.store.getLength() - offset; - long[] info = new long[] {offset, size}; - batchManager.physicalMapping.put(this.id, info); - } - if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.TRACE)) { - LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, batchManager.id, id, "batch written starting at:", offset); //$NON-NLS-1$ - } - } - if (softCache) { - this.batchReference = new SoftReference<TupleBatch>(batch); - } else if (useWeakReferences) { - this.batchReference = new WeakReference<TupleBatch>(batch); - } + ce = cache.get(batch, this); + if (ce == null) { + throw new AssertionError("Batch not found in storage " + batch); //$NON-NLS-1$ } - } catch (IOException e) { - throw new TeiidComponentException(e); - } catch (Throwable e) { - throw new TeiidComponentException(e); - } finally { - persistent = true; - activeBatch = null; - if (lockheld) { - this.batchManager.compactionLock.writeLock().unlock(); + if (!retain) { + cache.remove(this.id, batch); } - } + ce.setSerializer(this.ref); + if (retain) { + addMemoryEntry(ce); + } + } + return (List<List<?>>)ce.getObject(); } + + @Override + public void remove(Long batch) { + cleanSoftReferences(); + BufferManagerImpl.this.remove(id, batch, prefersMemory.get()); + } + @Override public void remove() { - cleanupManagedBatch(batchManager, beginRow, id); + removeCacheGroup(id, prefersMemory.get()); } - + @Override - public CleanupHook getCleanupHook() { - return new CleanupHook(id, beginRow, batchManager); + public String toString() { + return id.toString(); } + } + + private static class BatchSoftReference extends SoftReference<CacheEntry> { + + private int sizeEstimate; + private Long key; - @Override - public String toString() { - return "ManagedBatch " + batchManager.id + " " + this.beginRow + " " + activeBatch; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + public BatchSoftReference(CacheEntry referent, + ReferenceQueue<? super CacheEntry> q, int sizeEstimate) { + super(referent, q); + this.sizeEstimate = sizeEstimate; + this.key = referent.getId(); } } + + private static final int TARGET_BYTES_PER_ROW = 1 << 11; //2k bytes per row + private static ReferenceQueue<CacheEntry> SOFT_QUEUE = new ReferenceQueue<CacheEntry>(); // Configuration private int connectorBatchSize = BufferManager.DEFAULT_CONNECTOR_BATCH_SIZE; @@ -399,21 +243,44 @@ //set to acceptable defaults for testing private int maxProcessingKB = 1 << 11; private Integer maxProcessingKBOrig; - private int maxReserveKB = 1 << 25; + private AtomicInteger maxReserveKB = new AtomicInteger(1 << 18); private volatile int reserveBatchKB; private int maxActivePlans = DQPConfiguration.DEFAULT_MAX_ACTIVE_PLANS; //used as a hint to set the reserveBatchKB private boolean useWeakReferences = true; + private boolean inlineLobs = true; + private int targetBytesPerRow = TARGET_BYTES_PER_ROW; + private int maxSoftReferences; private ReentrantLock lock = new ReentrantLock(true); private Condition batchesFreed = lock.newCondition(); - private volatile int activeBatchKB = 0; - private Map<String, TupleBufferInfo> activeBatches = new LinkedHashMap<String, TupleBufferInfo>(); + private AtomicInteger activeBatchKB = new AtomicInteger(); + + //tiered memory entries. the first tier is just a queue of adds/gets. once accessed again, the entry moves to the tenured tier. + private LinkedHashMap<Long, CacheEntry> memoryEntries = new LinkedHashMap<Long, CacheEntry>(16, .75f, false); + private LinkedHashMap<Long, CacheEntry> tenuredMemoryEntries = new LinkedHashMap<Long, CacheEntry>(16, .75f, true); + + //limited size reference caches based upon the memory settings + private WeakReferenceHashedValueCache<CacheEntry> weakReferenceCache; + private Map<Long, BatchSoftReference> softCache = Collections.synchronizedMap(new LinkedHashMap<Long, BatchSoftReference>(16, .75f, false) { + private static final long serialVersionUID = 1L; + + protected boolean removeEldestEntry(Map.Entry<Long,BatchSoftReference> eldest) { + if (size() > maxSoftReferences) { + BatchSoftReference bsr = eldest.getValue(); + maxReserveKB.addAndGet(bsr.sizeEstimate); + bsr.clear(); + return true; + } + return false; + }; + }); + + Cache cache; + private Map<String, TupleReference> tupleBufferMap = new ConcurrentHashMap<String, TupleReference>(); private ReferenceQueue<TupleBuffer> tupleBufferQueue = new ReferenceQueue<TupleBuffer>(); - private StorageManager diskMgr; - private AtomicLong tsId = new AtomicLong(); private AtomicLong batchAdded = new AtomicLong(); private AtomicLong readCount = new AtomicLong(); @@ -463,58 +330,51 @@ public void setConnectorBatchSize(int connectorBatchSize) { this.connectorBatchSize = connectorBatchSize; } + + public void setTargetBytesPerRow(int targetBytesPerRow) { + this.targetBytesPerRow = targetBytesPerRow; + } public void setProcessorBatchSize(int processorBatchSize) { this.processorBatchSize = processorBatchSize; } - /** - * Add a storage manager to this buffer manager, order is unimportant - * @param storageManager Storage manager to add - */ - public void setStorageManager(StorageManager storageManager) { - Assertion.isNotNull(storageManager); - Assertion.isNull(diskMgr); - this.diskMgr = storageManager; - } - - public StorageManager getStorageManager() { - return diskMgr; - } - @Override public TupleBuffer createTupleBuffer(final List elements, String groupName, TupleSourceType tupleSourceType) { - final String newID = String.valueOf(this.tsId.getAndIncrement()); + final Long newID = this.tsId.getAndIncrement(); int[] lobIndexes = LobManager.getLobIndexes(elements); - BatchManager batchManager = new BatchManagerImpl(newID, lobIndexes); - TupleBuffer tupleBuffer = new TupleBuffer(batchManager, newID, elements, lobIndexes, getProcessorBatchSize()); + String[] types = TupleBuffer.getTypeNames(elements); + BatchManagerImpl batchManager = createBatchManager(newID, types); + LobManager lobManager = null; + FileStore lobStore = null; + if (lobIndexes != null) { + lobStore = createFileStore(newID + "_lobs"); //$NON-NLS-1$ + lobManager = new LobManager(lobIndexes, lobStore); + batchManager.setLobManager(lobManager); + } + TupleBuffer tupleBuffer = new TupleBuffer(batchManager, String.valueOf(newID), elements, lobManager, getProcessorBatchSize(elements)); + if (lobStore != null) { + AutoCleanupUtil.setCleanupReference(batchManager, lobStore); + } if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.DETAIL)) { - LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, "Creating TupleBuffer:", newID, elements, Arrays.toString(tupleBuffer.getTypes()), "of type", tupleSourceType); //$NON-NLS-1$ //$NON-NLS-2$ + LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, "Creating TupleBuffer:", newID, elements, Arrays.toString(types), "of type", tupleSourceType); //$NON-NLS-1$ //$NON-NLS-2$ } + tupleBuffer.setInlineLobs(inlineLobs); return tupleBuffer; } - private void cleanupManagedBatch(BatchManagerImpl batchManager, int beginRow, long id) { - synchronized (activeBatches) { - TupleBufferInfo tbi = activeBatches.get(batchManager.id); - if (tbi != null && tbi.removeBatch(beginRow) != null) { - if (tbi.batches.isEmpty()) { - activeBatches.remove(batchManager.id); - } - } - } - long[] info = batchManager.physicalMapping.remove(id); - if (info != null) { - batchManager.unusedSpace.addAndGet(info[1]); - } - } - public STree createSTree(final List elements, String groupName, int keyLength) { - String newID = String.valueOf(this.tsId.getAndIncrement()); + Long newID = this.tsId.getAndIncrement(); int[] lobIndexes = LobManager.getLobIndexes(elements); - BatchManager bm = new BatchManagerImpl(newID, lobIndexes); - BatchManager keyManager = new BatchManagerImpl(String.valueOf(this.tsId.getAndIncrement()), null); + String[] types = TupleBuffer.getTypeNames(elements); + BatchManagerImpl bm = createBatchManager(newID, types); + LobManager lobManager = null; + if (lobIndexes != null) { + lobManager = new LobManager(lobIndexes, null); //persistence is not expected yet - later we might utilize storage for out-of-line lob values + bm.setLobManager(lobManager); + } + BatchManager keyManager = createBatchManager(this.tsId.getAndIncrement(), Arrays.copyOf(types, keyLength)); int[] compareIndexes = new int[keyLength]; for (int i = 1; i < compareIndexes.length; i++) { compareIndexes[i] = i; @@ -522,16 +382,33 @@ if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.DETAIL)) { LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, "Creating STree:", newID); //$NON-NLS-1$ } - return new STree(keyManager, bm, new ListNestedSortComparator(compareIndexes), getProcessorBatchSize(), keyLength, TupleBuffer.getTypeNames(elements)); + return new STree(keyManager, bm, new ListNestedSortComparator(compareIndexes), getProcessorBatchSize(elements), getProcessorBatchSize(elements.subList(0, keyLength)), keyLength, lobManager); } + private BatchManagerImpl createBatchManager(final Long newID, String[] types) { + BatchManagerImpl bm = new BatchManagerImpl(newID, types); + final AtomicBoolean prefersMemory = bm.prefersMemory; + AutoCleanupUtil.setCleanupReference(bm, new Removable() { + + @Override + public void remove() { + BufferManagerImpl.this.removeCacheGroup(newID, prefersMemory.get()); + } + }); + return bm; + } + @Override public FileStore createFileStore(String name) { if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.DETAIL)) { LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, "Creating FileStore:", name); //$NON-NLS-1$ } - return this.diskMgr.createFileStore(name); + return this.cache.createFileStore(name); } + + public Cache getCache() { + return cache; + } public void setMaxActivePlans(int maxActivePlans) { this.maxActivePlans = maxActivePlans; @@ -542,23 +419,23 @@ } public void setMaxReserveKB(int maxReserveBatchKB) { - this.maxReserveKB = maxReserveBatchKB; + this.maxReserveKB.set(maxReserveBatchKB); } @Override public void initialize() throws TeiidComponentException { int maxMemory = (int)Math.min(Runtime.getRuntime().maxMemory() / 1024, Integer.MAX_VALUE); maxMemory -= 300 * 1024; //assume 300 megs of overhead for the AS/system stuff - if (maxReserveKB < 0) { - this.maxReserveKB = 0; + if (getMaxReserveKB() < 0) { + this.setMaxReserveKB(0); int one_gig = 1024 * 1024; if (maxMemory > one_gig) { //assume 75% of the memory over the first gig - this.maxReserveKB += (int)Math.max(0, (maxMemory - one_gig) * .75); + this.maxReserveKB.addAndGet(((int)Math.max(0, (maxMemory - one_gig) * .75))); } - this.maxReserveKB += Math.max(0, Math.min(one_gig, maxMemory) * .5); + this.maxReserveKB.addAndGet(((int)Math.max(0, Math.min(one_gig, maxMemory) * .5))); } - this.reserveBatchKB = this.maxReserveKB; + this.reserveBatchKB = this.getMaxReserveKB(); if (this.maxProcessingKBOrig == null) { //store the config value so that we can be reinitialized (this is not a clean approach) this.maxProcessingKBOrig = this.maxProcessingKB; @@ -566,6 +443,12 @@ if (this.maxProcessingKBOrig < 0) { this.maxProcessingKB = Math.max(Math.min(8 * processorBatchSize, Integer.MAX_VALUE), (int)(.1 * maxMemory)/maxActivePlans); } + int memoryBatches = (this.maxProcessingKB * maxActivePlans + this.getMaxReserveKB()) / (processorBatchSize * targetBytesPerRow / 1024); + int logSize = 39 - Integer.numberOfLeadingZeros(memoryBatches); + if (useWeakReferences) { + weakReferenceCache = new WeakReferenceHashedValueCache<CacheEntry>(Math.min(20, logSize)); + } + this.maxSoftReferences = 1 << Math.max(28, logSize+2); } @Override @@ -594,7 +477,7 @@ try { if (mode == BufferReserveMode.WAIT) { //don't wait for more than is available - int waitCount = Math.min(count, this.maxReserveKB); + int waitCount = Math.min(count, this.getMaxReserveKB()); while (waitCount > 0 && waitCount > this.reserveBatchKB) { try { batchesFreed.await(100, TimeUnit.MILLISECONDS); @@ -618,60 +501,233 @@ } void persistBatchReferences() { - if (activeBatchKB == 0 || activeBatchKB <= reserveBatchKB) { - int memoryCount = activeBatchKB + maxReserveKB - reserveBatchKB; + if (activeBatchKB.get() == 0 || activeBatchKB.get() <= reserveBatchKB) { + int memoryCount = activeBatchKB.get() + getMaxReserveKB() - reserveBatchKB; if (DataTypeManager.isValueCacheEnabled()) { - if (memoryCount < maxReserveKB / 8) { + if (memoryCount < getMaxReserveKB() / 8) { DataTypeManager.setValueCacheEnabled(false); } - } else if (memoryCount > maxReserveKB / 4) { + } else if (memoryCount > getMaxReserveKB() / 4) { DataTypeManager.setValueCacheEnabled(true); } return; } + boolean first = true; while (true) { - ManagedBatchImpl mb = null; - synchronized (activeBatches) { - if (activeBatchKB == 0 || activeBatchKB < reserveBatchKB * .8) { + CacheEntry ce = null; + synchronized (memoryEntries) { + if (activeBatchKB.get() == 0 || activeBatchKB.get() < reserveBatchKB * .8) { break; } - Iterator<TupleBufferInfo> iter = activeBatches.values().iterator(); - TupleBufferInfo tbi = iter.next(); - Map.Entry<Integer, ManagedBatchImpl> entry = null; - if (tbi.lastUsed != null) { - entry = tbi.batches.floorEntry(tbi.lastUsed - 1); + if (first) { //let one entry per persist cycle loose its tenure. this helps us be more write avoident. + first = false; + if (!tenuredMemoryEntries.isEmpty()) { + Iterator<Map.Entry<Long, CacheEntry>> iter = tenuredMemoryEntries.entrySet().iterator(); + Map.Entry<Long, CacheEntry> entry = iter.next(); + iter.remove(); + memoryEntries.put(entry.getKey(), entry.getValue()); + } } - if (entry == null) { - entry = tbi.batches.lastEntry(); + LinkedHashMap<Long, CacheEntry> toDrain = memoryEntries; + if (memoryEntries.isEmpty()) { + toDrain = tenuredMemoryEntries; + if (tenuredMemoryEntries.isEmpty()) { + break; + } + } + Iterator<CacheEntry> iter = toDrain.values().iterator(); + ce = iter.next(); + iter.remove(); + activeBatchKB.addAndGet(-ce.getSizeEstimate()); + } + persist(ce); + } + } + + void persist(CacheEntry ce) { + Serializer<?> s = ce.getSerializer().get(); + if (s == null) { + return; + } + if (!ce.isPersistent()) { + long count = writeCount.incrementAndGet(); + if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.DETAIL)) { + LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, ce.getId(), "writing batch to storage, total writes: ", count); //$NON-NLS-1$ + } + cache.add(ce, s); + ce.setPersistent(true); + } + if (s.useSoftCache()) { + createSoftReference(ce); + } else if (useWeakReferences) { + weakReferenceCache.getValue(ce); //a get will set the value + } + } + + private void createSoftReference(CacheEntry ce) { + BatchSoftReference ref = new BatchSoftReference(ce, SOFT_QUEUE, ce.getSizeEstimate()/2); + softCache.put(ce.getId(), ref); + maxReserveKB.addAndGet(- ce.getSizeEstimate()/2); + } + + /** + * Get a CacheEntry without hitting the cache + */ + CacheEntry fastGet(Long batch, boolean prefersMemory, boolean retain) { + CacheEntry ce = null; + synchronized (memoryEntries) { + if (retain) { + ce = tenuredMemoryEntries.get(batch); + if (ce == null) { + ce = memoryEntries.remove(batch); + if (ce != null) { + tenuredMemoryEntries.put(batch, ce); + } } - tbi.removeBatch(entry.getKey()); - if (tbi.batches.isEmpty()) { - iter.remove(); + } else { + ce = tenuredMemoryEntries.remove(batch); + if (ce == null) { + ce = memoryEntries.remove(batch); } - mb = entry.getValue(); } - try { - mb.persist(); - } catch (TeiidComponentException e) { - LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, e, "Error persisting batch, attempts to read that batch later will result in an exception"); //$NON-NLS-1$ + } + if (ce != null) { + if (!retain) { + BufferManagerImpl.this.remove(ce, true); } + return ce; } + if (prefersMemory) { + BatchSoftReference bsr = softCache.remove(batch); + if (bsr != null) { + ce = bsr.get(); + if (ce != null) { + maxReserveKB.addAndGet(bsr.sizeEstimate); + } + } + } else if (useWeakReferences) { + ce = weakReferenceCache.getByHash(batch); + if (ce == null || !ce.getId().equals(batch)) { + return null; + } + } + if (ce != null && ce.getObject() != null) { + referenceHit.getAndIncrement(); + if (retain) { + addMemoryEntry(ce); + } else { + BufferManagerImpl.this.remove(ce, false); + } + return ce; + } + return null; } + void remove(Long gid, Long batch, boolean prefersMemory) { + if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.TRACE)) { + LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, "Removing batch from BufferManager", batch); //$NON-NLS-1$ + } + CacheEntry ce = fastGet(batch, prefersMemory, false); + if (ce == null) { + cache.remove(gid, batch); + } else { + ce.nullOut(); + } + } + + private void remove(CacheEntry ce, boolean inMemory) { + if (inMemory) { + activeBatchKB.addAndGet(-ce.getSizeEstimate()); + } + Serializer<?> s = ce.getSerializer().get(); + if (s != null) { + cache.remove(s.getId(), ce.getId()); + } + } + + Long addCacheEntry(CacheEntry ce, Serializer<?> s) { + if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.TRACE)) { + LogManager.logTrace(LogConstants.CTX_BUFFER_MGR, "Add batch to BufferManager", ce.getId(), "with size estimate", ce.getSizeEstimate()); //$NON-NLS-1$ //$NON-NLS-2$ + } + cache.addToCacheGroup(s.getId(), ce.getId()); + addMemoryEntry(ce); + return ce.getId(); + } + + void addMemoryEntry(CacheEntry ce) { + persistBatchReferences(); + synchronized (memoryEntries) { + memoryEntries.put(ce.getId(), ce); + } + activeBatchKB.getAndAdd(ce.getSizeEstimate()); + } + + void removeCacheGroup(Long id, boolean prefersMemory) { + cleanSoftReferences(); + Collection<Long> vals = cache.removeCacheGroup(id); + for (Long val : vals) { + fastGet(val, prefersMemory, false); + } + } + + void cleanSoftReferences() { + for (int i = 0; i < 10; i++) { + BatchSoftReference ref = (BatchSoftReference)SOFT_QUEUE.poll(); + if (ref == null) { + break; + } + softCache.remove(ref.key); + maxReserveKB.addAndGet(ref.sizeEstimate); + ref.clear(); + } + } + @Override - public int getSchemaSize(List<? extends Expression> elements) { + public int getProcessorBatchSize(List<? extends Expression> schema) { + return getSizeEstimates(schema)[0]; + } + + private int[] getSizeEstimates(List<? extends Expression> elements) { int total = 0; boolean isValueCacheEnabled = DataTypeManager.isValueCacheEnabled(); - //we make a assumption that the average column size under 64bits is approximately 128bytes - //this includes alignment, row/array, and reference overhead for (Expression element : elements) { Class<?> type = element.getType(); total += SizeUtility.getSize(isValueCacheEnabled, type); } + //assume 64-bit total += 8*elements.size() + 36; // column list / row overhead - total *= processorBatchSize; - return Math.max(1, total / 1024); + + //nominal targetBytesPerRow but can scale up or down + + int totalCopy = total; + boolean less = totalCopy < targetBytesPerRow; + int rowCount = processorBatchSize; + + for (int i = 0; i < 3; i++) { + if (less) { + totalCopy <<= 1; + } else { + totalCopy >>= 2; + } + if (less && totalCopy > targetBytesPerRow + || !less && totalCopy < targetBytesPerRow) { + break; + } + if (less) { + rowCount <<= 1; + } else { + rowCount >>= 1; + } + } + rowCount = Math.max(1, rowCount); + total *= rowCount; + return new int[]{rowCount, Math.max(1, total / 1024)}; } + + @Override + public int getSchemaSize(List<? extends Expression> elements) { + return getSizeEstimates(elements)[1]; + } public void shutdown() { } @@ -758,11 +814,9 @@ out.writeInt(buffer.getRowCount()); out.writeInt(buffer.getBatchSize()); out.writeObject(buffer.getTypes()); - out.writeBoolean(buffer.isPrefersMemory()); for (int row = 1; row <= buffer.getRowCount(); row+=buffer.getBatchSize()) { TupleBatch b = buffer.getBatch(row); - b.preserveTypes(); - out.writeObject(b); + out.writeObject(new SerializableTupleBatch(b, buffer.getTypes())); } } @@ -812,7 +866,6 @@ int rowCount = in.readInt(); int batchSize = in.readInt(); String[] types = (String[])in.readObject(); - boolean prefersMemory = in.readBoolean(); List<ElementSymbol> schema = new ArrayList<ElementSymbol>(types.length); for (String type : types) { @@ -823,7 +876,6 @@ TupleBuffer buffer = createTupleBuffer(schema, "cached", TupleSourceType.FINAL); //$NON-NLS-1$ buffer.setBatchSize(batchSize); buffer.setId(state_id); - buffer.setPrefersMemory(prefersMemory); for (int row = 1; row <= rowCount; row+=batchSize) { TupleBatch batch = (TupleBatch)in.readObject(); @@ -844,4 +896,16 @@ @Override public void droppedMembers(Collection<Serializable> addresses) { } + + public void setInlineLobs(boolean inlineLobs) { + this.inlineLobs = inlineLobs; + } + + private int getMaxReserveKB() { + return maxReserveKB.get(); + } + + public void setCache(Cache cache) { + this.cache = cache; + } } Copied: branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/ExtensibleBufferedOutputStream.java (from rev 3506, trunk/engine/src/main/java/org/teiid/common/buffer/impl/ExtensibleBufferedOutputStream.java) =================================================================== --- branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/ExtensibleBufferedOutputStream.java (rev 0) +++ branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/ExtensibleBufferedOutputStream.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -0,0 +1,80 @@ +/* + * JBoss, Home of Professional Open Source. + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA. + */ + +package org.teiid.common.buffer.impl; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; + +public abstract class ExtensibleBufferedOutputStream extends OutputStream { + + protected ByteBuffer buf; + + public ExtensibleBufferedOutputStream() { + } + + public void write(int b) throws IOException { + ensureBuffer(); + if (buf.remaining() == 0) { + flush(); + } + buf.put((byte)b); + } + + private void ensureBuffer() { + if (buf == null) { + buf = newBuffer(); + } + } + + public void write(byte b[], int off, int len) throws IOException { + while (true) { + ensureBuffer(); + int toCopy = Math.min(buf.remaining(), len); + buf.put(b, off, toCopy); + len -= toCopy; + off += toCopy; + if (buf.remaining() > 0) { + break; + } + flush(); + } + } + + public void flush() throws IOException { + if (buf != null && buf.position() > 0) { + flushDirect(); + } + buf = null; + } + + protected abstract ByteBuffer newBuffer(); + + protected abstract void flushDirect() throws IOException; + + @Override + public void close() throws IOException { + flush(); + } + +} Modified: branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/FileStorageManager.java =================================================================== --- branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/FileStorageManager.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/FileStorageManager.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -29,13 +29,11 @@ import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; -import java.util.TreeMap; import java.util.concurrent.atomic.AtomicLong; import org.teiid.common.buffer.FileStore; import org.teiid.common.buffer.StorageManager; import org.teiid.core.TeiidComponentException; -import org.teiid.core.util.Assertion; import org.teiid.logging.LogManager; import org.teiid.logging.MessageLevel; import org.teiid.query.QueryPlugin; @@ -47,7 +45,6 @@ public class FileStorageManager implements StorageManager { public static final int DEFAULT_MAX_OPEN_FILES = 64; - public static final long DEFAULT_MAX_FILESIZE = 2L * 1024L; public static final long DEFAULT_MAX_BUFFERSPACE = 50L * 1024L * 1024L * 1024L; private static final String FILE_PREFIX = "b_"; //$NON-NLS-1$ @@ -97,87 +94,85 @@ public class DiskStore extends FileStore { private String name; - private TreeMap<Long, FileInfo> storageFiles = new TreeMap<Long, FileInfo>(); + private FileInfo fileInfo; public DiskStore(String name) { this.name = name; } - /** - * Concurrent reads are possible, but only after writing is complete. - */ - public int readDirect(long fileOffset, byte[] b, int offSet, int length) throws TeiidComponentException { - Map.Entry<Long, FileInfo> entry = storageFiles.floorEntry(fileOffset); - Assertion.isNotNull(entry); - FileInfo fileInfo = entry.getValue(); - synchronized (fileInfo) { + @Override + public synchronized long getLength() { + if (fileInfo == null) { + return 0; + } + return fileInfo.file.length(); + } + + @Override + protected synchronized int readWrite(long fileOffset, byte[] b, int offSet, + int length, boolean write) throws IOException { + if (!write) { try { RandomAccessFile fileAccess = fileInfo.open(); - fileAccess.seek(fileOffset - entry.getKey()); + fileAccess.seek(fileOffset); return fileAccess.read(b, offSet, length); - } catch (IOException e) { - throw new TeiidComponentException(e, QueryPlugin.Util.getString("FileStoreageManager.error_reading", fileInfo.file.getAbsoluteFile())); //$NON-NLS-1$ } finally { fileInfo.close(); } - } - } - - /** - * Concurrent writes are prevented by FileStore, but in general should not happen since processing is single threaded. - */ - public void writeDirect(byte[] bytes, int offset, int length) throws TeiidComponentException { - long used = usedBufferSpace.addAndGet(length); - if (used > maxBufferSpace) { - usedBufferSpace.addAndGet(-length); - throw new TeiidComponentException(QueryPlugin.Util.getString("FileStoreageManager.space_exhausted", maxBufferSpace)); //$NON-NLS-1$ - } - Map.Entry<Long, FileInfo> entry = this.storageFiles.lastEntry(); - boolean createNew = false; - FileInfo fileInfo = null; - long fileOffset = 0; - if (entry == null) { - createNew = true; - } else { - fileInfo = entry.getValue(); - fileOffset = entry.getKey(); - createNew = entry.getValue().file.length() + length > getMaxFileSize(); - } - if (createNew) { - FileInfo newFileInfo = new FileInfo(createFile(name, storageFiles.size())); - if (fileInfo != null) { - fileOffset += fileInfo.file.length(); + } + if (fileInfo == null) { + fileInfo = new FileInfo(createFile(name)); + } + long bytesUsed = 0; + try { + RandomAccessFile fileAccess = fileInfo.open(); + long newLength = fileOffset + length; + bytesUsed = newLength - fileAccess.length(); + if (bytesUsed > 0) { + long used = usedBufferSpace.addAndGet(bytesUsed); + if (used > maxBufferSpace) { + usedBufferSpace.addAndGet(-bytesUsed); + //TODO: trigger a compaction before this is thrown + throw new IOException(QueryPlugin.Util.getString("FileStoreageManager.space_exhausted", maxBufferSpace)); //$NON-NLS-1$ + } + fileAccess.setLength(bytesUsed); + bytesUsed = 0; } - storageFiles.put(fileOffset, newFileInfo); - fileInfo = newFileInfo; + fileAccess.seek(fileOffset); + fileAccess.write(b, offSet, length); + } finally { + if (bytesUsed > 0) { + usedBufferSpace.addAndGet(-bytesUsed); + } + fileInfo.close(); + } + return length; + } + + @Override + public synchronized void setLength(long length) throws IOException { + if (fileInfo == null) { + fileInfo = new FileInfo(createFile(name)); } - synchronized (fileInfo) { - try { - RandomAccessFile fileAccess = fileInfo.open(); - long pointer = fileAccess.length(); - fileAccess.setLength(pointer + length); - fileAccess.seek(pointer); - fileAccess.write(bytes, offset, length); - } catch(IOException e) { - throw new TeiidComponentException(e, QueryPlugin.Util.getString("FileStoreageManager.error_reading", fileInfo.file.getAbsoluteFile())); //$NON-NLS-1$ - } finally { - fileInfo.close(); - } - } - } + try { + fileInfo.open().setLength(length); + } finally { + fileInfo.close(); + } + } + @Override public synchronized void removeDirect() { - usedBufferSpace.addAndGet(-len); - for (FileInfo info : storageFiles.values()) { - info.delete(); + usedBufferSpace.addAndGet(-getLength()); + if (fileInfo != null){ + fileInfo.delete(); } } - + } // Initialization private int maxOpenFiles = DEFAULT_MAX_OPEN_FILES; - private long maxFileSize = DEFAULT_MAX_FILESIZE * 1024L * 1024L; // 2GB private String directory; private File dirFile; @@ -216,14 +211,6 @@ } } - public void setMaxFileSize(long maxFileSize) { - this.maxFileSize = maxFileSize * 1024L * 1024L; - } - - void setMaxFileSizeDirect(long maxFileSize) { - this.maxFileSize = maxFileSize; - } - public void setMaxOpenFiles(int maxOpenFiles) { this.maxOpenFiles = maxOpenFiles; } @@ -232,26 +219,18 @@ this.directory = directory; } - File createFile(String name, int fileNumber) throws TeiidComponentException { - try { - File storageFile = File.createTempFile(FILE_PREFIX + name + "_" + String.valueOf(fileNumber) + "_", null, this.dirFile); //$NON-NLS-1$ //$NON-NLS-2$ - if (LogManager.isMessageToBeRecorded(org.teiid.logging.LogConstants.CTX_BUFFER_MGR, MessageLevel.DETAIL)) { - LogManager.logDetail(org.teiid.logging.LogConstants.CTX_BUFFER_MGR, "Created temporary storage area file " + storageFile.getAbsoluteFile()); //$NON-NLS-1$ - } - return storageFile; - } catch(IOException e) { - throw new TeiidComponentException(e, QueryPlugin.Util.getString("FileStoreageManager.error_creating", name + "_" + fileNumber)); //$NON-NLS-1$ //$NON-NLS-2$ + File createFile(String name) throws IOException { + File storageFile = File.createTempFile(FILE_PREFIX + name + "_", null, this.dirFile); //$NON-NLS-1$ + if (LogManager.isMessageToBeRecorded(org.teiid.logging.LogConstants.CTX_BUFFER_MGR, MessageLevel.DETAIL)) { + LogManager.logDetail(org.teiid.logging.LogConstants.CTX_BUFFER_MGR, "Created temporary storage area file " + storageFile.getAbsoluteFile()); //$NON-NLS-1$ } + return storageFile; } public FileStore createFileStore(String name) { return new DiskStore(name); } - public long getMaxFileSize() { - return maxFileSize; - } - public String getDirectory() { return directory; } Copied: branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/FileStoreCache.java (from rev 3506, trunk/engine/src/main/java/org/teiid/common/buffer/impl/FileStoreCache.java) =================================================================== --- branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/FileStoreCache.java (rev 0) +++ branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/FileStoreCache.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -0,0 +1,746 @@ +/* + * JBoss, Home of Professional Open Source. + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA. + */ + +package org.teiid.common.buffer.impl; + +import java.io.IOException; +import java.io.InputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.ReentrantLock; + +import org.teiid.common.buffer.Cache; +import org.teiid.common.buffer.CacheEntry; +import org.teiid.common.buffer.FileStore; +import org.teiid.common.buffer.Serializer; +import org.teiid.common.buffer.StorageManager; +import org.teiid.core.TeiidComponentException; +import org.teiid.core.TeiidRuntimeException; +import org.teiid.logging.LogConstants; +import org.teiid.logging.LogManager; +import org.teiid.logging.MessageLevel; +import org.teiid.query.QueryPlugin; + +/** + * Implements storage against a {@link FileStore} abstraction using a filesystem paradigm. + * The filesystem uses a 31bit address space on top of 2^14 byte blocks. + * + * Therefore there is 2^31*2^14 = 2^45 or 32 terabytes max of addressable space. + * + * Some amount of the space is taken up by system information (inodes and use flags). + * This is held in a separate file. + * + * The 64 byte inode format is: + * 14 32 bit direct block pointers + * 1 32 bit block indirect pointer + * 1 32 bit block doubly indirect pointer + * + * The data block format is: + * data bytes | long gid | long oid | three byte int block num + * + * The gid/oid are stored with the block so that defrag/compaction can be performed + * with minimal blocking/lookups. + * + * This means that the maximum number of blocks available to a "file" is + * 14 + (2^14-18)/4 + ((2^14-18)/4)^2 ~= 2^24 + * + * Thus the max serialized object size is: 2^24*(2^14-18) ~= 256GB. + * + * The root directory "physicalMapping" is held in memory for performance, + * but could itself be switched to using a block map. It will grow in + * proportion to the number of tables/tuplebuffers in use. + * + * TODO: defragment + * TODO: lobs could also be supported in this structure. + */ +public class FileStoreCache implements Cache, StorageManager { + + //TODO allow the block size to be configurable + static final int LOG_BLOCK_SIZE = 14; + static final int ADDRESS_BITS = 31; + static final int SYSTEM_MASK = 1<<ADDRESS_BITS; + static final int BLOCK_SIZE = 1 << LOG_BLOCK_SIZE; + static final int BLOCK_MASK = BLOCK_SIZE - 1; + static final int BLOCK_OVERHEAD = 8+8+3; + static final int BLOCK_DATA_BYTES = BLOCK_SIZE - BLOCK_OVERHEAD; + static final int BYTES_PER_BLOCK_ADDRESS = 4; + static final int ADDRESSES_PER_BLOCK = BLOCK_DATA_BYTES/BYTES_PER_BLOCK_ADDRESS; + static final int INODE_BYTES = 16*BYTES_PER_BLOCK_ADDRESS; + static final int DIRECT_POINTERS = 14; + static final int MAX_INDIRECT = DIRECT_POINTERS + ADDRESSES_PER_BLOCK; + static final int MAX_DOUBLE_INDIRECT = MAX_INDIRECT + ADDRESSES_PER_BLOCK * ADDRESSES_PER_BLOCK; + static final int EMPTY_ADDRESS = -1; + + private final class BlockOutputStream extends + ExtensibleBufferedOutputStream { + private final BlockManager blockManager; + int blockNum = -1; + BlockInfo bi; + + private BlockOutputStream(BlockManager blockManager) { + this.blockManager = blockManager; + } + + @Override + protected ByteBuffer newBuffer() { + bi = blockManager.allocateBlock(++blockNum); + return bi.buf; + } + + @Override + protected void flushDirect() throws IOException { + blockManager.updateBlock(bi); + bi = null; + } + } + + private final class BitSetBlockManager implements BlockManager { + final int offset; + + BitSetBlockManager(int offset) { + this.offset = offset; + } + + @Override + public void updateBlock(BlockInfo info) { + updatePhysicalBlock(info); + } + + @Override + public BlockInfo getBlock(int index) { + return getPhysicalBlock(index + offset, true, false); + } + + @Override + public BlockInfo allocateBlock(int index) { + return getPhysicalBlock(index + offset, true, true); + } + + @Override + public int getInode() { + throw new AssertionError(); + } + + @Override + public void freeBlock(int index) { + throw new AssertionError(); + } + + @Override + public void free() { + throw new AssertionError(); + } + + } + + private enum Mode { + GET, + UPDATE, + ALLOCATE + } + + static final class BlockInfo { + final boolean system; + final int inodeOffset; + final ByteBuffer buf; + final int physicalAddress; + boolean dirty; + + BlockInfo(boolean system, ByteBuffer ib, int index, int inodeOffset) { + this.system = system; + this.buf = ib; + this.physicalAddress = index; + this.inodeOffset = inodeOffset; + } + } + + private final class InodeBlockManager implements BlockManager { + private final int inode; + private final long gid; + private final long oid; + private int lastBlock = -1; + boolean trackLast; + + InodeBlockManager(long gid, long oid, int inode) { + if (inode == EMPTY_ADDRESS) { + synchronized (inodesInuse) { + inode = inodesInuse.nextClearBit(0); + if (inode == -1) { + throw new TeiidRuntimeException("no inodes available"); //$NON-NLS-1$ + } + if (LogManager.isMessageToBeRecorded(LogConstants.CTX_DQP, MessageLevel.TRACE)) { + LogManager.logTrace(LogConstants.CTX_DQP, "Allocating inode", inode, "to", gid, oid); //$NON-NLS-1$ //$NON-NLS-2$ + } + inodesInuse.set(inode, true); + } + int inodeBlock = inode/inodesPerBlock; + int inodePosition = INODE_BYTES*(inode%inodesPerBlock); + BlockInfo bb = getInodeSubBlock(inodeBlock, inodePosition); + bb.buf.putInt(EMPTY_ADDRESS); + updatePhysicalBlock(bb); + } + this.inode = inode; + this.gid = gid; + this.oid = oid; + } + + @Override + public int getInode() { + return inode; + } + + @Override + public void updateBlock(BlockInfo info) { + updatePhysicalBlock(info); + } + + @Override + public BlockInfo getBlock(int index) { + int dataBlock = getOrUpdateDataBlockIndex(index, EMPTY_ADDRESS, Mode.GET); + BlockInfo bb = getPhysicalBlock(dataBlock, false, false); + bb.buf.position(0); + bb.buf.limit(BLOCK_DATA_BYTES); + return bb; + } + + private int getOrUpdateDataBlockIndex(int index, int value, Mode mode) { + if (index >= MAX_DOUBLE_INDIRECT) { + throw new TeiidRuntimeException("Max block number exceeded"); //$NON-NLS-1$ + } + int dataBlock = 0; + int position = 0; + int inodePosition = INODE_BYTES*(inode%inodesPerBlock); + BlockInfo info = getInodeSubBlock(inode/inodesPerBlock, inodePosition); + if (index >= MAX_INDIRECT) { + position = BYTES_PER_BLOCK_ADDRESS*(DIRECT_POINTERS+1); + BlockInfo next = updateIndirectBlockInfo(info, index, position, MAX_INDIRECT, MAX_DOUBLE_INDIRECT+1, value, mode); + if (next != info) { + info = next; + //should have traversed to the secondary + int indirectAddressBlock = (index - MAX_INDIRECT) / ADDRESSES_PER_BLOCK; + position = indirectAddressBlock * BYTES_PER_BLOCK_ADDRESS; + if (mode == Mode.ALLOCATE && position + BYTES_PER_BLOCK_ADDRESS < BLOCK_DATA_BYTES) { + info.buf.putInt(position + BYTES_PER_BLOCK_ADDRESS, EMPTY_ADDRESS); + } + next = updateIndirectBlockInfo(info, index, position, MAX_INDIRECT + indirectAddressBlock * ADDRESSES_PER_BLOCK, MAX_DOUBLE_INDIRECT + 2 + indirectAddressBlock, value, mode); + if (next != info) { + info = next; + position = ((index - MAX_INDIRECT)%ADDRESSES_PER_BLOCK) * BYTES_PER_BLOCK_ADDRESS; + if (mode == Mode.ALLOCATE && position + BYTES_PER_BLOCK_ADDRESS < BLOCK_DATA_BYTES) { + info.buf.putInt(position + BYTES_PER_BLOCK_ADDRESS, EMPTY_ADDRESS); + } + } + } + } else if (index >= DIRECT_POINTERS) { + //indirect + position = BYTES_PER_BLOCK_ADDRESS*DIRECT_POINTERS; + BlockInfo next = updateIndirectBlockInfo(info, index, position, DIRECT_POINTERS, MAX_DOUBLE_INDIRECT, value, mode); + if (next != info) { + info = next; + position = (index - DIRECT_POINTERS) * BYTES_PER_BLOCK_ADDRESS; + if (mode == Mode.ALLOCATE && position + BYTES_PER_BLOCK_ADDRESS < BLOCK_DATA_BYTES) { + info.buf.putInt(position + BYTES_PER_BLOCK_ADDRESS, EMPTY_ADDRESS); + } + } + } else { + position = BYTES_PER_BLOCK_ADDRESS*index; + if (mode == Mode.ALLOCATE) { + info.buf.putInt(position + BYTES_PER_BLOCK_ADDRESS, EMPTY_ADDRESS); + } + } + if (mode == Mode.ALLOCATE) { + dataBlock = nextBlock(); + info.buf.putInt(position, dataBlock); + updatePhysicalBlock(info); + } else { + dataBlock = info.buf.getInt(position); + if (mode == Mode.UPDATE) { + info.buf.putInt(position, value); + updatePhysicalBlock(info); + } + } + return dataBlock; + } + + private BlockInfo updateIndirectBlockInfo(BlockInfo info, int index, int position, int cutOff, int blockId, int value, Mode mode) { + int sib_index = info.buf.getInt(position); + boolean newBlock = false; + if (index == cutOff) { + if (mode == Mode.ALLOCATE) { + sib_index = nextBlock(); + info.buf.putInt(position, sib_index); + updatePhysicalBlock(info); + newBlock = true; + } else if (mode == Mode.UPDATE && value == EMPTY_ADDRESS) { + freeDataBlock(sib_index); + return info; + } + } + info = getPhysicalBlock(sib_index, false, false); + if (newBlock) { + putBlockId(blockId, info.buf); + } + return info; + } + + private int nextBlock() { + int next = -1; + synchronized (blocksInuse) { + next = blocksInuse.nextClearBit(lastBlock + 1); + if (next == -1) { + throw new TeiidRuntimeException("no freespace available"); //$NON-NLS-1$ + } + blocksInuse.set(next, true); + } + if (LogManager.isMessageToBeRecorded(LogConstants.CTX_DQP, MessageLevel.TRACE)) { + LogManager.logTrace(LogConstants.CTX_DQP, "Allocating block", next, "to", gid, oid); //$NON-NLS-1$ //$NON-NLS-2$ + } + if (trackLast) { + lastBlock = next; + } + return next; + } + + @Override + public void freeBlock(int index) { + int dataBlock = getOrUpdateDataBlockIndex(index, EMPTY_ADDRESS, Mode.UPDATE); + freeDataBlock(dataBlock); + } + + private void freeDataBlock(int dataBlock) { + if (LogManager.isMessageToBeRecorded(LogConstants.CTX_DQP, MessageLevel.TRACE)) { + LogManager.logTrace(LogConstants.CTX_DQP, "freeing data block", dataBlock, "for", gid, oid); //$NON-NLS-1$ //$NON-NLS-2$ + } + blockCache.remove(dataBlock); + blocksInuse.set(dataBlock, false); + } + + BlockInfo getInodeSubBlock(int inodeBlock, int inodePosition) { + BlockInfo bi = getPhysicalBlock(inodeBlock + blocksInUseBlocks + inodesInUseBlocks, true, false); + ByteBuffer bb = bi.buf.duplicate(); + bb.position(inodePosition); + bb.limit(inodePosition + INODE_BYTES); + bb = bb.slice(); + return new BlockInfo(true, bb, inodeBlock + blocksInUseBlocks + inodesInUseBlocks, inodePosition); + } + + @Override + public void free() { + int inodeBlock = inode/inodesPerBlock; + int inodePosition = INODE_BYTES*(inode%inodesPerBlock); + BlockInfo bi = getInodeSubBlock(inodeBlock, inodePosition); + ByteBuffer ib = bi.buf; + int indirectIndexBlock = ib.getInt(ib.position() + BYTES_PER_BLOCK_ADDRESS*DIRECT_POINTERS); + int doublyIndirectIndexBlock = ib.getInt(ib.position() + BYTES_PER_BLOCK_ADDRESS*(DIRECT_POINTERS+1)); + boolean freedAll = freeBlock(ib, DIRECT_POINTERS, true); + if (LogManager.isMessageToBeRecorded(LogConstants.CTX_DQP, MessageLevel.TRACE)) { + LogManager.logTrace(LogConstants.CTX_DQP, "freeing inode", inode, "for", gid, oid); //$NON-NLS-1$ //$NON-NLS-2$ + } + inodesInuse.set(inode, false); + if (!freedAll || indirectIndexBlock == EMPTY_ADDRESS) { + return; + } + freedAll = freeIndirectBlock(indirectIndexBlock); + if (!freedAll || doublyIndirectIndexBlock == EMPTY_ADDRESS) { + return; + } + BlockInfo bb = getPhysicalBlock(doublyIndirectIndexBlock, false, false); + freeBlock(bb.buf, ADDRESSES_PER_BLOCK, false); + freeDataBlock(doublyIndirectIndexBlock); + } + + private boolean freeIndirectBlock(int indirectIndexBlock) { + BlockInfo bb = getPhysicalBlock(indirectIndexBlock, false, false); + bb.buf.position(0); + boolean freedAll = freeBlock(bb.buf, ADDRESSES_PER_BLOCK, true); + freeDataBlock(indirectIndexBlock); + return freedAll; + } + + private boolean freeBlock(ByteBuffer ib, int numPointers, boolean primary) { + for (int i = 0; i < numPointers; i++) { + int dataBlock = ib.getInt(); + if (dataBlock == EMPTY_ADDRESS) { + return false; + } + if (primary) { + freeDataBlock(dataBlock); + } else { + freeIndirectBlock(dataBlock); + } + } + return true; + } + + @Override + public BlockInfo allocateBlock(int blockNum) { + int dataBlock = getOrUpdateDataBlockIndex(blockNum, EMPTY_ADDRESS, Mode.ALLOCATE); + BlockInfo bb = getPhysicalBlock(dataBlock, false, true); + putBlockId(blockNum, bb.buf); + bb.buf.position(0); + bb.buf.limit(BLOCK_DATA_BYTES); + return bb; + } + + private void putBlockId(int blockNum, ByteBuffer bb) { + bb.position(BLOCK_DATA_BYTES); + bb.putLong(gid); + bb.putLong(oid); + bb.put((byte)(blockNum >> 16)); + bb.putShort((short)blockNum); + } + + } + + private StorageManager storageManager; + private long maxBufferSpace = FileStorageManager.DEFAULT_MAX_BUFFERSPACE; + private int inodes; + private int blocks; + private int inodesPerBlock; + private int inodesInUseBlocks; + private int blocksInUseBlocks; + private FileStore store; + private FileStore systemStore; + private BlockBitSetTree blocksInuse; + private BlockBitSetTree inodesInuse; + + //root directory + ConcurrentHashMap<Long, BlockClosedLongIntHashTable> physicalMapping = new ConcurrentHashMap<Long, BlockClosedLongIntHashTable>(); + + //block caching + int blockCacheSize = 128; //2 MB + ReentrantLock blockLock = new ReentrantLock(); + LinkedHashMap<Integer, BlockInfo> blockCache = new LinkedHashMap<Integer, BlockInfo>() { + private static final long serialVersionUID = -4240291744435552008L; + BlockInfo eldestEntry = null; + + protected boolean removeEldestEntry(Map.Entry<Integer,BlockInfo> eldest) { + if (size() > blockCacheSize) { + BlockInfo bi = eldest.getValue(); + if (bi.dirty) { + eldestEntry = bi; + } + return true; + } + return false; + } + + public BlockInfo put(Integer key, BlockInfo value) { + blockLock.lock(); + value.dirty = true; + try { + return super.put(key, value); + } finally { + BlockInfo toUpdate = eldestEntry; + eldestEntry = null; + blockLock.unlock(); + if (toUpdate != null) { + updatePhysicalBlockDirect(toUpdate); + } + } + } + + public BlockInfo get(Object key) { + blockLock.lock(); + try { + return super.get(key); + } finally { + blockLock.unlock(); + } + } + + public BlockInfo remove(Object key) { + blockLock.lock(); + try { + return super.remove(key); + } finally { + blockLock.unlock(); + } + } + }; + + BlockInfo getPhysicalBlock(int block, boolean system, boolean allocate) { + if (block < 0) { + throw new AssertionError("invalid block address " + block); //$NON-NLS-1$ + } + try { + int key = block; + if (system) { + key |= SYSTEM_MASK; + } + BlockInfo result = blockCache.get(key); + assert result == null || !allocate; + if (result == null) { + ByteBuffer bb = null; + if (system) { + bb = systemStore.getBuffer(block<<LOG_BLOCK_SIZE, BLOCK_SIZE, allocate); + } else { + bb = store.getBuffer(block<<LOG_BLOCK_SIZE, BLOCK_SIZE, allocate); + } + result = new BlockInfo(system, bb, block, -1); + blockLock.lock(); + try { + BlockInfo existing = blockCache.get(key); + if (existing != null) { + return existing; + } + blockCache.put(key, result); + } finally { + blockLock.unlock(); + } + return result; + } + return result; + } catch (IOException e) { + throw new TeiidRuntimeException(e); + } + } + + void updatePhysicalBlock(BlockInfo bi) { + int key = bi.physicalAddress; + if (bi.system) { + key |= SYSTEM_MASK; + } + if (bi.inodeOffset >= 0) { + blockLock.lock(); + try { + BlockInfo actual = blockCache.get(key); + if (actual == null) { + //we're not in the cache, so just update storage + updatePhysicalBlockDirect(bi); + } else { + //TODO: check to see we're sharing the same buffer + for (int i = 0; i < INODE_BYTES; i++) { + actual.buf.put(bi.inodeOffset + i, bi.buf.get(i)); + } + } + } finally { + blockLock.unlock(); + } + return; + } + blockCache.put(key, bi); + } + + private void updatePhysicalBlockDirect(BlockInfo bi) { + try { + bi.buf.rewind(); + if (!bi.system) { + store.updateFromBuffer(bi.buf, bi.physicalAddress<<LOG_BLOCK_SIZE); + } else { + systemStore.updateFromBuffer(bi.buf, bi.physicalAddress<<LOG_BLOCK_SIZE); + } + } catch (IOException e) { + throw new TeiidRuntimeException(e); + } + } + + InodeBlockManager getBlockManager(long gid, long oid, int inode) { + return new InodeBlockManager(gid, oid, inode); + } + + @SuppressWarnings("unchecked") + @Override + public void add(CacheEntry entry, Serializer s) { + boolean success = false; + InodeBlockManager blockManager = null; + try { + BlockClosedLongIntHashTable map = physicalMapping.get(s.getId()); + if (map == null) { + return; + } + blockManager = getBlockManager(s.getId(), entry.getId(), EMPTY_ADDRESS); + blockManager.trackLast = true; + ExtensibleBufferedOutputStream fsos = new BlockOutputStream(blockManager); + ObjectOutputStream oos = new ObjectOutputStream(fsos); + oos.writeInt(entry.getSizeEstimate()); + s.serialize(entry.getObject(), oos); + oos.close(); + map.put(entry.getId(), blockManager.getInode()); + success = true; + } catch (Throwable e) { + LogManager.logError(LogConstants.CTX_BUFFER_MGR, e, "Error persisting batch, attempts to read batch "+ entry.getId() +" later will result in an exception"); //$NON-NLS-1$ //$NON-NLS-2$ + } finally { + if (!success && blockManager != null) { + blockManager.free(); + } + } + } + + @Override + public CacheEntry get(Long oid, Serializer<?> serializer) throws TeiidComponentException { + try { + BlockClosedLongIntHashTable map = physicalMapping.get(serializer.getId()); + if (map == null) { + return null; + } + final int inode = map.get(oid); + if (inode == EMPTY_ADDRESS) { + return null; + } + final BlockManager manager = getBlockManager(serializer.getId(), oid, inode); + ObjectInputStream ois = new ObjectInputStream(new InputStream() { + + int blockIndex; + BlockInfo buf; + + @Override + public int read() throws IOException { + ensureBytes(); + return buf.buf.get() & 0xff; + } + + private void ensureBytes() { + if (buf == null || buf.buf.remaining() == 0) { + if (buf != null) { + buf = null; + } + buf = manager.getBlock(blockIndex++); + } + } + + @Override + public int read(byte[] b, int off, int len) + throws IOException { + ensureBytes(); + len = Math.min(len, buf.buf.remaining()); + buf.buf.get(b, off, len); + return len; + } + }); + CacheEntry ce = new CacheEntry(oid); + ce.setSizeEstimate(ois.readInt()); + ce.setObject(serializer.deserialize(ois)); + ce.setPersistent(true); + return ce; + } catch(IOException e) { + throw new TeiidComponentException(e, QueryPlugin.Util.getString("FileStoreageManager.error_reading", oid)); //$NON-NLS-1$ + } catch (ClassNotFoundException e) { + throw new TeiidComponentException(e, QueryPlugin.Util.getString("FileStoreageManager.error_reading", oid)); //$NON-NLS-1$ + } + } + + @Override + public FileStore createFileStore(String name) { + return storageManager.createFileStore(name); + } + + @Override + public void initialize() throws TeiidComponentException { + storageManager.initialize(); + int logSpace = Math.min(45, 63 - Long.numberOfLeadingZeros(this.maxBufferSpace)); + + blocks = 1 << Math.min(Math.max(12, logSpace -LOG_BLOCK_SIZE +1), ADDRESS_BITS); //blocks per segment + + inodes = blocks>>1; + inodesPerBlock = BLOCK_SIZE/INODE_BYTES; + + inodesInUseBlocks = computeInuseBlocks(inodes); + blocksInUseBlocks = computeInuseBlocks(blocks); + + inodesInuse = new BlockBitSetTree(inodes - 1, new BitSetBlockManager(0)); + blocksInuse = new BlockBitSetTree(blocks - 1, new BitSetBlockManager(inodesInUseBlocks)); + + store = storageManager.createFileStore("data_store"); //$NON-NLS-1$ + systemStore = storageManager.createFileStore("system_store"); //$NON-NLS-1$ + } + + static int computeInuseBlocks(int number) { + int blockCount = (number>>LOG_BLOCK_SIZE) + ((number&BLOCK_MASK)>0?1:0); + return (blockCount>>3) + ((blockCount&7)>0?1:0); + } + + @Override + public void addToCacheGroup(Long gid, Long oid) { + BlockClosedLongIntHashTable map = physicalMapping.get(gid); + if (map == null) { + return; + } + map.put(oid, EMPTY_ADDRESS); + } + + @Override + public void createCacheGroup(Long gid) { + BlockClosedLongIntHashTable map = new BlockClosedLongIntHashTable(getBlockManager(gid, -1, EMPTY_ADDRESS)); + physicalMapping.put(gid, map); + } + + @Override + public void remove(Long gid, Long id) { + BlockClosedLongIntHashTable map = physicalMapping.get(gid); + if (map == null) { + return; + } + int inode = map.remove(id); + free(gid, id, inode); + } + + @Override + public Collection<Long> removeCacheGroup(Long gid) { + BlockClosedLongIntHashTable map = physicalMapping.remove(gid); + if (map == null) { + return Collections.emptySet(); + } + Map<Long, Integer> values = map.remove(); + for (Map.Entry<Long, Integer> entry : values.entrySet()) { + if (entry.getValue() != null) { + free(gid, entry.getKey(), entry.getValue()); + } + } + return values.keySet(); + } + + void free(Long gid, Long oid, int inode) { + if (inode == EMPTY_ADDRESS) { + return; + } + BlockManager bm = getBlockManager(gid, oid, inode); + bm.free(); + } + + public void setStorageManager(StorageManager storageManager) { + this.storageManager = storageManager; + } + + public StorageManager getStorageManager() { + return storageManager; + } + + public void setMaxBufferSpace(long maxBufferSpace) { + this.maxBufferSpace = maxBufferSpace; + } + + public int getInodesInUse() { + return this.inodesInuse.getBitsSet(); + } + + public int getDataBlocksInUse() { + return this.blocksInuse.getBitsSet(); + } + +} Modified: branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/MemoryStorageManager.java =================================================================== --- branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/MemoryStorageManager.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/MemoryStorageManager.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -22,16 +22,89 @@ package org.teiid.common.buffer.impl; +import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; +import org.teiid.common.buffer.Cache; +import org.teiid.common.buffer.CacheEntry; import org.teiid.common.buffer.FileStore; -import org.teiid.common.buffer.StorageManager; +import org.teiid.common.buffer.Serializer; import org.teiid.core.TeiidComponentException; -public class MemoryStorageManager implements StorageManager { +public class MemoryStorageManager implements Cache { + + public static final int MAX_FILE_SIZE = 1 << 17; + private final class MemoryFileStore extends FileStore { + private ByteBuffer buffer = ByteBuffer.allocate(MAX_FILE_SIZE); + + public MemoryFileStore() { + buffer.limit(0); + } + + @Override + public synchronized void removeDirect() { + removed.incrementAndGet(); + buffer = ByteBuffer.allocate(0); + } + + @Override + protected synchronized int readWrite(long fileOffset, byte[] b, int offSet, + int length, boolean write) { + if (!write) { + if (fileOffset >= getLength()) { + return -1; + } + int position = (int)fileOffset; + buffer.position(position); + length = Math.min(length, (int)getLength() - position); + buffer.get(b, offSet, length); + return length; + } + int requiredLength = (int)(fileOffset + length); + if (requiredLength > buffer.limit()) { + buffer.limit(requiredLength); + } + buffer.position((int)fileOffset); + buffer.put(b, offSet, length); + return length; + } + + @Override + public synchronized void setLength(long length) { + buffer.limit((int)length); + } + + @Override + public synchronized long getLength() { + return buffer.limit(); + } + + @Override + public synchronized ByteBuffer getBuffer(long start, int length, boolean allocate) { + int position = (int)start; + buffer.limit(position + length); + buffer.position(position); + return buffer.slice(); + } + + @Override + public void updateFromBuffer(ByteBuffer bb, long start) + throws IOException { + //do nothing we are sharing the bytes + } + + } + + private Map<Long, Map<Long, CacheEntry>> groups = new ConcurrentHashMap<Long, Map<Long, CacheEntry>>(); private AtomicInteger created = new AtomicInteger(); private AtomicInteger removed = new AtomicInteger(); @@ -41,40 +114,7 @@ @Override public FileStore createFileStore(String name) { created.incrementAndGet(); - return new FileStore() { - private ByteBuffer buffer = ByteBuffer.allocate(1 << 16); - - @Override - public void writeDirect(byte[] bytes, int offset, int length) throws TeiidComponentException { - if (getLength() + length > buffer.capacity()) { - ByteBuffer newBuffer = ByteBuffer.allocate(buffer.capacity() * 2 + length); - buffer.position(0); - newBuffer.put(buffer); - buffer = newBuffer; - } - buffer.position((int)getLength()); - buffer.put(bytes, offset, length); - } - - @Override - public synchronized void removeDirect() { - removed.incrementAndGet(); - buffer = ByteBuffer.allocate(0); - } - - @Override - public synchronized int readDirect(long fileOffset, byte[] b, int offset, int length) - throws TeiidComponentException { - if (fileOffset >= getLength()) { - return -1; - } - int position = (int)fileOffset; - buffer.position(position); - length = Math.min(length, (int)getLength() - position); - buffer.get(b, offset, length); - return length; - } - }; + return new MemoryFileStore(); } public int getCreated() { @@ -84,4 +124,55 @@ public int getRemoved() { return removed.get(); } + + @Override + public void add(CacheEntry entry, Serializer<?> s) { + Map<Long, CacheEntry> group = groups.get(s.getId()); + if (group != null) { + group.put(entry.getId(), entry); + } + } + + @Override + public void addToCacheGroup(Long gid, Long oid) { + Map<Long, CacheEntry> group = groups.get(gid); + if (group != null) { + group.put(oid, null); + } + } + + @Override + public void createCacheGroup(Long gid) { + groups.put(gid, Collections.synchronizedMap(new HashMap<Long, CacheEntry>())); + } + + @Override + public CacheEntry get(Long id, Serializer<?> serializer) + throws TeiidComponentException { + Map<Long, CacheEntry> group = groups.get(serializer.getId()); + if (group != null) { + return group.get(id); + } + return null; + } + + @Override + public void remove(Long gid, Long id) { + Map<Long, CacheEntry> group = groups.get(gid); + if (group != null) { + group.remove(id); + } + } + + @Override + public Collection<Long> removeCacheGroup(Long gid) { + Map<Long, CacheEntry> group = groups.remove(gid); + if (group == null) { + return Collections.emptySet(); + } + synchronized (group) { + return new ArrayList<Long>(group.keySet()); + } + } + } \ No newline at end of file Modified: branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/SizeUtility.java =================================================================== --- branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/SizeUtility.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/SizeUtility.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -25,8 +25,10 @@ import java.lang.reflect.Array; import java.math.BigDecimal; import java.math.BigInteger; +import java.util.HashMap; +import java.util.List; +import java.util.Map; -import org.teiid.common.buffer.TupleBatch; import org.teiid.core.types.DataTypeManager; @@ -40,29 +42,52 @@ public final class SizeUtility { public static final int REFERENCE_SIZE = 8; + private static Map<Class<?>, int[]> SIZE_ESTIMATES = new HashMap<Class<?>, int[]>(128); + + static { + SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.STRING, new int[] {100, 256}); + SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.DATE, new int[] {20, 28}); + SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.TIME, new int[] {20, 28}); + SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.TIMESTAMP, new int[] {20, 28}); + SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.LONG, new int[] {12, 16}); + SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.DOUBLE, new int[] {12, 16}); + SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.INTEGER, new int[] {6, 12}); + SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.FLOAT, new int[] {6, 12}); + SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.CHAR, new int[] {4, 10}); + SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.SHORT, new int[] {4, 10}); + SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.OBJECT, new int[] {1024, 1024}); + SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.NULL, new int[] {0, 0}); + SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.BYTE, new int[] {1, 1}); + SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.BOOLEAN, new int[] {1, 1}); + SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.BIG_INTEGER, new int[] {75, 100}); + SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.BIG_DECIMAL, new int[] {150, 200}); + } + private long bigIntegerEstimate; private long bigDecimalEstimate; + private String[] types; - public SizeUtility() { + public SizeUtility(String[] types) { boolean isValueCacheEnabled = DataTypeManager.isValueCacheEnabled(); bigIntegerEstimate = getSize(isValueCacheEnabled, DataTypeManager.DefaultDataClasses.BIG_INTEGER); bigDecimalEstimate = getSize(isValueCacheEnabled, DataTypeManager.DefaultDataClasses.BIG_DECIMAL); + this.types = types; } - public long getBatchSize(TupleBatch data) { + public long getBatchSize(List<? extends List<?>> data) { return getBatchSize(DataTypeManager.isValueCacheEnabled(), data); } - private long getBatchSize(boolean accountForValueCache, TupleBatch data) { - int colLength = data.getDataTypes().length; - int rowLength = data.getRowCount(); + private long getBatchSize(boolean accountForValueCache, List<? extends List<?>> data) { + int colLength = types.length; + int rowLength = data.size(); // Array overhead for row array long size = 16 + alignMemory(rowLength * REFERENCE_SIZE); // array overhead for all the columns ( 8 object overhead + 4 ref + 4 int) size += (rowLength * (48 + alignMemory(colLength * REFERENCE_SIZE))); for (int col = 0; col < colLength; col++) { - Class<?> type = DataTypeManager.getDataTypeClass(data.getDataTypes()[col]); + Class<?> type = DataTypeManager.getDataTypeClass(types[col]); if (type == DataTypeManager.DefaultDataClasses.STRING || type == DataTypeManager.DefaultDataClasses.OBJECT @@ -71,7 +96,7 @@ int estRow = 0; for (int row = 0; row < rowLength; row++) { boolean updateEst = row == estRow; - size += getSize(data.getTuples().get(row).get(col), updateEst, accountForValueCache); + size += getSize(data.get(row).get(col), updateEst, accountForValueCache); if (updateEst) { estRow = estRow * 2 + 1; } @@ -85,36 +110,12 @@ static int getSize(boolean isValueCacheEnabled, Class<?> type) { - if (type == DataTypeManager.DefaultDataClasses.STRING) { - return isValueCacheEnabled?100:256; //assumes an "average" string length of approximately 100 chars - } else if (type == DataTypeManager.DefaultDataClasses.DATE - || type == DataTypeManager.DefaultDataClasses.TIME - || type == DataTypeManager.DefaultDataClasses.TIMESTAMP) { - return isValueCacheEnabled?20:28; - } else if (type == DataTypeManager.DefaultDataClasses.LONG - || type == DataTypeManager.DefaultDataClasses.DOUBLE) { - return isValueCacheEnabled?12:16; - } else if (type == DataTypeManager.DefaultDataClasses.INTEGER - || type == DataTypeManager.DefaultDataClasses.FLOAT) { - return isValueCacheEnabled?6:12; - } else if (type == DataTypeManager.DefaultDataClasses.CHAR - || type == DataTypeManager.DefaultDataClasses.SHORT) { - return isValueCacheEnabled?4:10; - } else if (type == DataTypeManager.DefaultDataClasses.OBJECT) { - return 1024; - } else if (type == DataTypeManager.DefaultDataClasses.NULL) { - return 0; //it's free - } else if (type == DataTypeManager.DefaultDataClasses.BYTE - || type == DataTypeManager.DefaultDataClasses.BOOLEAN) { - return 1; //should always be value cached, but there's a small chance it's not - } else if (type == DataTypeManager.DefaultDataClasses.BIG_INTEGER){ - return isValueCacheEnabled?75:100; - } else if (type == DataTypeManager.DefaultDataClasses.BIG_DECIMAL) { - return isValueCacheEnabled?150:200; - } - return 512; //assumes buffer overhead in the case of lobs - //however the account for lobs is misleading as the lob - //references are not actually removed from memory + int[] vals = SIZE_ESTIMATES.get(type); + if (vals == null) { + return 512; //this is is misleading for lobs + //most references are not actually removed from memory + } + return vals[isValueCacheEnabled?0:1]; } /** Copied: branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/SplittableStorageManager.java (from rev 3506, trunk/engine/src/main/java/org/teiid/common/buffer/impl/SplittableStorageManager.java) =================================================================== --- branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/SplittableStorageManager.java (rev 0) +++ branches/as7/engine/src/main/java/org/teiid/common/buffer/impl/SplittableStorageManager.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -0,0 +1,190 @@ +/* + * JBoss, Home of Professional Open Source. + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA. + */ + +package org.teiid.common.buffer.impl; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import org.teiid.common.buffer.FileStore; +import org.teiid.common.buffer.StorageManager; +import org.teiid.core.TeiidComponentException; + +/** + * A storage manager that combines smaller files into a larger + * logical file. + * + * The buffer methods assume that buffers cannot go beyond single + * file boundaries. + */ +public class SplittableStorageManager implements StorageManager { + + public static final long DEFAULT_MAX_FILESIZE = 2 * 1024l; + private long maxFileSize = DEFAULT_MAX_FILESIZE * 1024l * 1024l; // 2GB + private StorageManager storageManager; + + public SplittableStorageManager(StorageManager storageManager) { + this.storageManager = storageManager; + } + + @Override + public FileStore createFileStore(String name) { + return new SplittableFileStore(name); + } + + @Override + public void initialize() throws TeiidComponentException { + storageManager.initialize(); + } + + public class SplittableFileStore extends FileStore { + private String name; + private List<FileStore> storageFiles = new ArrayList<FileStore>(); + + private volatile long len; + + public SplittableFileStore(String name) { + this.name = name; + } + + @Override + public long getLength() { + return len; + } + + @Override + protected int readWrite(long fileOffset, byte[] b, int offSet, + int length, boolean write) throws IOException { + FileStore store = null; + if (!write) { + synchronized (this) { + if (fileOffset + length > len) { + throw new IOException("Invalid file position " + fileOffset + " length " + length); //$NON-NLS-1$ //$NON-NLS-2$ + } + store = storageFiles.get((int)(fileOffset/maxFileSize)); + } + return store.read(fileOffset%maxFileSize, b, offSet, length); + } + synchronized (this) { + ensureLength(fileOffset + length); + store = storageFiles.get((int)(fileOffset/maxFileSize)); + } + long fileBegin = fileOffset%maxFileSize; + length = Math.min(length, (int)Math.min(Integer.MAX_VALUE, maxFileSize - fileBegin)); + store.write(fileBegin, b, offSet, length); + return length; + } + + private void ensureLength(long length) throws IOException { + if (length <= len) { + return; + } + int numFiles = (int)(length/maxFileSize); + long lastFileSize = length%maxFileSize; + if (lastFileSize > 0) { + numFiles++; + } + for (int i = storageFiles.size(); i < numFiles; i++) { + FileStore newFileInfo = storageManager.createFileStore(name + "_" + storageFiles.size()); //$NON-NLS-1$ + storageFiles.add(newFileInfo); + if (lastFileSize == 0 || i != numFiles - 1) { + newFileInfo.setLength(maxFileSize); + } + } + if (lastFileSize > 0) { + storageFiles.get(storageFiles.size() - 1).setLength(lastFileSize); + } + len = length; + } + + @Override + public ByteBuffer getBuffer(long start, int length, boolean allocate) throws IOException { + FileStore store = null; + synchronized (this) { + ensureLength(start + length); + store = storageFiles.get((int)(start/maxFileSize)); + } + long fileBegin = start%maxFileSize; + return store.getBuffer(fileBegin, length, allocate); + } + + @Override + public void updateFromBuffer(ByteBuffer bb, long start) + throws IOException { + FileStore store = null; + synchronized (this) { + store = storageFiles.get((int)(start/maxFileSize)); + } + long fileBegin = start%maxFileSize; + store.updateFromBuffer(bb, fileBegin); + } + + @Override + public synchronized void setLength(long length) throws IOException { + if (length > len) { + ensureLength(length); + } else { + int numFiles = (int)(length/maxFileSize); + long lastFileSize = length%maxFileSize; + if (lastFileSize > 0) { + numFiles++; + } + int toRemove = storageFiles.size() - numFiles; + for (int i = 0; i < toRemove; i++) { + FileStore store = storageFiles.remove(storageFiles.size() -1); + store.remove(); + } + if (lastFileSize > 0) { + storageFiles.get(storageFiles.size() - 1).setLength(lastFileSize); + } + } + len = length; + } + + public synchronized void removeDirect() { + for (FileStore info : storageFiles) { + info.remove(); + } + storageFiles.clear(); + } + + } + + public long getMaxFileSize() { + return maxFileSize; + } + + public void setMaxFileSize(long maxFileSize) { + this.maxFileSize = maxFileSize * 1024l * 1024l; + } + + public void setMaxFileSizeDirect(long maxFileSize) { + this.maxFileSize = maxFileSize; + } + + public StorageManager getStorageManager() { + return storageManager; + } + +} Modified: branches/as7/engine/src/main/java/org/teiid/dqp/internal/datamgr/ConnectorWorkItem.java =================================================================== --- branches/as7/engine/src/main/java/org/teiid/dqp/internal/datamgr/ConnectorWorkItem.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/main/java/org/teiid/dqp/internal/datamgr/ConnectorWorkItem.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -31,7 +31,6 @@ import org.teiid.adminapi.impl.VDBMetaData; import org.teiid.common.buffer.BlockedException; -import org.teiid.common.buffer.TupleBuffer; import org.teiid.core.TeiidProcessingException; import org.teiid.core.util.Assertion; import org.teiid.dqp.message.AtomicRequestID; @@ -332,7 +331,7 @@ LogManager.logWarning(LogConstants.CTX_CONNECTOR, QueryPlugin.Util.getString("ConnectorWorker.zero_size_non_last_batch", requestMsg.getConnectorName())); //$NON-NLS-1$ } - AtomicResultsMessage response = createResultsMessage(rows.toArray(new List[currentRowCount]), requestMsg.getCommand().getProjectedSymbols()); + AtomicResultsMessage response = createResultsMessage(rows.toArray(new List[currentRowCount])); // if we need to keep the execution alive, then we can not support implicit close. response.setSupportsImplicitClose(!this.securityContext.keepExecutionAlive()); @@ -346,9 +345,8 @@ return response; } - public static AtomicResultsMessage createResultsMessage(List[] batch, List columnSymbols) { - String[] dataTypes = TupleBuffer.getTypeNames(columnSymbols); - return new AtomicResultsMessage(batch, dataTypes); + public static AtomicResultsMessage createResultsMessage(List[] batch) { + return new AtomicResultsMessage(batch); } boolean isCancelled() { Modified: branches/as7/engine/src/main/java/org/teiid/dqp/internal/process/DataTierTupleSource.java =================================================================== --- branches/as7/engine/src/main/java/org/teiid/dqp/internal/process/DataTierTupleSource.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/main/java/org/teiid/dqp/internal/process/DataTierTupleSource.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -398,7 +398,7 @@ fullyCloseSource(); } if(workItem.requestMsg.supportsPartialResults()) { - AtomicResultsMessage emptyResults = new AtomicResultsMessage(new List[0], null); + AtomicResultsMessage emptyResults = new AtomicResultsMessage(new List[0]); emptyResults.setWarnings(Arrays.asList((Exception)exception)); emptyResults.setFinalRow(this.rowsProcessed); return emptyResults; Modified: branches/as7/engine/src/main/java/org/teiid/dqp/internal/process/Request.java =================================================================== --- branches/as7/engine/src/main/java/org/teiid/dqp/internal/process/Request.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/main/java/org/teiid/dqp/internal/process/Request.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -233,7 +233,6 @@ props, this.requestMsg.getShowPlan() != ShowPlan.OFF); this.context.setProcessorBatchSize(bufferManager.getProcessorBatchSize()); - this.context.setConnectorBatchSize(bufferManager.getConnectorBatchSize()); this.context.setGlobalTableStore(this.globalTables); if (multiSourceModels != null) { MultiSourcePlanToProcessConverter modifier = new MultiSourcePlanToProcessConverter( Modified: branches/as7/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java =================================================================== --- branches/as7/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/main/java/org/teiid/dqp/internal/process/RequestWorkItem.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -25,12 +25,13 @@ import java.sql.ResultSet; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.concurrent.Callable; -import java.util.concurrent.ConcurrentHashMap; import org.teiid.client.RequestMessage; import org.teiid.client.ResultsMessage; @@ -144,7 +145,7 @@ * maintained during processing */ private Throwable processingException; - private Map<AtomicRequestID, DataTierTupleSource> connectorInfo = new ConcurrentHashMap<AtomicRequestID, DataTierTupleSource>(4); + private Map<AtomicRequestID, DataTierTupleSource> connectorInfo = Collections.synchronizedMap(new HashMap<AtomicRequestID, DataTierTupleSource>(4)); // This exception contains details of all the atomic requests that failed when query is run in partial results mode. private List<TeiidException> warnings = new LinkedList<TeiidException>(); private volatile boolean doneProducingBatches; @@ -157,7 +158,7 @@ private int begin; private int end; private TupleBatch savedBatch; - private Map<Integer, LobWorkItem> lobStreams = new ConcurrentHashMap<Integer, LobWorkItem>(4); + private Map<Integer, LobWorkItem> lobStreams = Collections.synchronizedMap(new HashMap<Integer, LobWorkItem>(4)); /**The time when command begins processing on the server.*/ private long processingTimestamp = System.currentTimeMillis(); @@ -362,7 +363,7 @@ * break the read of a lob from a transactional source under a transaction * if the source does not support holding the clob open after commit */ - for (DataTierTupleSource connectorRequest : this.connectorInfo.values()) { + for (DataTierTupleSource connectorRequest : getConnectorRequests()) { if (connectorRequest.isTransactional()) { connectorRequest.fullyCloseSource(); } @@ -401,15 +402,21 @@ } } - for (DataTierTupleSource connectorRequest : this.connectorInfo.values()) { + for (DataTierTupleSource connectorRequest : getConnectorRequests()) { connectorRequest.fullyCloseSource(); } } this.resultsBuffer = null; - for (LobWorkItem lobWorkItem : this.lobStreams.values()) { - lobWorkItem.close(); + if (!this.lobStreams.isEmpty()) { + List<LobWorkItem> lobs = null; + synchronized (lobStreams) { + lobs = new ArrayList<LobWorkItem>(this.lobStreams.values()); + } + for (LobWorkItem lobWorkItem : lobs) { + lobWorkItem.close(); + } } } @@ -592,7 +599,7 @@ } int finalRowCount = this.resultsBuffer.isFinal()?this.resultsBuffer.getRowCount():(batch.getTerminationFlag()?batch.getEndRow():-1); - response = createResultsMessage(batch.getAllTuples(), this.originalCommand.getProjectedSymbols()); + response = createResultsMessage(batch.getTuples(), this.originalCommand.getProjectedSymbols()); response.setFirstRow(batch.getBeginRow()); response.setLastRow(batch.getEndRow()); response.setUpdateResult(this.returnsUpdateCount); @@ -631,7 +638,7 @@ return result; } - public ResultsMessage createResultsMessage(List[] batch, List columnSymbols) { + public ResultsMessage createResultsMessage(List<? extends List<?>> batch, List columnSymbols) { String[] columnNames = new String[columnSymbols.size()]; String[] dataTypes = new String[columnSymbols.size()]; @@ -640,7 +647,7 @@ columnNames[i] = SingleElementSymbol.getShortName(symbol.getOutputName()); dataTypes[i] = DataTypeManager.getDataTypeName(symbol.getType()); } - ResultsMessage result = new ResultsMessage(requestMsg, batch, columnNames, dataTypes); + ResultsMessage result = new ResultsMessage(batch, columnNames, dataTypes); setAnalysisRecords(result); return result; } @@ -672,7 +679,7 @@ } } LogManager.logDetail(LogConstants.CTX_DQP, processingException, "Sending error to client", requestID); //$NON-NLS-1$ - ResultsMessage response = new ResultsMessage(requestMsg); + ResultsMessage response = new ResultsMessage(); Throwable exception = this.processingException; if (isCanceled) { exception = addCancelCode(exception); @@ -737,7 +744,7 @@ // Cancel Connector atomic requests try { - for (DataTierTupleSource connectorRequest : this.connectorInfo.values()) { + for (DataTierTupleSource connectorRequest : getConnectorRequests()) { connectorRequest.cancelRequest(); } } finally { @@ -837,7 +844,9 @@ Collection<DataTierTupleSource> getConnectorRequests() { - return new LinkedList<DataTierTupleSource>(this.connectorInfo.values()); + synchronized (this.connectorInfo) { + return new ArrayList<DataTierTupleSource>(this.connectorInfo.values()); + } } DataTierTupleSource getConnectorRequest(AtomicRequestID id) { @@ -872,6 +881,12 @@ private void doneProducingBatches() { this.doneProducingBatches = true; + //TODO: we could perform more tracking to know what source lobs are in use + if (this.resultsBuffer.getLobCount() == 0) { + for (DataTierTupleSource connectorRequest : getConnectorRequests()) { + connectorRequest.fullyCloseSource(); + } + } dqpCore.finishProcessing(this); } Copied: branches/as7/engine/src/main/java/org/teiid/dqp/internal/process/SerializableTupleBatch.java (from rev 3506, trunk/engine/src/main/java/org/teiid/dqp/internal/process/SerializableTupleBatch.java) =================================================================== --- branches/as7/engine/src/main/java/org/teiid/dqp/internal/process/SerializableTupleBatch.java (rev 0) +++ branches/as7/engine/src/main/java/org/teiid/dqp/internal/process/SerializableTupleBatch.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -0,0 +1,65 @@ +/* + * JBoss, Home of Professional Open Source. + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA. + */ + +package org.teiid.dqp.internal.process; + +import java.io.Externalizable; +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.util.List; + +import org.teiid.client.BatchSerializer; +import org.teiid.common.buffer.TupleBatch; +import org.teiid.core.util.ExternalizeUtil; + +public class SerializableTupleBatch extends TupleBatch implements Externalizable { + + private String[] types; + + public SerializableTupleBatch() { + //for Externalizable + } + + public SerializableTupleBatch(TupleBatch batch, String[] types) { + super(batch.getBeginRow(), batch.getTuples()); + this.types = types; + } + + @Override + public void readExternal(ObjectInput in) throws IOException, + ClassNotFoundException { + String[] types = ExternalizeUtil.readStringArray(in); + this.setRowOffset(in.readInt()); + this.setTerminationFlag(in.readBoolean()); + this.tuples = (List)BatchSerializer.readBatch(in, types); + } + + @Override + public void writeExternal(ObjectOutput out) throws IOException { + ExternalizeUtil.writeArray(out, types); + out.writeInt(this.getBeginRow()); + out.writeBoolean(this.getTerminationFlag()); + BatchSerializer.writeBatch(out, types, this.getTuples()); + } + +} Modified: branches/as7/engine/src/main/java/org/teiid/dqp/message/AtomicResultsMessage.java =================================================================== --- branches/as7/engine/src/main/java/org/teiid/dqp/message/AtomicResultsMessage.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/main/java/org/teiid/dqp/message/AtomicResultsMessage.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -22,20 +22,12 @@ package org.teiid.dqp.message; -import java.io.Externalizable; -import java.io.IOException; -import java.io.ObjectInput; -import java.io.ObjectOutput; import java.util.List; -import org.teiid.client.BatchSerializer; -import org.teiid.core.util.ExternalizeUtil; +public class AtomicResultsMessage { -public class AtomicResultsMessage implements Externalizable { - private List[] results; - private String[] dataTypes; // Final row index in complete result set, if known private int finalRow = -1; @@ -53,8 +45,7 @@ public AtomicResultsMessage() { } - public AtomicResultsMessage(List[] results, String[] dataTypes) { - this.dataTypes = dataTypes; + public AtomicResultsMessage(List[] results) { this.results = results; } @@ -86,26 +77,6 @@ return results; } - public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { - dataTypes = ExternalizeUtil.readStringArray(in); - results = BatchSerializer.readBatch(in, dataTypes); - finalRow = in.readInt(); - supportsImplicitClose = in.readBoolean(); - warnings = (List<Exception>)in.readObject(); - isTransactional = in.readBoolean(); - supportsCloseWithLobs = in.readBoolean(); - } - - public void writeExternal(ObjectOutput out) throws IOException { - ExternalizeUtil.writeArray(out, dataTypes); - BatchSerializer.writeBatch(out, dataTypes, results); - out.writeInt(finalRow); - out.writeBoolean(supportsImplicitClose); - out.writeObject(warnings); - out.writeBoolean(isTransactional); - out.writeBoolean(supportsCloseWithLobs); - } - public boolean isTransactional() { return isTransactional; } Property changes on: branches/as7/engine/src/main/java/org/teiid/query/function/FunctionDescriptor.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/main/java/org/teiid/query/function/FunctionDescriptor.java:3149-3171 /trunk/engine/src/main/java/org/teiid/query/function/FunctionDescriptor.java:3188-3450 + /branches/7.4.x/engine/src/main/java/org/teiid/query/function/FunctionDescriptor.java:3149-3171 /trunk/engine/src/main/java/org/teiid/query/function/FunctionDescriptor.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/main/java/org/teiid/query/function/FunctionTree.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/main/java/org/teiid/query/function/FunctionTree.java:3149-3171 /trunk/engine/src/main/java/org/teiid/query/function/FunctionTree.java:3188-3450 + /branches/7.4.x/engine/src/main/java/org/teiid/query/function/FunctionTree.java:3149-3171 /trunk/engine/src/main/java/org/teiid/query/function/FunctionTree.java:3188-3450,3452-3506 Modified: branches/as7/engine/src/main/java/org/teiid/query/function/source/SecuritySystemFunctions.java =================================================================== --- branches/as7/engine/src/main/java/org/teiid/query/function/source/SecuritySystemFunctions.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/main/java/org/teiid/query/function/source/SecuritySystemFunctions.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -31,7 +31,7 @@ public class SecuritySystemFunctions { - public static boolean hasRole(CommandContext context, Object roleName) throws FunctionExecutionException { + public static boolean hasRole(CommandContext context, String roleName) throws FunctionExecutionException { SecurityFunctionEvaluator eval = context.getSecurityFunctionEvaluator(); if (eval == null) { @@ -39,13 +39,13 @@ } try { - return eval.hasRole(SecurityFunctionEvaluator.DATA_ROLE, (String)roleName); + return eval.hasRole(SecurityFunctionEvaluator.DATA_ROLE, roleName); } catch (TeiidComponentException err) { throw new FunctionExecutionException(err, err.getMessage()); } } - public static boolean hasRole(CommandContext context, Object roleType, Object roleName) throws FunctionExecutionException { + public static boolean hasRole(CommandContext context, String roleType, String roleName) throws FunctionExecutionException { SecurityFunctionEvaluator eval = context.getSecurityFunctionEvaluator(); @@ -54,7 +54,7 @@ } try { - return eval.hasRole((String)roleType, (String)roleName); + return eval.hasRole(roleType, roleName); } catch (TeiidComponentException err) { throw new FunctionExecutionException(err, err.getMessage()); } Modified: branches/as7/engine/src/main/java/org/teiid/query/metadata/TransformationMetadata.java =================================================================== --- branches/as7/engine/src/main/java/org/teiid/query/metadata/TransformationMetadata.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/main/java/org/teiid/query/metadata/TransformationMetadata.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -105,6 +105,11 @@ } catch (IOException e) { } } + + @Override + public StorageMode getStorageMode() { + return StorageMode.PERSISTENT; + } } public static class Resource { Property changes on: branches/as7/engine/src/main/java/org/teiid/query/optimizer/relational/PlanToProcessConverter.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/main/java/org/teiid/query/optimizer/relational/PlanToProcessConverter.java:3149-3217 /trunk/engine/src/main/java/org/teiid/query/optimizer/relational/PlanToProcessConverter.java:3188-3450 + /branches/7.4.x/engine/src/main/java/org/teiid/query/optimizer/relational/PlanToProcessConverter.java:3149-3217 /trunk/engine/src/main/java/org/teiid/query/optimizer/relational/PlanToProcessConverter.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java:3149-3217 /trunk/engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java:3188-3450 + /branches/7.4.x/engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java:3149-3217 /trunk/engine/src/main/java/org/teiid/query/optimizer/relational/RelationalPlanner.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/main/java/org/teiid/query/processor/QueryProcessor.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/main/java/org/teiid/query/processor/QueryProcessor.java:3220-3275,3340-3349 /trunk/engine/src/main/java/org/teiid/query/processor/QueryProcessor.java:3188-3450 + /branches/7.4.x/engine/src/main/java/org/teiid/query/processor/QueryProcessor.java:3220-3275,3340-3349 /trunk/engine/src/main/java/org/teiid/query/processor/QueryProcessor.java:3188-3450,3452-3506 Modified: branches/as7/engine/src/main/java/org/teiid/query/processor/proc/ProcedurePlan.java =================================================================== --- branches/as7/engine/src/main/java/org/teiid/query/processor/proc/ProcedurePlan.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/main/java/org/teiid/query/processor/proc/ProcedurePlan.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -163,7 +163,7 @@ */ public void initialize(CommandContext context, ProcessorDataManager dataMgr, BufferManager bufferMgr) { this.bufferMgr = bufferMgr; - this.batchSize = bufferMgr.getProcessorBatchSize(); + this.batchSize = bufferMgr.getProcessorBatchSize(getOutputElements()); setContext(context.clone()); this.dataMgr = new ProcessorDataManager() { Modified: branches/as7/engine/src/main/java/org/teiid/query/processor/relational/EnhancedSortMergeJoinStrategy.java =================================================================== --- branches/as7/engine/src/main/java/org/teiid/query/processor/relational/EnhancedSortMergeJoinStrategy.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/main/java/org/teiid/query/processor/relational/EnhancedSortMergeJoinStrategy.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -141,6 +141,7 @@ List<?> lastTuple = null; boolean sortedDistinct = sorted && !state.isDistinct(); int sizeHint = index.getExpectedHeight(state.getTupleBuffer().getRowCount()); + index.setBatchInsert(sorted); outer: while (its.hasNext()) { //detect if sorted and distinct List<?> originalTuple = its.nextTuple(); @@ -162,6 +163,8 @@ } if (!sorted) { index.compact(); + } else { + index.setBatchInsert(false); } its.closeSource(); this.reverseIndexes = new int[elements.size()]; Modified: branches/as7/engine/src/main/java/org/teiid/query/processor/relational/NullNode.java =================================================================== --- branches/as7/engine/src/main/java/org/teiid/query/processor/relational/NullNode.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/main/java/org/teiid/query/processor/relational/NullNode.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -22,6 +22,9 @@ package org.teiid.query.processor.relational; +import java.util.Collections; +import java.util.List; + import org.teiid.common.buffer.TupleBatch; import org.teiid.core.TeiidComponentException; @@ -38,11 +41,12 @@ this.terminateBatches(); return pullBatch(); } + + @Override + public List getOutputElements() { + return Collections.emptyList(); + } - protected void getNodeString(StringBuffer str) { - super.getNodeString(str); - } - public Object clone(){ NullNode clonedNode = new NullNode(super.getID()); super.copy(this, clonedNode); Modified: branches/as7/engine/src/main/java/org/teiid/query/processor/relational/RelationalNode.java =================================================================== --- branches/as7/engine/src/main/java/org/teiid/query/processor/relational/RelationalNode.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/main/java/org/teiid/query/processor/relational/RelationalNode.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -114,7 +114,11 @@ this.getProcessingState().nodeStatistics = new RelationalNodeStatistics(); } - this.getProcessingState().batchSize = bufferManager.getProcessorBatchSize(); + if (getOutputElements() != null) { + this.getProcessingState().batchSize = bufferManager.getProcessorBatchSize(getOutputElements()); + } else { + this.getProcessingState().batchSize = bufferManager.getProcessorBatchSize(); + } } public CommandContext getContext() { Modified: branches/as7/engine/src/main/java/org/teiid/query/processor/relational/SortUtility.java =================================================================== --- branches/as7/engine/src/main/java/org/teiid/query/processor/relational/SortUtility.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/main/java/org/teiid/query/processor/relational/SortUtility.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -94,6 +94,7 @@ private String groupName; private List<? extends Expression> schema; private int schemaSize; + private int batchSize; private ListNestedSortComparator comparator; private TupleBuffer output; @@ -164,6 +165,7 @@ this.groupName = groupName; this.schema = schema; this.schemaSize = bufferManager.getSchemaSize(this.schema); + this.batchSize = bufferManager.getProcessorBatchSize(this.schema); this.comparator = new ListNestedSortComparator(cols, sortTypes); int distinctIndex = cols.length - 1; this.comparator.setDistinctIndex(distinctIndex); @@ -226,7 +228,7 @@ int totalReservedBuffers = 0; try { - int maxRows = this.bufferManager.getProcessorBatchSize(); + int maxRows = this.batchSize; while(!doneReading) { //attempt to reserve more working memory if there are additional rows available before blocking if (workingTuples.size() >= maxRows) { @@ -236,7 +238,7 @@ break; } totalReservedBuffers += reserved; - maxRows += bufferManager.getProcessorBatchSize(); + maxRows += this.batchSize; } try { List<?> tuple = source.nextTuple(); @@ -249,7 +251,7 @@ this.collected++; } } catch(BlockedException e) { - if (workingTuples.size() >= bufferManager.getProcessorBatchSize()) { + if (workingTuples.size() >= this.batchSize) { break; } if (mode != Mode.DUP_REMOVE Modified: branches/as7/engine/src/main/java/org/teiid/query/processor/xml/RelationalPlanExecutor.java =================================================================== --- branches/as7/engine/src/main/java/org/teiid/query/processor/xml/RelationalPlanExecutor.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/main/java/org/teiid/query/processor/xml/RelationalPlanExecutor.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -144,11 +144,8 @@ if (this.resultInfo.isAutoStaged() && tempTable != null) { AlterTempTable att = new AlterTempTable(tempTable); - int size = (Integer)this.currentRow.get(0); - if (size > this.bufferMgr.getProcessorBatchSize() * 2) { - //TODO: if the parent is small, then this is not necessary - att.setIndexColumns(this.resultInfo.getFkColumns()); - } + //TODO: if the parent is small, then this is not necessary + att.setIndexColumns(this.resultInfo.getFkColumns()); this.dataManager.registerRequest(this.internalProcessor.getContext(), att, TempMetadataAdapter.TEMP_MODEL.getName(), null, 0, -1); } Property changes on: branches/as7/engine/src/main/java/org/teiid/query/sql/LanguageObject.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/main/java/org/teiid/query/sql/LanguageObject.java:3149-3217 /trunk/engine/src/main/java/org/teiid/query/sql/LanguageObject.java:3188-3450 + /branches/7.4.x/engine/src/main/java/org/teiid/query/sql/LanguageObject.java:3149-3217 /trunk/engine/src/main/java/org/teiid/query/sql/LanguageObject.java:3188-3450,3452-3506 Modified: branches/as7/engine/src/main/java/org/teiid/query/tempdata/GlobalTableStoreImpl.java =================================================================== --- branches/as7/engine/src/main/java/org/teiid/query/tempdata/GlobalTableStoreImpl.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/main/java/org/teiid/query/tempdata/GlobalTableStoreImpl.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -290,7 +290,7 @@ if (tempTable != null) { TempMetadataID id = tableStore.getMetadataStore().getTempGroupID(matTableName); synchronized (id) { - boolean clone = tempTable.getActiveReaders().get() != 0; + boolean clone = tempTable.getActive().get() != 0; if (clone) { tempTable = tempTable.clone(); } Modified: branches/as7/engine/src/main/java/org/teiid/query/tempdata/TempTable.java =================================================================== --- branches/as7/engine/src/main/java/org/teiid/query/tempdata/TempTable.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/main/java/org/teiid/query/tempdata/TempTable.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -36,6 +36,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantReadWriteLock; import org.teiid.api.exception.query.ExpressionEvaluationException; @@ -89,6 +90,19 @@ this.addRowId = addRowId; this.indexes = indexes; } + + @Override + int process() throws ExpressionEvaluationException, + TeiidComponentException, TeiidProcessingException { + tree.setBatchInsert(addRowId); + return super.process(); + } + + @Override + protected void afterCompletion() throws TeiidComponentException { + tree.setBatchInsert(false); + } + @Override protected void tuplePassed(List tuple) throws BlockedException, @@ -236,6 +250,7 @@ success = true; } finally { try { + afterCompletion(); if (!success && undoLog != null) { undoLog.setFinal(true); TupleSource undoTs = undoLog.createIndexedTupleSource(); @@ -259,6 +274,14 @@ return updateCount; } + /** + * + * @throws TeiidComponentException + */ + protected void afterCompletion() throws TeiidComponentException { + + } + @SuppressWarnings("unused") void success() throws TeiidComponentException, ExpressionEvaluationException, TeiidProcessingException {} @@ -275,9 +298,9 @@ } } - private static AtomicInteger ID_GENERATOR = new AtomicInteger(); + private static AtomicLong ID_GENERATOR = new AtomicLong(); - private int id = ID_GENERATOR.getAndIncrement(); + private Long id = ID_GENERATOR.getAndIncrement(); private STree tree; private AtomicInteger rowId; private List<ElementSymbol> columns; @@ -361,7 +384,7 @@ } } - public AtomicInteger getActiveReaders() { + public AtomicInteger getActive() { return activeReaders; } @@ -803,13 +826,13 @@ return tid; } - public int getId() { + public Long getId() { return id; } @Override public int hashCode() { - return id; + return id.hashCode(); } @Override @@ -821,7 +844,7 @@ return false; } TempTable other = (TempTable)obj; - return id == other.id; + return id.equals(other.id); } } \ No newline at end of file Modified: branches/as7/engine/src/main/java/org/teiid/query/tempdata/TempTableDataManager.java =================================================================== --- branches/as7/engine/src/main/java/org/teiid/query/tempdata/TempTableDataManager.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/main/java/org/teiid/query/tempdata/TempTableDataManager.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -213,7 +213,7 @@ TempTable tt = contextStore.getTempTable(att.getTempTable().toUpperCase()); Assertion.isNotNull(tt, "Table doesn't exist"); //$NON-NLS-1$ tt.setUpdatable(false); - if (att.getIndexColumns() != null) { + if (att.getIndexColumns() != null && tt.getRowCount() > 2*tt.getTree().getPageSize(true)) { tt.addIndex(att.getIndexColumns(), false); } return CollectionTupleSource.createUpdateCountTupleSource(0); Modified: branches/as7/engine/src/main/java/org/teiid/query/tempdata/TempTableStore.java =================================================================== --- branches/as7/engine/src/main/java/org/teiid/query/tempdata/TempTableStore.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/main/java/org/teiid/query/tempdata/TempTableStore.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -82,7 +82,7 @@ public class TempTableSynchronization implements Synchronization { private String id; - Set<Integer> existingTables = new HashSet<Integer>(); + Set<Long> existingTables = new HashSet<Long>(); ConcurrentHashMap<String, TempTable> tables = new ConcurrentHashMap<String, TempTable>(); private List<TransactionCallback> callbacks = new LinkedList<TransactionCallback>(); @@ -131,10 +131,21 @@ @Override public synchronized void afterCompletion(int status) { - //TODO: cleanup tables completed = true; synchronizations.remove(id); - for (TransactionCallback callback : callbacks) { + if (transactionMode == TransactionMode.ISOLATE_READS) { + for (TempTable table : tables.values()) { + table.getActive().decrementAndGet(); + } + } else { + HashSet<TempTable> current = new HashSet<TempTable>(tempTables.values()); + current.retainAll(tables.values()); + for (TempTable table : current) { + table.getActive().set(0); + table.getTree().clearClonedFlags(); + } + } + for (TransactionCallback callback : callbacks) { if (status == Status.STATUS_COMMITTED) { callback.commit(); } else { @@ -144,6 +155,10 @@ callbacks.clear(); } + public boolean isCompleted() { + return completed; + } + @Override public void beforeCompletion() { @@ -307,11 +322,21 @@ if (synch != null && synch.existingTables.contains(tempTable.getId())) { TempTable result = synch.tables.get(tempTableID); if (result == null) { - synch.tables.put(tempTableID, tempTable.clone()); + synchronized (synch) { + if (synch.isCompleted()) { + throw new AssertionError("Expected active transaction"); //$NON-NLS-1$ + } + if (!tempTable.getActive().compareAndSet(0, 1)) { + throw new TeiidProcessingException(QueryPlugin.Util.getString("TempTableStore.pending_update", tempTableID)); //$NON-NLS-1$ + } + synch.tables.put(tempTableID, tempTable.clone()); + } } return tempTable; } - } + } else if (tempTable.getActive().get() != 0) { + throw new TeiidProcessingException(QueryPlugin.Util.getString("TempTableStore.pending_update", tempTableID)); //$NON-NLS-1$ + } } } else if (transactionMode == TransactionMode.ISOLATE_READS) { TransactionContext tc = context.getTransactionContext(); @@ -320,24 +345,13 @@ if (synch != null) { TempTable result = synch.tables.get(tempTableID); if (result == null) { - synch.tables.put(tempTableID, tempTable); result = tempTable; - result.getActiveReaders().getAndIncrement(); - TransactionCallback callback = new TransactionCallback() { - - @Override - public void rollback() { - tempTable.getActiveReaders().getAndDecrement(); - } - - @Override - public void commit() { - tempTable.getActiveReaders().getAndDecrement(); - } - }; - if (!synch.addCallback(callback)) { - callback.rollback(); - } + synchronized (synch) { + if (!synch.isCompleted()) { + synch.tables.put(tempTableID, tempTable); + result.getActive().getAndIncrement(); + } + } } return result; } Modified: branches/as7/engine/src/main/java/org/teiid/query/util/CommandContext.java =================================================================== --- branches/as7/engine/src/main/java/org/teiid/query/util/CommandContext.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/main/java/org/teiid/query/util/CommandContext.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -26,6 +26,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.LinkedList; +import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Random; @@ -78,8 +79,6 @@ private int processorBatchSize = BufferManager.DEFAULT_PROCESSOR_BATCH_SIZE; - private int connectorBatchSize = BufferManager.DEFAULT_CONNECTOR_BATCH_SIZE; - private String userName; private Serializable commandPayload; @@ -312,20 +311,15 @@ return this.globalState.collectNodeStatistics; } - public int getConnectorBatchSize() { - return this.globalState.connectorBatchSize; - } - - public void setConnectorBatchSize(int connectorBatchSize) { - this.globalState.connectorBatchSize = connectorBatchSize; - } - - + @Override public int getProcessorBatchSize() { return this.globalState.processorBatchSize; } - + public int getProcessorBatchSize(List<Expression> schema) { + return this.globalState.bufferManager.getProcessorBatchSize(schema); + } + public void setProcessorBatchSize(int processorBatchSize) { this.globalState.processorBatchSize = processorBatchSize; } Modified: branches/as7/engine/src/main/resources/org/teiid/query/i18n.properties =================================================================== --- branches/as7/engine/src/main/resources/org/teiid/query/i18n.properties 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/main/resources/org/teiid/query/i18n.properties 2011-09-29 16:46:38 UTC (rev 3507) @@ -791,6 +791,7 @@ RulePlanJoins.cantSatisfy=Join region with unsatisfied access patterns cannot be satisfied by the join criteria, Access patterns: {0} TempTableStore.table_exist_error=Temporary table "{0}" already exists. TempTableStore.table_doesnt_exist_error=Temporary table "{0}" does not exist. +TempTableStore.pending_update=Table {0} is locked by pending transaction update. XMLQueryPlanner.cannot_plan=Cannot create a query for MappingClass with user criteria {0} XMLQueryPlanner.invalid_relationship=Conjunct "{0}" has no relationship with target context {1}. Modified: branches/as7/engine/src/test/java/org/teiid/common/buffer/BufferManagerFactory.java =================================================================== --- branches/as7/engine/src/test/java/org/teiid/common/buffer/BufferManagerFactory.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/test/java/org/teiid/common/buffer/BufferManagerFactory.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -22,9 +22,10 @@ package org.teiid.common.buffer; -import org.teiid.common.buffer.BufferManager; import org.teiid.common.buffer.impl.BufferManagerImpl; +import org.teiid.common.buffer.impl.FileStoreCache; import org.teiid.common.buffer.impl.MemoryStorageManager; +import org.teiid.common.buffer.impl.SplittableStorageManager; import org.teiid.core.TeiidComponentException; @@ -81,12 +82,18 @@ public static BufferManagerImpl initBufferManager(BufferManagerImpl bufferManager) { try { bufferManager.initialize(); - } catch (TeiidComponentException e) { + MemoryStorageManager storageManager = new MemoryStorageManager(); + SplittableStorageManager ssm = new SplittableStorageManager(storageManager); + ssm.setMaxFileSizeDirect(MemoryStorageManager.MAX_FILE_SIZE); + FileStoreCache fsc = new FileStoreCache(); + fsc.setMaxBufferSpace(Runtime.getRuntime().maxMemory()/4); + fsc.setStorageManager(ssm); + fsc.initialize(); + bufferManager.setCache(fsc); + return bufferManager; + } catch (TeiidComponentException e) { throw new RuntimeException(e); } - - bufferManager.setStorageManager(new MemoryStorageManager()); - return bufferManager; } } Modified: branches/as7/engine/src/test/java/org/teiid/common/buffer/TestLobManager.java =================================================================== --- branches/as7/engine/src/test/java/org/teiid/common/buffer/TestLobManager.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/test/java/org/teiid/common/buffer/TestLobManager.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -28,8 +28,10 @@ import java.io.StringReader; import java.nio.charset.Charset; import java.util.Arrays; +import java.util.List; import org.junit.Test; +import org.teiid.common.buffer.LobManager.ReferenceMode; import org.teiid.common.buffer.impl.BufferManagerImpl; import org.teiid.core.types.BlobImpl; import org.teiid.core.types.BlobType; @@ -65,9 +67,11 @@ })); - LobManager lobManager = new LobManager(); - lobManager.updateReferences(new int[] {0,1}, Arrays.asList(clob, blob)); - lobManager.persist(fs); + LobManager lobManager = new LobManager(new int[] {0, 1}, fs); + lobManager.setMaxMemoryBytes(4); + List<Streamable<? extends Object>> tuple = Arrays.asList(clob, blob); + lobManager.updateReferences(tuple, ReferenceMode.CREATE); + lobManager.persist(); Streamable<?>lob = lobManager.getLobReference(clob.getReferenceStreamId()); assertTrue(lob.getClass().isAssignableFrom(ClobType.class)); @@ -80,6 +84,10 @@ BlobType blobRead = (BlobType)lob; assertTrue(Arrays.equals(ObjectConverterUtil.convertToByteArray(blob.getBinaryStream()), ObjectConverterUtil.convertToByteArray(blobRead.getBinaryStream()))); + lobManager.updateReferences(tuple, ReferenceMode.REMOVE); + + assertEquals(0, lobManager.getLobCount()); + } } Modified: branches/as7/engine/src/test/java/org/teiid/common/buffer/TestSTree.java =================================================================== --- branches/as7/engine/src/test/java/org/teiid/common/buffer/TestSTree.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/test/java/org/teiid/common/buffer/TestSTree.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -79,7 +79,7 @@ @Test public void testOrderedInsert() throws TeiidComponentException { BufferManagerImpl bm = BufferManagerFactory.createBufferManager(); - bm.setProcessorBatchSize(16); + bm.setProcessorBatchSize(4); ElementSymbol e1 = new ElementSymbol("x"); e1.setType(Integer.class); @@ -100,5 +100,33 @@ } } - + + /** + * Forces the logic through several compaction cycles by using large strings + * @throws TeiidComponentException + */ + @Test public void testCompaction() throws TeiidComponentException { + BufferManagerImpl bm = BufferManagerFactory.createBufferManager(); + bm.setProcessorBatchSize(32); + bm.setMaxReserveKB(0);//force all to disk + bm.initialize(); + + ElementSymbol e1 = new ElementSymbol("x"); + e1.setType(String.class); + List<ElementSymbol> elements = Arrays.asList(e1); + STree map = bm.createSTree(elements, "1", 1); + + int size = 1000; + + for (int i = 0; i < size; i++) { + assertNull(map.insert(Arrays.asList(new String(new byte[1000])), InsertMode.ORDERED, size)); + assertEquals(i + 1, map.getRowCount()); + } + + for (int i = 0; i < size; i++) { + assertNotNull(map.remove(Arrays.asList(new String(new byte[1000])))); + } + + } + } Modified: branches/as7/engine/src/test/java/org/teiid/common/buffer/TestTupleBuffer.java =================================================================== --- branches/as7/engine/src/test/java/org/teiid/common/buffer/TestTupleBuffer.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/test/java/org/teiid/common/buffer/TestTupleBuffer.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -30,60 +30,18 @@ import javax.sql.rowset.serial.SerialClob; import org.junit.Test; -import org.mockito.Mockito; -import org.teiid.core.TeiidComponentException; +import org.teiid.common.buffer.BufferManager.TupleSourceType; import org.teiid.core.types.ClobType; import org.teiid.core.types.DataTypeManager; import org.teiid.query.sql.symbol.ElementSymbol; - public class TestTupleBuffer { - public static final class FakeBatchManager implements BatchManager { - @Override - public void remove() { - - } - - @Override - public ManagedBatch createManagedBatch(final TupleBatch batch, boolean softCache) - throws TeiidComponentException { - return new ManagedBatch() { - - @Override - public void remove() { - - } - - @Override - public TupleBatch getBatch(boolean cache, String[] types) - throws TeiidComponentException { - return batch; - } - - @Override - public void setPrefersMemory(boolean prefers) { - - } - - @Override - public CleanupHook getCleanupHook() { - return null; - } - }; - } - - @Override - public FileStore createStorage(String prefix) { - return Mockito.mock(FileStore.class); - } - } - @Test public void testForwardOnly() throws Exception { ElementSymbol x = new ElementSymbol("x"); //$NON-NLS-1$ x.setType(DataTypeManager.DefaultDataClasses.INTEGER); List<ElementSymbol> schema = Arrays.asList(x); - TupleBuffer tb = new TupleBuffer(new FakeBatchManager(), "x", schema, null, 32); //$NON-NLS-1$ + TupleBuffer tb = BufferManagerFactory.getStandaloneBufferManager().createTupleBuffer(schema, "x", TupleSourceType.PROCESSOR); //$NON-NLS-1$ tb.setForwardOnly(true); tb.addTuple(Arrays.asList(1)); TupleBatch batch = tb.getBatch(1); @@ -106,7 +64,8 @@ ElementSymbol x = new ElementSymbol("x"); //$NON-NLS-1$ x.setType(DataTypeManager.DefaultDataClasses.CLOB); List<ElementSymbol> schema = Arrays.asList(x); - TupleBuffer tb = new TupleBuffer(new FakeBatchManager(), "x", schema, LobManager.getLobIndexes(schema), 32); //$NON-NLS-1$ + TupleBuffer tb = BufferManagerFactory.getStandaloneBufferManager().createTupleBuffer(schema, "x", TupleSourceType.PROCESSOR); //$NON-NLS-1$ + tb.setInlineLobs(false); ClobType c = new ClobType(new SerialClob(new char[0])); TupleBatch batch = new TupleBatch(1, new List[] {Arrays.asList(c)}); tb.addTupleBatch(batch, false); Copied: branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestBlockBitSetTree.java (from rev 3506, trunk/engine/src/test/java/org/teiid/common/buffer/impl/TestBlockBitSetTree.java) =================================================================== --- branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestBlockBitSetTree.java (rev 0) +++ branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestBlockBitSetTree.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -0,0 +1,66 @@ +/* + * JBoss, Home of Professional Open Source. + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA. + */ + + package org.teiid.common.buffer.impl; + +import static org.junit.Assert.*; + +import java.util.BitSet; +import java.util.Random; + +import org.junit.Test; + +public class TestBlockBitSetTree { + + @Test public void testBitsSet() { + BlockBitSetTree bst = new BlockBitSetTree((1 << 20) -1, new TestBlockClosedLongIntHashTable.DummyBlockManager()); + bst.set(1, true); + bst.set(100, true); + bst.set(10000, true); + bst.set(1000000, true); + assertEquals(4, bst.getBitsSet()); + bst.set(1, false); + assertEquals(3, bst.getBitsSet()); + assertFalse(bst.get(1)); + } + + @Test public void testNextClearSet() { + BlockBitSetTree bst = new BlockBitSetTree((1 << 20) -1, new TestBlockClosedLongIntHashTable.DummyBlockManager()); + BitSet bst1 = new BitSet(); + Random r = new Random(1); + for (int i = 0; i < 1000; i++) { + int rand = r.nextInt() & bst.getMaxIndex(); + bst.set(rand, true); + bst1.set(rand, true); + assertTrue(bst.get(rand)); + assertEquals(bst1.nextSetBit(rand), bst.nextSetBit(rand)); + assertEquals(String.valueOf(i), bst1.nextSetBit(rand), bst.nextSetBit(rand)); + } + + for (int i = 0; i < 10000; i++) { + int rand = r.nextInt() & bst.getMaxIndex(); + assertEquals(bst1.nextClearBit(rand), bst.nextClearBit(rand)); + assertEquals(bst1.nextSetBit(rand), bst.nextSetBit(rand)); + } + } + +} Copied: branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestBlockClosedLongIntHashTable.java (from rev 3506, trunk/engine/src/test/java/org/teiid/common/buffer/impl/TestBlockClosedLongIntHashTable.java) =================================================================== --- branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestBlockClosedLongIntHashTable.java (rev 0) +++ branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestBlockClosedLongIntHashTable.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -0,0 +1,92 @@ +/* + * JBoss, Home of Professional Open Source. + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA. + */ + +package org.teiid.common.buffer.impl; + +import static org.junit.Assert.*; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Random; + +import org.junit.Test; +import org.teiid.common.buffer.impl.FileStoreCache.BlockInfo; + +public class TestBlockClosedLongIntHashTable { + + public static final class DummyBlockManager implements BlockManager { + List<BlockInfo> blocks = new ArrayList<BlockInfo>(); + + @Override + public int getInode() { + return 0; + } + + @Override + public void updateBlock(BlockInfo block) { + + } + + @Override + public void free() { + blocks.clear(); + } + + @Override + public BlockInfo getBlock(int index) { + BlockInfo block = blocks.get(index); + block.buf.rewind(); + return block; + } + + @Override + public void freeBlock(int index) { + blocks.remove(index); + } + + @Override + public BlockInfo allocateBlock(int index) { + assertEquals(index, blocks.size()); + ByteBuffer result = ByteBuffer.wrap(new byte[FileStoreCache.BLOCK_SIZE]); + blocks.add(new BlockInfo(false, result, index, -1)); + return blocks.get(blocks.size() - 1); + } + } + + @Test public void testAgainstHashMap() { + BlockClosedLongIntHashTable table = new BlockClosedLongIntHashTable(new DummyBlockManager()); + HashMap<Long, Integer> table1 = new HashMap<Long, Integer>(16); + for (long i = 1; i < 200000; i++) { + table.put(i, (int)i); + table1.put(i, (int)i); + } + Random r = new Random(0); + for (int i = 1; i < 2000000; i++) { + long toRemove = r.nextInt(i); + boolean removed = table.remove(toRemove) != BlockClosedLongIntHashTable.EMPTY; + assertEquals(table1.remove(toRemove) != null, removed); + } + } + +} Modified: branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestFileStorageManager.java =================================================================== --- branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestFileStorageManager.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestFileStorageManager.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -25,11 +25,10 @@ import static org.junit.Assert.*; import java.io.File; +import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.io.RandomAccessFile; import java.util.Arrays; -import java.util.Map; import java.util.Random; import org.junit.Test; @@ -41,12 +40,9 @@ @SuppressWarnings("nls") public class TestFileStorageManager { - public FileStorageManager getStorageManager(Integer maxFileSize, Integer openFiles, String dir) throws TeiidComponentException { + public FileStorageManager getStorageManager(Integer openFiles, String dir) throws TeiidComponentException { FileStorageManager sm = new FileStorageManager(); sm.setStorageDirectory(UnitTestUtil.getTestScratchPath() + (dir != null ? File.separator + dir : "")); //$NON-NLS-1$ - if (maxFileSize != null) { - sm.setMaxFileSizeDirect(maxFileSize); - } if (openFiles != null) { sm.setMaxOpenFiles(openFiles); } @@ -55,7 +51,7 @@ } @Test public void testWrite() throws Exception { - FileStorageManager sm = getStorageManager(null, null, null); + FileStorageManager sm = getStorageManager(null, null); String tsID = "0"; //$NON-NLS-1$ FileStore store = sm.createFileStore(tsID); writeBytes(store); @@ -64,27 +60,8 @@ assertEquals(0, sm.getUsedBufferSpace()); } - @Test public void testCreatesSpillFiles() throws Exception { - FileStorageManager sm = getStorageManager(1024, null, null); // 1KB - String tsID = "0"; //$NON-NLS-1$ - // Add one batch - FileStore store = sm.createFileStore(tsID); - writeBytes(store); - - Map<File, RandomAccessFile> cache = sm.getFileCache(); - assertEquals(1, cache.size()); - - writeBytes(store); - - assertEquals(2, cache.size()); - - store.remove(); - - assertEquals(0, cache.size()); - } - - @Test(expected=TeiidComponentException.class) public void testMaxSpace() throws Exception { - FileStorageManager sm = getStorageManager(null, null, null); + @Test(expected=IOException.class) public void testMaxSpace() throws Exception { + FileStorageManager sm = getStorageManager(null, null); sm.setMaxBufferSpace(1); String tsID = "0"; //$NON-NLS-1$ // Add one batch @@ -93,7 +70,7 @@ } @Test public void testFlush() throws Exception { - FileStorageManager sm = getStorageManager(null, null, null); + FileStorageManager sm = getStorageManager(null, null); FileStore store = sm.createFileStore("0"); FileStoreOutputStream fsos = store.createOutputStream(2); fsos.write(new byte[3]); @@ -104,19 +81,19 @@ static Random r = new Random(); - private void writeBytes(FileStore store) - throws TeiidComponentException { + static void writeBytes(FileStore store) + throws IOException { byte[] bytes = new byte[2048]; r.nextBytes(bytes); - long start = store.write(bytes, 0, bytes.length); + long start = store.getLength(); + store.write(bytes, 0, bytes.length); byte[] bytesRead = new byte[2048]; store.readFully(start, bytesRead, 0, bytesRead.length); assertTrue(Arrays.equals(bytes, bytesRead)); } - @Test public void testWritingMultipleFiles() throws Exception { - FileStorageManager sm = getStorageManager(1024, null, null); + FileStorageManager sm = getStorageManager(null, null); String tsID = "0"; //$NON-NLS-1$ // Add one batch FileStore store = sm.createFileStore(tsID); Copied: branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestFileStoreCache.java (from rev 3506, trunk/engine/src/test/java/org/teiid/common/buffer/impl/TestFileStoreCache.java) =================================================================== --- branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestFileStoreCache.java (rev 0) +++ branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestFileStoreCache.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -0,0 +1,119 @@ +/* + * JBoss, Home of Professional Open Source. + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA. + */ + +package org.teiid.common.buffer.impl; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; + +import org.junit.Test; +import org.teiid.common.buffer.CacheEntry; +import org.teiid.common.buffer.Serializer; +import org.teiid.core.util.UnitTestUtil; + +public class TestFileStoreCache { + + private final class SimpleSerializer implements Serializer<Integer> { + @Override + public Integer deserialize(ObjectInputStream ois) + throws IOException, ClassNotFoundException { + Integer result = ois.readInt(); + for (int i = 0; i < result; i++) { + assertEquals(i, ois.readInt()); + } + return result; + } + + @Override + public Long getId() { + return 1l; + } + + @Override + public void serialize(Integer obj, ObjectOutputStream oos) + throws IOException { + oos.writeInt(obj); + for (int i = 0; i < obj; i++) { + oos.writeInt(i); + } + } + + @Override + public boolean useSoftCache() { + return false; + } + } + + @Test public void testAddGetMultiBlock() throws Exception { + FileStoreCache fsc = new FileStoreCache(); + fsc.setMaxBufferSpace(1 << 28); + SplittableStorageManager ssm = new SplittableStorageManager(new MemoryStorageManager()); + ssm.setMaxFileSizeDirect(MemoryStorageManager.MAX_FILE_SIZE); + fsc.setStorageManager(ssm); + fsc.initialize(); + + UnitTestUtil.enableTraceLogging("org.teiid"); //$NON-NLS-1$ + + CacheEntry ce = new CacheEntry(2l); + Serializer<Integer> s = new SimpleSerializer(); + fsc.createCacheGroup(s.getId()); + Integer cacheObject = Integer.valueOf(2); + ce.setObject(cacheObject); + fsc.add(ce, s); + + ce = fsc.get(2l, s); + assertEquals(cacheObject, ce.getObject()); + + //test something that exceeds the direct inode data blocks + ce = new CacheEntry(3l); + cacheObject = Integer.valueOf(80000); + ce.setObject(cacheObject); + fsc.add(ce, s); + + ce = fsc.get(3l, s); + assertEquals(cacheObject, ce.getObject()); + + fsc.removeCacheGroup(1l); + + assertEquals(0, fsc.getDataBlocksInUse()); + assertEquals(0, fsc.getInodesInUse()); + + //test something that exceeds the indirect inode data blocks + ce = new CacheEntry(3l); + fsc.createCacheGroup(s.getId()); + cacheObject = Integer.valueOf(5000000); + ce.setObject(cacheObject); + fsc.add(ce, s); + + ce = fsc.get(3l, s); + assertEquals(cacheObject, ce.getObject()); + + fsc.removeCacheGroup(1l); + + assertEquals(0, fsc.getDataBlocksInUse()); + assertEquals(0, fsc.getInodesInUse()); + } + +} Modified: branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestSizeUtility.java =================================================================== --- branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestSizeUtility.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestSizeUtility.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -34,7 +34,6 @@ import java.util.List; import org.junit.Test; -import org.teiid.common.buffer.TupleBatch; public class TestSizeUtility { @@ -43,7 +42,7 @@ } public void helpTestGetSize(Object obj, long expectedSize) { - long actualSize = new SizeUtility().getSize(obj, true, false); + long actualSize = new SizeUtility(null).getSize(obj, true, false); assertEquals("Got unexpected size: ", expectedSize, actualSize); //$NON-NLS-1$ } @@ -99,18 +98,6 @@ helpTestGetSize("abcdefghij", 64); //$NON-NLS-1$ } - public void XtestGetSizeLongString() { - // There is no clear way of figuring out the actual size of a string that is created - // from a StringBuffer because the buffer can sometimes be twice as big as the actual length of the string - // Since the data comin from the connector is not created this way, this test is an inaccurate setup - int size = 10000; - StringBuffer str = new StringBuffer(); - for(int i=0; i<size; i++) { - str.append("a"); //$NON-NLS-1$ - } - helpTestGetSize(str.toString(), size+3); - } - @Test public void testGetSizeRow1() { List<Object> row = new ArrayList<Object>(1); row.add(new Integer(0)); @@ -176,9 +163,7 @@ String[] types = {"string", "integer", "boolean", "double", "string", "integer"}; //$NON-NLS-1$//$NON-NLS-2$//$NON-NLS-3$//$NON-NLS-4$ //$NON-NLS-5$//$NON-NLS-6$ - TupleBatch tb = new TupleBatch(1, expected); - tb.setDataTypes(types); - long actualSize = new SizeUtility().getBatchSize(tb); + long actualSize = new SizeUtility(types).getBatchSize(Arrays.asList(expected)); assertEquals("Got unexpected size: ", 2667, actualSize); //$NON-NLS-1$ } Copied: branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestSplittableStorageManager.java (from rev 3506, trunk/engine/src/test/java/org/teiid/common/buffer/impl/TestSplittableStorageManager.java) =================================================================== --- branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestSplittableStorageManager.java (rev 0) +++ branches/as7/engine/src/test/java/org/teiid/common/buffer/impl/TestSplittableStorageManager.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -0,0 +1,81 @@ +/* + * JBoss, Home of Professional Open Source. + * See the COPYRIGHT.txt file distributed with this work for information + * regarding copyright ownership. Some portions may be licensed + * to Red Hat, Inc. under one or more contributor license agreements. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301 USA. + */ + +package org.teiid.common.buffer.impl; + +import static org.junit.Assert.*; + +import org.junit.Test; +import org.teiid.common.buffer.FileStore; + +public class TestSplittableStorageManager { + + @Test public void testCreatesSpillFiles() throws Exception { + MemoryStorageManager msm = new MemoryStorageManager(); + SplittableStorageManager ssm = new SplittableStorageManager(msm); + ssm.setMaxFileSizeDirect(2048); + String tsID = "0"; //$NON-NLS-1$ + // Add one batch + FileStore store = ssm.createFileStore(tsID); + TestFileStorageManager.writeBytes(store); + + assertEquals(1, msm.getCreated()); + + TestFileStorageManager.writeBytes(store); + + assertEquals(2, msm.getCreated()); + + store.setLength(10000); + + assertEquals(5, msm.getCreated()); + + store.setLength(100); + + assertEquals(4, msm.getRemoved()); + + store.remove(); + + assertEquals(5, msm.getRemoved()); + } + + @Test public void testTruncate() throws Exception { + MemoryStorageManager msm = new MemoryStorageManager(); + SplittableStorageManager ssm = new SplittableStorageManager(msm); + ssm.setMaxFileSizeDirect(2048); + String tsID = "0"; //$NON-NLS-1$ + // Add one batch + FileStore store = ssm.createFileStore(tsID); + TestFileStorageManager.writeBytes(store); + + assertEquals(1, msm.getCreated()); + + TestFileStorageManager.writeBytes(store); + + assertEquals(2, msm.getCreated()); + + store.setLength(100); + + assertEquals(1, msm.getRemoved()); + + } + +} Modified: branches/as7/engine/src/test/java/org/teiid/dqp/internal/process/TestCachedResults.java =================================================================== --- branches/as7/engine/src/test/java/org/teiid/dqp/internal/process/TestCachedResults.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/test/java/org/teiid/dqp/internal/process/TestCachedResults.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -23,7 +23,6 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import java.util.Arrays; @@ -33,8 +32,8 @@ import org.teiid.cache.Cache; import org.teiid.cache.DefaultCache; import org.teiid.common.buffer.BufferManager; +import org.teiid.common.buffer.BufferManager.TupleSourceType; import org.teiid.common.buffer.BufferManagerFactory; -import org.teiid.common.buffer.TestTupleBuffer.FakeBatchManager; import org.teiid.common.buffer.TupleBuffer; import org.teiid.core.types.DataTypeManager; import org.teiid.core.util.UnitTestUtil; @@ -62,7 +61,7 @@ ElementSymbol x = new ElementSymbol("x"); //$NON-NLS-1$ x.setType(DataTypeManager.DefaultDataClasses.INTEGER); List<ElementSymbol> schema = Arrays.asList(x); - TupleBuffer tb = new TupleBuffer(new FakeBatchManager(), "x", schema, null, 4); //$NON-NLS-1$ + TupleBuffer tb = BufferManagerFactory.getStandaloneBufferManager().createTupleBuffer(schema, "x", TupleSourceType.PROCESSOR); //$NON-NLS-1$ tb.setForwardOnly(false); tb.addTuple(Arrays.asList(1)); Modified: branches/as7/engine/src/test/java/org/teiid/dqp/internal/process/TestDQPCore.java =================================================================== --- branches/as7/engine/src/test/java/org/teiid/dqp/internal/process/TestDQPCore.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/test/java/org/teiid/dqp/internal/process/TestDQPCore.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -25,6 +25,7 @@ import static org.junit.Assert.*; import java.sql.ResultSet; +import java.util.HashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; @@ -35,6 +36,8 @@ import org.junit.Ignore; import org.junit.Test; import org.mockito.Mockito; +import org.teiid.adminapi.DataPolicy; +import org.teiid.adminapi.impl.DataPolicyMetadata; import org.teiid.api.exception.query.QueryResolverException; import org.teiid.cache.CacheConfiguration; import org.teiid.cache.DefaultCacheFactory; @@ -104,6 +107,9 @@ DQPWorkContext context = RealMetadataFactory.buildWorkContext(RealMetadataFactory.createTransformationMetadata(RealMetadataFactory.exampleBQTCached().getMetadataStore(), "bqt")); context.getVDB().getModel("BQT3").setVisible(false); //$NON-NLS-1$ context.getVDB().getModel("VQT").setVisible(false); //$NON-NLS-1$ + HashMap<String, DataPolicy> policies = new HashMap<String, DataPolicy>(); + policies.put("foo", new DataPolicyMetadata()); + context.setPolicies(policies); ConnectorManagerRepository repo = Mockito.mock(ConnectorManagerRepository.class); context.getVDB().addAttchment(ConnectorManagerRepository.class, repo); @@ -112,7 +118,9 @@ @Override public BufferManager getBufferManager() { - return BufferManagerFactory.createBufferManager(); + BufferManagerImpl bm = BufferManagerFactory.createBufferManager(); + bm.setInlineLobs(false); + return bm; } }; core = new DQPCore(); @@ -124,6 +132,9 @@ config = new DQPConfiguration(); config.setMaxActivePlans(1); config.setUserRequestSourceConcurrency(2); + DefaultAuthorizationValidator daa = new DefaultAuthorizationValidator(); + daa.setPolicyDecider(new DataRolePolicyDecider()); + config.setAuthorizationValidator(daa); core.start(config); core.getPrepPlanCache().setModTime(1); core.getRsCache().setBufferManager(bs.getBufferManager()); @@ -151,6 +162,20 @@ @Test public void testRequest1() throws Exception { helpExecute("SELECT IntKey FROM BQT1.SmallA", "a"); //$NON-NLS-1$ //$NON-NLS-2$ } + + @Test public void testHasRole() throws Exception { + String sql = "SELECT hasRole('foo')"; //$NON-NLS-1$ + String userName = "logon"; //$NON-NLS-1$ + ResultsMessage rm = helpExecute(sql, userName); + assertTrue((Boolean)rm.getResults()[0].get(0)); + } + + @Test public void testNotHasRole() throws Exception { + String sql = "SELECT hasRole('bar')"; //$NON-NLS-1$ + String userName = "logon"; //$NON-NLS-1$ + ResultsMessage rm = helpExecute(sql, userName); + assertFalse((Boolean)rm.getResults()[0].get(0)); + } @Test public void testUser1() throws Exception { String sql = "SELECT IntKey FROM BQT1.SmallA WHERE user() = 'logon'"; //$NON-NLS-1$ @@ -277,8 +302,8 @@ } @Test public void testBufferLimit() throws Exception { - //the sql should return 100 rows - String sql = "SELECT A.IntKey FROM BQT1.SmallA as A, BQT1.SmallA as B"; //$NON-NLS-1$ + //the sql should return 400 rows + String sql = "SELECT A.IntKey FROM BQT1.SmallA as A, BQT1.SmallA as B, (select intkey from BQT1.SmallA limit 4) as C"; //$NON-NLS-1$ String userName = "1"; //$NON-NLS-1$ String sessionid = "1"; //$NON-NLS-1$ @@ -286,31 +311,33 @@ reqMsg.setCursorType(ResultSet.TYPE_FORWARD_ONLY); DQPWorkContext.getWorkContext().getSession().setSessionId(sessionid); DQPWorkContext.getWorkContext().getSession().setUserName(userName); - ((BufferManagerImpl)core.getBufferManager()).setProcessorBatchSize(2); + ((BufferManagerImpl)core.getBufferManager()).setProcessorBatchSize(1); Future<ResultsMessage> message = core.executeRequest(reqMsg.getExecutionId(), reqMsg); ResultsMessage rm = message.get(500000, TimeUnit.MILLISECONDS); assertNull(rm.getException()); - assertEquals(2, rm.getResults().length); + + int rowsPerBatch = 8; + assertEquals(rowsPerBatch, rm.getResults().length); RequestWorkItem item = core.getRequestWorkItem(DQPWorkContext.getWorkContext().getRequestID(reqMsg.getExecutionId())); - message = core.processCursorRequest(reqMsg.getExecutionId(), 3, 2); + message = core.processCursorRequest(reqMsg.getExecutionId(), 9, rowsPerBatch); rm = message.get(500000, TimeUnit.MILLISECONDS); assertNull(rm.getException()); - assertEquals(2, rm.getResults().length); + assertEquals(rowsPerBatch, rm.getResults().length); //ensure that we are idle for (int i = 0; i < 10 && item.getThreadState() != ThreadState.IDLE; i++) { Thread.sleep(100); } assertEquals(ThreadState.IDLE, item.getThreadState()); - assertTrue(item.resultsBuffer.getManagedRowCount() <= 46); + assertTrue(item.resultsBuffer.getManagedRowCount() <= rowsPerBatch*23); //pull the rest of the results for (int j = 0; j < 48; j++) { item = core.getRequestWorkItem(DQPWorkContext.getWorkContext().getRequestID(reqMsg.getExecutionId())); - message = core.processCursorRequest(reqMsg.getExecutionId(), j * 2 + 5, 2); + message = core.processCursorRequest(reqMsg.getExecutionId(), (j + 2) * rowsPerBatch + 1, rowsPerBatch); rm = message.get(5000, TimeUnit.MILLISECONDS); assertNull(rm.getException()); - assertEquals(2, rm.getResults().length); + assertEquals(rowsPerBatch, rm.getResultsList().size()); } } @@ -324,11 +351,11 @@ reqMsg.setCursorType(ResultSet.TYPE_FORWARD_ONLY); DQPWorkContext.getWorkContext().getSession().setSessionId(sessionid); DQPWorkContext.getWorkContext().getSession().setUserName(userName); - ((BufferManagerImpl)core.getBufferManager()).setProcessorBatchSize(2); + ((BufferManagerImpl)core.getBufferManager()).setProcessorBatchSize(1); Future<ResultsMessage> message = core.executeRequest(reqMsg.getExecutionId(), reqMsg); ResultsMessage rm = message.get(500000, TimeUnit.MILLISECONDS); assertNull(rm.getException()); - assertEquals(2, rm.getResults().length); + assertEquals(8, rm.getResultsList().size()); RequestWorkItem item = core.getRequestWorkItem(DQPWorkContext.getWorkContext().getRequestID(reqMsg.getExecutionId())); assertEquals(100, item.resultsBuffer.getRowCount()); } Modified: branches/as7/engine/src/test/java/org/teiid/dqp/service/AutoGenDataService.java =================================================================== --- branches/as7/engine/src/test/java/org/teiid/dqp/service/AutoGenDataService.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/test/java/org/teiid/dqp/service/AutoGenDataService.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -95,7 +95,7 @@ results = new List[] {Arrays.asList(1)}; } - final AtomicResultsMessage msg = ConnectorWorkItem.createResultsMessage(results, projectedSymbols); + final AtomicResultsMessage msg = ConnectorWorkItem.createResultsMessage(results); msg.setFinalRow(rows); return new ConnectorWork() { Property changes on: branches/as7/engine/src/test/java/org/teiid/query/function/TestFunction.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/function/TestFunction.java:3281-3325 /trunk/engine/src/test/java/org/teiid/query/function/TestFunction.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/function/TestFunction.java:3281-3325 /trunk/engine/src/test/java/org/teiid/query/function/TestFunction.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/test/java/org/teiid/query/function/TestFunctionLibrary.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/function/TestFunctionLibrary.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/function/TestFunctionLibrary.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/function/TestFunctionLibrary.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/function/TestFunctionLibrary.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/test/java/org/teiid/query/function/TestFunctionTree.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/function/TestFunctionTree.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/function/TestFunctionTree.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/function/TestFunctionTree.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/function/TestFunctionTree.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/test/java/org/teiid/query/function/TestResolvedFunctions.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/function/TestResolvedFunctions.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/function/TestResolvedFunctions.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/function/TestResolvedFunctions.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/function/TestResolvedFunctions.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestAccessPatterns.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestAccessPatterns.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestAccessPatterns.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestAccessPatterns.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestAccessPatterns.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestAggregatePushdown.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestAggregatePushdown.java:3149-3217,3220-3275 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestAggregatePushdown.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestAggregatePushdown.java:3149-3217,3220-3275 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestAggregatePushdown.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestAnsiJoinPushdown.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestAnsiJoinPushdown.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestAnsiJoinPushdown.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestAnsiJoinPushdown.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestAnsiJoinPushdown.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestBatchedUpdatePlanner.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestBatchedUpdatePlanner.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestBatchedUpdatePlanner.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestBatchedUpdatePlanner.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestBatchedUpdatePlanner.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestDependentJoins.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestDependentJoins.java:3149-3217,3220-3275 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestDependentJoins.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestDependentJoins.java:3149-3217,3220-3275 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestDependentJoins.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestExpressionsInGroupBy.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestExpressionsInGroupBy.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestExpressionsInGroupBy.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestExpressionsInGroupBy.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestExpressionsInGroupBy.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestInlineView.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestInlineView.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestInlineView.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestInlineView.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestInlineView.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestJoinOptimization.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestJoinOptimization.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestJoinOptimization.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestJoinOptimization.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestJoinOptimization.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestJoinPushdownRestrictions.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestJoinPushdownRestrictions.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestJoinPushdownRestrictions.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestJoinPushdownRestrictions.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestJoinPushdownRestrictions.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestLimit.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestLimit.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestLimit.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestLimit.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestLimit.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestOptimizer.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestOptimizer.java:3149-3217,3220-3275 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestOptimizer.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestOptimizer.java:3149-3217,3220-3275 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestOptimizer.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestOptionalJoins.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestOptionalJoins.java:3149-3217,3220-3275 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestOptionalJoins.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestOptionalJoins.java:3149-3217,3220-3275 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestOptionalJoins.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestPartitionedJoinPlanning.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestPartitionedJoinPlanning.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestPartitionedJoinPlanning.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestPartitionedJoinPlanning.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestPartitionedJoinPlanning.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestRuleMergeVirtual.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestRuleMergeVirtual.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestRuleMergeVirtual.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestRuleMergeVirtual.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestRuleMergeVirtual.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestRuleRaiseNull.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestRuleRaiseNull.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestRuleRaiseNull.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestRuleRaiseNull.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestRuleRaiseNull.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestRuleRemoveSorts.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestRuleRemoveSorts.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestRuleRemoveSorts.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestRuleRemoveSorts.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestRuleRemoveSorts.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestSortOptimization.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestSortOptimization.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestSortOptimization.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestSortOptimization.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestSortOptimization.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestStoredProcedurePlanning.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestStoredProcedurePlanning.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestStoredProcedurePlanning.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestStoredProcedurePlanning.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestStoredProcedurePlanning.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestSubqueryPushdown.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestSubqueryPushdown.java:3149-3217,3281-3325 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestSubqueryPushdown.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestSubqueryPushdown.java:3149-3217,3281-3325 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestSubqueryPushdown.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/TestUnionPlanning.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestUnionPlanning.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestUnionPlanning.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/TestUnionPlanning.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/optimizer/TestUnionPlanning.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/test/java/org/teiid/query/optimizer/relational/TestAliasGenerator.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/relational/TestAliasGenerator.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/optimizer/relational/TestAliasGenerator.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/optimizer/relational/TestAliasGenerator.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/optimizer/relational/TestAliasGenerator.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/test/java/org/teiid/query/processor/FakeDataManager.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/FakeDataManager.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/processor/FakeDataManager.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/FakeDataManager.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/processor/FakeDataManager.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/test/java/org/teiid/query/processor/FakeDataStore.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/FakeDataStore.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/processor/FakeDataStore.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/FakeDataStore.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/processor/FakeDataStore.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/test/java/org/teiid/query/processor/TestAggregateProcessing.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestAggregateProcessing.java:3149-3217,3220-3275 /trunk/engine/src/test/java/org/teiid/query/processor/TestAggregateProcessing.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestAggregateProcessing.java:3149-3217,3220-3275 /trunk/engine/src/test/java/org/teiid/query/processor/TestAggregateProcessing.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/test/java/org/teiid/query/processor/TestDependentJoins.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestDependentJoins.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/processor/TestDependentJoins.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestDependentJoins.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/processor/TestDependentJoins.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/test/java/org/teiid/query/processor/TestInsertProcessing.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestInsertProcessing.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/processor/TestInsertProcessing.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestInsertProcessing.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/processor/TestInsertProcessing.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/test/java/org/teiid/query/processor/TestJoinWithFunction.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestJoinWithFunction.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/processor/TestJoinWithFunction.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestJoinWithFunction.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/processor/TestJoinWithFunction.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/test/java/org/teiid/query/processor/TestOptionalJoins.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestOptionalJoins.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/processor/TestOptionalJoins.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestOptionalJoins.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/processor/TestOptionalJoins.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/test/java/org/teiid/query/processor/TestOrderByProcessing.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestOrderByProcessing.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/processor/TestOrderByProcessing.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestOrderByProcessing.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/processor/TestOrderByProcessing.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/test/java/org/teiid/query/processor/TestProcedureRelational.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestProcedureRelational.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/processor/TestProcedureRelational.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestProcedureRelational.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/processor/TestProcedureRelational.java:3188-3450,3452-3506 Modified: branches/as7/engine/src/test/java/org/teiid/query/processor/TestProcessor.java =================================================================== --- branches/as7/engine/src/test/java/org/teiid/query/processor/TestProcessor.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/test/java/org/teiid/query/processor/TestProcessor.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -365,7 +365,6 @@ props.setProperty("soap_port", "12345"); //$NON-NLS-1$ //$NON-NLS-2$ CommandContext context = new CommandContext("0", "test", "user", null, "myvdb", 1, props, DEBUG); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-4$ context.setProcessorBatchSize(BufferManager.DEFAULT_PROCESSOR_BATCH_SIZE); - context.setConnectorBatchSize(BufferManager.DEFAULT_CONNECTOR_BATCH_SIZE); context.setBufferManager(BufferManagerFactory.getStandaloneBufferManager()); context.setPreparedPlanCache(new SessionAwareCache<PreparedPlan>()); return context; @@ -1080,7 +1079,6 @@ CommandContext context = createCommandContext(); context.setProcessorBatchSize(2); - context.setConnectorBatchSize(2); context.setMetadata(RealMetadataFactory.example1Cached()); // Plan query Property changes on: branches/as7/engine/src/test/java/org/teiid/query/processor/TestProcessor.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestProcessor.java:3149-3217,3220-3275 /trunk/engine/src/test/java/org/teiid/query/processor/TestProcessor.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestProcessor.java:3149-3217,3220-3275 /trunk/engine/src/test/java/org/teiid/query/processor/TestProcessor.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/test/java/org/teiid/query/processor/TestSQLXMLProcessing.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestSQLXMLProcessing.java:3149-3217,3220-3275,3281-3325,3340-3349 /trunk/engine/src/test/java/org/teiid/query/processor/TestSQLXMLProcessing.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestSQLXMLProcessing.java:3149-3217,3220-3275,3281-3325,3340-3349 /trunk/engine/src/test/java/org/teiid/query/processor/TestSQLXMLProcessing.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/test/java/org/teiid/query/processor/TestSecurityFunctions.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestSecurityFunctions.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/processor/TestSecurityFunctions.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestSecurityFunctions.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/processor/TestSecurityFunctions.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/test/java/org/teiid/query/processor/TestSetProcessing.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestSetProcessing.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/processor/TestSetProcessing.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestSetProcessing.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/processor/TestSetProcessing.java:3188-3450,3452-3506 Modified: branches/as7/engine/src/test/java/org/teiid/query/processor/TestTempTables.java =================================================================== --- branches/as7/engine/src/test/java/org/teiid/query/processor/TestTempTables.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/test/java/org/teiid/query/processor/TestTempTables.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -135,6 +135,34 @@ execute("select * from x", new List[] {}); } + @Test public void testCommitExistingRemoved() throws Exception { + execute("create local temporary table x (e1 string, e2 integer)", new List[] {Arrays.asList(0)}); //$NON-NLS-1$ + setupTransaction(Connection.TRANSACTION_SERIALIZABLE); + execute("drop table x", new List[] {Arrays.asList(0)}); //$NON-NLS-1$ + synch.afterCompletion(Status.STATUS_COMMITTED); + try { + execute("select * from x", new List[] {}); + fail(); + } catch (Exception e) { + + } + } + + @Test public void testUpdateLock() throws Exception { + execute("create local temporary table x (e1 string, e2 integer)", new List[] {Arrays.asList(0)}); //$NON-NLS-1$ + setupTransaction(Connection.TRANSACTION_SERIALIZABLE); + execute("insert into x (e2, e1) select e2, e1 from pm1.g1", new List[] {Arrays.asList(6)}); //$NON-NLS-1$ + tc = null; + try { + execute("insert into x (e2, e1) select e2, e1 from pm1.g1", new List[] {Arrays.asList(6)}); //$NON-NLS-1$ + fail(); + } catch (Exception e) { + + } + synch.afterCompletion(Status.STATUS_COMMITTED); + execute("insert into x (e2, e1) select e2, e1 from pm1.g1", new List[] {Arrays.asList(6)}); //$NON-NLS-1$ + } + @Test public void testRollbackExisting1() throws Exception { execute("create local temporary table x (e1 string, e2 integer)", new List[] {Arrays.asList(0)}); //$NON-NLS-1$ for (int i = 0; i < 86; i++) { @@ -160,19 +188,19 @@ tempStore = gtsi.getTempTableStore(); metadata = new TempMetadataAdapter(RealMetadataFactory.example1Cached(), tempStore.getMetadataStore()); execute("create local temporary table x (e1 string, e2 integer)", new List[] {Arrays.asList(0)}); //$NON-NLS-1$ - for (int i = 0; i < 86; i++) { + for (int i = 0; i < 300; i++) { execute("insert into x (e2, e1) select e2, e1 from pm1.g1", new List[] {Arrays.asList(6)}); //$NON-NLS-1$ } setupTransaction(Connection.TRANSACTION_SERIALIZABLE); - execute("select count(*) from x", new List[] {Arrays.asList(516)}); - gtsi.updateMatViewRow("X", Arrays.asList(1), true); + execute("select count(e1) from x", new List[] {Arrays.asList(1500)}); + gtsi.updateMatViewRow("X", Arrays.asList(2), true); tc=null; //outside of the transaction we can see the row removed - execute("select count(*) from x", new List[] {Arrays.asList(515)}); + execute("select count(e1) from x", new List[] {Arrays.asList(1499)}); //back in the transaction we see the original state setupTransaction(Connection.TRANSACTION_SERIALIZABLE); - execute("select count(*) from x", new List[] {Arrays.asList(516)}); + execute("select count(e1) from x", new List[] {Arrays.asList(1500)}); synch.afterCompletion(Status.STATUS_COMMITTED); } Property changes on: branches/as7/engine/src/test/java/org/teiid/query/processor/TestTempTables.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestTempTables.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/processor/TestTempTables.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestTempTables.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/processor/TestTempTables.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/test/java/org/teiid/query/processor/TestTextTable.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestTextTable.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/processor/TestTextTable.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestTextTable.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/processor/TestTextTable.java:3188-3450,3452-3506 Modified: branches/as7/engine/src/test/java/org/teiid/query/processor/TestVirtualDepJoin.java =================================================================== --- branches/as7/engine/src/test/java/org/teiid/query/processor/TestVirtualDepJoin.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/test/java/org/teiid/query/processor/TestVirtualDepJoin.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -33,9 +33,8 @@ import org.junit.Test; import org.teiid.common.buffer.BufferManager; +import org.teiid.common.buffer.BufferManagerFactory; import org.teiid.common.buffer.TupleBuffer; -import org.teiid.common.buffer.impl.BufferManagerImpl; -import org.teiid.common.buffer.impl.MemoryStorageManager; import org.teiid.core.TeiidComponentException; import org.teiid.core.TeiidException; import org.teiid.core.types.DataTypeManager; @@ -461,16 +460,8 @@ TestProcessor.examineResults((List[])expected.toArray(new List[expected.size()]), bufferMgr, id); } - private BufferManager createCustomBufferMgr(int batchSize) throws TeiidComponentException { - BufferManagerImpl bufferMgr = new BufferManagerImpl(); - bufferMgr.setConnectorBatchSize(batchSize); - bufferMgr.setProcessorBatchSize(batchSize); - bufferMgr.initialize(); - - // Add unmanaged memory storage manager - bufferMgr.setStorageManager(new MemoryStorageManager()); - - return bufferMgr; + private BufferManager createCustomBufferMgr(int batchSize) { + return BufferManagerFactory.getTestBufferManager(200000, batchSize); } public void helpTestVirtualDepJoin(boolean pushCriteria) throws Exception { Property changes on: branches/as7/engine/src/test/java/org/teiid/query/processor/TestVirtualDepJoin.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestVirtualDepJoin.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/processor/TestVirtualDepJoin.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestVirtualDepJoin.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/processor/TestVirtualDepJoin.java:3188-3450,3452-3506 Property changes on: branches/as7/engine/src/test/java/org/teiid/query/processor/TestWithClauseProcessing.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestWithClauseProcessing.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/processor/TestWithClauseProcessing.java:3188-3450 + /branches/7.4.x/engine/src/test/java/org/teiid/query/processor/TestWithClauseProcessing.java:3149-3217 /trunk/engine/src/test/java/org/teiid/query/processor/TestWithClauseProcessing.java:3188-3450,3452-3506 Modified: branches/as7/engine/src/test/java/org/teiid/query/processor/relational/TestJoinNode.java =================================================================== --- branches/as7/engine/src/test/java/org/teiid/query/processor/relational/TestJoinNode.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/test/java/org/teiid/query/processor/relational/TestJoinNode.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -33,9 +33,9 @@ import org.junit.Before; import org.junit.Test; import org.teiid.common.buffer.BlockedException; -import org.teiid.common.buffer.BufferManager; import org.teiid.common.buffer.BufferManagerFactory; import org.teiid.common.buffer.TupleBatch; +import org.teiid.common.buffer.impl.BufferManagerImpl; import org.teiid.core.TeiidComponentException; import org.teiid.core.TeiidProcessingException; import org.teiid.core.types.DataTypeManager; @@ -232,7 +232,8 @@ } public void helpTestJoinDirect(List[] expectedResults, int batchSize, int processingBytes) throws TeiidComponentException, TeiidProcessingException { - BufferManager mgr = BufferManagerFactory.getTestBufferManager(processingBytes, batchSize); + BufferManagerImpl mgr = BufferManagerFactory.getTestBufferManager(processingBytes, batchSize); + mgr.setTargetBytesPerRow(100); CommandContext context = new CommandContext("pid", "test", null, null, 1); //$NON-NLS-1$ //$NON-NLS-2$ join.addChild(leftNode); Modified: branches/as7/engine/src/test/java/org/teiid/query/processor/relational/TestProjectIntoNode.java =================================================================== --- branches/as7/engine/src/test/java/org/teiid/query/processor/relational/TestProjectIntoNode.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/test/java/org/teiid/query/processor/relational/TestProjectIntoNode.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -72,10 +72,9 @@ ElementSymbol elementSymbol_2 = new ElementSymbol("myGroup.myElement2"); //$NON-NLS-1$ elementSymbol_1.setType(Integer.class); elementSymbol_2.setType(String.class); - ArrayList elements = new ArrayList(); - elements.add(elementSymbol_1); - elements.add(elementSymbol_2); + List<ElementSymbol> elements = Arrays.asList(elementSymbol_1, elementSymbol_2); node.setIntoElements(elements); + child.setElements(elements); node.setMode(mode); node.setModelName("myModel"); //$NON-NLS-1$ Modified: branches/as7/engine/src/test/java/org/teiid/query/sql/symbol/TestElementSymbol.java =================================================================== --- branches/as7/engine/src/test/java/org/teiid/query/sql/symbol/TestElementSymbol.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/engine/src/test/java/org/teiid/query/sql/symbol/TestElementSymbol.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -127,22 +127,22 @@ @Test public void testEquals5() { ElementSymbol es = sampleElement1(); - helpEquals(es, (ElementSymbol)es.clone(), true); + helpEquals(es, es.clone(), true); } @Test public void testEquals6() { ElementSymbol es = sampleElement2(); - helpEquals(es, (ElementSymbol)es.clone(), true); + helpEquals(es, es.clone(), true); } @Test public void testEquals7() { ElementSymbol es = sampleElement3(); - helpEquals(es, (ElementSymbol)es.clone(), true); + helpEquals(es, es.clone(), true); } @Test public void testEquals8() { ElementSymbol es = sampleElement4(); - helpEquals(es, (ElementSymbol)es.clone(), true); + helpEquals(es, es.clone(), true); } // Compare fully-qualified to short versions @@ -190,7 +190,7 @@ @Test public void testCloneEquivalence(){ ElementSymbol s1 = sampleElement1(); - ElementSymbol s2 = (ElementSymbol)s1.clone(); + ElementSymbol s2 = s1.clone(); int equals = 0; UnitTestUtil.helpTestEquivalence(equals, s1, s2); } @@ -225,31 +225,31 @@ @Test public void testClone1() { ElementSymbol e1 = sampleElement1(); - ElementSymbol copy = (ElementSymbol) e1.clone(); + ElementSymbol copy = e1.clone(); helpEquals(e1, copy, true); } @Test public void testClone2() { ElementSymbol e1 = sampleElement2(); - ElementSymbol copy = (ElementSymbol) e1.clone(); + ElementSymbol copy = e1.clone(); helpEquals(e1, copy, true); } @Test public void testClone3() { ElementSymbol e1 = sampleElement3(); - ElementSymbol copy = (ElementSymbol) e1.clone(); + ElementSymbol copy = e1.clone(); helpEquals(e1, copy, true); } @Test public void testClone4() { ElementSymbol e1 = sampleElement4(); - ElementSymbol copy = (ElementSymbol) e1.clone(); + ElementSymbol copy = e1.clone(); helpEquals(e1, copy, true); } @Test public void testClone5() { ElementSymbol e1 = sampleElement1(); - ElementSymbol copy = (ElementSymbol) e1.clone(); + ElementSymbol copy = e1.clone(); helpEquals(e1, copy, true); // Change original, clone shouldn't change Property changes on: branches/as7/engine/src/test/resources/text/cdm_dos_win.txt ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/engine/src/test/resources/text/cdm_dos_win.txt:3149-3164,3172-3217 /trunk/engine/src/test/resources/text/cdm_dos_win.txt:3188-3450 + /branches/7.4.x/engine/src/test/resources/text/cdm_dos_win.txt:3149-3164,3172-3217 /trunk/engine/src/test/resources/text/cdm_dos_win.txt:3188-3450,3452-3506 Modified: branches/as7/jboss-integration/pom.xml =================================================================== --- branches/as7/jboss-integration/pom.xml 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/jboss-integration/pom.xml 2011-09-29 16:46:38 UTC (rev 3507) @@ -203,7 +203,12 @@ <scope>test</scope> </dependency> --> - + <dependency> + <groupId>org.jboss.teiid</groupId> + <artifactId>teiid-common-core</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> </dependencies> </project> Modified: branches/as7/metadata/src/main/java/org/teiid/internal/core/index/BlocksIndexInput.java =================================================================== --- branches/as7/metadata/src/main/java/org/teiid/internal/core/index/BlocksIndexInput.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/metadata/src/main/java/org/teiid/internal/core/index/BlocksIndexInput.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -60,9 +60,12 @@ //System.out.println(" Closing Index File: " + this.indexFile.getName()); summary= null; setOpen(false); - if (raf != null) + if (raf != null) { raf.close(); - vraf.close(); + } + if (vraf != null) { + vraf.close(); + } } } Modified: branches/as7/runtime/src/main/java/org/teiid/services/BufferServiceImpl.java =================================================================== --- branches/as7/runtime/src/main/java/org/teiid/services/BufferServiceImpl.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/runtime/src/main/java/org/teiid/services/BufferServiceImpl.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -29,7 +29,9 @@ import org.teiid.common.buffer.BufferManager; import org.teiid.common.buffer.impl.BufferManagerImpl; import org.teiid.common.buffer.impl.FileStorageManager; +import org.teiid.common.buffer.impl.FileStoreCache; import org.teiid.common.buffer.impl.MemoryStorageManager; +import org.teiid.common.buffer.impl.SplittableStorageManager; import org.teiid.core.TeiidComponentException; import org.teiid.core.TeiidRuntimeException; import org.teiid.core.util.FileUtils; @@ -56,10 +58,11 @@ private int processorBatchSize = BufferManager.DEFAULT_PROCESSOR_BATCH_SIZE; private int connectorBatchSize = BufferManager.DEFAULT_CONNECTOR_BATCH_SIZE; private int maxOpenFiles = FileStorageManager.DEFAULT_MAX_OPEN_FILES; - private long maxFileSize = FileStorageManager.DEFAULT_MAX_FILESIZE; // 2GB + private long maxFileSize = SplittableStorageManager.DEFAULT_MAX_FILESIZE; // 2GB private int maxProcessingKb = BufferManager.DEFAULT_MAX_PROCESSING_KB; private int maxReserveKb = BufferManager.DEFAULT_RESERVE_BUFFER_KB; - private long maxBufferSpace = FileStorageManager.DEFAULT_MAX_BUFFERSPACE; + private long maxBufferSpace = FileStorageManager.DEFAULT_MAX_BUFFERSPACE>>20; + private boolean inlineLobs = true; private FileStorageManager fsm; /** @@ -94,13 +97,17 @@ // Get the properties for FileStorageManager and create. fsm = new FileStorageManager(); fsm.setStorageDirectory(bufferDir.getCanonicalPath()); - fsm.setMaxFileSize(maxFileSize); fsm.setMaxOpenFiles(maxOpenFiles); fsm.setMaxBufferSpace(maxBufferSpace*MB); - fsm.initialize(); - this.bufferMgr.setStorageManager(fsm); + SplittableStorageManager ssm = new SplittableStorageManager(fsm); + ssm.setMaxFileSize(maxFileSize); + FileStoreCache fsc = new FileStoreCache(); + fsc.setStorageManager(ssm); + fsc.setMaxBufferSpace(maxBufferSpace*MB); + fsc.initialize(); + this.bufferMgr.setCache(fsc); } else { - this.bufferMgr.setStorageManager(new MemoryStorageManager()); + this.bufferMgr.setCache(new MemoryStorageManager()); } } catch(TeiidComponentException e) { @@ -121,7 +128,7 @@ } } - public BufferManager getBufferManager() { + public BufferManagerImpl getBufferManager() { return this.bufferMgr; } @@ -139,6 +146,10 @@ public void setConnectorBatchSize(int size) { this.connectorBatchSize = size; } + + public void setInlineLobs(boolean inlineLobs) { + this.inlineLobs = inlineLobs; + } public File getBufferDirectory() { return bufferDir; @@ -147,6 +158,10 @@ public boolean isUseDisk() { return this.useDisk; } + + public boolean isInlineLobs() { + return inlineLobs; + } public int getProcessorBatchSize() { return this.processorBatchSize; Modified: branches/as7/runtime/src/test/java/org/teiid/dqp/service/buffer/TestLocalBufferService.java =================================================================== --- branches/as7/runtime/src/test/java/org/teiid/dqp/service/buffer/TestLocalBufferService.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/runtime/src/test/java/org/teiid/dqp/service/buffer/TestLocalBufferService.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -37,6 +37,8 @@ import org.teiid.common.buffer.BufferManager.TupleSourceType; import org.teiid.common.buffer.impl.BufferManagerImpl; import org.teiid.common.buffer.impl.FileStorageManager; +import org.teiid.common.buffer.impl.FileStoreCache; +import org.teiid.common.buffer.impl.SplittableStorageManager; import org.teiid.core.types.DataTypeManager; import org.teiid.core.types.DataTypeManager.DefaultDataTypes; import org.teiid.core.util.UnitTestUtil; @@ -60,8 +62,9 @@ assertTrue("does not end with one", svc.getBufferDirectory().getParent().endsWith("1")); //$NON-NLS-1$ //$NON-NLS-2$ assertTrue(svc.isUseDisk()); - BufferManagerImpl mgr = (BufferManagerImpl) svc.getBufferManager(); - assertTrue(((FileStorageManager)mgr.getStorageManager()).getDirectory().endsWith(svc.getBufferDirectory().getName())); + BufferManagerImpl mgr = svc.getBufferManager(); + SplittableStorageManager ssm = (SplittableStorageManager)((FileStoreCache)mgr.getCache()).getStorageManager(); + assertTrue(((FileStorageManager)ssm.getStorageManager()).getDirectory().endsWith(svc.getBufferDirectory().getName())); } @Test public void testCheckMemPropertyGotSet2() throws Exception { @@ -99,7 +102,8 @@ svc.start(); BufferManager mgr = svc.getBufferManager(); - assertEquals(13141, mgr.getSchemaSize(schema)); + assertEquals(6570, mgr.getSchemaSize(schema)); + assertEquals(256, mgr.getProcessorBatchSize(schema)); } @Test Modified: branches/as7/test-integration/common/src/test/java/org/teiid/systemmodel/TestMatViewReplication.java =================================================================== --- branches/as7/test-integration/common/src/test/java/org/teiid/systemmodel/TestMatViewReplication.java 2011-09-29 01:08:46 UTC (rev 3506) +++ branches/as7/test-integration/common/src/test/java/org/teiid/systemmodel/TestMatViewReplication.java 2011-09-29 16:46:38 UTC (rev 3507) @@ -30,9 +30,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.HashMap; -import java.util.logging.Handler; -import java.util.logging.Level; -import java.util.logging.Logger; import org.jboss.as.clustering.jgroups.ChannelFactory; import org.jgroups.Channel; @@ -60,16 +57,7 @@ @Test public void testReplication() throws Exception { if (DEBUG) { - Logger logger = Logger.getLogger("org.teiid"); - logger.setLevel(Level.FINEST); - for (Handler h : logger.getHandlers()) { - h.setLevel(Level.FINEST); - } - /*org.apache.log4j.Logger l = LogManager.getLogger("org.jgroups"); - l.setLevel(org.apache.log4j.Level.TRACE); - ConsoleAppender ca = new ConsoleAppender(new PatternLayout()); - ca.setName("x"); - l.addAppender(ca);*/ + UnitTestUtil.enableTraceLogging("org.teiid"); } FakeServer server1 = createServer(); Property changes on: branches/as7/test-integration/common/src/test/java/org/teiid/transport/TestODBCSocketTransport.java ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/test-integration/common/src/test/java/org/teiid/transport/TestODBCSocketTransport.java:3149-3217,3281-3325 /trunk/test-integration/common/src/test/java/org/teiid/transport/TestODBCSocketTransport.java:3188-3450 + /branches/7.4.x/test-integration/common/src/test/java/org/teiid/transport/TestODBCSocketTransport.java:3149-3217,3281-3325 /trunk/test-integration/common/src/test/java/org/teiid/transport/TestODBCSocketTransport.java:3188-3450,3452-3506 Property changes on: branches/as7/test-integration/common/src/test/resources/TestODBCSocketTransport/testPk.expected ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/test-integration/common/src/test/resources/TestODBCSocketTransport/testPk.expected:3281-3325 /trunk/test-integration/common/src/test/resources/TestODBCSocketTransport/testPk.expected:3188-3450 + /branches/7.4.x/test-integration/common/src/test/resources/TestODBCSocketTransport/testPk.expected:3281-3325 /trunk/test-integration/common/src/test/resources/TestODBCSocketTransport/testPk.expected:3188-3450,3452-3506 Property changes on: branches/as7/test-integration/common/src/test/resources/TestODBCSocketTransport/testPkPrepared.expected ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/test-integration/common/src/test/resources/TestODBCSocketTransport/testPkPrepared.expected:3281-3325 /trunk/test-integration/common/src/test/resources/TestODBCSocketTransport/testPkPrepared.expected:3188-3450 + /branches/7.4.x/test-integration/common/src/test/resources/TestODBCSocketTransport/testPkPrepared.expected:3281-3325 /trunk/test-integration/common/src/test/resources/TestODBCSocketTransport/testPkPrepared.expected:3188-3450,3452-3506 Property changes on: branches/as7/test-integration/common/src/test/resources/TestSystemVirtualModel/testProcedureParams.expected ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/test-integration/common/src/test/resources/TestSystemVirtualModel/testProcedureParams.expected:3220-3275 /trunk/test-integration/common/src/test/resources/TestSystemVirtualModel/testProcedureParams.expected:3188-3450 + /branches/7.4.x/test-integration/common/src/test/resources/TestSystemVirtualModel/testProcedureParams.expected:3220-3275 /trunk/test-integration/common/src/test/resources/TestSystemVirtualModel/testProcedureParams.expected:3188-3450,3452-3506 Property changes on: branches/as7/test-integration/common/src/test/resources/TestSystemVirtualModel/testProcedures.expected ___________________________________________________________________ Modified: svn:mergeinfo - /branches/7.4.x/test-integration/common/src/test/resources/TestSystemVirtualModel/testProcedures.expected:3220-3275 /trunk/test-integration/common/src/test/resources/TestSystemVirtualModel/testProcedures.expected:3188-3450 + /branches/7.4.x/test-integration/common/src/test/resources/TestSystemVirtualModel/testProcedures.expected:3220-3275 /trunk/test-integration/common/src/test/resources/TestSystemVirtualModel/testProcedures.expected:3188-3450,3452-3506