[teiid-commits] teiid SVN: r3006 - in trunk: build/kits/jboss-container/deploy/teiid and 9 other directories.

Author: shawkins Date: 2011-03-18 10:45:33 -0400 (Fri, 18 Mar 2011) New Revision: 3006 Modified: trunk/build/kits/jboss-container/deploy/teiid/teiid-jboss-beans.xml trunk/build/kits/jboss-container/teiid-releasenotes.html trunk/common-core/src/main/java/org/teiid/core/types/DataTypeManager.java trunk/documentation/admin-guide/src/main/docbook/en-US/content/appendix-c.xml trunk/documentation/admin-guide/src/main/docbook/en-US/content/performance.xml trunk/engine/src/main/java/org/teiid/common/buffer/BufferManager.java trunk/engine/src/main/java/org/teiid/common/buffer/impl/BufferManagerImpl.java trunk/engine/src/main/java/org/teiid/dqp/internal/process/DQPCore.java trunk/engine/src/main/java/org/teiid/dqp/internal/process/MetaDataProcessor.java trunk/engine/src/main/java/org/teiid/query/metadata/QueryMetadataInterface.java trunk/engine/src/main/java/org/teiid/query/processor/relational/PartitionedSortJoin.java trunk/engine/src/main/java/org/teiid/query/processor/relational/SortUtility.java trunk/engine/src/test/java/org/teiid/dqp/internal/datamgr/TestSubqueryCompareCriteriaImpl.java trunk/runtime/src/test/java/org/teiid/dqp/service/buffer/TestLocalBufferService.java Log: TEIID-1509 changing the default for value caching and cleaning up internal memory handling to use more precise byte estimates Modified: trunk/build/kits/jboss-container/deploy/teiid/teiid-jboss-beans.xml =================================================================== --- trunk/build/kits/jboss-container/deploy/teiid/teiid-jboss-beans.xml 2011-03-18 13:03:31 UTC (rev 3005) +++ trunk/build/kits/jboss-container/deploy/teiid/teiid-jboss-beans.xml 2011-03-18 14:45:33 UTC (rev 3006) @@ -25,18 +25,15 @@ <!-- The max row count of a batch from a connector. Should be even multiple of processorBatchSize. (default 1024) --> <property name="connectorBatchSize">1024</property> <!-- - The number of batch columns to allow in memory (default 16384). - This value should be set lower or higher depending on the available memory to Teiid in the VM. - 16384 is considered a good default for a dedicated 32-bit VM running Teiid with a 1 gig heap. + The number of batch columns to allow in buffer memory. -1 means to automatically calculate a value (default -1). See the admin guide for more. --> - <property name="maxReserveBatchColumns">16384</property> + <property name="maxReserveBatchColumns">-1</property> <!-- - The number of batch columns guaranteed to a processing operation. Set this value lower if the workload typically - processes larger numbers of concurrent queries with large intermediate results from operations such as sorting, - grouping, etc. (default 128) + The number of batch columns guaranteed to a processing operation. -1 means to automatically calculate a value (default -1). + See the admin guide for more. --> - <property name="maxProcessingBatchesColumns">128</property> + <property name="maxProcessingBatchesColumns">-1</property> <!-- Max File size in MB (default 2GB)--> <property name="maxFileSize">2048</property> <!-- Max storage space, in MB, to be used for buffer files (default 50G) --> Modified: trunk/build/kits/jboss-container/teiid-releasenotes.html =================================================================== --- trunk/build/kits/jboss-container/teiid-releasenotes.html 2011-03-18 13:03:31 UTC (rev 3005) +++ trunk/build/kits/jboss-container/teiid-releasenotes.html 2011-03-18 14:45:33 UTC (rev 3006) @@ -41,6 +41,7 @@ <LI><B>InterSystems Cache</B> - InterSystems Cache database translator is now available to use as supported source under Teiid. <LI><B>userRequestSourceConcurrency</B> - was added to control the number of concurrent source queries allowed for each user request. <LI><B>Dependent Join Improvements</B> - dependent join analysis and costing in general was improved to consider dependent joins earlier in planning. + <LI><B>Memory Management Improvements</B> - maxReserveBatchColumns and maxProcessingBatchesColumns will be default be determined automatically and will more reliably prevent memory issues. See the admin guide for more. </UL> <h2><a name="Compatibility">Compatibility Issues</a></h2> @@ -108,7 +109,9 @@ <h4>from 7.3</h4> <ul> - <LI>SocketConfiguration.maxSocketThreads will interpret a setting of 0 to mean use the system default of max available processors. + <LI>SocketConfiguration.maxSocketThreads will interpret a setting of 0 to mean use the system default of max available processors. + <LI>maxReserveBatchColumns and maxProcessingBatchesColumns will interpret a setting of -1 to mean auto-calculate acceptable values given the max heap and other information. See the admin guide for more. + <LI>The default for org.teiid.useValueCache has changed to false, since typical installations will not greatly benefit from the additional lookup cost. </ul> <h4>from 7.2</h4> Modified: trunk/common-core/src/main/java/org/teiid/core/types/DataTypeManager.java =================================================================== --- trunk/common-core/src/main/java/org/teiid/core/types/DataTypeManager.java 2011-03-18 13:03:31 UTC (rev 3005) +++ trunk/common-core/src/main/java/org/teiid/core/types/DataTypeManager.java 2011-03-18 14:45:33 UTC (rev 3006) @@ -78,7 +78,7 @@ */ public class DataTypeManager { - private static final boolean USE_VALUE_CACHE = PropertiesUtils.getBooleanProperty(System.getProperties(), "org.teiid.useValueCache", true); //$NON-NLS-1$ + private static final boolean USE_VALUE_CACHE = PropertiesUtils.getBooleanProperty(System.getProperties(), "org.teiid.useValueCache", false); //$NON-NLS-1$ private static boolean valueCacheEnabled; @@ -488,19 +488,20 @@ dataTypeNames.put(DataTypeAliases.TINYINT, DefaultDataClasses.BYTE); dataTypeNames.put(DataTypeAliases.VARCHAR, DefaultDataClasses.STRING); + valueMaps.put(DefaultDataClasses.BOOLEAN, new ValueCache<Boolean>() { + @Override + public Boolean getValue(Boolean value) { + return Boolean.valueOf(value); + } + }); + valueMaps.put(DefaultDataClasses.BYTE, new ValueCache<Byte>() { + @Override + public Byte getValue(Byte value) { + return Byte.valueOf(value); + } + }); + if (USE_VALUE_CACHE) { - valueMaps.put(DefaultDataClasses.BOOLEAN, new ValueCache<Boolean>() { - @Override - public Boolean getValue(Boolean value) { - return Boolean.valueOf(value); - } - }); - valueMaps.put(DefaultDataClasses.BYTE, new ValueCache<Byte>() { - @Override - public Byte getValue(Byte value) { - return Byte.valueOf(value); - } - }); valueMaps.put(DefaultDataClasses.SHORT, new HashedValueCache<Short>(13)); valueMaps.put(DefaultDataClasses.CHAR, new HashedValueCache<Character>(13)); valueMaps.put(DefaultDataClasses.INTEGER, new HashedValueCache<Integer>(14)); @@ -797,13 +798,13 @@ valueCacheEnabled = enabled; } - public static boolean isValueCacheEnabled() { + public static final boolean isValueCacheEnabled() { return valueCacheEnabled; } @SuppressWarnings("unchecked") public static <T> T getCanonicalValue(T value) { - if (USE_VALUE_CACHE && valueCacheEnabled) { + if (valueCacheEnabled) { if (value == null) { return null; } Modified: trunk/documentation/admin-guide/src/main/docbook/en-US/content/appendix-c.xml =================================================================== --- trunk/documentation/admin-guide/src/main/docbook/en-US/content/appendix-c.xml 2011-03-18 13:03:31 UTC (rev 3005) +++ trunk/documentation/admin-guide/src/main/docbook/en-US/content/appendix-c.xml 2011-03-18 14:45:33 UTC (rev 3006) @@ -12,10 +12,11 @@ <para><emphasis>org.teiid.allowNanInfinity</emphasis> - defaults to false. Set to true to allow numeric functions to return NaN (Not A Number) and +-Infinity. Note that these values are not covered by the SQL specification.</para> </listitem> - <listitem> - <para><emphasis>org.teiid.useValueCache</emphasis> - defaults to true. Set to false to disable the value cache. - Value caching is used dynamically when buffer memory is running low to reuse identical values, but at a computational cost. - If there is memory available, you should first increase the buffer memory rather than disabling value caching.</para> + <listitem id="value_caching"> + <para><emphasis>org.teiid.useValueCache</emphasis> - defaults to false. Set to true to enable the canonical value cache. + Value caching is used dynamically when buffer memory is consumed to reuse identical values and thus reduce the memory consumed by Teiid. + There is a computation cost associated with the cache lookup, so enabling this setting is not appropriate for installations handling large volumes of dissimilar data. + </para> </listitem> <listitem> <para><emphasis>org.teiid.ansiQuotedIdentifiers</emphasis> - defaults to true. Modified: trunk/documentation/admin-guide/src/main/docbook/en-US/content/performance.xml =================================================================== --- trunk/documentation/admin-guide/src/main/docbook/en-US/content/performance.xml 2011-03-18 13:03:31 UTC (rev 3005) +++ trunk/documentation/admin-guide/src/main/docbook/en-US/content/performance.xml 2011-03-18 14:45:33 UTC (rev 3006) @@ -19,33 +19,44 @@ </para> <para> The <code>maxReserveBatchColumns</code> - setting determines the total number of batches (with a max of - <code>processorBatchSize</code> rows) multiplied by their column width - that can be held in memory directly by the BufferManager. + setting determines the total size of batches that can be held by the BufferManager in memory. This number does not include persistent batches held by soft (such as - index pages) or weak references. When your installation can dedicate - more memory to Teiid, consider increasing this value in proportion to the number of gigabytes you - wish Teiid to use - e.g. 2GB on a 32 bit VM would double the value to - 32768. For 64 bit VMs you should use a value of approximately 11000 - per GB. - The BufferManager automatically triggers the use of a canonical - value cache when more than 25% of the reserve is in use. + index pages) or weak references. + The value is treated internally as an approximation of bytes using the conversion + <code>maxReserveBatchColumns</code> * <code>processorBatchSize</code> * (64bytes per column value). + The default value of -1 will auto-calculate a typical max based upon the max heap available to the VM. + The auto-calculated value assumes a 64bit architecture and will limit buffer usage to 50% of the first + gigabyte of memory beyond the first 300 megabytes (which are assumed for use by the AS and other Teiid purposes) + and 75% of the memory beyond that. + </para> + <para> + The BufferManager automatically triggers the use of a canonical + value cache if enabled when more than 25% of the reserve is in use. This can dramatically cut the memory usage in situations where similar value sets are being read through Teiid, but does introduce a lookup cost. - If you are processing large (100s of MBs) of highly unique datasets - through Teiid, you should consider <link linkend="system_properties">disabling value caching</link> since it - will not significantly reduce memory consumption. - <note> - <para>The formula (average bytes per value)*(processor batch size)*(maxReserveBatchColumns) approximates the amount of - reserve memory used by the BufferManager. The defaults with an assumed value (8 bytes in 32 bit mode) for bytes per column - works out to be: 8 bytes * 512 * 16384 = 64MB. Memory consumption can be significantly more or less - depending upon actual column values and whether value caching is enabled. The 8 bytes per column includes row/batch list overhead and - represents the Java type memory foot print (e.g. java.lang.Integer not a raw 4 byte integer). - The nominal target of 64MB per 1 gigabyte of may be too low based upon the particulars of your usage. Raising the maxReserveBatchColumns - value too high though may result in out of memory errors.</para> - </note> + If you are processing small or highly similar datasets + through Teiid, and wish to conserve memory, you should consider <link linkend="value_caching">enabling value caching</link>. </para> + <note> + <para>Memory consumption can be significantly more or less than the nominal target + depending upon actual column values and whether value caching is enabled. Large strings, bigintegers, bigdecimals, or values typed as object can exceed their default size estimate. + If an out of memory errors occur, then set a lower the maxReserveBatchColumns value. + </para> + </note> <para> + The <code>maxProcessingBatchesColumns</code> + setting determines the total size of batches that can be used by active plans regardless of the memory held based on <code>maxReserveBatchColumns</code>. + The value is treated internally as an approximation of bytes using the conversion + <code>maxProcessingBatchesColumns</code> * <code>processorBatchSize</code> * (64bytes per column value). + The default value of -1 will auto-calculate a typical max based upon the max heap available to the VM and max active plans. + The auto-calculated value assumes a 64bit architecture and will limit processing batch usage to 10% of memory + beyond the first 300 megabytes (which are assumed for use by the AS and other Teiid purposes). + </para> + <para> + In systems where large intermediate results are normal (scrolling cursors or sorting over millions of rows) you can consider increasing the <code>maxProcessingBatchColumns</code> and decreasing + the <code>maxReserveBatchColumns</code> so that each request has access to an effectively smaller buffer space. + </para> + <para> Each intermediate result buffer, temporary LOB, and temporary table is stored in its own set of buffer files, where an individual file is @@ -59,10 +70,11 @@ <section> <title>Threading</title> <para>Socket threads are configured for each <link linkend="socket_transport">transport</link>. - They handle NIO non-blocking IO operations as well as directly servicing any operation that can run without blocking.</para> - <para>For longer running operations, the socket threads queue with work the query engine. - The query engine has two settings that determine its thread utilization. + They handle NIO non-blocking IO operations as well as directly servicing any operation that can run without blocking. + For longer running operations, the socket threads queue with work the query engine.</para> + <para>The query engine has several settings that determine its thread utilization. + <code>maxThreads</code> sets the total number of threads available for query engine work (processing plans, transaction control operations, processing source queries, etc.). You should consider increasing the maximum threads on systems with a large number of available processors and/or when it's common to issue non-transactional queries with that issue a large number of concurrent source requests. @@ -93,14 +105,16 @@ <title>Socket Transports</title> <para>Teiid separates the configuration of its socket transports for JDBC, ODBC, and Admin access. - Typical installations will not need to - adjust the default thread and buffer - size settings. + Typical installations will not need to adjust the default thread and buffer size settings. + The default input output buffer sizes are set to 0, which will use the system default. + Before adjusting this value keep in mind that each JDBC, ODBC, and Admin client will create a new socket connection. + Setting these values to a large buffer size should only be done if the number of client is constrained. All JDBC socket operations are non-blocking, so setting the number of maxThreads - higher than the maximum effective parallelism of the machine will not result in greater performance. The default value 0 for JDBC socket threads will - set the max to the number of available processors. - At this time, ODBC - queries are executed synchronously from the socket thread. + higher than the maximum effective parallelism of the machine should not result in greater performance. + The default value 0 for JDBC socket threads will set the max to the number of available processors. + </para> + <para> + At this time, ODBC queries are executed synchronously from the socket thread. Simultaneous long-running queries may exhaust the available threads. Consider increasing the default max threads (15) for ODBC if you expect a higher concurrent load of long-running ODBC client queries.</para> Modified: trunk/engine/src/main/java/org/teiid/common/buffer/BufferManager.java =================================================================== --- trunk/engine/src/main/java/org/teiid/common/buffer/BufferManager.java 2011-03-18 13:03:31 UTC (rev 3005) +++ trunk/engine/src/main/java/org/teiid/common/buffer/BufferManager.java 2011-03-18 14:45:33 UTC (rev 3006) @@ -25,6 +25,7 @@ import java.util.List; import org.teiid.core.TeiidComponentException; +import org.teiid.query.sql.symbol.Expression; /** @@ -50,21 +51,25 @@ } public enum BufferReserveMode { + /** + * Claim unused buffers up to the amount requested, using a progressive decaying algorithm + */ WAIT, + /** + * Claim all of the buffers requested, even if they are not available, without waiting + */ FORCE, + /** + * Claim unused buffers up to the amount requested witout waiting + */ NO_WAIT } public static int DEFAULT_CONNECTOR_BATCH_SIZE = 1024; public static int DEFAULT_PROCESSOR_BATCH_SIZE = 512; - public static int DEFAULT_MAX_PROCESSING_BATCHES = 128; + public static int DEFAULT_MAX_PROCESSING_BATCHES = -1; + public static int DEFAULT_RESERVE_BUFFERS = -1; - /** - * This is the maximum number of batch columns used for processing. - * See {@link #reserveBuffers(int, boolean)} - */ - public static int DEFAULT_RESERVE_BUFFERS = 16384; - /** * Get the batch size to use during query processing. * @return Batch size (# of rows) @@ -81,11 +86,11 @@ throws TeiidComponentException; /** - * Return the maximum number of batches that can be temporarily held potentially + * Return the maximum KB that can be temporarily held potentially * across even a blocked exception. * @return */ - int getMaxProcessingBatchColumns(); + int getMaxProcessingKB(); /** * Creates a new {@link FileStore}. See {@link FileStore#setCleanupReference(Object)} to @@ -110,13 +115,19 @@ void releaseBuffers(int count); /** - * Get the size estimate for the given schema. + * Get the size estimate in KB for the given schema. */ - int getSchemaSize(List elements); + int getSchemaSize(List<? extends Expression> elements); STree createSTree(final List elements, String groupName, int keyLength); void addTupleBuffer(TupleBuffer tb); - TupleBuffer getTupleBuffer(String id); + TupleBuffer getTupleBuffer(String id); + + /** + * Set the maxActivePlans as a hint at determining the maxProcessingKB + * @param maxActivePlans + */ + void setMaxActivePlans(int maxActivePlans); } Modified: trunk/engine/src/main/java/org/teiid/common/buffer/impl/BufferManagerImpl.java =================================================================== --- trunk/engine/src/main/java/org/teiid/common/buffer/impl/BufferManagerImpl.java 2011-03-18 13:03:31 UTC (rev 3005) +++ trunk/engine/src/main/java/org/teiid/common/buffer/impl/BufferManagerImpl.java 2011-03-18 14:45:33 UTC (rev 3006) @@ -61,6 +61,7 @@ import org.teiid.core.TeiidRuntimeException; import org.teiid.core.types.DataTypeManager; import org.teiid.core.util.Assertion; +import org.teiid.dqp.internal.process.DQPConfiguration; import org.teiid.logging.LogConstants; import org.teiid.logging.LogManager; import org.teiid.query.QueryPlugin; @@ -86,6 +87,7 @@ */ public class BufferManagerImpl implements BufferManager, StorageManager { + private static final double KB_PER_VALUE = 64d/1024; private static final int IO_BUFFER_SIZE = 1 << 14; private static final int COMPACTION_THRESHOLD = 1 << 25; //start checking at 32 megs @@ -384,8 +386,11 @@ private int processorBatchSize = BufferManager.DEFAULT_PROCESSOR_BATCH_SIZE; private int maxProcessingBatches = BufferManager.DEFAULT_MAX_PROCESSING_BATCHES; private int maxReserveBatchColumns = BufferManager.DEFAULT_RESERVE_BUFFERS; - private volatile int reserveBatchColumns = BufferManager.DEFAULT_RESERVE_BUFFERS; - + private int maxProcessingKB; + private int maxReserveBatchKB; + private volatile int reserveBatchKB; + private int maxActivePlans = DQPConfiguration.DEFAULT_MAX_ACTIVE_PLANS; //used as a hint to set the reserveBatchKB + private ReentrantLock lock = new ReentrantLock(true); private Condition batchesFreed = lock.newCondition(); @@ -421,12 +426,12 @@ } @Override - public int getMaxProcessingBatchColumns() { - return maxProcessingBatches; + public int getMaxProcessingKB() { + return maxProcessingKB; } public void setMaxProcessingBatchColumns(int maxProcessingBatches) { - this.maxProcessingBatches = Math.max(0, maxProcessingBatches); + this.maxProcessingBatches = maxProcessingBatches; } /** @@ -498,10 +503,32 @@ LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, "Creating FileStore:", name); //$NON-NLS-1$ return this.diskMgr.createFileStore(name); } + + public void setMaxActivePlans(int maxActivePlans) { + this.maxActivePlans = maxActivePlans; + } @Override public void initialize() throws TeiidComponentException { - + int maxMemory = (int)Math.min(Runtime.getRuntime().maxMemory() / 1024, Integer.MAX_VALUE); + if (maxReserveBatchColumns < 0) { + this.maxReserveBatchKB = 0; + maxMemory -= 300 * 1024; //assume 300 megs of overhead for the AS/system stuff + int one_gig = 1024 * 1024; + if (maxMemory > one_gig) { + //assume 75% of the memory over the first gig + this.maxReserveBatchKB += (int)Math.max(0, (maxMemory - one_gig) * .75); + } + this.maxReserveBatchKB += Math.max(0, Math.min(one_gig, maxMemory) * .5); + } else { + this.maxReserveBatchKB = Math.max(0, (int)Math.min(maxReserveBatchColumns * KB_PER_VALUE * processorBatchSize, Integer.MAX_VALUE)); + } + this.reserveBatchKB = this.maxReserveBatchKB; + if (this.maxProcessingBatches < 0) { + this.maxProcessingKB = (int)(.1 * maxMemory)/maxActivePlans; + } else { + this.maxProcessingKB = Math.max(0, (int)Math.min(maxProcessingBatches * KB_PER_VALUE * processorBatchSize, Integer.MAX_VALUE)); + } } @Override @@ -511,7 +538,7 @@ } lock.lock(); try { - this.reserveBatchColumns += count; + this.reserveBatchKB += count; batchesFreed.signalAll(); } finally { lock.unlock(); @@ -524,7 +551,7 @@ try { if (mode == BufferReserveMode.WAIT) { int waitCount = 0; - while (count - waitCount > this.reserveBatchColumns) { + while (count - waitCount > this.reserveBatchKB) { try { batchesFreed.await(100, TimeUnit.MILLISECONDS); } catch (InterruptedException e) { @@ -533,12 +560,12 @@ waitCount++; } } - if (this.reserveBatchColumns >= count || mode == BufferReserveMode.FORCE) { - this.reserveBatchColumns -= count; + if (this.reserveBatchKB >= count || mode == BufferReserveMode.FORCE) { + this.reserveBatchKB -= count; return count; } - int result = Math.max(0, this.reserveBatchColumns); - this.reserveBatchColumns -= result; + int result = Math.max(0, this.reserveBatchKB); + this.reserveBatchKB -= result; return result; } finally { lock.unlock(); @@ -547,8 +574,8 @@ } void persistBatchReferences() { - if (activeBatchColumnCount == 0 || activeBatchColumnCount <= reserveBatchColumns) { - int memoryCount = activeBatchColumnCount + maxReserveBatchColumns - reserveBatchColumns; + if (activeBatchColumnCount == 0 || activeBatchColumnCount <= reserveBatchKB) { + int memoryCount = activeBatchColumnCount + maxReserveBatchColumns - reserveBatchKB; if (DataTypeManager.isValueCacheEnabled()) { if (memoryCount < maxReserveBatchColumns / 8) { DataTypeManager.setValueCacheEnabled(false); @@ -561,7 +588,7 @@ while (true) { ManagedBatchImpl mb = null; synchronized (activeBatches) { - if (activeBatchColumnCount == 0 || activeBatchColumnCount * 5 < reserveBatchColumns * 4) { + if (activeBatchColumnCount == 0 || activeBatchColumnCount * 5 < reserveBatchKB * 4) { break; } Iterator<TupleBufferInfo> iter = activeBatches.values().iterator(); @@ -588,47 +615,49 @@ } @Override - public int getSchemaSize(List elements) { + public int getSchemaSize(List<? extends Expression> elements) { int total = 0; + boolean isValueCacheEnabled = DataTypeManager.isValueCacheEnabled(); //we make a assumption that the average column size under 64bits is approximately 128bytes //this includes alignment, row/array, and reference overhead - for (Object element : elements) { - Class<?> type = ((Expression)element).getType(); + for (Expression element : elements) { + Class<?> type = element.getType(); if (type == DataTypeManager.DefaultDataClasses.STRING) { - total += 256; //assumes an "average" string length of approximately 100 chars + total += isValueCacheEnabled?100:256; //assumes an "average" string length of approximately 100 chars } else if (type == DataTypeManager.DefaultDataClasses.DATE || type == DataTypeManager.DefaultDataClasses.TIME || type == DataTypeManager.DefaultDataClasses.TIMESTAMP) { - total += 32; + total += isValueCacheEnabled?20:28; } else if (type == DataTypeManager.DefaultDataClasses.LONG || type == DataTypeManager.DefaultDataClasses.DOUBLE) { - total += 20; + total += isValueCacheEnabled?12:16; } else if (type == DataTypeManager.DefaultDataClasses.INTEGER || type == DataTypeManager.DefaultDataClasses.FLOAT) { - total += 14; + total += isValueCacheEnabled?6:12; } else if (type == DataTypeManager.DefaultDataClasses.CHAR || type == DataTypeManager.DefaultDataClasses.SHORT) { - total += 10; - } else if (type == DataTypeManager.DefaultDataClasses.BOOLEAN - || type == DataTypeManager.DefaultDataClasses.BYTE - || type == DataTypeManager.DefaultDataClasses.NULL) { - //even if value caching is turned off we don't bother counting - //the additional references that may exist to boolean and byte values - total += 8; + total += isValueCacheEnabled?4:10; } else if (type == DataTypeManager.DefaultDataClasses.OBJECT) { total += 1024; + } else if (type == DataTypeManager.DefaultDataClasses.NULL) { + //it's free + } else if (type == DataTypeManager.DefaultDataClasses.BYTE) { + total += 2; //always value cached + } else if (type == DataTypeManager.DefaultDataClasses.BOOLEAN) { + total += 1; //always value cached } else { total += 512; //assumes buffer overhead in the case of lobs //however the account for lobs is misleading as the lob //references are not actually removed from memory } } - return Math.max(1, total/128); + total += 8*elements.size() + 36; // column list / row overhead + total *= processorBatchSize; + return Math.max(1, total / 1024); } public void setMaxReserveBatchColumns(int maxReserve) { this.maxReserveBatchColumns = maxReserve; - this.reserveBatchColumns = maxReserve; } public void shutdown() { @@ -652,7 +681,7 @@ private void cleanDefunctTupleBuffers() { while (true) { - Reference r = this.tupleBufferQueue.poll(); + Reference<?> r = this.tupleBufferQueue.poll(); if (r == null) { break; } Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/process/DQPCore.java =================================================================== --- trunk/engine/src/main/java/org/teiid/dqp/internal/process/DQPCore.java 2011-03-18 13:03:31 UTC (rev 3005) +++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/DQPCore.java 2011-03-18 14:45:33 UTC (rev 3006) @@ -58,6 +58,7 @@ import org.teiid.common.buffer.BufferManager; import org.teiid.core.TeiidComponentException; import org.teiid.core.TeiidProcessingException; +import org.teiid.core.TeiidRuntimeException; import org.teiid.core.types.Streamable; import org.teiid.dqp.internal.process.ThreadReuseExecutor.PrioritizedRunnable; import org.teiid.dqp.message.AtomicRequestMessage; @@ -683,6 +684,14 @@ LogManager.logWarning(LogConstants.CTX_DQP, QueryPlugin.Util.getString("DQPCore.invalid_max_active_plan", this.maxActivePlans, config.getMaxThreads())); //$NON-NLS-1$ this.maxActivePlans = config.getMaxThreads(); } + + //hack to set the max active plans + this.bufferManager.setMaxActivePlans(this.maxActivePlans); + try { + this.bufferManager.initialize(); + } catch (TeiidComponentException e) { + throw new TeiidRuntimeException(e); + } this.userRequestSourceConcurrency = config.getUserRequestSourceConcurrency(); if (this.userRequestSourceConcurrency < 1) { Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/process/MetaDataProcessor.java =================================================================== --- trunk/engine/src/main/java/org/teiid/dqp/internal/process/MetaDataProcessor.java 2011-03-18 13:03:31 UTC (rev 3005) +++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/MetaDataProcessor.java 2011-03-18 14:45:33 UTC (rev 3006) @@ -217,7 +217,7 @@ } private Map createXMLColumnMetadata(Query xmlCommand) { - GroupSymbol doc = (GroupSymbol) xmlCommand.getFrom().getGroups().get(0); + GroupSymbol doc = xmlCommand.getFrom().getGroups().get(0); Map xmlMetadata = getDefaultColumn(doc.getName(), ResultsMetadataDefaults.XML_COLUMN_NAME, XMLType.class); // Override size as XML may be big @@ -238,7 +238,7 @@ private Map createElementMetadata(String shortColumnName, ElementSymbol symbol) throws QueryMetadataException, TeiidComponentException { Object elementID = symbol.getMetadataID(); - Map column = new HashMap(); + Map<Integer, Object> column = new HashMap<Integer, Object>(); column.put(ResultsMetadataConstants.AUTO_INCREMENTING, Boolean.valueOf(metadata.elementSupports(elementID, SupportConstants.Element.AUTO_INCREMENT))); column.put(ResultsMetadataConstants.CASE_SENSITIVE, Boolean.valueOf(metadata.elementSupports(elementID, SupportConstants.Element.CASE_SENSITIVE))); column.put(ResultsMetadataConstants.CURRENCY, Boolean.FALSE); @@ -318,7 +318,7 @@ return getDefaultColumn(null, shortColumnName, symbol.getType()); } - private int getColumnPrecision(Class dataType, Object elementID) throws QueryMetadataException, TeiidComponentException { + private int getColumnPrecision(Class<?> dataType, Object elementID) throws QueryMetadataException, TeiidComponentException { if (!Number.class.isAssignableFrom(dataType)) { int length = metadata.getElementLength(elementID); if (length > 0) { @@ -346,7 +346,7 @@ * @param dataType A string representing the MetaMatrix data type of the column * @return An int value giving the displaysize of the column */ - private Integer getColumnDisplaySize(int precision, Class dataType, Object elementID) throws QueryMetadataException, TeiidComponentException { + private Integer getColumnDisplaySize(int precision, Class<?> dataType, Object elementID) throws QueryMetadataException, TeiidComponentException { if(elementID != null && dataType.equals(DataTypeManager.DefaultDataClasses.STRING)) { int length = metadata.getElementLength(elementID); Modified: trunk/engine/src/main/java/org/teiid/query/metadata/QueryMetadataInterface.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/metadata/QueryMetadataInterface.java 2011-03-18 13:03:31 UTC (rev 3005) +++ trunk/engine/src/main/java/org/teiid/query/metadata/QueryMetadataInterface.java 2011-03-18 14:45:33 UTC (rev 3006) @@ -685,7 +685,17 @@ boolean isMultiSourceElement(Object elementId) throws QueryMetadataException, TeiidComponentException; + /** + * Get the metadata without visibility and session tables + * @return + */ QueryMetadataInterface getDesignTimeMetadata(); + /** + * Return true if a procedure exists with the given name (partial or fqn) + * @param name + * @return + * @throws TeiidComponentException + */ boolean hasProcedure(String name) throws TeiidComponentException; } Modified: trunk/engine/src/main/java/org/teiid/query/processor/relational/PartitionedSortJoin.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/processor/relational/PartitionedSortJoin.java 2011-03-18 13:03:31 UTC (rev 3005) +++ trunk/engine/src/main/java/org/teiid/query/processor/relational/PartitionedSortJoin.java 2011-03-18 14:45:33 UTC (rev 3006) @@ -163,11 +163,11 @@ private boolean testAndSetPartitions(int rowCount, List elements) { int partitionCount = (rowCount / this.joinNode.getBatchSize() + rowCount % this.joinNode.getBatchSize() == 0 ? 0:1) * this.joinNode.getBufferManager().getSchemaSize(elements); - if (partitionCount > this.joinNode.getBufferManager().getMaxProcessingBatchColumns() * 8) { + if (partitionCount > this.joinNode.getBufferManager().getMaxProcessingKB() * 8) { return false; } int toReserve = Math.max(1, (int)(partitionCount * .75)); - int excess = Math.max(0, toReserve - this.joinNode.getBufferManager().getMaxProcessingBatchColumns()); + int excess = Math.max(0, toReserve - this.joinNode.getBufferManager().getMaxProcessingKB()); reserved = this.joinNode.getBufferManager().reserveBuffers(toReserve - excess, BufferReserveMode.FORCE); if (excess > 0) { reserved += this.joinNode.getBufferManager().reserveBuffers(toReserve, BufferReserveMode.NO_WAIT); Modified: trunk/engine/src/main/java/org/teiid/query/processor/relational/SortUtility.java =================================================================== --- trunk/engine/src/main/java/org/teiid/query/processor/relational/SortUtility.java 2011-03-18 13:03:31 UTC (rev 3005) +++ trunk/engine/src/main/java/org/teiid/query/processor/relational/SortUtility.java 2011-03-18 14:45:33 UTC (rev 3006) @@ -203,7 +203,7 @@ //attempt to reserve more working memory if there are additional rows available before blocking if (workingTuples.size() >= maxRows) { int reserved = bufferManager.reserveBuffers(schemaSize, - (totalReservedBuffers + schemaSize <= bufferManager.getMaxProcessingBatchColumns())?BufferReserveMode.FORCE:BufferReserveMode.NO_WAIT); + (totalReservedBuffers + schemaSize <= bufferManager.getMaxProcessingKB())?BufferReserveMode.FORCE:BufferReserveMode.NO_WAIT); if (reserved != schemaSize) { break; } @@ -268,7 +268,7 @@ TupleBuffer merged = createTupleBuffer(); int desiredSpace = activeTupleBuffers.size() * schemaSize; - int reserved = Math.min(desiredSpace, this.bufferManager.getMaxProcessingBatchColumns()); + int reserved = Math.min(desiredSpace, this.bufferManager.getMaxProcessingKB()); bufferManager.reserveBuffers(reserved, BufferReserveMode.FORCE); if (desiredSpace > reserved) { reserved += bufferManager.reserveBuffers(desiredSpace - reserved, BufferReserveMode.WAIT); Modified: trunk/engine/src/test/java/org/teiid/dqp/internal/datamgr/TestSubqueryCompareCriteriaImpl.java =================================================================== --- trunk/engine/src/test/java/org/teiid/dqp/internal/datamgr/TestSubqueryCompareCriteriaImpl.java 2011-03-18 13:03:31 UTC (rev 3005) +++ trunk/engine/src/test/java/org/teiid/dqp/internal/datamgr/TestSubqueryCompareCriteriaImpl.java 2011-03-18 14:45:33 UTC (rev 3006) @@ -53,7 +53,7 @@ } public static SubqueryComparison example() throws Exception { - return (SubqueryComparison)TstLanguageBridgeFactory.factory.translate(helpExample()); + return TstLanguageBridgeFactory.factory.translate(helpExample()); } public void testGetExpression() throws Exception { Modified: trunk/runtime/src/test/java/org/teiid/dqp/service/buffer/TestLocalBufferService.java =================================================================== --- trunk/runtime/src/test/java/org/teiid/dqp/service/buffer/TestLocalBufferService.java 2011-03-18 13:03:31 UTC (rev 3005) +++ trunk/runtime/src/test/java/org/teiid/dqp/service/buffer/TestLocalBufferService.java 2011-03-18 14:45:33 UTC (rev 3006) @@ -24,10 +24,17 @@ import static org.junit.Assert.*; +import java.util.ArrayList; +import java.util.List; + import org.junit.Test; +import org.teiid.common.buffer.BufferManager; import org.teiid.common.buffer.impl.BufferManagerImpl; import org.teiid.common.buffer.impl.FileStorageManager; +import org.teiid.core.types.DataTypeManager; import org.teiid.core.util.UnitTestUtil; +import org.teiid.query.sql.symbol.Constant; +import org.teiid.query.sql.symbol.Expression; import org.teiid.services.BufferServiceImpl; @SuppressWarnings("nls") @@ -59,4 +66,32 @@ assertFalse(svc.isUseDisk()); } + @Test public void testSchemaSize() throws Exception { + //82 strings of Total Length 2515 charcacters + //11 Dates + //1 Long + //1 short + //20 bigdecimal with 671 total integers in them. + List<Expression> schema = new ArrayList<Expression>(); + for (int i = 0; i <82; i++) { + schema.add(new Constant(null, DataTypeManager.DefaultDataClasses.STRING)); + } + for (int i = 0; i <11; i++) { + schema.add(new Constant(null, DataTypeManager.DefaultDataClasses.DATE)); + } + schema.add(new Constant(null, DataTypeManager.DefaultDataClasses.LONG)); + schema.add(new Constant(null, DataTypeManager.DefaultDataClasses.SHORT)); + for (int i = 0; i <20; i++) { + schema.add(new Constant(null, DataTypeManager.DefaultDataClasses.BIG_DECIMAL)); + } + + BufferServiceImpl svc = new BufferServiceImpl(); + svc.setDiskDirectory(UnitTestUtil.getTestScratchPath()+"/teiid/1"); + svc.setUseDisk(false); + svc.start(); + + BufferManager mgr = svc.getBufferManager(); + assertEquals(16261, mgr.getSchemaSize(schema)); + } + }