[teiid-commits] teiid SVN: r3006 - in trunk: build/kits/jboss-container/deploy/teiid and 9 other directories.
teiid-commits at lists.jboss.org
teiid-commits at lists.jboss.org
Fri Mar 18 10:45:34 EDT 2011
Author: shawkins
Date: 2011-03-18 10:45:33 -0400 (Fri, 18 Mar 2011)
New Revision: 3006
Modified:
trunk/build/kits/jboss-container/deploy/teiid/teiid-jboss-beans.xml
trunk/build/kits/jboss-container/teiid-releasenotes.html
trunk/common-core/src/main/java/org/teiid/core/types/DataTypeManager.java
trunk/documentation/admin-guide/src/main/docbook/en-US/content/appendix-c.xml
trunk/documentation/admin-guide/src/main/docbook/en-US/content/performance.xml
trunk/engine/src/main/java/org/teiid/common/buffer/BufferManager.java
trunk/engine/src/main/java/org/teiid/common/buffer/impl/BufferManagerImpl.java
trunk/engine/src/main/java/org/teiid/dqp/internal/process/DQPCore.java
trunk/engine/src/main/java/org/teiid/dqp/internal/process/MetaDataProcessor.java
trunk/engine/src/main/java/org/teiid/query/metadata/QueryMetadataInterface.java
trunk/engine/src/main/java/org/teiid/query/processor/relational/PartitionedSortJoin.java
trunk/engine/src/main/java/org/teiid/query/processor/relational/SortUtility.java
trunk/engine/src/test/java/org/teiid/dqp/internal/datamgr/TestSubqueryCompareCriteriaImpl.java
trunk/runtime/src/test/java/org/teiid/dqp/service/buffer/TestLocalBufferService.java
Log:
TEIID-1509 changing the default for value caching and cleaning up internal memory handling to use more precise byte estimates
Modified: trunk/build/kits/jboss-container/deploy/teiid/teiid-jboss-beans.xml
===================================================================
--- trunk/build/kits/jboss-container/deploy/teiid/teiid-jboss-beans.xml 2011-03-18 13:03:31 UTC (rev 3005)
+++ trunk/build/kits/jboss-container/deploy/teiid/teiid-jboss-beans.xml 2011-03-18 14:45:33 UTC (rev 3006)
@@ -25,18 +25,15 @@
<!-- The max row count of a batch from a connector. Should be even multiple of processorBatchSize. (default 1024) -->
<property name="connectorBatchSize">1024</property>
<!--
- The number of batch columns to allow in memory (default 16384).
- This value should be set lower or higher depending on the available memory to Teiid in the VM.
- 16384 is considered a good default for a dedicated 32-bit VM running Teiid with a 1 gig heap.
+ The number of batch columns to allow in buffer memory. -1 means to automatically calculate a value (default -1).
See the admin guide for more.
-->
- <property name="maxReserveBatchColumns">16384</property>
+ <property name="maxReserveBatchColumns">-1</property>
<!--
- The number of batch columns guaranteed to a processing operation. Set this value lower if the workload typically
- processes larger numbers of concurrent queries with large intermediate results from operations such as sorting,
- grouping, etc. (default 128)
+ The number of batch columns guaranteed to a processing operation. -1 means to automatically calculate a value (default -1).
+ See the admin guide for more.
-->
- <property name="maxProcessingBatchesColumns">128</property>
+ <property name="maxProcessingBatchesColumns">-1</property>
<!-- Max File size in MB (default 2GB)-->
<property name="maxFileSize">2048</property>
<!-- Max storage space, in MB, to be used for buffer files (default 50G) -->
Modified: trunk/build/kits/jboss-container/teiid-releasenotes.html
===================================================================
--- trunk/build/kits/jboss-container/teiid-releasenotes.html 2011-03-18 13:03:31 UTC (rev 3005)
+++ trunk/build/kits/jboss-container/teiid-releasenotes.html 2011-03-18 14:45:33 UTC (rev 3006)
@@ -41,6 +41,7 @@
<LI><B>InterSystems Cache</B> - InterSystems Cache database translator is now available to use as supported source under Teiid.
<LI><B>userRequestSourceConcurrency</B> - was added to control the number of concurrent source queries allowed for each user request.
<LI><B>Dependent Join Improvements</B> - dependent join analysis and costing in general was improved to consider dependent joins earlier in planning.
+ <LI><B>Memory Management Improvements</B> - maxReserveBatchColumns and maxProcessingBatchesColumns will be default be determined automatically and will more reliably prevent memory issues. See the admin guide for more.
</UL>
<h2><a name="Compatibility">Compatibility Issues</a></h2>
@@ -108,7 +109,9 @@
<h4>from 7.3</h4>
<ul>
- <LI>SocketConfiguration.maxSocketThreads will interpret a setting of 0 to mean use the system default of max available processors.
+ <LI>SocketConfiguration.maxSocketThreads will interpret a setting of 0 to mean use the system default of max available processors.
+ <LI>maxReserveBatchColumns and maxProcessingBatchesColumns will interpret a setting of -1 to mean auto-calculate acceptable values given the max heap and other information. See the admin guide for more.
+ <LI>The default for org.teiid.useValueCache has changed to false, since typical installations will not greatly benefit from the additional lookup cost.
</ul>
<h4>from 7.2</h4>
Modified: trunk/common-core/src/main/java/org/teiid/core/types/DataTypeManager.java
===================================================================
--- trunk/common-core/src/main/java/org/teiid/core/types/DataTypeManager.java 2011-03-18 13:03:31 UTC (rev 3005)
+++ trunk/common-core/src/main/java/org/teiid/core/types/DataTypeManager.java 2011-03-18 14:45:33 UTC (rev 3006)
@@ -78,7 +78,7 @@
*/
public class DataTypeManager {
- private static final boolean USE_VALUE_CACHE = PropertiesUtils.getBooleanProperty(System.getProperties(), "org.teiid.useValueCache", true); //$NON-NLS-1$
+ private static final boolean USE_VALUE_CACHE = PropertiesUtils.getBooleanProperty(System.getProperties(), "org.teiid.useValueCache", false); //$NON-NLS-1$
private static boolean valueCacheEnabled;
@@ -488,19 +488,20 @@
dataTypeNames.put(DataTypeAliases.TINYINT, DefaultDataClasses.BYTE);
dataTypeNames.put(DataTypeAliases.VARCHAR, DefaultDataClasses.STRING);
+ valueMaps.put(DefaultDataClasses.BOOLEAN, new ValueCache<Boolean>() {
+ @Override
+ public Boolean getValue(Boolean value) {
+ return Boolean.valueOf(value);
+ }
+ });
+ valueMaps.put(DefaultDataClasses.BYTE, new ValueCache<Byte>() {
+ @Override
+ public Byte getValue(Byte value) {
+ return Byte.valueOf(value);
+ }
+ });
+
if (USE_VALUE_CACHE) {
- valueMaps.put(DefaultDataClasses.BOOLEAN, new ValueCache<Boolean>() {
- @Override
- public Boolean getValue(Boolean value) {
- return Boolean.valueOf(value);
- }
- });
- valueMaps.put(DefaultDataClasses.BYTE, new ValueCache<Byte>() {
- @Override
- public Byte getValue(Byte value) {
- return Byte.valueOf(value);
- }
- });
valueMaps.put(DefaultDataClasses.SHORT, new HashedValueCache<Short>(13));
valueMaps.put(DefaultDataClasses.CHAR, new HashedValueCache<Character>(13));
valueMaps.put(DefaultDataClasses.INTEGER, new HashedValueCache<Integer>(14));
@@ -797,13 +798,13 @@
valueCacheEnabled = enabled;
}
- public static boolean isValueCacheEnabled() {
+ public static final boolean isValueCacheEnabled() {
return valueCacheEnabled;
}
@SuppressWarnings("unchecked")
public static <T> T getCanonicalValue(T value) {
- if (USE_VALUE_CACHE && valueCacheEnabled) {
+ if (valueCacheEnabled) {
if (value == null) {
return null;
}
Modified: trunk/documentation/admin-guide/src/main/docbook/en-US/content/appendix-c.xml
===================================================================
--- trunk/documentation/admin-guide/src/main/docbook/en-US/content/appendix-c.xml 2011-03-18 13:03:31 UTC (rev 3005)
+++ trunk/documentation/admin-guide/src/main/docbook/en-US/content/appendix-c.xml 2011-03-18 14:45:33 UTC (rev 3006)
@@ -12,10 +12,11 @@
<para><emphasis>org.teiid.allowNanInfinity</emphasis> - defaults to false. Set to true to allow numeric functions
to return NaN (Not A Number) and +-Infinity. Note that these values are not covered by the SQL specification.</para>
</listitem>
- <listitem>
- <para><emphasis>org.teiid.useValueCache</emphasis> - defaults to true. Set to false to disable the value cache.
- Value caching is used dynamically when buffer memory is running low to reuse identical values, but at a computational cost.
- If there is memory available, you should first increase the buffer memory rather than disabling value caching.</para>
+ <listitem id="value_caching">
+ <para><emphasis>org.teiid.useValueCache</emphasis> - defaults to false. Set to true to enable the canonical value cache.
+ Value caching is used dynamically when buffer memory is consumed to reuse identical values and thus reduce the memory consumed by Teiid.
+ There is a computation cost associated with the cache lookup, so enabling this setting is not appropriate for installations handling large volumes of dissimilar data.
+ </para>
</listitem>
<listitem>
<para><emphasis>org.teiid.ansiQuotedIdentifiers</emphasis> - defaults to true.
Modified: trunk/documentation/admin-guide/src/main/docbook/en-US/content/performance.xml
===================================================================
--- trunk/documentation/admin-guide/src/main/docbook/en-US/content/performance.xml 2011-03-18 13:03:31 UTC (rev 3005)
+++ trunk/documentation/admin-guide/src/main/docbook/en-US/content/performance.xml 2011-03-18 14:45:33 UTC (rev 3006)
@@ -19,33 +19,44 @@
</para>
<para>
The <code>maxReserveBatchColumns</code>
- setting determines the total number of batches (with a max of
- <code>processorBatchSize</code> rows) multiplied by their column width
- that can be held in memory directly by the BufferManager.
+ setting determines the total size of batches that can be held by the BufferManager in memory.
This number does not include persistent batches held by soft (such as
- index pages) or weak references. When your installation can dedicate
- more memory to Teiid, consider increasing this value in proportion to the number of gigabytes you
- wish Teiid to use - e.g. 2GB on a 32 bit VM would double the value to
- 32768. For 64 bit VMs you should use a value of approximately 11000
- per GB.
- The BufferManager automatically triggers the use of a canonical
- value cache when more than 25% of the reserve is in use.
+ index pages) or weak references.
+ The value is treated internally as an approximation of bytes using the conversion
+ <code>maxReserveBatchColumns</code> * <code>processorBatchSize</code> * (64bytes per column value).
+ The default value of -1 will auto-calculate a typical max based upon the max heap available to the VM.
+ The auto-calculated value assumes a 64bit architecture and will limit buffer usage to 50% of the first
+ gigabyte of memory beyond the first 300 megabytes (which are assumed for use by the AS and other Teiid purposes)
+ and 75% of the memory beyond that.
+ </para>
+ <para>
+ The BufferManager automatically triggers the use of a canonical
+ value cache if enabled when more than 25% of the reserve is in use.
This can dramatically cut the memory usage in situations where similar
value sets are being read through Teiid, but does introduce a lookup cost.
- If you are processing large (100s of MBs) of highly unique datasets
- through Teiid, you should consider <link linkend="system_properties">disabling value caching</link> since it
- will not significantly reduce memory consumption.
- <note>
- <para>The formula (average bytes per value)*(processor batch size)*(maxReserveBatchColumns) approximates the amount of
- reserve memory used by the BufferManager. The defaults with an assumed value (8 bytes in 32 bit mode) for bytes per column
- works out to be: 8 bytes * 512 * 16384 = 64MB. Memory consumption can be significantly more or less
- depending upon actual column values and whether value caching is enabled. The 8 bytes per column includes row/batch list overhead and
- represents the Java type memory foot print (e.g. java.lang.Integer not a raw 4 byte integer).
- The nominal target of 64MB per 1 gigabyte of may be too low based upon the particulars of your usage. Raising the maxReserveBatchColumns
- value too high though may result in out of memory errors.</para>
- </note>
+ If you are processing small or highly similar datasets
+ through Teiid, and wish to conserve memory, you should consider <link linkend="value_caching">enabling value caching</link>.
</para>
+ <note>
+ <para>Memory consumption can be significantly more or less than the nominal target
+ depending upon actual column values and whether value caching is enabled. Large strings, bigintegers, bigdecimals, or values typed as object can exceed their default size estimate.
+ If an out of memory errors occur, then set a lower the maxReserveBatchColumns value.
+ </para>
+ </note>
<para>
+ The <code>maxProcessingBatchesColumns</code>
+ setting determines the total size of batches that can be used by active plans regardless of the memory held based on <code>maxReserveBatchColumns</code>.
+ The value is treated internally as an approximation of bytes using the conversion
+ <code>maxProcessingBatchesColumns</code> * <code>processorBatchSize</code> * (64bytes per column value).
+ The default value of -1 will auto-calculate a typical max based upon the max heap available to the VM and max active plans.
+ The auto-calculated value assumes a 64bit architecture and will limit processing batch usage to 10% of memory
+ beyond the first 300 megabytes (which are assumed for use by the AS and other Teiid purposes).
+ </para>
+ <para>
+ In systems where large intermediate results are normal (scrolling cursors or sorting over millions of rows) you can consider increasing the <code>maxProcessingBatchColumns</code> and decreasing
+ the <code>maxReserveBatchColumns</code> so that each request has access to an effectively smaller buffer space.
+ </para>
+ <para>
Each intermediate result buffer, temporary LOB, and temporary table
is stored in its own set of buffer files,
where an individual file is
@@ -59,10 +70,11 @@
<section>
<title>Threading</title>
<para>Socket threads are configured for each <link linkend="socket_transport">transport</link>.
- They handle NIO non-blocking IO operations as well as directly servicing any operation that can run without blocking.</para>
- <para>For longer running operations, the socket threads queue with work the query engine.
- The query engine has two settings that determine its thread utilization.
+ They handle NIO non-blocking IO operations as well as directly servicing any operation that can run without blocking.
+ For longer running operations, the socket threads queue with work the query engine.</para>
+ <para>The query engine has several settings that determine its thread utilization.
+
<code>maxThreads</code> sets the total number of threads available for query engine work (processing plans, transaction control operations, processing source queries, etc.).
You should consider increasing the maximum threads on systems with a large number of available processors and/or when it's common to issue non-transactional queries with that
issue a large number of concurrent source requests.
@@ -93,14 +105,16 @@
<title>Socket Transports</title>
<para>Teiid separates the configuration of its socket transports for
JDBC, ODBC, and Admin access.
- Typical installations will not need to
- adjust the default thread and buffer
- size settings.
+ Typical installations will not need to adjust the default thread and buffer size settings.
+ The default input output buffer sizes are set to 0, which will use the system default.
+ Before adjusting this value keep in mind that each JDBC, ODBC, and Admin client will create a new socket connection.
+ Setting these values to a large buffer size should only be done if the number of client is constrained.
All JDBC socket operations are non-blocking, so setting the number of maxThreads
- higher than the maximum effective parallelism of the machine will not result in greater performance. The default value 0 for JDBC socket threads will
- set the max to the number of available processors.
- At this time, ODBC
- queries are executed synchronously from the socket thread.
+ higher than the maximum effective parallelism of the machine should not result in greater performance.
+ The default value 0 for JDBC socket threads will set the max to the number of available processors.
+ </para>
+ <para>
+ At this time, ODBC queries are executed synchronously from the socket thread.
Simultaneous long-running queries may exhaust the available threads.
Consider increasing the default max threads (15) for ODBC if you
expect a higher concurrent load of long-running ODBC client queries.</para>
Modified: trunk/engine/src/main/java/org/teiid/common/buffer/BufferManager.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/common/buffer/BufferManager.java 2011-03-18 13:03:31 UTC (rev 3005)
+++ trunk/engine/src/main/java/org/teiid/common/buffer/BufferManager.java 2011-03-18 14:45:33 UTC (rev 3006)
@@ -25,6 +25,7 @@
import java.util.List;
import org.teiid.core.TeiidComponentException;
+import org.teiid.query.sql.symbol.Expression;
/**
@@ -50,21 +51,25 @@
}
public enum BufferReserveMode {
+ /**
+ * Claim unused buffers up to the amount requested, using a progressive decaying algorithm
+ */
WAIT,
+ /**
+ * Claim all of the buffers requested, even if they are not available, without waiting
+ */
FORCE,
+ /**
+ * Claim unused buffers up to the amount requested witout waiting
+ */
NO_WAIT
}
public static int DEFAULT_CONNECTOR_BATCH_SIZE = 1024;
public static int DEFAULT_PROCESSOR_BATCH_SIZE = 512;
- public static int DEFAULT_MAX_PROCESSING_BATCHES = 128;
+ public static int DEFAULT_MAX_PROCESSING_BATCHES = -1;
+ public static int DEFAULT_RESERVE_BUFFERS = -1;
- /**
- * This is the maximum number of batch columns used for processing.
- * See {@link #reserveBuffers(int, boolean)}
- */
- public static int DEFAULT_RESERVE_BUFFERS = 16384;
-
/**
* Get the batch size to use during query processing.
* @return Batch size (# of rows)
@@ -81,11 +86,11 @@
throws TeiidComponentException;
/**
- * Return the maximum number of batches that can be temporarily held potentially
+ * Return the maximum KB that can be temporarily held potentially
* across even a blocked exception.
* @return
*/
- int getMaxProcessingBatchColumns();
+ int getMaxProcessingKB();
/**
* Creates a new {@link FileStore}. See {@link FileStore#setCleanupReference(Object)} to
@@ -110,13 +115,19 @@
void releaseBuffers(int count);
/**
- * Get the size estimate for the given schema.
+ * Get the size estimate in KB for the given schema.
*/
- int getSchemaSize(List elements);
+ int getSchemaSize(List<? extends Expression> elements);
STree createSTree(final List elements, String groupName, int keyLength);
void addTupleBuffer(TupleBuffer tb);
- TupleBuffer getTupleBuffer(String id);
+ TupleBuffer getTupleBuffer(String id);
+
+ /**
+ * Set the maxActivePlans as a hint at determining the maxProcessingKB
+ * @param maxActivePlans
+ */
+ void setMaxActivePlans(int maxActivePlans);
}
Modified: trunk/engine/src/main/java/org/teiid/common/buffer/impl/BufferManagerImpl.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/common/buffer/impl/BufferManagerImpl.java 2011-03-18 13:03:31 UTC (rev 3005)
+++ trunk/engine/src/main/java/org/teiid/common/buffer/impl/BufferManagerImpl.java 2011-03-18 14:45:33 UTC (rev 3006)
@@ -61,6 +61,7 @@
import org.teiid.core.TeiidRuntimeException;
import org.teiid.core.types.DataTypeManager;
import org.teiid.core.util.Assertion;
+import org.teiid.dqp.internal.process.DQPConfiguration;
import org.teiid.logging.LogConstants;
import org.teiid.logging.LogManager;
import org.teiid.query.QueryPlugin;
@@ -86,6 +87,7 @@
*/
public class BufferManagerImpl implements BufferManager, StorageManager {
+ private static final double KB_PER_VALUE = 64d/1024;
private static final int IO_BUFFER_SIZE = 1 << 14;
private static final int COMPACTION_THRESHOLD = 1 << 25; //start checking at 32 megs
@@ -384,8 +386,11 @@
private int processorBatchSize = BufferManager.DEFAULT_PROCESSOR_BATCH_SIZE;
private int maxProcessingBatches = BufferManager.DEFAULT_MAX_PROCESSING_BATCHES;
private int maxReserveBatchColumns = BufferManager.DEFAULT_RESERVE_BUFFERS;
- private volatile int reserveBatchColumns = BufferManager.DEFAULT_RESERVE_BUFFERS;
-
+ private int maxProcessingKB;
+ private int maxReserveBatchKB;
+ private volatile int reserveBatchKB;
+ private int maxActivePlans = DQPConfiguration.DEFAULT_MAX_ACTIVE_PLANS; //used as a hint to set the reserveBatchKB
+
private ReentrantLock lock = new ReentrantLock(true);
private Condition batchesFreed = lock.newCondition();
@@ -421,12 +426,12 @@
}
@Override
- public int getMaxProcessingBatchColumns() {
- return maxProcessingBatches;
+ public int getMaxProcessingKB() {
+ return maxProcessingKB;
}
public void setMaxProcessingBatchColumns(int maxProcessingBatches) {
- this.maxProcessingBatches = Math.max(0, maxProcessingBatches);
+ this.maxProcessingBatches = maxProcessingBatches;
}
/**
@@ -498,10 +503,32 @@
LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, "Creating FileStore:", name); //$NON-NLS-1$
return this.diskMgr.createFileStore(name);
}
+
+ public void setMaxActivePlans(int maxActivePlans) {
+ this.maxActivePlans = maxActivePlans;
+ }
@Override
public void initialize() throws TeiidComponentException {
-
+ int maxMemory = (int)Math.min(Runtime.getRuntime().maxMemory() / 1024, Integer.MAX_VALUE);
+ if (maxReserveBatchColumns < 0) {
+ this.maxReserveBatchKB = 0;
+ maxMemory -= 300 * 1024; //assume 300 megs of overhead for the AS/system stuff
+ int one_gig = 1024 * 1024;
+ if (maxMemory > one_gig) {
+ //assume 75% of the memory over the first gig
+ this.maxReserveBatchKB += (int)Math.max(0, (maxMemory - one_gig) * .75);
+ }
+ this.maxReserveBatchKB += Math.max(0, Math.min(one_gig, maxMemory) * .5);
+ } else {
+ this.maxReserveBatchKB = Math.max(0, (int)Math.min(maxReserveBatchColumns * KB_PER_VALUE * processorBatchSize, Integer.MAX_VALUE));
+ }
+ this.reserveBatchKB = this.maxReserveBatchKB;
+ if (this.maxProcessingBatches < 0) {
+ this.maxProcessingKB = (int)(.1 * maxMemory)/maxActivePlans;
+ } else {
+ this.maxProcessingKB = Math.max(0, (int)Math.min(maxProcessingBatches * KB_PER_VALUE * processorBatchSize, Integer.MAX_VALUE));
+ }
}
@Override
@@ -511,7 +538,7 @@
}
lock.lock();
try {
- this.reserveBatchColumns += count;
+ this.reserveBatchKB += count;
batchesFreed.signalAll();
} finally {
lock.unlock();
@@ -524,7 +551,7 @@
try {
if (mode == BufferReserveMode.WAIT) {
int waitCount = 0;
- while (count - waitCount > this.reserveBatchColumns) {
+ while (count - waitCount > this.reserveBatchKB) {
try {
batchesFreed.await(100, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
@@ -533,12 +560,12 @@
waitCount++;
}
}
- if (this.reserveBatchColumns >= count || mode == BufferReserveMode.FORCE) {
- this.reserveBatchColumns -= count;
+ if (this.reserveBatchKB >= count || mode == BufferReserveMode.FORCE) {
+ this.reserveBatchKB -= count;
return count;
}
- int result = Math.max(0, this.reserveBatchColumns);
- this.reserveBatchColumns -= result;
+ int result = Math.max(0, this.reserveBatchKB);
+ this.reserveBatchKB -= result;
return result;
} finally {
lock.unlock();
@@ -547,8 +574,8 @@
}
void persistBatchReferences() {
- if (activeBatchColumnCount == 0 || activeBatchColumnCount <= reserveBatchColumns) {
- int memoryCount = activeBatchColumnCount + maxReserveBatchColumns - reserveBatchColumns;
+ if (activeBatchColumnCount == 0 || activeBatchColumnCount <= reserveBatchKB) {
+ int memoryCount = activeBatchColumnCount + maxReserveBatchColumns - reserveBatchKB;
if (DataTypeManager.isValueCacheEnabled()) {
if (memoryCount < maxReserveBatchColumns / 8) {
DataTypeManager.setValueCacheEnabled(false);
@@ -561,7 +588,7 @@
while (true) {
ManagedBatchImpl mb = null;
synchronized (activeBatches) {
- if (activeBatchColumnCount == 0 || activeBatchColumnCount * 5 < reserveBatchColumns * 4) {
+ if (activeBatchColumnCount == 0 || activeBatchColumnCount * 5 < reserveBatchKB * 4) {
break;
}
Iterator<TupleBufferInfo> iter = activeBatches.values().iterator();
@@ -588,47 +615,49 @@
}
@Override
- public int getSchemaSize(List elements) {
+ public int getSchemaSize(List<? extends Expression> elements) {
int total = 0;
+ boolean isValueCacheEnabled = DataTypeManager.isValueCacheEnabled();
//we make a assumption that the average column size under 64bits is approximately 128bytes
//this includes alignment, row/array, and reference overhead
- for (Object element : elements) {
- Class<?> type = ((Expression)element).getType();
+ for (Expression element : elements) {
+ Class<?> type = element.getType();
if (type == DataTypeManager.DefaultDataClasses.STRING) {
- total += 256; //assumes an "average" string length of approximately 100 chars
+ total += isValueCacheEnabled?100:256; //assumes an "average" string length of approximately 100 chars
} else if (type == DataTypeManager.DefaultDataClasses.DATE
|| type == DataTypeManager.DefaultDataClasses.TIME
|| type == DataTypeManager.DefaultDataClasses.TIMESTAMP) {
- total += 32;
+ total += isValueCacheEnabled?20:28;
} else if (type == DataTypeManager.DefaultDataClasses.LONG
|| type == DataTypeManager.DefaultDataClasses.DOUBLE) {
- total += 20;
+ total += isValueCacheEnabled?12:16;
} else if (type == DataTypeManager.DefaultDataClasses.INTEGER
|| type == DataTypeManager.DefaultDataClasses.FLOAT) {
- total += 14;
+ total += isValueCacheEnabled?6:12;
} else if (type == DataTypeManager.DefaultDataClasses.CHAR
|| type == DataTypeManager.DefaultDataClasses.SHORT) {
- total += 10;
- } else if (type == DataTypeManager.DefaultDataClasses.BOOLEAN
- || type == DataTypeManager.DefaultDataClasses.BYTE
- || type == DataTypeManager.DefaultDataClasses.NULL) {
- //even if value caching is turned off we don't bother counting
- //the additional references that may exist to boolean and byte values
- total += 8;
+ total += isValueCacheEnabled?4:10;
} else if (type == DataTypeManager.DefaultDataClasses.OBJECT) {
total += 1024;
+ } else if (type == DataTypeManager.DefaultDataClasses.NULL) {
+ //it's free
+ } else if (type == DataTypeManager.DefaultDataClasses.BYTE) {
+ total += 2; //always value cached
+ } else if (type == DataTypeManager.DefaultDataClasses.BOOLEAN) {
+ total += 1; //always value cached
} else {
total += 512; //assumes buffer overhead in the case of lobs
//however the account for lobs is misleading as the lob
//references are not actually removed from memory
}
}
- return Math.max(1, total/128);
+ total += 8*elements.size() + 36; // column list / row overhead
+ total *= processorBatchSize;
+ return Math.max(1, total / 1024);
}
public void setMaxReserveBatchColumns(int maxReserve) {
this.maxReserveBatchColumns = maxReserve;
- this.reserveBatchColumns = maxReserve;
}
public void shutdown() {
@@ -652,7 +681,7 @@
private void cleanDefunctTupleBuffers() {
while (true) {
- Reference r = this.tupleBufferQueue.poll();
+ Reference<?> r = this.tupleBufferQueue.poll();
if (r == null) {
break;
}
Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/process/DQPCore.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/dqp/internal/process/DQPCore.java 2011-03-18 13:03:31 UTC (rev 3005)
+++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/DQPCore.java 2011-03-18 14:45:33 UTC (rev 3006)
@@ -58,6 +58,7 @@
import org.teiid.common.buffer.BufferManager;
import org.teiid.core.TeiidComponentException;
import org.teiid.core.TeiidProcessingException;
+import org.teiid.core.TeiidRuntimeException;
import org.teiid.core.types.Streamable;
import org.teiid.dqp.internal.process.ThreadReuseExecutor.PrioritizedRunnable;
import org.teiid.dqp.message.AtomicRequestMessage;
@@ -683,6 +684,14 @@
LogManager.logWarning(LogConstants.CTX_DQP, QueryPlugin.Util.getString("DQPCore.invalid_max_active_plan", this.maxActivePlans, config.getMaxThreads())); //$NON-NLS-1$
this.maxActivePlans = config.getMaxThreads();
}
+
+ //hack to set the max active plans
+ this.bufferManager.setMaxActivePlans(this.maxActivePlans);
+ try {
+ this.bufferManager.initialize();
+ } catch (TeiidComponentException e) {
+ throw new TeiidRuntimeException(e);
+ }
this.userRequestSourceConcurrency = config.getUserRequestSourceConcurrency();
if (this.userRequestSourceConcurrency < 1) {
Modified: trunk/engine/src/main/java/org/teiid/dqp/internal/process/MetaDataProcessor.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/dqp/internal/process/MetaDataProcessor.java 2011-03-18 13:03:31 UTC (rev 3005)
+++ trunk/engine/src/main/java/org/teiid/dqp/internal/process/MetaDataProcessor.java 2011-03-18 14:45:33 UTC (rev 3006)
@@ -217,7 +217,7 @@
}
private Map createXMLColumnMetadata(Query xmlCommand) {
- GroupSymbol doc = (GroupSymbol) xmlCommand.getFrom().getGroups().get(0);
+ GroupSymbol doc = xmlCommand.getFrom().getGroups().get(0);
Map xmlMetadata = getDefaultColumn(doc.getName(), ResultsMetadataDefaults.XML_COLUMN_NAME, XMLType.class);
// Override size as XML may be big
@@ -238,7 +238,7 @@
private Map createElementMetadata(String shortColumnName, ElementSymbol symbol) throws QueryMetadataException, TeiidComponentException {
Object elementID = symbol.getMetadataID();
- Map column = new HashMap();
+ Map<Integer, Object> column = new HashMap<Integer, Object>();
column.put(ResultsMetadataConstants.AUTO_INCREMENTING, Boolean.valueOf(metadata.elementSupports(elementID, SupportConstants.Element.AUTO_INCREMENT)));
column.put(ResultsMetadataConstants.CASE_SENSITIVE, Boolean.valueOf(metadata.elementSupports(elementID, SupportConstants.Element.CASE_SENSITIVE)));
column.put(ResultsMetadataConstants.CURRENCY, Boolean.FALSE);
@@ -318,7 +318,7 @@
return getDefaultColumn(null, shortColumnName, symbol.getType());
}
- private int getColumnPrecision(Class dataType, Object elementID) throws QueryMetadataException, TeiidComponentException {
+ private int getColumnPrecision(Class<?> dataType, Object elementID) throws QueryMetadataException, TeiidComponentException {
if (!Number.class.isAssignableFrom(dataType)) {
int length = metadata.getElementLength(elementID);
if (length > 0) {
@@ -346,7 +346,7 @@
* @param dataType A string representing the MetaMatrix data type of the column
* @return An int value giving the displaysize of the column
*/
- private Integer getColumnDisplaySize(int precision, Class dataType, Object elementID) throws QueryMetadataException, TeiidComponentException {
+ private Integer getColumnDisplaySize(int precision, Class<?> dataType, Object elementID) throws QueryMetadataException, TeiidComponentException {
if(elementID != null && dataType.equals(DataTypeManager.DefaultDataClasses.STRING)) {
int length = metadata.getElementLength(elementID);
Modified: trunk/engine/src/main/java/org/teiid/query/metadata/QueryMetadataInterface.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/query/metadata/QueryMetadataInterface.java 2011-03-18 13:03:31 UTC (rev 3005)
+++ trunk/engine/src/main/java/org/teiid/query/metadata/QueryMetadataInterface.java 2011-03-18 14:45:33 UTC (rev 3006)
@@ -685,7 +685,17 @@
boolean isMultiSourceElement(Object elementId) throws QueryMetadataException, TeiidComponentException;
+ /**
+ * Get the metadata without visibility and session tables
+ * @return
+ */
QueryMetadataInterface getDesignTimeMetadata();
+ /**
+ * Return true if a procedure exists with the given name (partial or fqn)
+ * @param name
+ * @return
+ * @throws TeiidComponentException
+ */
boolean hasProcedure(String name) throws TeiidComponentException;
}
Modified: trunk/engine/src/main/java/org/teiid/query/processor/relational/PartitionedSortJoin.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/query/processor/relational/PartitionedSortJoin.java 2011-03-18 13:03:31 UTC (rev 3005)
+++ trunk/engine/src/main/java/org/teiid/query/processor/relational/PartitionedSortJoin.java 2011-03-18 14:45:33 UTC (rev 3006)
@@ -163,11 +163,11 @@
private boolean testAndSetPartitions(int rowCount, List elements) {
int partitionCount = (rowCount / this.joinNode.getBatchSize() + rowCount % this.joinNode.getBatchSize() == 0 ? 0:1)
* this.joinNode.getBufferManager().getSchemaSize(elements);
- if (partitionCount > this.joinNode.getBufferManager().getMaxProcessingBatchColumns() * 8) {
+ if (partitionCount > this.joinNode.getBufferManager().getMaxProcessingKB() * 8) {
return false;
}
int toReserve = Math.max(1, (int)(partitionCount * .75));
- int excess = Math.max(0, toReserve - this.joinNode.getBufferManager().getMaxProcessingBatchColumns());
+ int excess = Math.max(0, toReserve - this.joinNode.getBufferManager().getMaxProcessingKB());
reserved = this.joinNode.getBufferManager().reserveBuffers(toReserve - excess, BufferReserveMode.FORCE);
if (excess > 0) {
reserved += this.joinNode.getBufferManager().reserveBuffers(toReserve, BufferReserveMode.NO_WAIT);
Modified: trunk/engine/src/main/java/org/teiid/query/processor/relational/SortUtility.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/query/processor/relational/SortUtility.java 2011-03-18 13:03:31 UTC (rev 3005)
+++ trunk/engine/src/main/java/org/teiid/query/processor/relational/SortUtility.java 2011-03-18 14:45:33 UTC (rev 3006)
@@ -203,7 +203,7 @@
//attempt to reserve more working memory if there are additional rows available before blocking
if (workingTuples.size() >= maxRows) {
int reserved = bufferManager.reserveBuffers(schemaSize,
- (totalReservedBuffers + schemaSize <= bufferManager.getMaxProcessingBatchColumns())?BufferReserveMode.FORCE:BufferReserveMode.NO_WAIT);
+ (totalReservedBuffers + schemaSize <= bufferManager.getMaxProcessingKB())?BufferReserveMode.FORCE:BufferReserveMode.NO_WAIT);
if (reserved != schemaSize) {
break;
}
@@ -268,7 +268,7 @@
TupleBuffer merged = createTupleBuffer();
int desiredSpace = activeTupleBuffers.size() * schemaSize;
- int reserved = Math.min(desiredSpace, this.bufferManager.getMaxProcessingBatchColumns());
+ int reserved = Math.min(desiredSpace, this.bufferManager.getMaxProcessingKB());
bufferManager.reserveBuffers(reserved, BufferReserveMode.FORCE);
if (desiredSpace > reserved) {
reserved += bufferManager.reserveBuffers(desiredSpace - reserved, BufferReserveMode.WAIT);
Modified: trunk/engine/src/test/java/org/teiid/dqp/internal/datamgr/TestSubqueryCompareCriteriaImpl.java
===================================================================
--- trunk/engine/src/test/java/org/teiid/dqp/internal/datamgr/TestSubqueryCompareCriteriaImpl.java 2011-03-18 13:03:31 UTC (rev 3005)
+++ trunk/engine/src/test/java/org/teiid/dqp/internal/datamgr/TestSubqueryCompareCriteriaImpl.java 2011-03-18 14:45:33 UTC (rev 3006)
@@ -53,7 +53,7 @@
}
public static SubqueryComparison example() throws Exception {
- return (SubqueryComparison)TstLanguageBridgeFactory.factory.translate(helpExample());
+ return TstLanguageBridgeFactory.factory.translate(helpExample());
}
public void testGetExpression() throws Exception {
Modified: trunk/runtime/src/test/java/org/teiid/dqp/service/buffer/TestLocalBufferService.java
===================================================================
--- trunk/runtime/src/test/java/org/teiid/dqp/service/buffer/TestLocalBufferService.java 2011-03-18 13:03:31 UTC (rev 3005)
+++ trunk/runtime/src/test/java/org/teiid/dqp/service/buffer/TestLocalBufferService.java 2011-03-18 14:45:33 UTC (rev 3006)
@@ -24,10 +24,17 @@
import static org.junit.Assert.*;
+import java.util.ArrayList;
+import java.util.List;
+
import org.junit.Test;
+import org.teiid.common.buffer.BufferManager;
import org.teiid.common.buffer.impl.BufferManagerImpl;
import org.teiid.common.buffer.impl.FileStorageManager;
+import org.teiid.core.types.DataTypeManager;
import org.teiid.core.util.UnitTestUtil;
+import org.teiid.query.sql.symbol.Constant;
+import org.teiid.query.sql.symbol.Expression;
import org.teiid.services.BufferServiceImpl;
@SuppressWarnings("nls")
@@ -59,4 +66,32 @@
assertFalse(svc.isUseDisk());
}
+ @Test public void testSchemaSize() throws Exception {
+ //82 strings of Total Length 2515 charcacters
+ //11 Dates
+ //1 Long
+ //1 short
+ //20 bigdecimal with 671 total integers in them.
+ List<Expression> schema = new ArrayList<Expression>();
+ for (int i = 0; i <82; i++) {
+ schema.add(new Constant(null, DataTypeManager.DefaultDataClasses.STRING));
+ }
+ for (int i = 0; i <11; i++) {
+ schema.add(new Constant(null, DataTypeManager.DefaultDataClasses.DATE));
+ }
+ schema.add(new Constant(null, DataTypeManager.DefaultDataClasses.LONG));
+ schema.add(new Constant(null, DataTypeManager.DefaultDataClasses.SHORT));
+ for (int i = 0; i <20; i++) {
+ schema.add(new Constant(null, DataTypeManager.DefaultDataClasses.BIG_DECIMAL));
+ }
+
+ BufferServiceImpl svc = new BufferServiceImpl();
+ svc.setDiskDirectory(UnitTestUtil.getTestScratchPath()+"/teiid/1");
+ svc.setUseDisk(false);
+ svc.start();
+
+ BufferManager mgr = svc.getBufferManager();
+ assertEquals(16261, mgr.getSchemaSize(schema));
+ }
+
}
More information about the teiid-commits
mailing list