[teiid-commits] teiid SVN: r3544 - in trunk/engine/src: test/java/org/teiid/common/buffer/impl and 1 other directory.
teiid-commits at lists.jboss.org
teiid-commits at lists.jboss.org
Fri Oct 7 12:52:06 EDT 2011
Author: shawkins
Date: 2011-10-07 12:52:06 -0400 (Fri, 07 Oct 2011)
New Revision: 3544
Modified:
trunk/engine/src/main/java/org/teiid/common/buffer/impl/BufferManagerImpl.java
trunk/engine/src/main/java/org/teiid/common/buffer/impl/SizeUtility.java
trunk/engine/src/test/java/org/teiid/common/buffer/impl/TestSizeUtility.java
Log:
TEIID-1750 optimizing sizeutility and correcting the reference initialization logic
Modified: trunk/engine/src/main/java/org/teiid/common/buffer/impl/BufferManagerImpl.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/common/buffer/impl/BufferManagerImpl.java 2011-10-07 15:25:46 UTC (rev 3543)
+++ trunk/engine/src/main/java/org/teiid/common/buffer/impl/BufferManagerImpl.java 2011-10-07 16:52:06 UTC (rev 3544)
@@ -34,6 +34,7 @@
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
+import java.util.ListIterator;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
@@ -87,10 +88,13 @@
String[] types;
private LobManager lobManager;
- private BatchManagerImpl(Long newID, String[] types) {
+ private BatchManagerImpl(Long newID, Class<?>[] types) {
this.id = newID;
this.sizeUtility = new SizeUtility(types);
- this.types = types;
+ this.types = new String[types.length];
+ for (int i = 0; i < types.length; i++) {
+ this.types[i] = DataTypeManager.getDataTypeName(types[i]);
+ }
cache.createCacheGroup(newID);
}
@@ -381,7 +385,7 @@
TupleSourceType tupleSourceType) {
final Long newID = this.tsId.getAndIncrement();
int[] lobIndexes = LobManager.getLobIndexes(elements);
- String[] types = TupleBuffer.getTypeNames(elements);
+ Class<?>[] types = getTypeClasses(elements);
BatchManagerImpl batchManager = createBatchManager(newID, types);
LobManager lobManager = null;
FileStore lobStore = null;
@@ -404,7 +408,7 @@
public STree createSTree(final List elements, String groupName, int keyLength) {
Long newID = this.tsId.getAndIncrement();
int[] lobIndexes = LobManager.getLobIndexes(elements);
- String[] types = TupleBuffer.getTypeNames(elements);
+ Class<?>[] types = getTypeClasses(elements);
BatchManagerImpl bm = createBatchManager(newID, types);
LobManager lobManager = null;
if (lobIndexes != null) {
@@ -422,7 +426,16 @@
return new STree(keyManager, bm, new ListNestedSortComparator(compareIndexes), getProcessorBatchSize(elements), getProcessorBatchSize(elements.subList(0, keyLength)), keyLength, lobManager);
}
- private BatchManagerImpl createBatchManager(final Long newID, String[] types) {
+ private static Class<?>[] getTypeClasses(final List elements) {
+ Class<?>[] types = new Class[elements.size()];
+ for (ListIterator<? extends Expression> i = elements.listIterator(); i.hasNext();) {
+ Expression expr = i.next();
+ types[i.previousIndex()] = expr.getType();
+ }
+ return types;
+ }
+
+ private BatchManagerImpl createBatchManager(final Long newID, Class<?>[] types) {
BatchManagerImpl bm = new BatchManagerImpl(newID, types);
final AtomicBoolean prefersMemory = bm.prefersMemory;
AutoCleanupUtil.setCleanupReference(bm, new Removable() {
@@ -480,12 +493,14 @@
if (this.maxProcessingKBOrig < 0) {
this.maxProcessingKB = Math.max(Math.min(8 * processorBatchSize, Integer.MAX_VALUE), (int)(.1 * maxMemory)/maxActivePlans);
}
- int memoryBatches = (this.maxProcessingKB * maxActivePlans + this.getMaxReserveKB()) / (processorBatchSize * targetBytesPerRow / 1024);
- int logSize = 39 - Integer.numberOfLeadingZeros(memoryBatches);
+ //make a guess at the max number of batches
+ int memoryBatches = maxMemory / (processorBatchSize * targetBytesPerRow / 1024);
+ //memoryBatches represents a full batch, so assume that most will be smaller
+ int logSize = 35 - Integer.numberOfLeadingZeros(memoryBatches);
if (useWeakReferences) {
- weakReferenceCache = new WeakReferenceHashedValueCache<CacheEntry>(Math.min(20, logSize));
+ weakReferenceCache = new WeakReferenceHashedValueCache<CacheEntry>(Math.min(30, logSize));
}
- this.maxSoftReferences = 1 << Math.max(28, logSize+2);
+ this.maxSoftReferences = 1 << Math.min(30, logSize);
}
@Override
@@ -538,8 +553,9 @@
}
void persistBatchReferences() {
- if (activeBatchKB.get() == 0 || activeBatchKB.get() <= reserveBatchKB) {
- int memoryCount = activeBatchKB.get() + getMaxReserveKB() - reserveBatchKB;
+ int activeBatch = activeBatchKB.get();
+ if (activeBatch <= reserveBatchKB) {
+ int memoryCount = activeBatch + getMaxReserveKB() - reserveBatchKB;
if (DataTypeManager.isValueCacheEnabled()) {
if (memoryCount < getMaxReserveKB() / 8) {
DataTypeManager.setValueCacheEnabled(false);
@@ -551,11 +567,11 @@
}
int maxToFree = Math.max(maxProcessingKB>>1, reserveBatchKB>>3);
int freed = 0;
- while (true) {
- if (freed > maxToFree || activeBatchKB.get() == 0 || activeBatchKB.get() < reserveBatchKB * .8) {
+ while (freed <= maxToFree && activeBatchKB.get() > reserveBatchKB * .8) {
+ CacheEntry ce = memoryEntries.evict();
+ if (ce == null) {
break;
}
- CacheEntry ce = memoryEntries.evict();
freed += ce.getSizeEstimate();
activeBatchKB.addAndGet(-ce.getSizeEstimate());
synchronized (ce) {
Modified: trunk/engine/src/main/java/org/teiid/common/buffer/impl/SizeUtility.java
===================================================================
--- trunk/engine/src/main/java/org/teiid/common/buffer/impl/SizeUtility.java 2011-10-07 15:25:46 UTC (rev 3543)
+++ trunk/engine/src/main/java/org/teiid/common/buffer/impl/SizeUtility.java 2011-10-07 16:52:06 UTC (rev 3544)
@@ -26,8 +26,10 @@
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.teiid.core.types.DataTypeManager;
@@ -43,7 +45,7 @@
public static final int REFERENCE_SIZE = 8;
private static Map<Class<?>, int[]> SIZE_ESTIMATES = new HashMap<Class<?>, int[]>(128);
-
+ private static Set<Class<?>> VARIABLE_SIZE_TYPES = new HashSet<Class<?>>();
static {
SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.STRING, new int[] {100, 256});
SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.DATE, new int[] {20, 28});
@@ -61,13 +63,17 @@
SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.BOOLEAN, new int[] {1, 1});
SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.BIG_INTEGER, new int[] {75, 100});
SIZE_ESTIMATES.put(DataTypeManager.DefaultDataClasses.BIG_DECIMAL, new int[] {150, 200});
+ VARIABLE_SIZE_TYPES.add(DataTypeManager.DefaultDataClasses.STRING);
+ VARIABLE_SIZE_TYPES.add(DataTypeManager.DefaultDataClasses.OBJECT);
+ VARIABLE_SIZE_TYPES.add(DataTypeManager.DefaultDataClasses.BIG_INTEGER);
+ VARIABLE_SIZE_TYPES.add(DataTypeManager.DefaultDataClasses.BIG_DECIMAL);
}
private long bigIntegerEstimate;
private long bigDecimalEstimate;
- private String[] types;
+ private Class<?>[] types;
- public SizeUtility(String[] types) {
+ public SizeUtility(Class<?>[] types) {
boolean isValueCacheEnabled = DataTypeManager.isValueCacheEnabled();
bigIntegerEstimate = getSize(isValueCacheEnabled, DataTypeManager.DefaultDataClasses.BIG_INTEGER);
bigDecimalEstimate = getSize(isValueCacheEnabled, DataTypeManager.DefaultDataClasses.BIG_DECIMAL);
@@ -83,12 +89,8 @@
// array overhead for all the columns ( 8 object overhead + 4 ref + 4 int)
size += (rowLength * (48 + alignMemory(colLength * REFERENCE_SIZE)));
for (int col = 0; col < colLength; col++) {
- Class<?> type = DataTypeManager.getDataTypeClass(types[col]);
-
- if (type == DataTypeManager.DefaultDataClasses.STRING
- || type == DataTypeManager.DefaultDataClasses.OBJECT
- || type == DataTypeManager.DefaultDataClasses.BIG_INTEGER
- || type == DataTypeManager.DefaultDataClasses.BIG_DECIMAL) {
+ Class<?> type = types[col];
+ if (VARIABLE_SIZE_TYPES.contains(type)) {
int estRow = 0;
for (int row = 0; row < rowLength; row++) {
boolean updateEst = row == estRow;
Modified: trunk/engine/src/test/java/org/teiid/common/buffer/impl/TestSizeUtility.java
===================================================================
--- trunk/engine/src/test/java/org/teiid/common/buffer/impl/TestSizeUtility.java 2011-10-07 15:25:46 UTC (rev 3543)
+++ trunk/engine/src/test/java/org/teiid/common/buffer/impl/TestSizeUtility.java 2011-10-07 16:52:06 UTC (rev 3544)
@@ -34,6 +34,7 @@
import java.util.List;
import org.junit.Test;
+import org.teiid.core.types.DataTypeManager;
public class TestSizeUtility {
@@ -161,7 +162,12 @@
Arrays.asList(new Object[] { "c", new Integer(1), Boolean.FALSE, new Double(0.0), "c", new Integer(1) }) //$NON-NLS-1$ //$NON-NLS-2$
};
- String[] types = {"string", "integer", "boolean", "double", "string", "integer"}; //$NON-NLS-1$//$NON-NLS-2$//$NON-NLS-3$//$NON-NLS-4$ //$NON-NLS-5$//$NON-NLS-6$
+ Class<?>[] types = {DataTypeManager.DefaultDataClasses.STRING,
+ DataTypeManager.DefaultDataClasses.INTEGER,
+ DataTypeManager.DefaultDataClasses.BOOLEAN,
+ DataTypeManager.DefaultDataClasses.DOUBLE,
+ DataTypeManager.DefaultDataClasses.STRING,
+ DataTypeManager.DefaultDataClasses.INTEGER};
long actualSize = new SizeUtility(types).getBatchSize(false, Arrays.asList(expected));
assertEquals("Got unexpected size: ", 2667, actualSize); //$NON-NLS-1$
More information about the teiid-commits
mailing list