[teiid-commits] teiid SVN: r1764 - in trunk/engine/src/main/java/com/metamatrix/common/buffer: impl and 1 other directory.
teiid-commits at lists.jboss.org
teiid-commits at lists.jboss.org
Thu Jan 21 00:11:37 EST 2010
Author: shawkins
Date: 2010-01-21 00:11:37 -0500 (Thu, 21 Jan 2010)
New Revision: 1764
Modified:
trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManager.java
trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferManagerImpl.java
trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/FileStorageManager.java
Log:
TEIID-913 adding back the ability to reserve buffers in the buffermanager and increasing the default for max open files.
Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManager.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManager.java 2010-01-21 03:59:24 UTC (rev 1763)
+++ trunk/engine/src/main/java/com/metamatrix/common/buffer/BufferManager.java 2010-01-21 05:11:37 UTC (rev 1764)
@@ -72,8 +72,7 @@
/**
* Optional property - this values specifies how many open file descriptors should be cached
* in the storage directory. Increasing this value in heavy load may improve performance
- * but will use more file descriptors, which are a limited system resource. The default
- * is 32.
+ * but will use more file descriptors, which are a limited system resource.
*/
public static final String MAX_OPEN_FILES = "metamatrix.buffer.maxOpenFiles"; //$NON-NLS-1$
/**
@@ -88,7 +87,8 @@
public static int DEFAULT_CONNECTOR_BATCH_SIZE = 2048;
public static int DEFAULT_PROCESSOR_BATCH_SIZE = 1024;
- public static int DEFAULT_MAX_PROCESSING_BATCHES = 16;
+ public static int DEFAULT_MAX_PROCESSING_BATCHES = 8;
+ public static int DEFAULT_RESERVE_BUFFERS = 64;
/**
* Get the batch size to use during query processing.
@@ -105,8 +105,35 @@
TupleBuffer createTupleBuffer(List elements, String groupName, TupleSourceType tupleSourceType)
throws MetaMatrixComponentException;
+ /**
+ * Return the maximum number of batches that can be temporarily held potentially
+ * across even a blocked exception.
+ * @return
+ */
int getMaxProcessingBatches();
+ /**
+ * Creates a new {@link FileStore}. See {@link FileStore#setCleanupReference(Object)} to
+ * automatically cleanup the underlying resources.
+ * @param name
+ * @return
+ */
FileStore createFileStore(String name);
+ /**
+ * Reserve up to count buffers for use. Wait will cause the process to block until
+ * all of the requested or half of the total buffers are available.
+ * @param count
+ * @param wait
+ * @return
+ * @throws MetaMatrixComponentException
+ */
+ int reserveBuffers(int count, boolean wait) throws MetaMatrixComponentException;
+
+ /**
+ * Releases the buffers reserved by a call to {@link BufferManager#reserveBuffers(int, boolean)}
+ * @param count
+ */
+ void releaseBuffers(int count);
+
}
Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferManagerImpl.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferManagerImpl.java 2010-01-21 03:59:24 UTC (rev 1763)
+++ trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/BufferManagerImpl.java 2010-01-21 05:11:37 UTC (rev 1764)
@@ -26,6 +26,8 @@
import java.util.ListIterator;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantLock;
import javax.xml.transform.Source;
@@ -60,6 +62,10 @@
private int connectorBatchSize = BufferManager.DEFAULT_CONNECTOR_BATCH_SIZE;
private int processorBatchSize = BufferManager.DEFAULT_PROCESSOR_BATCH_SIZE;
private int maxProcessingBatches = BufferManager.DEFAULT_MAX_PROCESSING_BATCHES;
+ private int reserveBatches = BufferManager.DEFAULT_RESERVE_BUFFERS;
+ private int maxReserveBatches = BufferManager.DEFAULT_RESERVE_BUFFERS;
+ private ReentrantLock lock = new ReentrantLock(true);
+ private Condition batchesFreed = lock.newCondition();
private StorageManager diskMgr;
@@ -169,6 +175,40 @@
}
return types;
}
+
+ @Override
+ public void releaseBuffers(int count) {
+ lock.lock();
+ try {
+ this.reserveBatches += count;
+ batchesFreed.signalAll();
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ @Override
+ public int reserveBuffers(int count, boolean wait) throws MetaMatrixComponentException {
+ lock.lock();
+ try {
+ while (wait && count > this.reserveBatches && this.reserveBatches < this.maxReserveBatches / 2) {
+ try {
+ batchesFreed.await();
+ } catch (InterruptedException e) {
+ throw new MetaMatrixComponentException(e);
+ }
+ }
+ this.reserveBatches -= count;
+ if (this.reserveBatches >= 0) {
+ return count;
+ }
+ int result = count + this.reserveBatches;
+ this.reserveBatches = 0;
+ return result;
+ } finally {
+ lock.unlock();
+ }
+ }
public void shutdown() {
}
Modified: trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/FileStorageManager.java
===================================================================
--- trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/FileStorageManager.java 2010-01-21 03:59:24 UTC (rev 1763)
+++ trunk/engine/src/main/java/com/metamatrix/common/buffer/impl/FileStorageManager.java 2010-01-21 05:11:37 UTC (rev 1764)
@@ -48,7 +48,8 @@
*/
public class FileStorageManager implements StorageManager {
- private static final String FILE_PREFIX = "b_"; //$NON-NLS-1$
+ private static final int DEFAULT_MAX_OPEN_FILES = 256;
+ private static final String FILE_PREFIX = "b_"; //$NON-NLS-1$
private class FileInfo {
private File file;
@@ -155,7 +156,7 @@
}
// Initialization
- private int maxOpenFiles = 32;
+ private int maxOpenFiles = DEFAULT_MAX_OPEN_FILES;
private long maxFileSize = 2L * 1024L * 1024L * 1024L; // 2GB
private String directory;
private File dirFile;
@@ -200,7 +201,7 @@
}
// Set up max number of open file descriptors
- maxOpenFiles = PropertiesUtils.getIntProperty(props, BufferManager.MAX_OPEN_FILES, 32);
+ maxOpenFiles = PropertiesUtils.getIntProperty(props, BufferManager.MAX_OPEN_FILES, DEFAULT_MAX_OPEN_FILES);
// Set the max file size
maxFileSize = PropertiesUtils.getIntProperty(props, BufferManager.MAX_FILE_SIZE, 2048) * 1024L * 1024L; // Multiply by 1MB
More information about the teiid-commits
mailing list