Author: jolee Date: 2013-02-05 14:59:24 -0500 (Tue, 05 Feb 2013) New Revision: 4549 Modified: branches/7.7.x/engine/src/main/java/org/teiid/common/buffer/impl/FileStorageManager.java branches/7.7.x/engine/src/test/java/org/teiid/common/buffer/impl/TestFileStorageManager.java Log: TEIID-2357: max buffer space is not honored Modified: branches/7.7.x/engine/src/main/java/org/teiid/common/buffer/impl/FileStorageManager.java =================================================================== --- branches/7.7.x/engine/src/main/java/org/teiid/common/buffer/impl/FileStorageManager.java 2013-02-05 19:36:43 UTC (rev 4548) +++ branches/7.7.x/engine/src/main/java/org/teiid/common/buffer/impl/FileStorageManager.java 2013-02-05 19:59:24 UTC (rev 4549) @@ -45,8 +45,9 @@ */ public class FileStorageManager implements StorageManager { + private static final long MB = 1024L * 1024L; public static final int DEFAULT_MAX_OPEN_FILES = 64; - public static final long DEFAULT_MAX_BUFFERSPACE = 50L * 1024L * 1024L * 1024L; + public static final long DEFAULT_MAX_BUFFERSPACE = 50L * 1024L * MB; private static final String FILE_PREFIX = "b_"; //$NON-NLS-1$ private long maxBufferSpace = DEFAULT_MAX_BUFFERSPACE; @@ -128,30 +129,46 @@ if (fileInfo == null) { fileInfo = new FileInfo(createFile(name)); } - long bytesUsed = 0; try { RandomAccessFile fileAccess = fileInfo.open(); long newLength = fileOffset + length; - bytesUsed = newLength - fileAccess.length(); - if (bytesUsed > 0) { - long used = usedBufferSpace.addAndGet(bytesUsed); - if (used > maxBufferSpace) { - //TODO: trigger a compaction before this is thrown - throw new IOException(QueryPlugin.Util.getString("FileStoreageManager.space_exhausted", maxBufferSpace)); //$NON-NLS-1$ - } - fileAccess.setLength(newLength); - bytesUsed = 0; - } + setLength(fileAccess, newLength, false); fileAccess.seek(fileOffset); fileAccess.write(b, offSet, length); } finally { - if (bytesUsed > 0) { - usedBufferSpace.addAndGet(-bytesUsed); - } fileInfo.close(); } return length; } + + private void setLength(RandomAccessFile fileAccess, long newLength, boolean truncate) + throws IOException { + long currentLength = fileAccess.length(); + long bytesUsed = newLength - currentLength; + if (bytesUsed == 0) { + return; + } + if (bytesUsed < 0) { + if (!truncate) { + return; + } + } else if (bytesUsed > MB) { + //this is a weak check, concurrent access may push us over the max. we are just trying to prevent large overage allocations + long used = usedBufferSpace.get() + bytesUsed; + if (used > maxBufferSpace) { + //TODO: trigger a compaction before this is thrown + throw new IOException(QueryPlugin.Util.getString("FileStoreageManager.space_exhausted", maxBufferSpace)); //$NON-NLS-1$ + } + } + fileAccess.setLength(newLength); + long used = usedBufferSpace.addAndGet(bytesUsed); + if (bytesUsed > 0 && used > maxBufferSpace) { + fileAccess.setLength(currentLength); + usedBufferSpace.addAndGet(-bytesUsed); + //TODO: trigger a compaction before this is thrown + throw new IOException(QueryPlugin.Util.getString("FileStoreageManager.space_exhausted", maxBufferSpace)); //$NON-NLS-1$ + } + } @Override public synchronized void setLength(long length) throws IOException { @@ -159,7 +176,7 @@ fileInfo = new FileInfo(createFile(name)); } try { - fileInfo.open().setLength(length); + setLength(fileInfo.open(), length, true); } finally { fileInfo.close(); } Modified: branches/7.7.x/engine/src/test/java/org/teiid/common/buffer/impl/TestFileStorageManager.java =================================================================== --- branches/7.7.x/engine/src/test/java/org/teiid/common/buffer/impl/TestFileStorageManager.java 2013-02-05 19:36:43 UTC (rev 4548) +++ branches/7.7.x/engine/src/test/java/org/teiid/common/buffer/impl/TestFileStorageManager.java 2013-02-05 19:59:24 UTC (rev 4549) @@ -95,6 +95,35 @@ writeBytes(store); } + @Test(expected=IOException.class) public void testMaxSpaceSplit() throws Exception { + FileStorageManager sm = getStorageManager(null, null); + sm.setMaxBufferSpace(1); + String tsID = "0"; //$NON-NLS-1$ + + SplittableStorageManager ssm = new SplittableStorageManager(sm); + FileStore store = ssm.createFileStore(tsID); + try { + writeBytes(store); + } finally { + assertEquals(0, sm.getUsedBufferSpace()); + } + } + + @Test public void testSetLength() throws Exception { + FileStorageManager sm = getStorageManager(null, null); + + String tsID = "0"; //$NON-NLS-1$ + FileStore store = sm.createFileStore(tsID); + store.setLength(1000); + assertEquals(1000, sm.getUsedBufferSpace()); + + store.setLength(200); + assertEquals(200, sm.getUsedBufferSpace()); + + store.setLength(1000); + assertEquals(1000, sm.getUsedBufferSpace()); + } + @Test public void testFlush() throws Exception { FileStorageManager sm = getStorageManager(null, null); FileStore store = sm.createFileStore("0");