[teiid-commits] teiid SVN: r4611 - in branches/7.7.x/engine/src: test/java/org/teiid/common/buffer/impl and 1 other directory.

[teiid-commits at lists.jboss.org]

Author: jolee
Date: 2013-11-07 10:55:23 -0500 (Thu, 07 Nov 2013)
New Revision: 4611

Modified:
   branches/7.7.x/engine/src/main/java/org/teiid/common/buffer/impl/BufferFrontedFileStoreCache.java
   branches/7.7.x/engine/src/test/java/org/teiid/common/buffer/impl/TestBufferFrontedFileStoreCache.java
Log:
TEIID-2714: Buffer defrag not working, adding an option to make the buffer files more compact and
defaulting to a somewhat more conservative default

Modified: branches/7.7.x/engine/src/main/java/org/teiid/common/buffer/impl/BufferFrontedFileStoreCache.java
===================================================================
--- branches/7.7.x/engine/src/main/java/org/teiid/common/buffer/impl/BufferFrontedFileStoreCache.java	2013-11-06 21:18:40 UTC (rev 4610)
+++ branches/7.7.x/engine/src/main/java/org/teiid/common/buffer/impl/BufferFrontedFileStoreCache.java	2013-11-07 15:55:23 UTC (rev 4611)
@@ -59,6 +59,7 @@
 import org.teiid.core.TeiidComponentException;
 import org.teiid.core.TeiidRuntimeException;
 import org.teiid.core.util.ExecutorUtils;
+import org.teiid.core.util.PropertiesUtils;
 import org.teiid.logging.LogConstants;
 import org.teiid.logging.LogManager;
 import org.teiid.logging.MessageLevel;
@@ -377,7 +378,9 @@
 	private AtomicBoolean defragRunning = new AtomicBoolean();
 	private AtomicInteger freedCounter = new AtomicInteger();
 	
-	private int truncateInterval = 10;
+	private boolean compactBufferFiles = PropertiesUtils.getBooleanProperty(System.getProperties(), "org.teiid.compactBufferFiles", false); //$NON-NLS-1$
+	
+	private int truncateInterval = 4;
 	//defrag to release freespace held by storage files
 	final class DefragTask implements Runnable {
 		private AtomicInteger runs = new AtomicInteger();
@@ -398,6 +401,7 @@
 		}
 		
 		private long truncate(boolean anySpace) {
+			anySpace |= compactBufferFiles;
 			long freed = 0;
 			for (int i = 0; i < sizeBasedStores.length; i++) {
 				BlockStore blockStore = sizeBasedStores[i];
@@ -412,6 +416,7 @@
 		}
 
 		private void defrag(boolean all) {
+			all |= compactBufferFiles;
 			if (LogManager.isMessageToBeRecorded(LogConstants.CTX_BUFFER_MGR, MessageLevel.DETAIL)) {
 				LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, "Running defrag"); //$NON-NLS-1$ 
 			}
@@ -585,6 +590,7 @@
 			}
 		} while ((size>>1) < maxStorageObjectSize);
 		this.sizeBasedStores = stores.toArray(new BlockStore[stores.size()]);
+		this.truncateInterval = compactBufferFiles?1:8;
 	}
 	
 	boolean lowBlocks(boolean critical) {
@@ -666,7 +672,7 @@
             	}
 			}
 		} catch (Throwable e) {
-			if ((e == BlockOutputStream.exceededMax && newEntry) || e == PhysicalInfo.sizeChanged) {
+			if (e == PhysicalInfo.sizeChanged) {
 				//entries are mutable after adding, the original should be removed shortly so just ignore
 				LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, "Object "+ entry.getId() +" changed size since first persistence, keeping the original."); //$NON-NLS-1$ //$NON-NLS-2$
 			} else if (e == BlockOutputStream.exceededMax){
@@ -1018,7 +1024,7 @@
 							LogManager.logDetail(LogConstants.CTX_BUFFER_MGR, "Freed storage data block", info.block, "of size", blockStore.blockSize); //$NON-NLS-1$ //$NON-NLS-2$
 						}
 						if (!defragRunning.get() 
-								&& (freedCounter.getAndIncrement()&Short.MAX_VALUE)==Short.MAX_VALUE //should be several gigabytes of turn over
+								&& (freedCounter.getAndIncrement()&0x3fff)==0x3fff //should be several hundred megs of turn over
 								&& defragRunning.compareAndSet(false, true)) {
 							this.asynchPool.execute(defragTask);
 						}
@@ -1196,4 +1202,8 @@
 		this.asynchPool.shutdownNow();
 	}
 	
+	public void setCompactBufferFiles(boolean compactBufferFiles) {
+		this.compactBufferFiles = compactBufferFiles;
+	}
+	
 }
\ No newline at end of file

Modified: branches/7.7.x/engine/src/test/java/org/teiid/common/buffer/impl/TestBufferFrontedFileStoreCache.java
===================================================================
--- branches/7.7.x/engine/src/test/java/org/teiid/common/buffer/impl/TestBufferFrontedFileStoreCache.java	2013-11-06 21:18:40 UTC (rev 4610)
+++ branches/7.7.x/engine/src/test/java/org/teiid/common/buffer/impl/TestBufferFrontedFileStoreCache.java	2013-11-07 15:55:23 UTC (rev 4611)
@@ -349,6 +349,38 @@
 		assertEquals(131072, cache.getDiskUsage());
 	}
 	
+	@Test public void testDefragTruncateCompact() throws Exception {
+		cache = createLayeredCache(1<<15, 1<<15, true);
+		cache.setCompactBufferFiles(true);
+		cache.setTruncateInterval(1);
+		cache.setMinDefrag(10000000);
+		Serializer<Integer> s = new SimpleSerializer();
+		WeakReference<? extends Serializer<?>> ref = new WeakReference<Serializer<?>>(s);
+		cache.createCacheGroup(s.getId());
+		Integer cacheObject = Integer.valueOf(5000);
+
+		for (int i = 0; i < 30; i++) {
+			CacheEntry ce = new CacheEntry((long)i);
+			ce.setSerializer(ref);
+			ce.setObject(cacheObject);
+
+			cache.addToCacheGroup(s.getId(), ce.getId());
+			cache.add(ce, s);
+		}
+		assertEquals(950272, cache.getDiskUsage());
+		for (int i = 0; i < 25; i++) {
+			cache.remove(1l, (long)i);
+		}
+		assertEquals(950272, cache.getDiskUsage());
+		cache.setMinDefrag(0);
+		cache.setTruncateInterval(1);
+		cache.defragTask.run();
+		assertEquals(131072, cache.getDiskUsage());
+		cache.defragTask.run();
+		//we've reached a stable size
+		assertEquals(131072, cache.getDiskUsage());
+	}
+	
 	@Test public void testDefragMin() throws Exception {
 		cache = createLayeredCache(1<<15, 1<<15, true);
 		cache.setMinDefrag(10000000);