[jbosscache-commits] JBoss Cache SVN: r7233 - in core/branches/flat/src: main/java/org/jboss/starobrno/util/concurrent/locks and 3 other directories.

Author: manik.surtani(a)jboss.com Date: 2008-12-02 06:00:56 -0500 (Tue, 02 Dec 2008) New Revision: 7233 Added: core/branches/flat/src/test/java/org/jboss/starobrno/api/tree/TreeStructureHashCodeTest.java core/branches/flat/src/test/java/org/jboss/starobrno/profiling/TreeProfileTest.java Modified: core/branches/flat/src/main/java/org/jboss/starobrno/tree/NodeKey.java core/branches/flat/src/main/java/org/jboss/starobrno/tree/TreeCacheImpl.java core/branches/flat/src/main/java/org/jboss/starobrno/util/concurrent/locks/LockContainer.java core/branches/flat/src/main/java/org/jboss/starobrno/util/concurrent/locks/OwnableReentrantLockContainer.java core/branches/flat/src/main/java/org/jboss/starobrno/util/concurrent/locks/ReentrantLockContainer.java core/branches/flat/src/test/java/org/jboss/starobrno/lock/LockContainerHashingTest.java core/branches/flat/src/test/java/org/jboss/starobrno/profiling/ProfileTest.java Log: Fixed tree related stuff Modified: core/branches/flat/src/main/java/org/jboss/starobrno/tree/NodeKey.java =================================================================== --- core/branches/flat/src/main/java/org/jboss/starobrno/tree/NodeKey.java 2008-12-02 10:45:00 UTC (rev 7232) +++ core/branches/flat/src/main/java/org/jboss/starobrno/tree/NodeKey.java 2008-12-02 11:00:56 UTC (rev 7233) @@ -37,7 +37,7 @@ Fqn fqn; Type contents; - enum Type + public static enum Type { DATA, STRUCTURE } @@ -63,9 +63,12 @@ public int hashCode() { - int hc = (contents != null ? contents.hashCode() : 1); - hc = 31 * hc + fqn.hashCode(); - return hc; + int h = fqn != null ? fqn.hashCode() : 1; + h += ~(h << 9); + h ^= (h >>> 14); + h += (h << 4); + h ^= (h >>> 10); + return h; } public String toString() Modified: core/branches/flat/src/main/java/org/jboss/starobrno/tree/TreeCacheImpl.java =================================================================== --- core/branches/flat/src/main/java/org/jboss/starobrno/tree/TreeCacheImpl.java 2008-12-02 10:45:00 UTC (rev 7232) +++ core/branches/flat/src/main/java/org/jboss/starobrno/tree/TreeCacheImpl.java 2008-12-02 11:00:56 UTC (rev 7233) @@ -84,7 +84,8 @@ startAtomic(); try { - return getNode(fqn).remove(key); + Node<K, V> n = getNode(fqn); + return n == null ? null : n.remove(key); } finally { @@ -103,8 +104,8 @@ startAtomic(); try { - Node n = getNode(fqn.getParent()); - return n == null ? false : n.removeChild(fqn.getLastElement()); + Node<K, V> n = getNode(fqn.getParent()); + return n != null && n.removeChild(fqn.getLastElement()); } finally { @@ -400,7 +401,7 @@ private void createRoot() { - createNodeInCache(Fqn.ROOT); + if (!exists(Fqn.ROOT)) createNodeInCache(Fqn.ROOT); } public String toString() Modified: core/branches/flat/src/main/java/org/jboss/starobrno/util/concurrent/locks/LockContainer.java =================================================================== --- core/branches/flat/src/main/java/org/jboss/starobrno/util/concurrent/locks/LockContainer.java 2008-12-02 10:45:00 UTC (rev 7232) +++ core/branches/flat/src/main/java/org/jboss/starobrno/util/concurrent/locks/LockContainer.java 2008-12-02 11:00:56 UTC (rev 7233) @@ -68,8 +68,15 @@ final int hash(E object) { int h = object.hashCode(); - h ^= (h >>> 20) ^ (h >>> 12); - return h ^ (h >>> 7) ^ (h >>> 4); +// h ^= (h >>> 20) ^ (h >>> 12); +// return h ^ (h >>> 7) ^ (h >>> 4); + + h += ~(h << 9); + h ^= (h >>> 14); + h += (h << 4); + h ^= (h >>> 10); + return h; + } protected abstract void initLocks(int numLocks); @@ -101,6 +108,11 @@ public abstract int getNumLocksHeld(); /** + * @return the size of the shared lock pool + */ + public abstract int size(); + + /** * Clears all locks held and re-initialises stripes. */ public abstract void reset(); Modified: core/branches/flat/src/main/java/org/jboss/starobrno/util/concurrent/locks/OwnableReentrantLockContainer.java =================================================================== --- core/branches/flat/src/main/java/org/jboss/starobrno/util/concurrent/locks/OwnableReentrantLockContainer.java 2008-12-02 10:45:00 UTC (rev 7232) +++ core/branches/flat/src/main/java/org/jboss/starobrno/util/concurrent/locks/OwnableReentrantLockContainer.java 2008-12-02 11:00:56 UTC (rev 7233) @@ -98,4 +98,9 @@ { initLocks(sharedLocks.length); } + + public int size() + { + return sharedLocks.length; + } } Modified: core/branches/flat/src/main/java/org/jboss/starobrno/util/concurrent/locks/ReentrantLockContainer.java =================================================================== --- core/branches/flat/src/main/java/org/jboss/starobrno/util/concurrent/locks/ReentrantLockContainer.java 2008-12-02 10:45:00 UTC (rev 7232) +++ core/branches/flat/src/main/java/org/jboss/starobrno/util/concurrent/locks/ReentrantLockContainer.java 2008-12-02 11:00:56 UTC (rev 7233) @@ -66,6 +66,11 @@ return i; } + public int size() + { + return sharedLocks.length; + } + public final boolean ownsLock(E object, Object owner) { ReentrantLock lock = getLock(object); Added: core/branches/flat/src/test/java/org/jboss/starobrno/api/tree/TreeStructureHashCodeTest.java =================================================================== --- core/branches/flat/src/test/java/org/jboss/starobrno/api/tree/TreeStructureHashCodeTest.java (rev 0) +++ core/branches/flat/src/test/java/org/jboss/starobrno/api/tree/TreeStructureHashCodeTest.java 2008-12-02 11:00:56 UTC (rev 7233) @@ -0,0 +1,64 @@ +package org.jboss.starobrno.api.tree; + +import org.jboss.starobrno.tree.Fqn; +import org.jboss.starobrno.tree.NodeKey; +import org.jboss.starobrno.util.concurrent.locks.LockContainer; +import org.jboss.starobrno.util.concurrent.locks.ReentrantLockContainer; +import org.testng.annotations.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.locks.Lock; + +/** + * Tests the degree to which hash codes get spread + */ +@Test +public class TreeStructureHashCodeTest +{ + public void testHashCodesAppendedCount() + { + List<Fqn> fqns = new ArrayList<Fqn>(); + fqns.add(Fqn.ROOT); + for (int i = 0; i < 256; i++) fqns.add(Fqn.fromString("/fqn" + i)); + doTest(fqns); + } + + public void testHashCodesAlpha() + { + List<Fqn> fqns = new ArrayList<Fqn>(); + fqns.add(Fqn.ROOT); + for (int i = 0; i < 256; i++) fqns.add(Fqn.fromString("/" + Integer.toString(i, 36))); + doTest(fqns); + } + + private void doTest(List<Fqn> fqns) + { + LockContainer<NodeKey> container = new ReentrantLockContainer<NodeKey>(512); + Map<Lock, Integer> distribution = new HashMap<Lock, Integer>(); + for (Fqn f : fqns) + { + NodeKey dataKey = new NodeKey(f, NodeKey.Type.DATA); + NodeKey structureKey = new NodeKey(f, NodeKey.Type.STRUCTURE); + addToDistribution(container.getLock(dataKey), distribution); + addToDistribution(container.getLock(structureKey), distribution); + } + + System.out.println("Distribution: " + distribution); + assert distribution.size() <= container.size() : "Cannot have more locks than the container size!"; + // assume at least a 2/3rd even distribution + // but also consider that data snd structure keys would typically provide the same hash code + // so we need to double this + assert distribution.size() * 1.5 * 2 >= container.size() : "Poorly distributed! Distribution size is just " + distribution.size() + " and there are " + container.size() + " shared locks"; + + } + + private void addToDistribution(Lock lock, Map<Lock, Integer> map) + { + int count = 1; + if (map.containsKey(lock)) count = map.get(lock) + 1; + map.put(lock, count); + } +} Modified: core/branches/flat/src/test/java/org/jboss/starobrno/lock/LockContainerHashingTest.java =================================================================== --- core/branches/flat/src/test/java/org/jboss/starobrno/lock/LockContainerHashingTest.java 2008-12-02 10:45:00 UTC (rev 7232) +++ core/branches/flat/src/test/java/org/jboss/starobrno/lock/LockContainerHashingTest.java 2008-12-02 11:00:56 UTC (rev 7233) @@ -26,11 +26,7 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; +import java.util.*; import java.util.concurrent.locks.Lock; @Test(groups = "unit") @@ -69,10 +65,10 @@ // cannot be larger than the number of locks System.out.println("dist size: " + distribution.size()); - System.out.println("num shared locks: " + stripedLock.getNumLocksHeld()); - assert distribution.size() <= stripedLock.getNumLocksHeld(); + System.out.println("num shared locks: " + stripedLock.size()); + assert distribution.size() <= stripedLock.size(); // assume at least a 2/3rd spread - assert distribution.size() * 1.5 >= stripedLock.getNumLocksHeld(); + assert distribution.size() * 1.5 >= stripedLock.size(); } private List<String> createRandomKeys(int number) Modified: core/branches/flat/src/test/java/org/jboss/starobrno/profiling/ProfileTest.java =================================================================== --- core/branches/flat/src/test/java/org/jboss/starobrno/profiling/ProfileTest.java 2008-12-02 10:45:00 UTC (rev 7232) +++ core/branches/flat/src/test/java/org/jboss/starobrno/profiling/ProfileTest.java 2008-12-02 11:00:56 UTC (rev 7233) @@ -49,12 +49,12 @@ Log log = LogFactory.getLog(ProfileTest.class); @Test(enabled = true) - public void testLocalModePess() throws Exception + public void testLocalMode() throws Exception { Cache c = (Cache) cache; c.getConfiguration().setCacheMode(Configuration.CacheMode.LOCAL); c.getConfiguration().setConcurrencyLevel(2000); - c.getConfiguration().setIsolationLevel(IsolationLevel.REPEATABLE_READ); + c.getConfiguration().setIsolationLevel(IsolationLevel.READ_COMMITTED); runCompleteTest(); } @@ -208,7 +208,7 @@ double nOps = (double) (NUM_OPERATIONS / 3); double avg = ((double) totalNanos) / nOps; double avgMicros = avg / 1000; - return avgMicros + " �s"; + return avgMicros + " µs"; } private double toMillis(long nanos) Copied: core/branches/flat/src/test/java/org/jboss/starobrno/profiling/TreeProfileTest.java (from rev 7228, core/branches/flat/src/test/java/org/jboss/starobrno/profiling/ProfileTest.java) =================================================================== --- core/branches/flat/src/test/java/org/jboss/starobrno/profiling/TreeProfileTest.java (rev 0) +++ core/branches/flat/src/test/java/org/jboss/starobrno/profiling/TreeProfileTest.java 2008-12-02 11:00:56 UTC (rev 7233) @@ -0,0 +1,338 @@ +package org.jboss.starobrno.profiling; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.jboss.cache.lock.IsolationLevel; +import org.jboss.starobrno.Cache; +import org.jboss.starobrno.UnitTestCacheFactory; +import org.jboss.starobrno.config.Configuration; +import org.jboss.starobrno.tree.Fqn; +import org.jboss.starobrno.tree.TreeCache; +import org.jboss.starobrno.tree.TreeCacheImpl; +import org.jboss.starobrno.util.TestingUtil; +import org.testng.annotations.AfterTest; +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Test to use with a profiler to profile replication. To be used in conjunction with ProfileSlaveTest. + * <p/> + * Typical usage pattern: + * <p/> + * 1. Start a single test method in ProfileSlaveTest. This will block until you kill it. + * 2. Start the corresponding test in this class, with the same name, in a different JVM, and attached to a profiler. + * 3. Profile away! + * <p/> + * + * @author Manik Surtani (<a href="mailto:manik@jboss.org">manik@jboss.org</a>) + * @since 2.1.0 + */ +@Test(groups = "profiling", sequential = true) +public class TreeProfileTest +{ + /* + Test configuration options + */ + protected static final long NUM_OPERATIONS = 100000; // DURATION is replaced with a fixed number of operations instead. + protected static final int NUM_THREADS = 25; + protected static final int MAX_RANDOM_SLEEP_MILLIS = 1; + protected static final int MAX_OVERALL_FQNS = 2000; + protected static final int TREE_DEPTH = 2; + protected static final int WARMUP_LOOPS = 20000; + protected static final boolean USE_SLEEP = false; // throttle generation a bit + + protected TreeCache cache; + + @BeforeTest + public void setUp() + { + Configuration cfg = new Configuration(); + cfg.setInvocationBatchingEnabled(true); + cfg.setCacheMode(Configuration.CacheMode.LOCAL); + cfg.setConcurrencyLevel(2000); + cfg.setLockAcquisitionTimeout(60000); + cfg.setIsolationLevel(IsolationLevel.REPEATABLE_READ); + Cache c = new UnitTestCacheFactory().createCache(cfg); + cache = new TreeCacheImpl(c); + } + + @AfterTest + public void tearDown() + { + TestingUtil.killTreeCaches(cache); + cache = null; + } + + private List<Fqn> fqns = new ArrayList<Fqn>(MAX_OVERALL_FQNS); + private Random r = new Random(); + + Log log = LogFactory.getLog(TreeProfileTest.class); + + @Test(enabled = true) + public void testLocalMode() throws Exception + { + runCompleteTest(); + } + + private void runCompleteTest() throws Exception + { + init(); + startup(); + warmup(); + doTest(); + + // wait for user exit + System.in.read(); + } + + /** + * Thr following test phases can be profiled individually using triggers in JProfiler. + */ + + protected void init() + { + long startTime = System.currentTimeMillis(); + log.warn("Starting init() phase"); + fqns.clear(); + for (int i = 0; i < MAX_OVERALL_FQNS; i++) + { + Fqn fqn = createRandomFqn(r); + while (fqns.contains(fqn)) fqn = createRandomFqn(r); + if (i % 100 == 0) + { + log.warn("Generated " + i + " fqns"); + } + fqns.add(fqn); + } + System.gc(); + long duration = System.currentTimeMillis() - startTime; + log.warn("Finished init() phase. " + printDuration(duration)); + } + + private Fqn createRandomFqn(Random r) + { + List<String> fqnElements = new ArrayList<String>(TREE_DEPTH); + for (int i = 0; i < TREE_DEPTH; i++) fqnElements.add(Integer.toHexString(r.nextInt(Integer.MAX_VALUE))); + return Fqn.fromList(fqnElements, true); + } + + + protected void startup() + { + long startTime = System.currentTimeMillis(); + log.warn("Starting cache"); + cache.start(); + long duration = System.currentTimeMillis() - startTime; + log.warn("Started cache. " + printDuration(duration)); + } + + private void warmup() throws InterruptedException + { + long startTime = System.currentTimeMillis(); + ExecutorService exec = Executors.newFixedThreadPool(NUM_THREADS); + log.warn("Starting warmup"); + // creates all the Fqns since this can be expensive and we don't really want to measure this (for now) + for (final Fqn fqn : fqns) + { + exec.execute(new Runnable() + { + public void run() + { + // this will create the necessary nodes. + cache.put(fqn, "key", Collections.emptyMap()); + } + }); + } + + // loop through WARMUP_LOOPS gets and puts for JVM optimisation + for (int i = 0; i < WARMUP_LOOPS; i++) + { + exec.execute(new Runnable() + { + public void run() + { + Fqn fqn = fqns.get(r.nextInt(MAX_OVERALL_FQNS)); + cache.get(fqn, "key"); + cache.put(fqn, "key", "Value"); + cache.remove(fqn, "key"); + } + }); + } + + exec.shutdown(); + exec.awaitTermination(360, TimeUnit.SECONDS); + + long duration = System.currentTimeMillis() - startTime; + log.warn("Finished warmup. " + printDuration(duration)); + //cache.removeNode(Fqn.ROOT); + cache.stop(); + + startup(); + } + + private void doTest() throws Exception + { + ExecutorService exec = Executors.newFixedThreadPool(NUM_THREADS); + log.warn("Starting test"); + int i; + long print = NUM_OPERATIONS / 10; + + AtomicLong durationPuts = new AtomicLong(); + AtomicLong durationGets = new AtomicLong(); + AtomicLong durationRemoves = new AtomicLong(); + + long stElapsed = System.nanoTime(); + for (i = 0; i < NUM_OPERATIONS; i++) + { + MyRunnable r = null; + switch (i % 3) + { + case 0: + r = new Putter(i, durationPuts); + break; + case 1: + r = new Getter(i, durationGets); + break; + case 2: + r = new Remover(i, durationRemoves); + break; + } + if (i % print == 0) + log.warn("processing iteration " + i); + exec.execute(r); +// if (USE_SLEEP) TestingUtil.sleepRandom(MAX_RANDOM_SLEEP_MILLIS); + if (USE_SLEEP) TestingUtil.sleepThread(MAX_RANDOM_SLEEP_MILLIS); + } + log.warn("Finished generating runnables; awaiting executor completion"); + // wait for executors to complete! + exec.shutdown(); + while (!exec.awaitTermination(((long) i), TimeUnit.SECONDS)) + { + Thread.sleep(1); + } + + // wait up to 1 sec for each call? + long elapsedTimeNanos = System.nanoTime() - stElapsed; + + log.warn("Finished test. " + printDuration((long) toMillis(elapsedTimeNanos))); + log.warn("Throughput: " + ((double) NUM_OPERATIONS * 1000 / toMillis(elapsedTimeNanos)) + " operations per second (roughly equal numbers of PUT, GET and REMOVE)"); + log.warn("Average GET time: " + printAvg(durationGets.get())); + log.warn("Average PUT time: " + printAvg(durationPuts.get())); + log.warn("Average REMOVE time: " + printAvg(durationRemoves.get())); + } + + private String printAvg(long totalNanos) + { + double nOps = (double) (NUM_OPERATIONS / 3); + double avg = ((double) totalNanos) / nOps; + double avgMicros = avg / 1000; + return avgMicros + " µs"; + } + + private double toMillis(long nanos) + { + return ((double) nanos / (double) 1000000); + } + + enum Mode + { + PUT, GET, REMOVE + } + + private abstract class MyRunnable implements Runnable + { + int id; + Mode mode; + AtomicLong duration; + + public void run() + { + Fqn fqn = fqns.get(r.nextInt(MAX_OVERALL_FQNS)); + long d = 0, st = 0; + switch (mode) + { + case PUT: + Object value = getRandomString(); + st = System.nanoTime(); + cache.put(fqn, "key", value); + d = System.nanoTime() - st; + break; + case GET: + st = System.nanoTime(); + cache.get(fqn, "key"); + d = System.nanoTime() - st; + break; + case REMOVE: + st = System.nanoTime(); + cache.remove(fqn, "key"); + d = System.nanoTime() - st; + break; + } + duration.getAndAdd(d); + } + } + + private class Putter extends MyRunnable + { + private Putter(int id, AtomicLong duration) + { + this.id = id; + this.duration = duration; + mode = Mode.PUT; + } + } + + private class Getter extends MyRunnable + { + private Getter(int id, AtomicLong duration) + { + this.id = id; + this.duration = duration; + mode = Mode.GET; + } + } + + private class Remover extends MyRunnable + { + private Remover(int id, AtomicLong duration) + { + this.id = id; + this.duration = duration; + mode = Mode.REMOVE; + } + } + + private String getRandomString() + { + StringBuilder sb = new StringBuilder(); + int len = r.nextInt(10); + + for (int i = 0; i < len; i++) + { + sb.append((char) (63 + r.nextInt(26))); + } + return sb.toString(); + } + + protected String printDuration(long duration) + { + if (duration > 2000) + { + double dSecs = ((double) duration / (double) 1000); + return "Duration: " + dSecs + " seconds"; + } + else + { + return "Duration: " + duration + " millis"; + } + } +} \ No newline at end of file Property changes on: core/branches/flat/src/test/java/org/jboss/starobrno/profiling/TreeProfileTest.java ___________________________________________________________________ Name: svn:keywords + Id Revision Name: svn:mergeinfo + Name: svn:eol-style + LF