Author: manik.surtani(a)jboss.com Date: 2008-04-25 07:05:20 -0400 (Fri, 25 Apr 2008) New Revision: 5686 Added: core/tags/2.1.1.GA/src/test/java/org/jboss/cache/buddyreplication/BuddyReplicationRejoinTest.java Modified: core/tags/2.1.1.GA/src/main/java/org/jboss/cache/RPCManagerImpl.java Log: Modified: core/tags/2.1.1.GA/src/main/java/org/jboss/cache/RPCManagerImpl.java =================================================================== --- core/tags/2.1.1.GA/src/main/java/org/jboss/cache/RPCManagerImpl.java 2008-04-25 11:01:33 UTC (rev 5685) +++ core/tags/2.1.1.GA/src/main/java/org/jboss/cache/RPCManagerImpl.java 2008-04-25 11:05:20 UTC (rev 5686) @@ -124,6 +124,7 @@ case INVALIDATION_ASYNC: case INVALIDATION_SYNC: isInLocalMode = false; + isUsingBuddyReplication = configuration.getBuddyReplicationConfig() != null && configuration.getBuddyReplicationConfig().isEnabled(); if (log.isDebugEnabled()) log.debug("Cache mode is " + configuration.getCacheMode()); initialiseChannelAndRpcDispatcher(); Added: core/tags/2.1.1.GA/src/test/java/org/jboss/cache/buddyreplication/BuddyReplicationRejoinTest.java =================================================================== --- core/tags/2.1.1.GA/src/test/java/org/jboss/cache/buddyreplication/BuddyReplicationRejoinTest.java (rev 0) +++ core/tags/2.1.1.GA/src/test/java/org/jboss/cache/buddyreplication/BuddyReplicationRejoinTest.java 2008-04-25 11:05:20 UTC (rev 5686) @@ -0,0 +1,235 @@ +package org.jboss.cache.buddyreplication; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.jboss.cache.Cache; +import org.jboss.cache.CacheStatus; +import org.jboss.cache.DefaultCacheFactory; +import org.jboss.cache.Fqn; +import org.jboss.cache.config.BuddyReplicationConfig; +import org.jboss.cache.config.Configuration; +import org.jboss.cache.misc.TestingUtil; +import org.jboss.cache.util.CachePrinter; +import org.testng.annotations.AfterTest; +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; + +/** + * Tests fail over scenario when using buddy replication. + * <p/> + * What we are looking for is the last log output 'RECOVERED DATA', if all counter values + * are zero then we have failed to recover the work done on the buddy node. + * <p/> + * Change the LATE_START_BUDDY_CACHE flag to trigger different startup behavior at the initial + * object creation. It does not seem to have an impact on the outcome though. + * + * @author Fredrik Johansson, Cubeia Ltd + */ +@Test +public class BuddyReplicationRejoinTest extends BuddyReplicationTestsBase +{ + private static Log log = LogFactory.getLog(BuddyReplicationRejoinTest.class); + + /** + * How many object we should insert to the cache. + */ + private final static int OBJECT_COUNT = 10; + + private Cache<String, Integer> cache1; + private Cache<String, Integer> cache2; + + @BeforeTest + public void setUp() throws Exception + { + BuddyReplicationConfig brc = new BuddyReplicationConfig(); + brc.setEnabled(true); + brc.setBuddyCommunicationTimeout(1000); + brc.setAutoDataGravitation(true); + brc.setDataGravitationRemoveOnFind(true); + brc.setDataGravitationSearchBackupTrees(true); + Configuration c = new Configuration(); + c.setCacheMode(Configuration.CacheMode.REPL_SYNC); + c.setBuddyReplicationConfig(brc); + + // Cache1 will be used only for recovery. + // Cache2 will perform some updates on the objects and then fail. + + cache1 = new DefaultCacheFactory().createCache(c, false); + cache2 = new DefaultCacheFactory().createCache(c.clone(), false); + } + + @AfterTest + public void tearDown() + { + TestingUtil.killCaches(cache1, cache2); + } + + /** + * Executes the test scenario which goes like this: + * <p/> + * 1. Start primary cache (cache1) + * <p/> + * 2. Add initial data to primary cache1. All counters are now 0. + * <p/> + * 3. Start secondary cache (cache2) + * <p/> + * 4. Let the secondary change data in half of the objects. The cache + * will simply update a counter value. The values should now be 1 + * for those objects. + * <p/> + * This will trigger a data-gravitation to the secondary cache and + * the objects should be removed from the primary cache's real data + * <p/> + * 5. Fail the secondary cache (cache.stop()). The primary cache should hold + * valid values for the objects updated by the secondary cache (values + * should be 2). + * <p/> + * 6. Print out all data in the primary cache. This will cause all objects + * gravitate to the primary cache. We should have the the same state as + * after #1 in list except that object 0-4 should have counter = 1. + * <p/> + * 7. Startup a secondary cache again. + * <p/> + * 8. Run the same update procedure. The secondary cache should gravitate objects + * 0-4 and update the counters with +1, making the counters = 2. + * <p/> + * 9. Fail the secondary cache. + * <p/> + * 10. Print out all data in the primary cache. This will cause all objects + * gravitate to the primary cache. We should have the the same state as + * after #1 in list except that object 0-4 should have counter = 2 this time + * since the second secondary cache should have update the values again. + * <p/> + * <p/> + * FAIL. + * <p/> + * The recovered values in the end are 1, not 2. + * It seems like the first buddy cache works fine, but he second time around it + * fails to remove the local objects on then primary cache on the remote gravitation. + * <p/> + * The log printout in the end *should* look like: + * <p/> + * *********** RECOVERED DATA *********** + * /0 counter = 2 + * /1 counter = 2 + * /2 counter = 2 + * /3 counter = 2 + * /4 counter = 2 + * /5 counter = 0 + * /6 counter = 0 + * /7 counter = 0 + * /8 counter = 0 + * /9 counter = 0 + * ********************************* + * <p/> + * But we get 1's instead of 2's, which means that the recovery failed. + */ + @SuppressWarnings("unchecked") + public void testGravitationAndFailover() throws CloneNotSupportedException + { + Configuration cfg = cache2.getConfiguration().clone(); + cache1.start(); + TestingUtil.sleepThread(100); + + addInitial(cache1); + printCacheDetails("INITIAL STATES"); + + cache2.start(); + printCacheDetails("CACHE2 STARTED"); + + runBuddyUpdatesAndFail(); + + checkRecoveredData(cache1, 1); + printCacheDetails("DATA GRAVITATED BACK TO CACHE1"); + + cache2 = new DefaultCacheFactory().createCache(cfg); + printCacheDetails("BUDDY BACK"); + + runBuddyUpdatesAndFail(); + + checkRecoveredData(cache1, 2); + printCacheDetails("DATA GRAVITATED BACK TO CACHE1 (AGAIN)"); + } + + + private void runBuddyUpdatesAndFail() + { + executeBuddy(cache2); + printCacheDetails("BUDDY UPDATED"); + + cache2.stop(); + printCacheDetails("BUDDY FAILED"); + } + + + /** + * Change some data for a select number of objects on the given cache. + * + * @param cache + */ + private void executeBuddy(Cache<String, Integer> cache) + { + for (int i = 0; i < OBJECT_COUNT / 2; i++) + { + Integer integer = cache.get(Fqn.fromString(String.valueOf(i)), "counter"); + cache.put(Fqn.fromString(String.valueOf(i)), "counter", integer.intValue() + 1); + } + } + + + /** + * Add inital state to a cache + * + * @param cache + */ + private void addInitial(Cache<String, Integer> cache) + { + for (int i = 0; i < OBJECT_COUNT; i++) + { + cache.put(new Fqn<String>(String.valueOf(i)), "counter", new Integer(0)); + } + } + + + /** + * Print all data to the log + * + * @param cache + */ + private void checkRecoveredData(Cache<String, Integer> cache, int expectedValue) + { + log.info("*********** RECOVERED DATA ***********"); + for (int i = 0; i < OBJECT_COUNT; i++) + { + Integer counter = cache.get(Fqn.fromString(String.valueOf(i)), "counter"); + log.info("/" + i + " counter = " + counter); + assert i < 5 ? counter == expectedValue : counter == 0; + } + log.info("*********************************"); + } + + + private void printCacheDetails(String state) + { + log.info("*********** " + state + " ***********"); + if (cache1.getCacheStatus() == CacheStatus.STARTED) + { + log.info("--------- CACHE 1 (" + cache1.getLocalAddress() + ") ---------"); + log.info(CachePrinter.printCacheDetails(cache1)); + } + else + { + log.info("--------- CACHE 1 STOPPED"); + } + + if (cache2.getCacheStatus() == CacheStatus.STARTED) + { + log.info("--------- CACHE 2 (" + cache2.getLocalAddress() + ") ---------"); + log.info(CachePrinter.printCacheDetails(cache2) + "\n\n"); + } + else + { + log.info("--------- CACHE 2 STOPPED"); + } + } +}